Merge main and recreate mint URL migration

This commit is contained in:
9qeklajc
2026-07-26 00:16:10 +02:00
52 changed files with 4609 additions and 569 deletions
+14 -2
View File
@@ -5,7 +5,20 @@ UPSTREAM_API_KEY=your-upstream-api-key
# Tinfoil (confidential inference enclaves, EHBP)
# TINFOIL_API_KEY=your-tinfoil-api-key
# ADMIN_PASSWORD=secure-admin-password
# Secret key used to encrypt node secrets at rest (optional). If unset, the node
# generates one on first start, writes it to routstr_secret.key (override the path
# with ROUTSTR_SECRET_KEY_FILE), and prints it once — back that file up, because
# losing the key makes previously encrypted secrets unreadable. Set it explicitly
# to manage the key yourself (recommended in production). Generate one with:
# uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
ROUTSTR_SECRET_KEY=
# The admin password and the Nostr identity (nsec) are NOT set here. The admin
# password is generated and logged once on first start (read it from the logs to
# sign in); both are managed afterwards from the admin UI and stored encrypted in
# the database. ADMIN_PASSWORD / NSEC are still read once as a legacy seed for
# existing deployments, but new nodes should set them in the UI — a value left in
# .env is ignored once the node has been configured.
# Database
# DATABASE_URL=sqlite+aiosqlite:///keys.db
@@ -13,7 +26,6 @@ UPSTREAM_API_KEY=your-upstream-api-key
# Node Information
# NAME=My Routstr Node
# DESCRIPTION=Fast AI API access with Bitcoin payments
# NSEC=nsec1...
# HTTP_URL=https://api.mynode.com
# ONION_URL=http://mynode.onion (auto fetched from compose)
# RELAYS="wss://relay.damus.io,wss://relay.nostr.band,wss://eden.nostr.land,wss://relay.routstr.com"
+1
View File
@@ -1,6 +1,7 @@
__pycache__
.env
keys.db
routstr_secret.key
wallet.sqlite3
# Python build artifacts
+25 -3
View File
@@ -55,19 +55,41 @@ If you are a node runner, start a Routstr Core instance using Docker Compose:
1. **Prepare your `.env`**:
```bash
ADMIN_PASSWORD=mysecretpassword
# Optional: encrypts node secrets at rest. If unset, the node generates a key
# on first start, writes it to routstr_secret.key, and prints it once — back
# up that file. Set it explicitly to manage the key yourself (recommended in
# production).
ROUTSTR_SECRET_KEY=<generated-key>
NAME="My AI Node"
DESCRIPTION="Fast access to models"
NSEC=yournsec
RECEIVE_LN_ADDRESS=yourname@wallet.com
```
Your Nostr identity (`nsec`) is not set in `.env` — configure it from the admin
UI after first start, where it's stored encrypted in the database. (`NSEC` in
`.env` is still read once as a legacy seed for existing deployments.)
If you don't set one, a key is generated and printed on first start — save it
somewhere safe (losing it makes previously encrypted secrets unreadable). To
supply your own, generate it once and keep it stable:
```bash
uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
```
2. **Start the services**:
```bash
docker compose up -d
```
3. **Configure**:
3. **Get your admin password**:
On first start the node generates an admin password and logs it once with the
`/admin` URL. Read it from the logs:
```bash
docker compose logs routstr | grep -i admin
```
(Lost it? Reset with `docker compose exec routstr /.venv/bin/python scripts/reset_admin_password.py --regenerate`.)
4. **Configure**:
Open [http://localhost:8000/admin/](http://localhost:8000/admin/) to connect your AI providers and set pricing.
For full instructions, see the **[Provider Quick Start Guide](https://docs.routstr.com/provider/quickstart/)**.
+25 -6
View File
@@ -13,7 +13,10 @@ Before running your node, you should create a `.env` file in the project root. T
### Example .env
```bash
ADMIN_PASSWORD=your-secure-password
# Encrypts node secrets at rest. Optional — if unset, the node generates a key on
# first start and prints it once (back it up). Set it to manage the key yourself
# (recommended in production). See "Secrets at Rest" below.
ROUTSTR_SECRET_KEY=
# Node Identity
NAME="My AI Node"
@@ -25,10 +28,10 @@ RECEIVE_LN_ADDRESS=yourname@wallet.com
### Setting the UI Password
There are two ways to set or change your Admin Dashboard password:
On first start the node generates an admin password and logs it once — read it from the container logs to sign in. You can then change it two ways:
1. **Via Environment Variable**: Set `ADMIN_PASSWORD` in your `.env` file before starting the container. This will be the password used for the first login.
2. **Via Dashboard**: Once logged in, go to **Settings****Security** to update your password. Dashboard settings override the `.env` file once saved.
1. **Via Dashboard**: Once logged in, go to **Settings****Security** to update your password.
2. **Via Environment Variable (legacy seed)**: Setting `ADMIN_PASSWORD` in `.env` before the first start seeds the initial password instead of generating one. It's read only once, for existing deployments; a value left in `.env` is ignored after the node has been configured.
---
@@ -123,12 +126,14 @@ Use environment variables for:
| -------------------- | --------------------------------- | ------------------------------------ |
| `UPSTREAM_BASE_URL` | Upstream API endpoint | — |
| `UPSTREAM_API_KEY` | Upstream API key | — |
| `ADMIN_PASSWORD` | Dashboard password | (none) |
| `ADMIN_PASSWORD` | Legacy seed for the dashboard password (otherwise generated + logged on first start) | (auto-generated) |
| `ROUTSTR_SECRET_KEY` | Master key encrypting node secrets at rest. Auto-generated to a key file if unset | (auto-generated) |
| `ROUTSTR_SECRET_KEY_FILE` | Path to the generated key file (used when `ROUTSTR_SECRET_KEY` is unset) | `routstr_secret.key` beside the database |
| `DATABASE_URL` | Database connection string | `sqlite+aiosqlite:///keys.db` |
| `NAME` | Node display name | `ARoutstrNode` |
| `DESCRIPTION` | Node description | `A Routstr Node` |
| `NPUB` | Nostr public key (bech32) | — |
| `NSEC` | Nostr private key | — |
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
@@ -142,6 +147,20 @@ Use environment variables for:
Environment variables are read on startup. Dashboard settings override them and persist in the database. Once you change a setting in the dashboard, the env var is ignored for that setting.
### Secrets at Rest
The node's Nostr private key (`nsec`) is encrypted in the database using
`ROUTSTR_SECRET_KEY`. You don't have to set it: if it's unset, the node generates a
key on first start, writes it **beside the database** (the file named by
`ROUTSTR_SECRET_KEY_FILE`, default `routstr_secret.key`) so it persists on the same
volume as your data, and prints it once.
**Back up that key** — it lives on the same volume as your database, so include it
in your backups. If it is lost or changed, previously encrypted secrets can't be
decrypted and must be re-entered — there is no rotation. To keep the key off the
data volume, set `ROUTSTR_SECRET_KEY` explicitly (an env value always takes
precedence over the file). See also [Deployment](deployment.md).
---
## Models
+24 -6
View File
@@ -38,7 +38,6 @@ services:
- routstr-data:/app/data
environment:
DATABASE_URL: "sqlite:////app/data/routstr.db"
ADMIN_KEY: "your-secure-admin-key"
LOG_LEVEL: "info"
volumes:
@@ -87,6 +86,8 @@ services:
- ./logs:/app/logs
environment:
- TOR_PROXY_URL=socks5://tor:9050
# Keep the database (and the key file generated beside it) on the volume.
- DATABASE_URL=sqlite:////app/data/routstr.db
depends_on:
- tor
@@ -125,8 +126,8 @@ services:
- UPSTREAM_BASE_URL=https://api.openai.com/v1
- UPSTREAM_API_KEY=sk-proj-...
# Secure the dashboard (recommended)
- ADMIN_PASSWORD=your-secure-password
# The admin password is generated and logged once on first start; set
# ADMIN_PASSWORD here only as a legacy seed for an existing deployment.
# Node identity
- NAME=My Provider Node
@@ -134,6 +135,9 @@ services:
# Lightning withdrawals
- RECEIVE_LN_ADDRESS=me@walletofsatoshi.com
# Keep the database (and the key file generated beside it) on the volume.
- DATABASE_URL=sqlite:////app/data/routstr.db
volumes:
- ./data:/app/data
```
@@ -155,22 +159,36 @@ Example `.env`:
```bash
UPSTREAM_BASE_URL=https://api.openai.com/v1
UPSTREAM_API_KEY=sk-proj-...
ADMIN_PASSWORD=change-me
# Keep the database (and the key file generated beside it) on the mounted volume.
DATABASE_URL=sqlite:////app/data/routstr.db
# Encrypts node secrets at rest. Optional — if unset, a key is generated next to
# your database (on the same volume) and its file is named once for backup. Set
# it explicitly to manage the key yourself.
ROUTSTR_SECRET_KEY=
NAME=My Provider Node
RECEIVE_LN_ADDRESS=me@walletofsatoshi.com
```
!!! note "Secret key persistence"
If you leave `ROUTSTR_SECRET_KEY` unset, the node generates one and stores it
as `routstr_secret.key` **next to your database**, so it persists on the same
volume as your data — just include that volume in your backups. For stronger
isolation (keeping the key off the data volume), set `ROUTSTR_SECRET_KEY` from
a secrets manager instead.
See [Configuration](configuration.md) for all available options.
---
## Persistence
Routstr stores all data in `/app/data`:
Point `DATABASE_URL` inside `/app/data` (as the examples above do) so everything
Routstr persists lands on the mounted volume:
| Path | Contents |
|------|----------|
| `keys.db` | SQLite database (settings, API keys, sessions) |
| `routstr.db` | SQLite database (settings, API keys, sessions) |
| `routstr_secret.key` | Auto-generated master key, written beside the database when `ROUTSTR_SECRET_KEY` is unset |
| `.wallet/` | Cashu wallet data (your Bitcoin!) |
!!! warning "Back Up Your Data"
+10 -4
View File
@@ -29,19 +29,25 @@ In future versions, you'll be able to run a node that connects to other Routstr
Create a `.env` file in the root of the project to store your secrets:
```bash
# Initial Admin Password
ADMIN_PASSWORD=mysecretpassword
# Encrypts node secrets at rest. Optional — if unset, the node generates a key on
# first start and prints it once (back it up).
ROUTSTR_SECRET_KEY=
# Node Identity
NAME="My AI Node"
DESCRIPTION="Fast access to models"
NSEC=yournsec
# Lightning Payouts
RECEIVE_LN_ADDRESS=yourname@wallet.com
```
The admin password is generated and logged once on first start (read it from the
logs to sign in), and your Nostr identity (`nsec`) is configured afterwards from
the admin UI — both are stored encrypted in the database, not in `.env`.
(`ADMIN_PASSWORD` / `NSEC` are still read once as a legacy seed for existing
deployments.)
## 2. Start the Node
The recommended way to run Routstr is using Docker Compose, which handles the node, the UI, and optional services like Tor.
@@ -72,7 +78,7 @@ docker compose up -d
Open the **Admin Dashboard** at [http://localhost:8000/admin/](http://localhost:8000/admin/).
!!! note "Login"
Use the `ADMIN_PASSWORD` you defined in your `.env` file to log in. If you didn't set one, the dashboard will prompt you to set one on first visit.
On first start the node generates an admin password and logs it once — read it from the container logs to sign in. You can change it afterwards from **Settings****Security**.
### Connect Your AI Providers
@@ -0,0 +1,62 @@
"""add reservation release idempotency records
Revision ID: 7f2843d3f4e4
Revises: fc4fa29630d2
Create Date: 2026-07-24 02:06:06.066726
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "7f2843d3f4e4"
down_revision = "fc4fa29630d2"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"reservation_releases",
sa.Column("id", sa.String(), nullable=False),
sa.Column("key_hash", sa.String(), nullable=False),
sa.Column("billing_key_hash", sa.String(), nullable=False),
sa.Column("reserved_msats", sa.Integer(), nullable=False),
sa.Column(
"status", sa.String(), nullable=False, server_default="active"
),
sa.Column("created_at", sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_reservation_releases_key_hash",
"reservation_releases",
["key_hash"],
)
op.create_index(
"ix_reservation_releases_billing_key_hash",
"reservation_releases",
["billing_key_hash"],
)
op.create_index(
"ix_reservation_releases_status_created_at",
"reservation_releases",
["status", "created_at"],
)
def downgrade() -> None:
op.drop_index(
"ix_reservation_releases_status_created_at",
table_name="reservation_releases",
)
op.drop_index(
"ix_reservation_releases_billing_key_hash",
table_name="reservation_releases",
)
op.drop_index(
"ix_reservation_releases_key_hash",
table_name="reservation_releases",
)
op.drop_table("reservation_releases")
@@ -0,0 +1,47 @@
"""repair missing fee payout checkpoint columns
Revision ID: 9c4d8e2f1a6b
Revises: 7f2843d3f4e4
Create Date: 2026-07-25 00:00:00.000000
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "9c4d8e2f1a6b"
down_revision = "7f2843d3f4e4"
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Repair databases stamped past the original checkpoint migration."""
conn = op.get_bind()
columns = {
column["name"] for column in sa.inspect(conn).get_columns("routstr_fees")
}
if "payout_in_progress_msats" not in columns:
op.add_column(
"routstr_fees",
sa.Column(
"payout_in_progress_msats",
sa.Integer(),
nullable=False,
server_default="0",
),
)
if "payout_started_at" not in columns:
op.add_column(
"routstr_fees",
sa.Column("payout_started_at", sa.Integer(), nullable=True),
)
def downgrade() -> None:
# The preceding revision already expects both columns. This migration only
# repairs schema drift, so downgrading it must preserve the expected schema.
pass
@@ -1,16 +1,16 @@
"""add mint url to lightning invoices
Revision ID: 11eaab843b49
Revises: d7e8f9a0b1c2
Create Date: 2026-07-22 22:25:45.278261
Revision ID: c7d5f8638599
Revises: 9c4d8e2f1a6b
Create Date: 2026-07-26 00:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "11eaab843b49"
down_revision = "d7e8f9a0b1c2"
revision = "c7d5f8638599"
down_revision = "9c4d8e2f1a6b"
branch_labels = None
depends_on = None
@@ -0,0 +1,51 @@
"""add secrets table
Revision ID: fc4fa29630d2
Revises: d7e8f9a0b1c2
Create Date: 2026-07-23 00:00:00.000000
Creates the node-level singleton secret store (issue #553). Schema only; moving
any legacy plaintext into the encrypted/hashed columns happens at bootstrap,
where the live ROUTSTR_SECRET_KEY is available. ``nsec_state`` records the vault's
ownership of the nsec (legacy | encrypted | cleared), so a cleared identity is
never resurrected from a stale legacy ``NSEC`` env var / settings blob on the next
boot.
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
revision = "fc4fa29630d2"
down_revision = "d7e8f9a0b1c2"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"secrets",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column(
"admin_password_hash",
sqlmodel.sql.sqltypes.AutoString(),
nullable=True,
),
sa.Column(
"encrypted_nsec",
sqlmodel.sql.sqltypes.AutoString(),
nullable=True,
),
sa.Column(
"nsec_state",
sqlmodel.sql.sqltypes.AutoString(),
nullable=False,
server_default="legacy",
),
sa.Column("updated_at", sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
def downgrade() -> None:
op.drop_table("secrets")
+371 -154
View File
@@ -3,16 +3,25 @@ import hashlib
import math
import random
import time
import uuid
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING, Optional
from fastapi import HTTPException
from sqlalchemy import case
from sqlalchemy import case, inspect
from sqlalchemy.exc import IntegrityError
from sqlmodel import col, select, update
from .core import get_logger
from .core.db import ApiKey, AsyncSession, accumulate_routstr_fee
from .core.db import (
ApiKey,
AsyncSession,
ReservationRelease,
accumulate_routstr_fee,
create_session,
)
from .core.settings import settings
from .payment.cost_calculation import (
CostData,
@@ -34,7 +43,9 @@ payments_logger = get_logger("routstr.payments")
# Routstr platform fee constants
ROUTSTR_FEE_PERCENT: float = 2.1
ROUTSTR_LN_ADDRESS: str = "npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
ROUTSTR_LN_ADDRESS: str = (
"npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
)
ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
@@ -57,6 +68,25 @@ def _model_balance_error(required: int, available: int) -> dict[str, dict[str, s
}
@dataclass(frozen=True)
class ReservationSnapshot:
release_id: str
key_hash: str
billing_key_hash: str
reserved_msats: int
_current_reservation: ContextVar[ReservationSnapshot | None] = ContextVar(
"current_billing_reservation", default=None
)
def _clear_current_reservation(snapshot: ReservationSnapshot) -> None:
current = _current_reservation.get()
if current is not None and current.release_id == snapshot.release_id:
_current_reservation.set(None)
# TODO: implement prepaid api key (not like it was before)
# PREPAID_API_KEY = os.environ.get("PREPAID_API_KEY", None)
# PREPAID_BALANCE = int(os.environ.get("PREPAID_BALANCE", "0")) * 1000 # Convert to msats
@@ -318,7 +348,9 @@ async def validate_bearer_key(
if min_cost > 0 and existing_key.total_balance < min_cost:
raise HTTPException(
status_code=402,
detail=_model_balance_error(min_cost, existing_key.total_balance),
detail=_model_balance_error(
min_cost, existing_key.total_balance
),
)
return existing_key
@@ -602,6 +634,16 @@ async def pay_for_request(
},
)
# Create the durable reservation identity before changing aggregate balances.
# The row and balance updates commit together, so every reserved amount has one
# owner that can reach exactly one terminal state.
reservation = ReservationSnapshot(
release_id=uuid.uuid4().hex,
key_hash=key.hashed_key,
billing_key_hash=billing_key.hashed_key,
reserved_msats=cost_per_request,
)
# Charge the base cost for the request atomically to avoid race conditions
reserved_at_now = int(time.time())
stmt = (
@@ -663,22 +705,83 @@ async def pay_for_request(
child_result = await session.exec(child_stmt) # type: ignore[call-overload]
if child_result.rowcount == 0:
# Build the error before rollback expires ORM attributes.
limit_message = (
f"Balance limit exceeded: {key.balance_limit} mSats limit. "
f"{key.total_spent} already spent ({key.reserved_balance} reserved), "
f"{cost_per_request} required for this request."
)
# The parent reservation update already ran in this transaction.
# Roll it back before failover code attempts to restore the previous
# reservation; otherwise that later commit can persist both updates.
await session.rollback()
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Balance limit exceeded: {key.balance_limit} mSats limit. {key.total_spent} already spent ({key.reserved_balance} reserved), {cost_per_request} required for this request.",
"message": limit_message,
"type": "insufficient_quota",
"code": "balance_limit_exceeded",
}
},
)
await session.commit()
session.add(
ReservationRelease(
id=reservation.release_id,
key_hash=reservation.key_hash,
billing_key_hash=reservation.billing_key_hash,
reserved_msats=reservation.reserved_msats,
status="active",
)
)
# Publish the identity before commit. If the commit succeeds but its
# acknowledgement is interrupted, exact cleanup can still recover the
# durable row. A definitely failed commit is harmless because every
# terminal transition validates that row before touching balances.
_current_reservation.set(reservation)
try:
await session.commit()
except BaseException:
# The database may have committed even if acknowledgement was cancelled
# or the connection failed. Reconcile using a fresh transaction and the
# exact durable identity; no upstream request has started yet.
try:
await session.rollback()
except Exception:
pass
try:
async with create_session() as cleanup_session:
record = await cleanup_session.get(
ReservationRelease, reservation.release_id
)
if record is not None and record.status == "active":
await _transition_reservation_to_released(
reservation,
cleanup_session,
decrement_requests=True,
idempotent_success=True,
)
except Exception:
logger.exception(
"Failed to reconcile ambiguous reservation commit",
extra={"reservation_id": reservation.release_id},
)
finally:
_clear_current_reservation(reservation)
raise
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
try:
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
except Exception:
# The reservation transaction is already committed and durable. Logging
# refresh failures must not make the caller treat it as unreserved.
logger.exception(
"Reservation committed but post-commit refresh failed",
extra={"reservation_id": reservation.release_id},
)
logger.info(
"Payment processed successfully",
@@ -708,81 +811,185 @@ async def pay_for_request(
async def revert_pay_for_request(
key: ApiKey, session: AsyncSession, cost_per_request: int
key: ApiKey,
session: AsyncSession,
cost_per_request: int,
reservation_snapshot: ReservationSnapshot | None = None,
) -> bool:
"""Revert a previously reserved payment. Returns True if revert succeeded,
False if the reservation was already released (prevents negative reserved_balance)."""
billing_key = await get_billing_key(key, session)
# Keep reserved_at while other reservations remain
cleared_reserved_at = case(
(col(ApiKey.reserved_balance) - cost_per_request > 0, col(ApiKey.reserved_at)),
else_=None,
"""Revert the current request's durable reservation exactly once."""
snapshot = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(key, snapshot, session, require_active=False)
if cost_per_request != snapshot.reserved_msats:
return False
return await _transition_reservation_to_released(
snapshot,
session,
decrement_requests=True,
idempotent_success=False,
)
stmt = (
async def _validate_reservation_snapshot(
key: ApiKey,
snapshot: ReservationSnapshot,
session: AsyncSession,
*,
require_active: bool = True,
) -> None:
"""Reject cross-request or forged reservation handles before any mutation."""
state = inspect(key)
identity = state.identity if state is not None else None
key_hash = str(identity[0]) if identity else key.__dict__.get("hashed_key")
if snapshot.key_hash != key_hash:
raise RuntimeError("Billing reservation does not belong to this key")
persisted_key = await session.get(ApiKey, snapshot.key_hash)
if persisted_key is None:
raise RuntimeError("Billing reservation key no longer exists")
expected_billing_hash = persisted_key.parent_key_hash or persisted_key.hashed_key
if snapshot.billing_key_hash != expected_billing_hash:
raise RuntimeError("Billing reservation does not belong to this billing key")
record = await session.get(ReservationRelease, snapshot.release_id)
if (
record is None
or (require_active and record.status != "active")
or record.key_hash != snapshot.key_hash
or record.billing_key_hash != snapshot.billing_key_hash
or record.reserved_msats != snapshot.reserved_msats
):
raise RuntimeError("Billing reservation record does not match the request")
async def get_reservation_snapshot(
key: ApiKey, session: AsyncSession
) -> ReservationSnapshot:
"""Return the durable reservation created for the current request."""
snapshot = _current_reservation.get()
if snapshot is None:
raise RuntimeError("No billing reservation is associated with this request")
await _validate_reservation_snapshot(key, snapshot, session)
return snapshot
async def _transition_reservation_to_released(
snapshot: ReservationSnapshot,
session: AsyncSession,
*,
decrement_requests: bool,
idempotent_success: bool,
) -> bool:
transition = (
update(ReservationRelease)
.where(col(ReservationRelease.id) == snapshot.release_id)
.where(col(ReservationRelease.status) == "active")
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
.values(status="released")
)
transition_result = await session.exec(transition) # type: ignore[call-overload]
if transition_result.rowcount != 1:
await session.rollback()
existing = await session.get(ReservationRelease, snapshot.release_id)
return bool(
idempotent_success
and existing is not None
and existing.status == "released"
and existing.key_hash == snapshot.key_hash
and existing.billing_key_hash == snapshot.billing_key_hash
and existing.reserved_msats == snapshot.reserved_msats
)
values: dict[str, object] = {
"reserved_balance": col(ApiKey.reserved_balance) - snapshot.reserved_msats,
"reserved_at": case(
(
col(ApiKey.reserved_balance) - snapshot.reserved_msats > 0,
col(ApiKey.reserved_at),
),
else_=None,
),
}
if decrement_requests:
values["total_requests"] = col(ApiKey.total_requests) - 1
release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.where(col(ApiKey.reserved_balance) >= cost_per_request)
.values(
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
reserved_at=cleared_reserved_at,
total_requests=col(ApiKey.total_requests) - 1,
)
.where(col(ApiKey.hashed_key) == snapshot.billing_key_hash)
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
.values(**values)
)
result = await session.exec(release_stmt) # type: ignore[call-overload]
if result.rowcount != 1:
await session.rollback()
return False
result = await session.exec(stmt) # type: ignore[call-overload]
# Also decrement total_requests and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_stmt = (
if snapshot.billing_key_hash != snapshot.key_hash:
child_release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.where(col(ApiKey.reserved_balance) >= cost_per_request)
.values(
total_requests=col(ApiKey.total_requests) - 1,
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
reserved_at=cleared_reserved_at,
)
.where(col(ApiKey.hashed_key) == snapshot.key_hash)
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
.values(**values)
)
await session.exec(child_stmt) # type: ignore[call-overload]
child_result = await session.exec( # type: ignore[call-overload]
child_release_stmt
)
if child_result.rowcount != 1:
await session.rollback()
return False
await session.commit()
if result.rowcount == 0:
logger.warning(
"Revert skipped - reservation already released (no-op to prevent negative reserved_balance)",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"cost_to_revert": cost_per_request,
"current_reserved_balance": billing_key.reserved_balance,
},
)
return False
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
payments_logger.info(
"REVERT",
extra={
"event": "revert",
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"cost_reverted": cost_per_request,
"balance": billing_key.balance,
"reserved_balance": billing_key.reserved_balance,
},
)
_clear_current_reservation(snapshot)
return True
async def release_reservation(
snapshot: ReservationSnapshot,
session: AsyncSession,
reserved_msats: int,
) -> bool:
"""Release one durable reservation exactly once without charging."""
if reserved_msats <= 0 or reserved_msats != snapshot.reserved_msats:
return False
return await _transition_reservation_to_released(
snapshot,
session,
decrement_requests=False,
idempotent_success=True,
)
async def _claim_reservation_for_charge(
snapshot: ReservationSnapshot, session: AsyncSession
) -> bool:
"""Claim an active reservation in the caller's charge transaction."""
statement = (
update(ReservationRelease)
.where(col(ReservationRelease.id) == snapshot.release_id)
.where(col(ReservationRelease.status) == "active")
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
.values(status="charged")
)
result = await session.exec(statement) # type: ignore[call-overload]
if result.rowcount == 1:
_clear_current_reservation(snapshot)
return True
await session.rollback()
return False
async def adjust_payment_for_tokens(
key: ApiKey,
response_data: dict,
session: AsyncSession,
deducted_max_cost: int,
model_obj: "Model | None",
provider_fee: float | None,
model_obj: "Model | None" = None,
provider_fee: float | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> dict:
"""
Adjusts the payment based on token usage in the response.
@@ -797,6 +1004,13 @@ async def adjust_payment_for_tokens(
``calculate_cost``.
"""
billing_key = await get_billing_key(key, session)
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(
key, reservation, session, require_active=False
)
# The persisted amount is authoritative if request-level minimum pricing
# changed the caller's original estimate.
deducted_max_cost = reservation.reserved_msats
model = response_data.get("model", "unknown")
logger.debug(
@@ -812,50 +1026,21 @@ async def adjust_payment_for_tokens(
)
async def release_reservation_only() -> None:
"""Fallback to release reservation without charging when main update fails."""
"""Fallback to release this request's reservation without charging."""
try:
release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
.values(
reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost
)
released = await release_reservation(
reservation, session, reservation.reserved_msats
)
logger.warning(
"Released reservation without charging (fallback)"
if released
else "Reservation was already finalized; fallback skipped",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"deducted_max_cost": deducted_max_cost,
},
)
result = await session.exec(release_stmt) # type: ignore[call-overload]
# Also release on child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
.values(
reserved_balance=col(ApiKey.reserved_balance)
- deducted_max_cost
)
)
await session.exec(child_release_stmt) # type: ignore[call-overload]
await session.commit()
if result.rowcount == 0: # type: ignore[union-attr]
logger.warning(
"Release reservation skipped - already released (no-op to prevent negative reserved_balance)",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"deducted_max_cost": deducted_max_cost,
},
)
else:
logger.warning(
"Released reservation without charging (fallback)",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"deducted_max_cost": deducted_max_cost,
},
)
except Exception as e:
logger.error(
"Failed to release reservation in fallback",
@@ -877,9 +1062,17 @@ async def adjust_payment_for_tokens(
extra={"error": str(e), "fee_msats": fee_msats},
)
match await calculate_cost(
calculated_cost = await calculate_cost(
response_data, deducted_max_cost, model_obj, provider_fee
):
)
if not isinstance(calculated_cost, CostDataError):
if not await _claim_reservation_for_charge(reservation, session):
# A prior charge or release already owns this reservation. Returning
# the calculated metadata is safe; the aggregate balances must not
# be modified a second time.
return calculated_cost.dict()
match calculated_cost:
case MaxCostData() as cost:
logger.debug(
"Using max cost data (no token adjustment)",
@@ -907,8 +1100,10 @@ async def adjust_payment_for_tokens(
)
safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
@@ -926,8 +1121,10 @@ async def adjust_payment_for_tokens(
# Also update total_spent and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
child_stmt = (
@@ -1037,8 +1234,10 @@ async def adjust_payment_for_tokens(
)
exact_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
@@ -1056,8 +1255,10 @@ async def adjust_payment_for_tokens(
# Also update total_spent and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_exact_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
child_stmt = (
@@ -1096,31 +1297,45 @@ async def adjust_payment_for_tokens(
# actual cost exceeded discounted reservation (due to tolerance_percentage)
if cost_difference > 0:
# Always release the reservation and charge min(actual_cost, balance).
# CASE expressions keep this atomic and safe even when the
# stale-reservation sweeper has already released the reservation.
chargeable = case(
(col(ApiKey.balance) >= total_cost_msats, total_cost_msats),
else_=col(ApiKey.balance),
)
overrun_safe_reserved = case(
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
finalize_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.values(
reserved_balance=overrun_safe_reserved,
balance=col(ApiKey.balance) - chargeable,
total_spent=col(ApiKey.total_spent) + chargeable,
# Lock the billing row so the parent and child record the same
# database-determined charge under concurrent finalizations.
actual_charge_msats = 0
for attempt in range(5):
locked_billing_key = (
await session.exec(
select(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.with_for_update()
.execution_options(populate_existing=True)
)
).one()
observed_balance = locked_billing_key.balance
actual_charge_msats = min(observed_balance, total_cost_msats)
overrun_safe_reserved = case(
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
)
await session.exec(finalize_stmt) # type: ignore[call-overload]
finalize_result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.where(col(ApiKey.balance) == observed_balance)
.values(
reserved_balance=overrun_safe_reserved,
balance=col(ApiKey.balance) - actual_charge_msats,
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
)
)
if finalize_result.rowcount == 1:
break
await session.rollback()
if not await _claim_reservation_for_charge(reservation, session):
return cost.dict()
else:
await session.rollback()
raise RuntimeError("Could not atomically finalize cost overrun")
if billing_key.hashed_key != key.hashed_key:
child_stmt = (
@@ -1128,7 +1343,7 @@ async def adjust_payment_for_tokens(
.where(col(ApiKey.hashed_key) == key.hashed_key)
.values(
reserved_balance=overrun_safe_reserved,
total_spent=col(ApiKey.total_spent) + min(billing_key.balance, total_cost_msats),
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
)
)
await session.exec(child_stmt) # type: ignore[call-overload]
@@ -1138,18 +1353,18 @@ async def adjust_payment_for_tokens(
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
cost.total_msats = total_cost_msats
cost.total_msats = actual_charge_msats
logger.info(
"Finalized payment with additional charge",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"charged_amount": total_cost_msats,
"charged_amount": actual_charge_msats,
"new_balance": billing_key.balance,
"model": model,
},
)
await _accumulate_fee(total_cost_msats)
await _accumulate_fee(actual_charge_msats)
payments_logger.info(
"FINALIZE",
extra={
@@ -1158,7 +1373,7 @@ async def adjust_payment_for_tokens(
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"model": model,
"cost_reserved": deducted_max_cost,
"cost_charged": total_cost_msats,
"cost_charged": actual_charge_msats,
"input_tokens": cost.input_tokens,
"output_tokens": cost.output_tokens,
"balance": billing_key.balance,
@@ -1198,8 +1413,10 @@ async def adjust_payment_for_tokens(
)
refund_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
@@ -1217,8 +1434,10 @@ async def adjust_payment_for_tokens(
# Also update total_spent and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_refund_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
child_stmt = (
@@ -1393,9 +1612,7 @@ async def periodic_dead_key_prune() -> None:
try:
async with create_session() as session:
await prune_dead_api_keys(
session, settings.dead_key_min_age_seconds
)
await prune_dead_api_keys(session, settings.dead_key_min_age_seconds)
except asyncio.CancelledError:
break
except Exception as e:
+53 -26
View File
@@ -7,7 +7,7 @@ from typing import Annotated, NoReturn
from fastapi import APIRouter, Depends, Header, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from sqlmodel import col, or_, select, update
from sqlmodel import col, select, update
from .auth import get_billing_key, validate_bearer_key
from .core.db import (
@@ -15,6 +15,7 @@ from .core.db import (
AsyncSession,
CashuTransaction,
get_session,
release_stale_reservations,
)
from .core.db import (
store_cashu_transaction_with_retry as store_cashu_transaction,
@@ -109,13 +110,19 @@ async def account_info(
# Note: validate_bearer_key already supports refund_address and key_expiry_time params
@router.get("/create")
async def create_balance(
class BalanceCreateRequest(BaseModel):
initial_balance_token: str
balance_limit: int | None = None
balance_limit_reset: str | None = None
validity_date: int | None = None
async def _create_balance(
initial_balance_token: str,
balance_limit: int | None = None,
balance_limit_reset: str | None = None,
validity_date: int | None = None,
session: AsyncSession = Depends(get_session),
balance_limit: int | None,
balance_limit_reset: str | None,
validity_date: int | None,
session: AsyncSession,
) -> dict:
key = await validate_bearer_key(initial_balance_token, session)
@@ -135,6 +142,37 @@ async def create_balance(
}
@router.post("/create")
async def create_balance_from_body(
payload: BalanceCreateRequest,
session: AsyncSession = Depends(get_session),
) -> dict:
return await _create_balance(
payload.initial_balance_token,
payload.balance_limit,
payload.balance_limit_reset,
payload.validity_date,
session,
)
@router.get("/create")
async def create_balance(
initial_balance_token: str,
balance_limit: int | None = None,
balance_limit_reset: str | None = None,
validity_date: int | None = None,
session: AsyncSession = Depends(get_session),
) -> dict:
return await _create_balance(
initial_balance_token,
balance_limit,
balance_limit_reset,
validity_date,
session,
)
@router.get("/info")
async def wallet_info(
key: ApiKey = Depends(get_key_from_header),
@@ -384,30 +422,19 @@ async def refund_wallet_endpoint(
)
if key.reserved_balance > 0:
# Release the reservation if it is stale
cutoff = int(time.time()) - settings.stale_reservation_timeout_seconds
stale_release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.where(col(ApiKey.reserved_balance) > 0)
.where(
or_(
col(ApiKey.reserved_at).is_(None),
col(ApiKey.reserved_at) < cutoff,
)
)
.values(reserved_balance=0, reserved_at=None)
# Release only durable reservations old enough to be stale. A newer
# request on the same aggregate balance must remain reserved.
await release_stale_reservations(
session,
settings.stale_reservation_timeout_seconds,
key_hash=key.hashed_key,
)
stale_result = await session.exec(stale_release_stmt) # type: ignore[call-overload]
await session.commit()
if stale_result.rowcount == 0:
await session.refresh(key)
if key.reserved_balance > 0:
raise HTTPException(
status_code=400,
detail="Cannot refund key. There are ongoing requests for this api key.",
)
await session.refresh(key)
logger.warning(
"refund_wallet_endpoint: released stale reservation before refund",
extra={
+64 -41
View File
@@ -20,6 +20,7 @@ from ..wallet import (
send_token,
slow_filter_spend_proofs,
)
from . import vault
from .db import (
ApiKey,
CashuTransaction,
@@ -28,6 +29,9 @@ from .db import (
ModelRow,
UpstreamProviderRow,
create_session,
get_secret,
set_admin_password,
set_nsec,
)
from .db import (
store_cashu_transaction_with_retry as store_cashu_transaction,
@@ -35,7 +39,7 @@ from .db import (
from .log_manager import log_manager
from .logging import get_logger
from .provider_slugs import allocate_unique_provider_slug
from .settings import SettingsService, settings
from .settings import SettingsService, derive_npub_from_nsec, settings
logger = get_logger(__name__)
@@ -206,8 +210,6 @@ async def get_settings(request: Request) -> dict:
data = settings.dict()
if "upstream_api_key" in data:
data["upstream_api_key"] = "[REDACTED]" if data["upstream_api_key"] else ""
if "admin_password" in data:
data["admin_password"] = "[REDACTED]" if data["admin_password"] else ""
if "nsec" in data:
data["nsec"] = "[REDACTED]" if data["nsec"] else ""
return data
@@ -224,9 +226,10 @@ class PasswordUpdate(BaseModel):
@admin_router.patch("/api/settings", dependencies=[Depends(require_admin_api)])
async def update_settings(request: Request, update: SettingsUpdate) -> dict:
# Remove sensitive fields from general settings update
# Secrets are not editable through the general settings endpoint; they have
# dedicated rotation paths and never reach the settings blob.
settings_data = update.root.copy()
sensitive_fields = ["admin_password", "upstream_api_key", "nsec"]
sensitive_fields = ["upstream_api_key", "nsec"]
for field in sensitive_fields:
if field in settings_data:
del settings_data[field]
@@ -241,8 +244,6 @@ async def update_settings(request: Request, update: SettingsUpdate) -> dict:
data = new_settings.dict()
if "upstream_api_key" in data:
data["upstream_api_key"] = "[REDACTED]" if data["upstream_api_key"] else ""
if "admin_password" in data:
data["admin_password"] = "[REDACTED]" if data["admin_password"] else ""
if "nsec" in data:
data["nsec"] = "[REDACTED]" if data["nsec"] else ""
return data
@@ -250,44 +251,63 @@ async def update_settings(request: Request, update: SettingsUpdate) -> dict:
@admin_router.patch("/api/password", dependencies=[Depends(require_admin_api)])
async def update_password(request: Request, password_update: PasswordUpdate) -> dict:
current_password = settings.admin_password
if not current_password:
raise HTTPException(status_code=500, detail="Admin password not configured")
if password_update.current_password != current_password:
raise HTTPException(status_code=401, detail="Current password is incorrect")
# Validate new password
new_password = password_update.new_password.strip()
if len(new_password) < 6:
raise HTTPException(
status_code=400, detail="New password must be at least 6 characters"
)
# Update password
async with create_session() as session:
await SettingsService.update({"admin_password": new_password}, session)
secret = await get_secret(session)
if not secret.admin_password_hash:
raise HTTPException(
status_code=500, detail="Admin password not configured"
)
if not vault.verify_password(
password_update.current_password, secret.admin_password_hash
):
raise HTTPException(
status_code=401, detail="Current password is incorrect"
)
# Validate new password
new_password = password_update.new_password.strip()
if len(new_password) < vault.MIN_PASSWORD_LENGTH:
raise HTTPException(
status_code=400,
detail=(
"New password must be at least "
f"{vault.MIN_PASSWORD_LENGTH} characters"
),
)
await set_admin_password(session, new_password)
return {"ok": True, "message": "Password updated successfully"}
class SetupRequest(BaseModel):
password: str
class NsecUpdate(BaseModel):
nsec: str
@admin_router.post("/api/setup")
async def initial_setup(request: Request, payload: SetupRequest) -> dict[str, object]:
if settings.admin_password:
raise HTTPException(status_code=409, detail="Admin password already set")
pw = (payload.password or "").strip()
if len(pw) < 8:
raise HTTPException(
status_code=400, detail="Password must be at least 8 characters"
)
@admin_router.patch("/api/nsec", dependencies=[Depends(require_admin_api)])
async def update_nsec(request: Request, payload: NsecUpdate) -> dict[str, object]:
# The node's Nostr identity is a secret: it is stored encrypted in the
# Secret store, never in the settings blob, so it gets its own endpoint
# rather than riding the general settings PATCH (which strips it). An empty
# nsec clears the identity.
nsec = payload.nsec.strip()
npub = ""
if nsec:
derived = derive_npub_from_nsec(nsec)
if not derived:
raise HTTPException(status_code=400, detail="Invalid nsec")
npub = derived
async with create_session() as session:
await SettingsService.update({"admin_password": pw}, session)
return {"ok": True}
await set_nsec(session, nsec)
# Reflect the change in the live runtime so Nostr signing/announcements pick
# it up without a restart (mirrors what bootstrap_secrets sets at boot).
settings.nsec = nsec
settings.npub = npub
return {"ok": True, "npub": npub}
class AdminLoginRequest(BaseModel):
@@ -298,12 +318,16 @@ class AdminLoginRequest(BaseModel):
async def admin_login(
request: Request, payload: AdminLoginRequest
) -> dict[str, object]:
admin_pw = settings.admin_password
async with create_session() as session:
secret = await get_secret(session)
# Read the hash while the session is open; the ORM object is detached
# once the context exits and its attributes can no longer be loaded.
password_hash = secret.admin_password_hash
if not admin_pw:
if not password_hash:
raise HTTPException(status_code=500, detail="Admin password not configured")
if payload.password != admin_pw:
if not vault.verify_password(payload.password, password_hash):
raise HTTPException(status_code=401, detail="Invalid password")
token = secrets.token_urlsafe(32)
@@ -482,7 +506,6 @@ class ModelCreate(BaseModel):
async def upsert_provider_model(
provider_id: str, payload: ModelCreate
) -> dict[str, object]:
print(payload)
logger.info(
f"UPSERT_PROVIDER_MODEL called: provider_id={provider_id}, model_id={payload.id}"
)
+214 -16
View File
@@ -6,12 +6,13 @@ import sqlite3
import time
import uuid
from contextlib import asynccontextmanager
from enum import Enum
from typing import AsyncGenerator
from alembic import command
from alembic.config import Config
from alembic.util.exc import CommandError
from sqlalchemy import UniqueConstraint, delete
from sqlalchemy import Index, UniqueConstraint, case, delete, or_
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.ext.asyncio.engine import create_async_engine
from sqlalchemy.orm import aliased
@@ -98,32 +99,130 @@ class ApiKey(SQLModel, table=True): # type: ignore
async def reset_all_reserved_balances(session: AsyncSession) -> None:
stmt = update(ApiKey).values(reserved_balance=0, reserved_at=None)
await session.exec(stmt) # type: ignore[call-overload]
"""Release every active durable reservation during explicit startup reset."""
await session.exec( # type: ignore[call-overload]
update(ReservationRelease)
.where(col(ReservationRelease.status) == "active")
.values(status="released")
)
await session.exec( # type: ignore[call-overload]
update(ApiKey).values(reserved_balance=0, reserved_at=None)
)
await session.commit()
logger.info("Reset reserved balances on startup")
async def release_stale_reservations(
session: AsyncSession, max_age_seconds: int
session: AsyncSession,
max_age_seconds: int,
*,
key_hash: str | None = None,
) -> int:
"""Release reservations whose last reserve is older than max_age_seconds.
"""
"""Release stale durable reservations without touching newer reservations."""
cutoff = int(time.time()) - max_age_seconds
stmt = (
update(ApiKey)
.where(col(ApiKey.reserved_balance) > 0)
.where(col(ApiKey.reserved_at).is_not(None))
.where(col(ApiKey.reserved_at) < cutoff)
.values(reserved_balance=0, reserved_at=None)
query = (
select(ReservationRelease)
.where(col(ReservationRelease.status) == "active")
.where(col(ReservationRelease.created_at) < cutoff)
)
result = await session.exec(stmt) # type: ignore[call-overload]
if key_hash is not None:
query = query.where(
or_(
col(ReservationRelease.key_hash) == key_hash,
col(ReservationRelease.billing_key_hash) == key_hash,
)
)
reservations = (await session.exec(query)).all()
released = 0
for reservation in reservations:
transition = await session.exec( # type: ignore[call-overload]
update(ReservationRelease)
.where(col(ReservationRelease.id) == reservation.id)
.where(col(ReservationRelease.status) == "active")
.values(status="released")
)
if transition.rowcount != 1:
continue
values = {
"reserved_balance": col(ApiKey.reserved_balance)
- reservation.reserved_msats,
"reserved_at": case(
(
col(ApiKey.reserved_balance) - reservation.reserved_msats > 0,
col(ApiKey.reserved_at),
),
else_=None,
),
}
parent_result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == reservation.billing_key_hash)
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
.values(**values)
)
if parent_result.rowcount != 1:
await session.rollback()
return 0
if reservation.billing_key_hash != reservation.key_hash:
child_result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == reservation.key_hash)
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
.values(**values)
)
if child_result.rowcount != 1:
await session.rollback()
return 0
released += 1
# Rolling upgrades can leave aggregate reservations created before durable
# reservation rows existed. Release only stale aggregates that have no active
# durable owner; targeted refund cleanup also heals legacy NULL timestamps.
legacy_query = select(ApiKey).where(col(ApiKey.reserved_balance) > 0)
if key_hash is None:
legacy_query = legacy_query.where(col(ApiKey.reserved_at).is_not(None)).where(
col(ApiKey.reserved_at) < cutoff
)
else:
legacy_query = legacy_query.where(
or_(
col(ApiKey.hashed_key) == key_hash,
col(ApiKey.parent_key_hash) == key_hash,
)
).where(
or_(col(ApiKey.reserved_at).is_(None), col(ApiKey.reserved_at) < cutoff)
)
for legacy_key in (await session.exec(legacy_query)).all():
active_owner = (
await session.exec(
select(ReservationRelease.id)
.where(col(ReservationRelease.status) == "active")
.where(
or_(
col(ReservationRelease.key_hash) == legacy_key.hashed_key,
col(ReservationRelease.billing_key_hash)
== legacy_key.hashed_key,
)
)
.limit(1)
)
).first()
if active_owner is not None:
continue
legacy_key.reserved_balance = 0
legacy_key.reserved_at = None
session.add(legacy_key)
released += 1
await session.commit()
released = int(result.rowcount or 0)
if released:
logger.warning(
"Released stale balance reservations",
extra={"released_keys": released, "max_age_seconds": max_age_seconds},
"Released stale reservations",
extra={"released_reservations": released, "max_age_seconds": max_age_seconds},
)
return released
@@ -436,6 +535,20 @@ class UpstreamProviderRow(SQLModel, table=True): # type: ignore
)
class ReservationRelease(SQLModel, table=True): # type: ignore
__tablename__ = "reservation_releases"
__table_args__ = (
Index("ix_reservation_releases_status_created_at", "status", "created_at"),
)
id: str = Field(primary_key=True)
key_hash: str = Field(index=True)
billing_key_hash: str = Field(index=True)
reserved_msats: int
status: str = Field(default="active")
created_at: int = Field(default_factory=lambda: int(time.time()))
class RoutstrFee(SQLModel, table=True): # type: ignore
__tablename__ = "routstr_fees"
id: int = Field(default=1, primary_key=True)
@@ -446,6 +559,42 @@ class RoutstrFee(SQLModel, table=True): # type: ignore
payout_started_at: int | None = Field(default=None)
class NsecState(str, Enum):
"""Ownership state of the node's nsec — an explicit 3-state machine.
The single ``encrypted_nsec`` column cannot distinguish "never migrated" from
"intentionally cleared" (both leave it empty), which let a cleared identity be
resurrected from a stale legacy ``NSEC``. This names the three states so the
bootstrap branches on ownership rather than inferring it:
* ``legacy`` — the vault has not taken ownership; a plaintext ``NSEC`` (env or
old settings blob) may still exist and should be migrated in once.
* ``encrypted`` — the vault owns a ciphertext; decrypt it, never re-read env.
* ``cleared`` — the vault owns it but the operator emptied it; stay empty,
never re-import from a stale legacy copy.
"""
legacy = "legacy"
encrypted = "encrypted"
cleared = "cleared"
class Secret(SQLModel, table=True): # type: ignore
"""Node-level secrets, stored encrypted/hashed at rest (singleton, id=1).
The asymmetric column names document the encoding: ``_hash`` is one-way
(scrypt, verify only) while ``encrypted_`` is reversible (Fernet). Per-provider
upstream keys live on ``upstream_providers``, not here. See ``routstr.core.vault``.
"""
__tablename__ = "secrets"
id: int = Field(default=1, primary_key=True)
admin_password_hash: str | None = Field(default=None)
encrypted_nsec: str | None = Field(default=None)
nsec_state: NsecState = Field(default=NsecState.legacy)
updated_at: int | None = Field(default=None)
class CliToken(SQLModel, table=True): # type: ignore
"""Long-lived authorization token for CLI/agent use against admin endpoints."""
@@ -484,6 +633,55 @@ async def get_routstr_fee(session: AsyncSession) -> RoutstrFee:
return fee
async def get_secret(session: AsyncSession) -> Secret:
secret = await session.get(Secret, 1)
if secret is None:
secret = Secret(id=1)
session.add(secret)
try:
await session.commit()
except IntegrityError:
# Another worker created the singleton row between our read and
# insert (multiple workers booting against one shared DB). Roll back
# and read the row they committed instead of failing startup.
await session.rollback()
secret = await session.get(Secret, 1)
if secret is None:
raise
return secret
await session.refresh(secret)
return secret
async def set_admin_password(session: AsyncSession, password: str) -> None:
"""Store the admin password as a one-way hash on the Secret singleton."""
from .vault import hash_password
secret = await get_secret(session)
secret.admin_password_hash = hash_password(password)
secret.updated_at = int(time.time())
session.add(secret)
await session.commit()
async def set_nsec(session: AsyncSession, nsec: str) -> None:
"""Store the node's nsec, Fernet-encrypted, on the Secret singleton.
An empty string clears it (the node then holds no Nostr identity and signs
no events). Either way the vault now owns the nsec, so the state moves off
``legacy``: a cleared identity (``cleared``) must not be resurrected from a
stale legacy ``NSEC`` on the next boot.
"""
from .vault import encrypt
secret = await get_secret(session)
secret.encrypted_nsec = encrypt(nsec) if nsec else None
secret.nsec_state = NsecState.encrypted if nsec else NsecState.cleared
secret.updated_at = int(time.time())
session.add(secret)
await session.commit()
async def reset_routstr_fee(session: AsyncSession, paid_msats: int) -> bool:
"""Checkpoint a fee payout before making the external payment."""
stmt = (
+9 -6
View File
@@ -37,7 +37,7 @@ from .exceptions import general_exception_handler, http_exception_handler
from .logging import get_logger, setup_logging
from .middleware import LoggingMiddleware
from .not_found import _NOT_FOUND_HTML, not_found_catch_all # noqa: F401
from .settings import SettingsService
from .settings import SettingsService, bootstrap_secrets
from .settings import settings as global_settings
from .version import __version__
@@ -85,17 +85,20 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
# Initialize application settings (env -> computed -> DB precedence)
async with create_session() as session:
# Move secrets into the encrypted/hashed store and decrypt the nsec
# into the in-memory settings BEFORE initializing settings: the
# initialize step strips secrets from the persisted blob, so legacy
# plaintext (env or old blob) must be migrated into the Secret store
# first or the only copy of a blob-only secret would be lost.
# Generates and logs an admin password on a fresh node; fails fast if
# a stored secret can't be decrypted.
await bootstrap_secrets(session)
s = await SettingsService.initialize(session)
if s.reset_reserved_balance_on_startup:
from .db import reset_all_reserved_balances
await reset_all_reserved_balances(session)
if not s.admin_password:
logger.warning(
f"Admin password is not set. Visit {s.http_url or 'http://localhost:8000'}/admin to set the password."
)
# Apply app metadata from settings
try:
app.title = s.name
+238 -26
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import asyncio
import json
import os
import secrets
import time
from datetime import datetime, timezone
from typing import Any
@@ -26,7 +28,6 @@ class Settings(BaseSettings):
# Core
upstream_base_url: str = Field(default="", env="UPSTREAM_BASE_URL")
upstream_api_key: str = Field(default="", env="UPSTREAM_API_KEY")
admin_password: str = Field(default="", env="ADMIN_PASSWORD")
# Node info
name: str = Field(default="ARoutstrNode", env="NAME")
@@ -145,10 +146,70 @@ def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
return normalized
# Secrets are credentials, not config: they live in the encrypted/hashed Secret
# store (and decrypted in-memory for runtime use), never in the persisted
# settings blob. ``admin_password`` is gone from the model entirely; ``nsec``
# remains a live field but is stripped from every blob write so it is never
# written back to plaintext. ``upstream_api_key`` is intentionally *not* here:
# it has no encrypted home yet (it is node-scoped today but really belongs on a
# provider), so stripping it would lose it on the next restart. It stays in the
# blob as before; encrypting it is follow-up work. See ``bootstrap_secrets`` and
# ``routstr.core.vault``.
SECRET_FIELDS = frozenset({"admin_password", "nsec"})
def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]:
"""Return a copy of ``data`` without any secret fields (for persistence)."""
return {k: v for k, v in data.items() if k not in SECRET_FIELDS}
def _apply_to_live_settings(data: dict[str, Any]) -> None:
"""Apply ``data`` onto the live ``settings`` for all in-process importers.
Secrets are owned exclusively by ``bootstrap_secrets``, which runs first and
has already decrypted the authoritative nsec into memory (importing any
legacy plaintext on the way). Never re-apply secret fields from env/blob
here: a non-empty but stale ``NSEC`` env var would otherwise override an nsec
the vault has taken ownership of (e.g. after the operator rotates it in the
UI), and an empty one would wipe the live value. Skip them entirely.
"""
for k, v in data.items():
if k in SECRET_FIELDS:
continue
setattr(settings, k, v)
def _compute_primary_mint(cashu_mints: list[str]) -> str:
return cashu_mints[0] if cashu_mints else "https://mint.minibits.cash/Bitcoin"
def derive_npub_from_nsec(nsec: str) -> str | None:
"""Derive the npub (bech32) from an nsec or 64-char hex private key, or None.
Parsing is delegated to :func:`routstr.nostr.listing.nsec_to_keypair`, the
single place that knows the nsec/hex formats (and already returns ``None`` on
any unusable input); this only bech32-encodes the resulting public key. The
contract stays "return None on unusable input", so a bad key never crashes
boot.
"""
try:
from nostr.key import PublicKey # type: ignore
from ..nostr.listing import nsec_to_keypair
except ImportError:
return None
keypair = nsec_to_keypair(nsec)
if keypair is None:
return None
_privkey_hex, pubkey_hex = keypair
try:
return PublicKey(bytes.fromhex(pubkey_hex)).bech32()
except (ValueError, AttributeError):
return None
def resolve_bootstrap() -> Settings:
base = Settings() # Reads env with custom parse_env_var
# Back-compat env mapping
@@ -203,23 +264,9 @@ def resolve_bootstrap() -> Settings:
pass
# Derive NPUB from NSEC if not provided
if not base.npub and base.nsec:
try:
from nostr.key import PrivateKey # type: ignore
if base.nsec.startswith("nsec"):
pk = PrivateKey.from_nsec(base.nsec)
elif len(base.nsec) == 64:
pk = PrivateKey(bytes.fromhex(base.nsec))
else:
pk = None
if pk is not None:
try:
base.npub = pk.public_key.bech32()
except Exception:
# Fallback to hex if bech32 not available
base.npub = pk.public_key.hex()
except Exception:
pass
npub = derive_npub_from_nsec(base.nsec)
if npub:
base.npub = npub
if not base.cors_origins:
base.cors_origins = ["*"]
if not base.primary_mint:
@@ -269,15 +316,14 @@ class SettingsService:
text(
"INSERT INTO settings (id, data, updated_at) VALUES (1, :data, :updated_at)"
).bindparams(
data=json.dumps(env_resolved.dict()),
data=json.dumps(_strip_secret_fields(env_resolved.dict())),
updated_at=datetime.now(timezone.utc),
)
)
await db_session.commit()
cls._current = settings
# Update the existing instance in-place for all live importers
for k, v in env_resolved.dict().items():
setattr(settings, k, v)
_apply_to_live_settings(env_resolved.dict())
return cls._current
db_id, db_data, _updated_at = row
@@ -308,20 +354,37 @@ class SettingsService:
merged_dict.get("cashu_mints", [])
)
if db_json_raw != merged_dict:
# Keep npub consistent with the live nsec. bootstrap_secrets has
# already run and holds the single authoritative nsec (decrypted from
# the encrypted store, or freshly imported). merged_dict starts from
# the env/blob, which may carry a STALE nsec — and therefore a stale
# derived npub — after the vault took ownership. Derive from the live
# value and OVERRIDE, not just fill: otherwise the node keeps the
# vault's private key but announces the old env key's npub (npub is a
# pure derivation of nsec, never configured independently of it).
if settings.nsec:
derived_npub = derive_npub_from_nsec(settings.nsec)
if derived_npub:
merged_dict["npub"] = derived_npub
# Persist without secrets; compare against the stripped target so a
# legacy blob that still carries plaintext secrets gets rewritten
# (and thereby sunset) even when its non-secret values are unchanged.
persisted = _strip_secret_fields(merged_dict)
if db_json_raw != persisted:
await db_session.exec( # type: ignore
text(
"UPDATE settings SET data = :data, updated_at = :updated_at WHERE id = 1"
).bindparams(
data=json.dumps(merged_dict),
data=json.dumps(persisted),
updated_at=datetime.now(timezone.utc),
)
)
await db_session.commit()
# Update the existing instance in-place for all live importers
for k, v in merged_dict.items():
setattr(settings, k, v)
# (keeps the decrypted nsec live in memory).
_apply_to_live_settings(merged_dict)
cls._current = settings
return cls._current
@@ -343,7 +406,7 @@ class SettingsService:
text(
"UPDATE settings SET data = :data, updated_at = :updated_at WHERE id = 1"
).bindparams(
data=json.dumps(candidate.dict()),
data=json.dumps(_strip_secret_fields(candidate.dict())),
updated_at=datetime.now(timezone.utc),
)
)
@@ -372,3 +435,152 @@ class SettingsService:
setattr(settings, k, v)
cls._current = settings
return settings
async def _read_raw_settings_blob(db_session: AsyncSession) -> dict[str, Any]:
"""Best-effort read of the raw persisted settings JSON (may not exist yet)."""
from sqlmodel import text
try:
result = await db_session.exec( # type: ignore
text("SELECT data FROM settings WHERE id = 1")
)
row = result.first()
except Exception:
return {}
if row is None:
return {}
(data_str,) = row
try:
data = json.loads(data_str) if isinstance(data_str, str) else dict(data_str)
except Exception:
return {}
return data if isinstance(data, dict) else {}
def _legacy_plaintext(
raw_blob: dict[str, Any], env_name: str, blob_key: str
) -> str | None:
"""Legacy plaintext for a secret: env first, then the old settings blob."""
env_value = os.environ.get(env_name)
if env_value:
return env_value
blob_value = raw_blob.get(blob_key)
if isinstance(blob_value, str) and blob_value:
return blob_value
return None
async def bootstrap_secrets(db_session: AsyncSession) -> None:
"""Move node secrets into the encrypted/hashed Secret store at startup.
Per secret:
* column already set -> use it (the nsec is decrypted into the in-memory
``settings``; a wrong ROUTSTR_SECRET_KEY surfaces as a clear fail-fast).
* column empty but legacy plaintext exists (env, or the old settings
blob) -> transform it (hash the password / encrypt the nsec) into the
column.
* nothing (admin password only) -> generate a strong random password,
hash it, and log it once with the /admin URL.
"""
from cryptography.fernet import InvalidToken
from sqlmodel import col, update
from . import vault
from .db import NsecState, Secret, get_secret
raw_blob = await _read_raw_settings_blob(db_session)
secret = await get_secret(db_session)
changed = False
# Admin password — one-way scrypt hash.
if secret.admin_password_hash is None:
legacy_password = _legacy_plaintext(
raw_blob, "ADMIN_PASSWORD", "admin_password"
)
if legacy_password:
secret.admin_password_hash = vault.hash_password(legacy_password)
changed = True
else:
generated = secrets.token_urlsafe(24)
# Claim the empty slot atomically: only the worker whose UPDATE flips
# NULL -> hash owns the generated password and announces it. On a
# shared DB a racing worker gets rowcount 0, so it neither clobbers
# the winner's hash (which the operator may already be using) nor
# prints a second password that would never work.
claim_stmt = (
update(Secret)
.where(col(Secret.id) == 1)
.where(col(Secret.admin_password_hash).is_(None))
.values(
admin_password_hash=vault.hash_password(generated),
updated_at=int(time.time()),
)
)
result = await db_session.exec(claim_stmt) # type: ignore[call-overload]
await db_session.commit()
await db_session.refresh(secret)
if result.rowcount == 1:
admin_url = (settings.http_url or "http://localhost:8000").rstrip("/")
# Print to stdout rather than the logger: the operator must see
# this once (e.g. `docker compose logs`), but it must not be
# persisted into the on-disk log files the logger also writes to.
print(
"No admin password set; generated a temporary one (shown "
f"only now): {generated}\nLog in at {admin_url}/admin and "
"change it from the dashboard settings.",
flush=True,
)
# Nostr nsec — reversible Fernet encryption. ``nsec_state`` is the single
# source of truth for ownership, so "intentionally cleared" is never
# conflated with "never migrated" (the bug the old bool could not encode).
if secret.nsec_state == NsecState.encrypted:
# The vault owns the identity: decrypt the ciphertext, never re-read
# env/blob. A missing ciphertext here means the row is inconsistent (a
# failed write or manual edit); fail fast rather than silently dropping
# the identity and falling back to a stale legacy copy.
if secret.encrypted_nsec is None:
raise RuntimeError(
"nsec_state is 'encrypted' but no ciphertext is stored; the "
"secrets row is inconsistent. Refusing to boot rather than "
"silently resurrecting a stale legacy NSEC."
)
try:
settings.nsec = vault.decrypt(secret.encrypted_nsec)
except InvalidToken as exc:
raise RuntimeError(
"Stored nsec cannot be decrypted with the current "
"ROUTSTR_SECRET_KEY. The key changed, or this database came from "
"another node. Restore the original ROUTSTR_SECRET_KEY to recover."
) from exc
elif secret.nsec_state == NsecState.cleared:
# The operator emptied the identity via the admin API. A fresh process
# has already reloaded a stale ``NSEC`` from env/blob into the live
# settings (and may have derived its npub); actively clear both so the
# cleared store wins rather than silently resurrecting the old identity.
settings.nsec = ""
settings.npub = ""
else: # NsecState.legacy — the vault has not taken ownership yet.
# Import any legacy plaintext (env, or the old settings blob) exactly
# once. Encryption at rest is mandatory, but a missing key is
# provisioned, not fatal: vault.encrypt generates and persists a master
# key (with a loud one-time operator notice) when none was supplied, so
# an upgrading node keeps running. The nsec is never stored in plaintext.
legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec")
if legacy_nsec:
secret.encrypted_nsec = vault.encrypt(legacy_nsec)
secret.nsec_state = NsecState.encrypted
settings.nsec = legacy_nsec
changed = True
# Derive npub from whatever nsec we now hold, if not already known.
if settings.nsec and not settings.npub:
npub = derive_npub_from_nsec(settings.nsec)
if npub:
settings.npub = npub
if changed:
secret.updated_at = int(time.time())
db_session.add(secret)
await db_session.commit()
+320
View File
@@ -0,0 +1,320 @@
"""Encrypt/hash/fingerprint helpers for secrets at rest (issue #553).
Thin wrapper over ``cryptography`` so nothing else in the codebase touches
Fernet/scrypt/HMAC directly:
- :func:`encrypt`/:func:`decrypt` — Fernet symmetric encryption, keyed by the
mandatory master key. Ciphertext is self-describing (``fernet:v1:`` prefix) so
a value can be told apart from legacy plaintext and so reading it under the
wrong key surfaces as a hard error rather than silent corruption.
- :func:`hash_password`/:func:`verify_password` — salted scrypt hashing. This is
*key-independent*: it never reads the master key, so password login and the
recovery script keep working even when the key is missing.
Key custody is flexible but encryption is not optional. The key comes from the
``ROUTSTR_SECRET_KEY`` env var, else a persisted key file
(``ROUTSTR_SECRET_KEY_FILE``, defaulting beside the SQLite database so it persists
on the same volume as the data); when neither is set, :func:`encrypt` generates
one to the key file and prints a one-time notice, so an existing node upgrades
without breaking instead of refusing to boot. Reading is strict — :func:`decrypt`
never generates a key (a new key could not match existing ciphertext) and fails
fast with the generation command when none is configured. A malformed
``ROUTSTR_SECRET_KEY`` is an operator error and always fails fast.
"""
import base64
import hashlib
import hmac
import os
import secrets
import tempfile
from pathlib import Path
from cryptography.fernet import Fernet
from sqlalchemy.engine import make_url
from sqlalchemy.exc import ArgumentError
_PREFIX = "fernet:v1:"
_GEN_COMMAND = (
'python -c "from cryptography.fernet import Fernet; '
'print(Fernet.generate_key().decode())"'
)
# Where an auto-generated master key is persisted when the operator supplies no
# ``ROUTSTR_SECRET_KEY``. Defaults beside the SQLite database so it rides whatever
# volume already persists the data (a container recreate would otherwise generate
# a fresh key and be unable to decrypt existing secrets); falls back to the
# working directory when the DB location is unknown. Override the exact path with
# ``ROUTSTR_SECRET_KEY_FILE``.
_KEY_FILE_ENV = "ROUTSTR_SECRET_KEY_FILE"
_DEFAULT_KEY_FILE = "routstr_secret.key"
# Minimum admin-password length, enforced wherever a password is set/changed
# (admin endpoints + the recovery script) so the policy lives in one place.
MIN_PASSWORD_LENGTH = 8
# scrypt parameters; packed into each hash so verification is parameter-free.
_SCRYPT_N = 2**14
_SCRYPT_R = 8
_SCRYPT_P = 1
_SCRYPT_DKLEN = 32
_SCRYPT_SALT_BYTES = 16
def _database_dir() -> Path | None:
"""Directory of the SQLite database file, or ``None`` when it has no on-disk
location (a non-SQLite URL or ``:memory:``).
Read from ``DATABASE_URL`` at call time and parsed here rather than importing
``routstr.core.db`` — that module builds the engine at import, which the
crypto layer must not drag in. Mirrors db.py's ``DATABASE_URL`` default.
"""
url_str = os.environ.get("DATABASE_URL", "sqlite+aiosqlite:///keys.db")
try:
url = make_url(url_str)
except ArgumentError:
return None
if url.get_backend_name() != "sqlite" or not url.database:
return None
if url.database == ":memory:":
return None
return Path(url.database).parent
def _key_file_path() -> Path:
"""Where the auto-generated master key is read from / written to.
``ROUTSTR_SECRET_KEY_FILE`` wins; otherwise the key sits beside the SQLite
database so it persists on the same volume as the data, falling back to the
working directory when the DB location is unknown.
"""
override = os.environ.get(_KEY_FILE_ENV)
if override:
return Path(override)
directory = _database_dir()
return (directory or Path()) / _DEFAULT_KEY_FILE
def _read_key_file(path: Path) -> str | None:
try:
stored = path.read_text().strip()
except OSError:
return None
if not stored:
return None
_repair_key_file_perms(path)
return stored
def _repair_key_file_perms(path: Path) -> None:
# A master key must never be group/other-readable. On POSIX, tighten loose
# permissions to owner-only (0600) rather than trust — or hard-fail on — a
# world-readable key; a friendlier repair keeps an upgrading node booting.
if os.name != "posix":
return
try:
mode = path.stat().st_mode
except OSError:
return
if mode & 0o077:
try:
os.chmod(path, 0o600)
except OSError:
pass
def _load_secret_key() -> str | None:
"""The configured key without provisioning: env var, then the key file."""
return os.environ.get("ROUTSTR_SECRET_KEY") or _read_key_file(_key_file_path())
def _warn_generated_key(path: Path) -> None:
# stdout, not the logger: the operator must see this once (e.g. in
# ``docker compose logs``), but it must never be persisted into the on-disk
# log files the logger also writes. Mirrors the generated-admin-password
# notice so an upgrade cannot silently create an unbacked key.
print(
"No ROUTSTR_SECRET_KEY was set; generated one to encrypt node secrets at "
f"rest and saved it to {path}.\n"
"!! BACK UP THIS FILE. If it is lost, the encrypted secrets cannot be "
"recovered and will have to be re-entered.\n"
"To manage the key yourself (e.g. from a secrets manager) set "
"ROUTSTR_SECRET_KEY in the environment instead; the value is in the file "
"above.",
flush=True,
)
def _generate_and_persist_key(path: Path) -> str:
"""Generate a Fernet key, persist it owner-only and atomically, warn once.
The key is written to a temp file in the same directory, flushed durably,
then ``os.link``-ed into place. ``os.link`` publishes the complete file in a
single atomic step — a crash mid-write leaves only the temp file (which is
removed), never a half-written or empty key at the final path that a later
boot would read as corrupt. It also refuses to overwrite an existing key, so
a racing worker that generated first keeps ownership (secrets may already be
encrypted under its key); the loser adopts that key instead of clobbering it.
"""
key = Fernet.generate_key().decode()
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_name = tempfile.mkstemp(
dir=path.parent, prefix=".routstr_secret.", suffix=".tmp"
)
tmp = Path(tmp_name)
try:
with os.fdopen(fd, "w") as handle:
handle.write(key) # mkstemp already created it 0600
handle.flush()
os.fsync(handle.fileno())
try:
os.link(tmp, path)
except FileExistsError:
# A concurrent worker linked its key in first; adopt theirs rather
# than clobber a key that secrets may already be encrypted under.
existing = _read_key_file(path)
if existing:
return existing
raise
_fsync_dir(path.parent)
finally:
tmp.unlink(missing_ok=True)
_warn_generated_key(path)
return key
def _fsync_dir(directory: Path) -> None:
# Persist the new directory entry so the linked key survives a crash right
# after publish. Best-effort: not every platform lets you fsync a directory.
try:
dir_fd = os.open(directory, os.O_RDONLY)
except OSError:
return
try:
os.fsync(dir_fd)
except OSError:
pass
finally:
os.close(dir_fd)
def ensure_secret_key() -> str:
"""Return the master key, provisioning one if the operator supplied none.
Precedence: the ``ROUTSTR_SECRET_KEY`` env var, then the persisted key file,
otherwise a freshly generated key written to the key file (with a one-time
operator notice). This keeps encryption at rest mandatory while letting an
existing node upgrade without setting a key first. A malformed env key is
left to fail at :func:`get_fernet` — it is an operator error, not an unset
key, so it must not trigger silent self-provisioning.
"""
env_key = os.environ.get("ROUTSTR_SECRET_KEY")
if env_key:
return env_key
path = _key_file_path()
return _read_key_file(path) or _generate_and_persist_key(path)
def _fernet_from_key(key: str) -> Fernet:
try:
return Fernet(key.encode())
except (ValueError, TypeError) as exc:
raise RuntimeError(
"ROUTSTR_SECRET_KEY is malformed; it must be a url-safe base64 "
"32-byte Fernet key. Generate one with:\n " + _GEN_COMMAND
) from exc
def get_fernet() -> Fernet:
"""Build a :class:`Fernet` from the configured key (env var or key file).
Strict: this never generates a key, so already-encrypted ciphertext is never
shadowed by a fresh key. A read with no key configured fails fast with the
generation command.
"""
key = _load_secret_key()
if not key:
raise RuntimeError(
"ROUTSTR_SECRET_KEY is not set. It is required to encrypt secrets at "
"rest. Generate one with:\n " + _GEN_COMMAND
)
return _fernet_from_key(key)
def encrypt(plaintext: str) -> str:
"""Encrypt ``plaintext`` into a self-describing ``fernet:v1:`` token.
Provisions a master key (env var, key file, or a freshly generated one) so an
upgrading node never has to set one before its first secret is stored; the
value is always encrypted, never persisted in plaintext.
"""
fernet = _fernet_from_key(ensure_secret_key())
return _PREFIX + fernet.encrypt(plaintext.encode()).decode()
def is_encrypted(value: str) -> bool:
"""True if ``value`` carries the ``fernet:v1:`` prefix this module emits."""
return value.startswith(_PREFIX)
def decrypt(ciphertext: str) -> str:
"""Decrypt a ``fernet:v1:`` token.
Raises ``ValueError`` for an unprefixed value (so legacy plaintext is never
mistaken for ciphertext) and ``InvalidToken`` when the value was written
under a different ``ROUTSTR_SECRET_KEY``.
"""
if not is_encrypted(ciphertext):
raise ValueError("value is not fernet:v1: ciphertext")
token = ciphertext[len(_PREFIX) :]
return get_fernet().decrypt(token.encode()).decode()
def hash_password(password: str) -> str:
"""Salted scrypt hash, self-describing as ``scrypt:n:r:p:salt:hash``."""
salt = secrets.token_bytes(_SCRYPT_SALT_BYTES)
derived = hashlib.scrypt(
password.encode(),
salt=salt,
n=_SCRYPT_N,
r=_SCRYPT_R,
p=_SCRYPT_P,
dklen=_SCRYPT_DKLEN,
)
return ":".join(
[
"scrypt",
str(_SCRYPT_N),
str(_SCRYPT_R),
str(_SCRYPT_P),
base64.b64encode(salt).decode(),
base64.b64encode(derived).decode(),
]
)
def verify_password(password: str, stored: str) -> bool:
"""Constant-time check of ``password`` against a :func:`hash_password` value."""
try:
scheme, n, r, p, salt_b64, hash_b64 = stored.split(":")
if scheme != "scrypt":
return False
n_int, r_int, p_int = int(n), int(r), int(p)
# Cap the work factor at the parameters this module emits. scrypt's
# memory cost grows with N*r, so an oversized N/r in a tampered or
# corrupt stored hash could turn a single login into an OOM/DoS.
if n_int > _SCRYPT_N or r_int > _SCRYPT_R or p_int > _SCRYPT_P:
return False
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(hash_b64)
derived = hashlib.scrypt(
password.encode(),
salt=salt,
n=n_int,
r=r_int,
p=p_int,
dklen=len(expected),
)
except (ValueError, TypeError):
return False
return hmac.compare_digest(derived, expected)
+30 -7
View File
@@ -7,7 +7,13 @@ from fastapi.responses import Response, StreamingResponse
from sqlmodel import select
from .algorithm import create_model_mappings
from .auth import pay_for_request, revert_pay_for_request, validate_bearer_key
from .auth import (
ReservationSnapshot,
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
validate_bearer_key,
)
from .core import get_logger
from .core.db import (
ApiKey,
@@ -440,8 +446,10 @@ async def proxy(
"upstream_error", "All upstreams failed", 502, request=request
)
reservation_snapshot: ReservationSnapshot | None = None
if is_ehbp or request_body_dict:
await pay_for_request(key, max_cost_for_model, session)
reservation_snapshot = await get_reservation_snapshot(key, session)
# Tracks request params already removed in response to upstream rejections,
# shared across providers so a stripped param stays stripped on failover and
@@ -463,14 +471,18 @@ async def proxy(
)
candidate_max = max(candidate_max, settings.min_request_msat)
if candidate_max > max_cost_for_model:
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
try:
await pay_for_request(key, candidate_max, session)
except HTTPException:
if i == len(candidates) - 1:
raise
await pay_for_request(key, max_cost_for_model, session)
reservation_snapshot = await get_reservation_snapshot(key, session)
continue
reservation_snapshot = await get_reservation_snapshot(key, session)
max_cost_for_model = candidate_max
headers = upstream.prepare_headers(dict(request.headers))
@@ -499,6 +511,7 @@ async def proxy(
max_cost_for_model=max_cost_for_model,
session=session,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
elif is_responses_api:
response = await upstream.forward_responses_request(
@@ -510,6 +523,7 @@ async def proxy(
max_cost_for_model,
session,
model_obj,
reservation_snapshot,
)
else:
response = await upstream.forward_request(
@@ -521,6 +535,7 @@ async def proxy(
max_cost_for_model,
session,
model_obj,
reservation_snapshot,
)
except UpstreamError:
# Let the outer UpstreamError handler manage retry/revert
@@ -537,7 +552,9 @@ async def proxy(
"max_cost_for_model": max_cost_for_model,
},
)
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
raise
# Reactive recovery: some models reject one specific request
@@ -606,7 +623,9 @@ async def proxy(
continue
# 4xx error (user error), or other non-retryable error, or last provider failed
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
logger.warning(
"Upstream request failed, revert payment "
"(provider=%s model=%s status=%s path=%s)",
@@ -638,8 +657,10 @@ async def proxy(
"max_cost_for_model": max_cost_for_model,
},
)
await asyncio.shield(
revert_pay_for_request(key, session, max_cost_for_model)
# The cancellation has been caught, so complete exact cleanup in
# this task before the request-scoped session can be torn down.
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
raise
@@ -659,7 +680,9 @@ async def proxy(
# If this was the last provider
if i == len(candidates) - 1:
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
return create_upstream_error_response(e, request)
# Otherwise loop continues to next provider
+254 -115
View File
@@ -14,7 +14,12 @@ from fastapi import BackgroundTasks, HTTPException, Request
from fastapi.responses import Response, StreamingResponse
from pydantic.v1 import BaseModel
from ..auth import adjust_payment_for_tokens
from ..auth import (
ReservationSnapshot,
adjust_payment_for_tokens,
get_reservation_snapshot,
release_reservation,
)
from ..core import get_logger
from ..core.db import (
ApiKey,
@@ -66,8 +71,7 @@ logger = get_logger(__name__)
def _is_json_content_type(content_type: str | None) -> bool:
"""Return True when the upstream response should be parsed as JSON.
"""
"""Return True when the upstream response should be parsed as JSON."""
if not content_type:
return False
main = content_type.split(";", 1)[0].strip().lower()
@@ -230,9 +234,7 @@ class BaseUpstreamProvider:
pass
if "prompt_tokens" in usage:
try:
usage["prompt_tokens"] = (
int(usage.get("prompt_tokens") or 0) + extra
)
usage["prompt_tokens"] = int(usage.get("prompt_tokens") or 0) + extra
except (TypeError, ValueError):
pass
@@ -734,7 +736,9 @@ class BaseUpstreamProvider:
and rate_limit.retry_after_seconds is not None
and "retry-after" not in {k.lower() for k in headers}
):
headers["Retry-After"] = str(max(1, math.ceil(rate_limit.retry_after_seconds)))
headers["Retry-After"] = str(
max(1, math.ceil(rate_limit.retry_after_seconds))
)
if is_json_body:
if not content_type:
@@ -793,6 +797,56 @@ class BaseUpstreamProvider:
media_type="application/json",
)
async def _release_failed_streaming_reservation(
self,
key: ApiKey,
session: AsyncSession,
reservation_snapshot: ReservationSnapshot | None,
) -> bool:
"""Attempt exact release and suppress unsafe settlement retries."""
try:
await session.rollback()
snapshot = reservation_snapshot
if snapshot is None:
snapshot = await get_reservation_snapshot(key, session)
released = await release_reservation(
snapshot,
session,
snapshot.reserved_msats,
)
if not released:
logger.critical(
"Billing reservation could not be released",
extra={
"key_hash": key.hashed_key[:8] + "...",
"reserved_balance": snapshot.reserved_msats,
},
)
# A failed release remains recoverable by the stale-reservation
# sweep. Retrying settlement here could charge after an ambiguous
# database failure or replace the original stream exception.
return True
except asyncio.CancelledError:
# Preserve the exception that triggered billing cleanup. The stream
# propagates it immediately after this helper returns, and stale
# reservation cleanup can recover an interrupted release.
logger.critical(
"Billing reservation release was cancelled",
extra={"key_hash": key.hashed_key[:8] + "..."},
exc_info=True,
)
return True
except Exception as release_error:
logger.critical(
"Billing reservation release failed",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(release_error),
},
exc_info=True,
)
return True
async def handle_streaming_chat_completion(
self,
response: httpx.Response,
@@ -801,6 +855,7 @@ class BaseUpstreamProvider:
background_tasks: BackgroundTasks,
requested_model: str | None = None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> StreamingResponse:
"""Handle streaming chat completion responses with token usage tracking and cost adjustment.
@@ -812,6 +867,15 @@ class BaseUpstreamProvider:
Returns:
StreamingResponse with cost data injected at the end
"""
if reservation_snapshot is None:
async with create_session() as snapshot_session:
snapshot_key = await snapshot_session.get(key.__class__, key.hashed_key)
if snapshot_key is None:
raise RuntimeError("Billing key disappeared before streaming")
reservation_snapshot = await get_reservation_snapshot(
snapshot_key, snapshot_session
)
logger.debug(
"Processing streaming chat completion",
extra={
@@ -845,6 +909,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
usage_finalized = True
except Exception:
@@ -969,9 +1034,7 @@ class BaseUpstreamProvider:
# line so multi-line ``data`` stays valid SSE framing - a bare
# second line would otherwise reach the client without its
# ``data:`` field and break naive parsers.
body = b"".join(
b"data: " + ln + b"\n" for ln in data.split(b"\n")
)
body = b"".join(b"data: " + ln + b"\n" for ln in data.split(b"\n"))
yield prefix + body + b"\n"
try:
@@ -1019,48 +1082,44 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
usage_finalized = True
except Exception as e:
logger.exception(
"Error during usage finalization",
except BaseException as e:
logger.critical(
"Error during usage finalization — CRITICAL",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(e),
},
exc_info=True,
)
# Fall back so we still emit a non-zero sats cost downstream.
cost_data = {
"base_msats": 0,
"input_msats": 0,
"output_msats": 0,
"total_msats": 0,
"total_usd": 0.0,
"input_tokens": 0,
"output_tokens": 0,
}
# Release is a terminal billing state. Do not enqueue
# finalize_db_only from the generator's finally block
# and charge this request later.
usage_finalized = (
await self._release_failed_streaming_reservation(
fresh_key,
session,
reservation_snapshot,
)
)
raise
if usage_chunk_data is None:
if not hasattr(self, "_current_stream_id"):
self._current_stream_id = (
f"chatcmpl-{uuid.uuid4()}"
)
self._current_stream_id = f"chatcmpl-{uuid.uuid4()}"
usage_chunk_data = {
"id": self._current_stream_id,
"object": "chat.completion.chunk",
"model": last_model_seen or "unknown",
"choices": [],
"usage": {
"prompt_tokens": cost_data.get(
"input_tokens", 0
),
"prompt_tokens": cost_data.get("input_tokens", 0),
"completion_tokens": cost_data.get(
"output_tokens", 0
),
"total_tokens": cost_data.get(
"input_tokens", 0
)
"total_tokens": cost_data.get("input_tokens", 0)
+ cost_data.get("output_tokens", 0),
},
}
@@ -1116,6 +1175,7 @@ class BaseUpstreamProvider:
deducted_max_cost: int,
requested_model: str | None = None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response:
"""Handle non-streaming chat completion responses with token usage tracking and cost adjustment.
@@ -1164,6 +1224,7 @@ class BaseUpstreamProvider:
deducted_max_cost,
model_obj,
self.provider_fee,
reservation_snapshot,
)
await session.refresh(key)
@@ -1260,6 +1321,7 @@ class BaseUpstreamProvider:
max_cost_for_model: int,
requested_model: str | None = None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> StreamingResponse:
"""Handle streaming Responses API responses with token usage tracking and cost adjustment.
@@ -1305,6 +1367,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
usage_finalized = True
except Exception:
@@ -1389,9 +1452,7 @@ class BaseUpstreamProvider:
return
# Re-prefix each line so multi-line ``data`` stays valid SSE
# framing for the client.
body = b"".join(
b"data: " + ln + b"\n" for ln in data.split(b"\n")
)
body = b"".join(b"data: " + ln + b"\n" for ln in data.split(b"\n"))
yield prefix + body + b"\n"
try:
@@ -1436,25 +1497,26 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
usage_finalized = True
except Exception as e:
logger.exception(
"Error during Responses API usage finalization",
except BaseException as e:
logger.critical(
"Error during Responses API usage finalization — CRITICAL",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(e),
},
exc_info=True,
)
cost_data = {
"base_msats": 0,
"input_msats": 0,
"output_msats": 0,
"total_msats": 0,
"total_usd": 0.0,
"input_tokens": 0,
"output_tokens": 0,
}
usage_finalized = (
await self._release_failed_streaming_reservation(
fresh_key,
session,
reservation_snapshot,
)
)
raise
if usage_chunk_data is None:
usage_chunk_data = {
@@ -1468,22 +1530,14 @@ class BaseUpstreamProvider:
"output_tokens": cost_data.get(
"output_tokens", 0
),
"total_tokens": cost_data.get(
"input_tokens", 0
)
"total_tokens": cost_data.get("input_tokens", 0)
+ cost_data.get("output_tokens", 0),
},
},
"usage": {
"input_tokens": cost_data.get(
"input_tokens", 0
),
"output_tokens": cost_data.get(
"output_tokens", 0
),
"total_tokens": cost_data.get(
"input_tokens", 0
)
"input_tokens": cost_data.get("input_tokens", 0),
"output_tokens": cost_data.get("output_tokens", 0),
"total_tokens": cost_data.get("input_tokens", 0)
+ cost_data.get("output_tokens", 0),
},
}
@@ -1499,9 +1553,9 @@ class BaseUpstreamProvider:
usage_chunk_data["response"]["usage"]["cost"] = (
cost_data.get("total_usd", 0.0)
)
usage_chunk_data["response"]["usage"][
"cost_sats"
] = sats_cost
usage_chunk_data["response"]["usage"]["cost_sats"] = (
sats_cost
)
usage_chunk_data["response"]["usage"][
"remaining_balance_msats"
] = remaining_balance_msats
@@ -1555,6 +1609,7 @@ class BaseUpstreamProvider:
deducted_max_cost: int,
requested_model: str | None = None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response:
"""Handle non-streaming Responses API responses with token usage tracking and cost adjustment.
@@ -1606,6 +1661,7 @@ class BaseUpstreamProvider:
deducted_max_cost,
model_obj,
self.provider_fee,
reservation_snapshot,
)
await session.refresh(key)
@@ -1696,7 +1752,13 @@ class BaseUpstreamProvider:
raise
async def _finalize_generic_streaming_payment(
self, key_hash: str, max_cost: int, path: str
self,
key_hash: str,
max_cost: int,
path: str,
model_obj: Model | None,
provider_fee: float | None,
reservation_snapshot: ReservationSnapshot,
) -> None:
"""Background task to finalize payment for generic streaming requests."""
async with create_session() as session:
@@ -1717,8 +1779,9 @@ class BaseUpstreamProvider:
{"model": "unknown", "usage": None},
session,
max_cost,
model_obj=None,
provider_fee=None,
model_obj=model_obj,
provider_fee=provider_fee,
reservation_snapshot=reservation_snapshot,
)
logger.debug(
"Finalized generic streaming payment in background",
@@ -1744,6 +1807,7 @@ class BaseUpstreamProvider:
max_cost_for_model: int,
requested_model: str | None = None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> StreamingResponse:
async def stream_with_cost(
max_cost_for_model: int,
@@ -1786,9 +1850,7 @@ class BaseUpstreamProvider:
_coerce_usd(cd.get("output_cost")),
)
for field in ("total_cost", "cost"):
total_cost = max(
total_cost, _coerce_usd(usage_or_root.get(field))
)
total_cost = max(total_cost, _coerce_usd(usage_or_root.get(field)))
async def finalize_without_usage() -> bytes | None:
nonlocal usage_finalized
@@ -1811,12 +1873,27 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
usage_finalized = True
return f"event: cost\ndata: {json.dumps({'cost': cost_data})}\n\n".encode()
except Exception:
usage_finalized = True
return None
except BaseException as e:
logger.critical(
"Error during Messages API usage finalization — CRITICAL",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(e),
},
exc_info=True,
)
usage_finalized = (
await self._release_failed_streaming_reservation(
fresh_key,
new_session,
reservation_snapshot,
)
)
raise
try:
async for chunk in response.aiter_bytes():
@@ -1834,9 +1911,7 @@ class BaseUpstreamProvider:
if msg and msg.get("model"):
last_model_seen = str(msg.get("model"))
provider_added = (
"provider" not in data
)
provider_added = "provider" not in data
self._apply_provider_field(data)
if requested_model:
@@ -1964,6 +2039,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
self.inject_cost_metadata(
@@ -1973,8 +2049,23 @@ class BaseUpstreamProvider:
usage_finalized = True
# Emit the full combined_data as the cost
yield f"event: cost\ndata: {json.dumps(combined_data)}\n\n".encode()
except Exception:
pass
except BaseException as e:
logger.critical(
"Error during Messages API usage finalization — CRITICAL",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(e),
},
exc_info=True,
)
usage_finalized = (
await self._release_failed_streaming_reservation(
fresh_key,
new_session,
reservation_snapshot,
)
)
raise
if not usage_finalized:
maybe_cost_event = await finalize_without_usage()
@@ -2012,6 +2103,7 @@ class BaseUpstreamProvider:
path: str,
requested_model: str | None = None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response:
try:
content = await response.aread()
@@ -2038,6 +2130,7 @@ class BaseUpstreamProvider:
deducted_max_cost,
model_obj,
self.provider_fee,
reservation_snapshot,
)
self.inject_cost_metadata(response_json, cost_data, key)
@@ -2085,9 +2178,7 @@ class BaseUpstreamProvider:
async def _aggregate_anthropic_events_to_message(
self, iterator: AsyncIterator[Any]
) -> dict:
return await messages_dispatch.aggregate_anthropic_events_to_message(
iterator
)
return await messages_dispatch.aggregate_anthropic_events_to_message(iterator)
async def _dispatch_anthropic_messages(
self,
@@ -2113,6 +2204,7 @@ class BaseUpstreamProvider:
session: AsyncSession,
max_cost_for_model: int,
model_obj: Model,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response | StreamingResponse:
"""Translate /v1/messages to upstream chat/completions via litellm.
@@ -2133,6 +2225,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
requested_model,
model_obj,
reservation_snapshot,
)
response_json = messages_dispatch.coerce_litellm_payload(result)
@@ -2146,6 +2239,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
self.inject_cost_metadata(response_json, cost_data, key)
@@ -2197,8 +2291,10 @@ class BaseUpstreamProvider:
response_json, max_cost_for_model, model_obj
)
if cost_data and "usage" in response_json and isinstance(
response_json["usage"], dict
if (
cost_data
and "usage" in response_json
and isinstance(response_json["usage"], dict)
):
response_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
self._fold_cache_into_input_tokens(response_json["usage"])
@@ -2241,6 +2337,7 @@ class BaseUpstreamProvider:
max_cost_for_model: int,
requested_model: str | None,
model_obj: Model | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> StreamingResponse:
"""Re-emit a litellm Anthropic-event iterator as live SSE bytes
with cost reconciliation appended at end of stream."""
@@ -2275,9 +2372,7 @@ class BaseUpstreamProvider:
},
)
async with create_session() as new_session:
fresh_key = await new_session.get(
key.__class__, key.hashed_key
)
fresh_key = await new_session.get(key.__class__, key.hashed_key)
if not fresh_key:
usage_finalized = True
return None
@@ -2293,15 +2388,29 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
usage_finalized = True
return (
f"event: cost\ndata: "
f"{json.dumps({'cost': cost_data})}\n\n"
f"event: cost\ndata: {json.dumps({'cost': cost_data})}\n\n"
).encode()
except Exception:
usage_finalized = True
return None
except BaseException as e:
logger.critical(
"Error during LiteLLM Messages usage finalization — CRITICAL",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(e),
},
exc_info=True,
)
usage_finalized = (
await self._release_failed_streaming_reservation(
fresh_key,
new_session,
reservation_snapshot,
)
)
raise
try:
async for annotated in messages_dispatch.stream_annotated_events(
@@ -2336,9 +2445,7 @@ class BaseUpstreamProvider:
or total_cost > 0
):
async with create_session() as new_session:
fresh_key = await new_session.get(
key.__class__, key.hashed_key
)
fresh_key = await new_session.get(key.__class__, key.hashed_key)
if fresh_key:
try:
rebuilt_usage: dict = {
@@ -2368,6 +2475,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
model_obj,
self.provider_fee,
reservation_snapshot,
)
self.inject_cost_metadata(
combined_data, cost_data, fresh_key
@@ -2377,8 +2485,23 @@ class BaseUpstreamProvider:
f"event: cost\ndata: "
f"{json.dumps({'cost': cost_data})}\n\n"
).encode()
except Exception:
pass
except BaseException as e:
logger.critical(
"Error during LiteLLM Messages usage finalization — CRITICAL",
extra={
"key_hash": key.hashed_key[:8] + "...",
"error": str(e),
},
exc_info=True,
)
usage_finalized = (
await self._release_failed_streaming_reservation(
fresh_key,
new_session,
reservation_snapshot,
)
)
raise
if not usage_finalized:
cost_event = await finalize_without_usage()
@@ -2389,6 +2512,9 @@ class BaseUpstreamProvider:
if not usage_finalized:
await finalize_without_usage()
raise
finally:
if not usage_finalized:
await finalize_without_usage()
return StreamingResponse(
stream_with_cost(),
@@ -2512,8 +2638,7 @@ class BaseUpstreamProvider:
)
response_headers["X-Cashu"] = refund_token
logger.info(
"Refund processed for streaming /v1/messages "
"via litellm",
"Refund processed for streaming /v1/messages via litellm",
extra={
"refund_amount": refund_amount,
"unit": unit,
@@ -2551,6 +2676,7 @@ class BaseUpstreamProvider:
max_cost_for_model: int,
session: AsyncSession,
model_obj: Model,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response | StreamingResponse:
"""Forward authenticated request to upstream service with cost tracking.
@@ -2585,6 +2711,7 @@ class BaseUpstreamProvider:
session=session,
max_cost_for_model=max_cost_for_model,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
url = self.build_request_url(path, model_obj)
@@ -2715,6 +2842,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
@@ -2732,6 +2860,7 @@ class BaseUpstreamProvider:
path,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
finally:
await response.aclose()
@@ -2748,6 +2877,7 @@ class BaseUpstreamProvider:
path,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
finally:
await response.aclose()
@@ -2798,6 +2928,7 @@ class BaseUpstreamProvider:
background_tasks,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
result.background = background_tasks
return result
@@ -2812,11 +2943,15 @@ class BaseUpstreamProvider:
max_cost_for_model,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
finally:
await response.aclose()
await client.aclose()
if reservation_snapshot is None:
reservation_snapshot = await get_reservation_snapshot(key, session)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
background_tasks.add_task(client.aclose)
@@ -2825,6 +2960,9 @@ class BaseUpstreamProvider:
key.hashed_key,
max_cost_for_model,
path,
model_obj,
self.provider_fee,
reservation_snapshot,
)
logger.debug(
@@ -2899,7 +3037,9 @@ class BaseUpstreamProvider:
supports_ehbp: bool = False
def get_confidential_inference_profile(self) -> "ConfidentialInferenceProfile | None":
def get_confidential_inference_profile(
self,
) -> "ConfidentialInferenceProfile | None":
"""Return provider policy for encrypted/confidential inference forwarding."""
return None
@@ -2927,6 +3067,7 @@ class BaseUpstreamProvider:
max_cost_for_model: int,
session: AsyncSession,
model_obj: Model,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response | StreamingResponse:
"""Forward authenticated Responses API request to upstream service with cost tracking.
@@ -3065,6 +3206,7 @@ class BaseUpstreamProvider:
max_cost_for_model,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
@@ -3081,11 +3223,15 @@ class BaseUpstreamProvider:
max_cost_for_model,
requested_model=original_model_id,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
finally:
await response.aclose()
await client.aclose()
if reservation_snapshot is None:
reservation_snapshot = await get_reservation_snapshot(key, session)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
background_tasks.add_task(client.aclose)
@@ -3094,6 +3240,9 @@ class BaseUpstreamProvider:
key.hashed_key,
max_cost_for_model,
path,
model_obj,
self.provider_fee,
reservation_snapshot,
)
logger.debug(
@@ -3574,14 +3723,8 @@ class BaseUpstreamProvider:
if "provider" not in data_json:
self._apply_provider_field(data_json)
changed = True
if (
cost_data
and "usage" in data_json
and data_json["usage"]
):
data_json["usage"]["cost_sats"] = (
cost_data.total_msats // 1000
)
if cost_data and "usage" in data_json and data_json["usage"]:
data_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
changed = True
if changed:
lines[i] = "data: " + json.dumps(data_json)
@@ -4561,14 +4704,8 @@ class BaseUpstreamProvider:
if "provider" not in data_json:
self._apply_provider_field(data_json)
changed = True
if (
cost_data
and "usage" in data_json
and data_json["usage"]
):
data_json["usage"]["cost_sats"] = (
cost_data.total_msats // 1000
)
if cost_data and "usage" in data_json and data_json["usage"]:
data_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
changed = True
if changed:
lines[i] = "data: " + json.dumps(data_json)
@@ -5060,7 +5197,9 @@ class BaseUpstreamProvider:
except Exception:
self._models_cache = models_with_fees
self._models_by_id = {m.forwarded_model_id or m.id: m for m in self._models_cache}
self._models_by_id = {
m.forwarded_model_id or m.id: m for m in self._models_cache
}
except Exception as e:
logger.error(
+28 -2
View File
@@ -15,7 +15,11 @@ from sqlmodel import col, update
from ..auth import (
ROUTSTR_FEE_PERCENT,
ReservationSnapshot,
_claim_reservation_for_charge,
_validate_reservation_snapshot,
get_billing_key,
get_reservation_snapshot,
payments_logger,
)
from ..core import get_logger
@@ -502,8 +506,14 @@ async def finalize_ehbp_actual_cost_payment(
reserved_cost_for_model: int,
model_id: str,
cost_info: dict,
reservation_snapshot: ReservationSnapshot | None = None,
) -> None:
"""Finalize an EHBP bearer request using clamped provider usage metrics."""
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(key, reservation, session)
if not await _claim_reservation_for_charge(reservation, session):
return
reserved_cost_for_model = reservation.reserved_msats
billing_key = await get_billing_key(key, session)
key_hash = key.hashed_key
billing_key_hash = billing_key.hashed_key
@@ -606,6 +616,7 @@ async def finalize_ehbp_max_cost_payment(
session: AsyncSession,
max_cost_for_model: int,
model_id: str,
reservation_snapshot: ReservationSnapshot | None = None,
) -> None:
"""Finalize an EHBP bearer request by charging the reserved max cost.
@@ -613,6 +624,11 @@ async def finalize_ehbp_max_cost_payment(
normal completion handlers, this intentionally charges the pre-reserved max
cost and releases the reservation.
"""
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(key, reservation, session)
if not await _claim_reservation_for_charge(reservation, session):
return
max_cost_for_model = reservation.reserved_msats
billing_key = await get_billing_key(key, session)
key_hash = key.hashed_key
billing_key_hash = billing_key.hashed_key
@@ -766,6 +782,7 @@ async def forward_ehbp_request(
max_cost_for_model: int,
session: AsyncSession,
model_obj: Model,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response | StreamingResponse:
"""Forward an EHBP bearer-auth request and finalize billing.
@@ -883,7 +900,12 @@ async def forward_ehbp_request(
# the requested model.
billing_model = cost_info.pop("actual_model", None) or model_obj.id
await finalize_ehbp_actual_cost_payment(
key, session, max_cost_for_model, billing_model, cost_info
key,
session,
max_cost_for_model,
billing_model,
cost_info,
reservation_snapshot,
)
cost_data = {**cost_info, "total_usd": 0.0}
else:
@@ -897,7 +919,11 @@ async def forward_ehbp_request(
},
)
await finalize_ehbp_max_cost_payment(
key, session, max_cost_for_model, model_obj.id
key,
session,
max_cost_for_model,
model_obj.id,
reservation_snapshot,
)
cost_data = {
"total_msats": max_cost_for_model,
+6 -7
View File
@@ -603,6 +603,8 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int
wallet, effective_mint_url, unit, not_reserved=True
)
proofs_for_mint = sum(proof.amount for proof in proofs)
all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved)
all_mint_urls = list({k.mint_url for k in wallet.keysets.values()})
proof_summary = {
@@ -618,7 +620,8 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int
raw_proofs_by_keyset[p.id] = raw_proofs_by_keyset.get(p.id, 0) + p.amount
logger.info(
f"send: proof inventory | mint={effective_mint_url} unit={unit} amount={amount} "
f"primary_mint={settings.primary_mint} proofs_for_mint={proofs_for_mint} "
f"primary_mint={settings.primary_mint} liquid_proofs_for_mint={proofs_for_mint} "
f"reserved_proofs_for_mint={reserved_for_mint} "
f"all_mints={all_mint_urls} by_keyset={proof_summary} "
f"raw_proofs_by_keyset_id={raw_proofs_by_keyset} "
f"total_wallet_proofs={sum(p.amount for p in wallet.proofs)}"
@@ -1750,9 +1753,7 @@ async def fetch_all_balances(
"unit": unit,
"wallet_balance": proofs_balance,
"user_balance": user_balance,
"owner_balance": proofs_balance - user_balance
if proofs_balance != 0
else 0,
"owner_balance": proofs_balance - user_balance,
}
# Build the set of mints to inspect. Received tokens are stored against
@@ -1825,9 +1826,7 @@ async def fetch_all_balances(
total_wallet_balance_sats += proofs_balance_sats
total_user_balance_sats += user_balance_sats
owner_balance = 0
if total_wallet_balance_sats != 0:
owner_balance = total_wallet_balance_sats - total_user_balance_sats
owner_balance = total_wallet_balance_sats - total_user_balance_sats
return (
balance_details,
+1
View File
@@ -0,0 +1 @@
"""Operational and recovery scripts for routstr-core (not a shipped package)."""
+104
View File
@@ -0,0 +1,104 @@
"""Recover admin access by resetting the stored admin password (issue #553).
The lockout escape hatch for an operator who has lost the admin password. It
talks to the ``secrets`` table directly and deliberately does *not* require
``ROUTSTR_SECRET_KEY``: the admin password is scrypt-hashed (key-independent),
so recovery works even when the encryption key is missing or has changed.
Two explicit, mutually exclusive actions — running with no arguments only prints
help, so the password can't be reset by accident:
python scripts/reset_admin_password.py --password <new-password>
Hash and store <new-password> now.
python scripts/reset_admin_password.py --regenerate
Clear the stored hash; the next node startup generates a fresh random
password and logs it once (with the /admin URL).
"""
import argparse
import asyncio
import sys
import time
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import create_session, get_secret, set_admin_password
from routstr.core.vault import MIN_PASSWORD_LENGTH
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="reset_admin_password",
description="Reset the node's admin password (recovery from lockout).",
)
action = parser.add_mutually_exclusive_group()
action.add_argument(
"--password",
metavar="NEW_PASSWORD",
help=f"set this as the new admin password (min {MIN_PASSWORD_LENGTH} chars)",
)
action.add_argument(
"--regenerate",
action="store_true",
help="clear the password so the next startup generates and logs a new one",
)
return parser
async def apply_reset(
session: AsyncSession,
*,
password: str | None = None,
regenerate: bool = False,
) -> str:
"""Perform the requested reset against ``session``; return a status message."""
if password is not None:
if len(password) < MIN_PASSWORD_LENGTH:
raise ValueError(
f"New password must be at least {MIN_PASSWORD_LENGTH} characters"
)
await set_admin_password(session, password)
return "Admin password updated."
if regenerate:
secret = await get_secret(session)
secret.admin_password_hash = None
secret.updated_at = int(time.time())
session.add(secret)
await session.commit()
return (
"Admin password cleared. The next node startup will generate a new "
"one and log it once with the /admin URL."
)
return ""
async def _run(password: str | None, regenerate: bool) -> str:
async with create_session() as session:
return await apply_reset(
session, password=password, regenerate=regenerate
)
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
if args.password is None and not args.regenerate:
parser.print_help()
return 0
try:
message = asyncio.run(_run(args.password, args.regenerate))
except ValueError as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
print(message)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+15
View File
@@ -0,0 +1,15 @@
"""Shared pytest configuration for the whole suite.
A fixed, valid ``ROUTSTR_SECRET_KEY`` is set before any app import so that
secret encryption is deterministic across the suite and the mandatory-key
fail-fast does not break app-boot tests. Tests that need a different key (or an
absent one) override this per-test via ``monkeypatch``.
"""
import os
# Valid Fernet keys; KEY_A is the suite default, KEY_B is for wrong-key tests.
TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU="
TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ="
os.environ.setdefault("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY)
+121
View File
@@ -0,0 +1,121 @@
"""Tests for admin password auth backed by the hashed Secret store (issue #553).
Login and password change verify against the one-way ``Secret.admin_password_hash``
(scrypt) instead of a plaintext settings field, which also closes the old ``!=``
timing-attack comparison. The first hash is written by ``bootstrap_secrets`` at
startup (generated, or migrated from a legacy password); here it is seeded
directly into the store, then login checks against it and a password change
re-hashes so the old password stops working and the new one starts.
"""
from __future__ import annotations
import pytest
from httpx import AsyncClient, Response
from routstr.core.db import create_session, set_admin_password
async def _seed_password(password: str) -> None:
# bootstrap_secrets owns first-password creation at startup; tests seed the
# hash straight into the store the same way, then exercise login/change.
async with create_session() as session:
await set_admin_password(session, password)
async def _login(client: AsyncClient, password: str) -> Response:
return await client.post("/admin/api/login", json={"password": password})
# --- login -----------------------------------------------------------------
@pytest.mark.integration
@pytest.mark.asyncio
async def test_login_500_when_no_password_configured(
integration_client: AsyncClient,
) -> None:
resp = await _login(integration_client, "anything")
assert resp.status_code == 500
@pytest.mark.integration
@pytest.mark.asyncio
async def test_login_succeeds_with_correct_password(
integration_client: AsyncClient,
) -> None:
await _seed_password("correct horse")
resp = await _login(integration_client, "correct horse")
assert resp.status_code == 200
body = resp.json()
assert body["ok"] is True
assert isinstance(body["token"], str) and body["token"]
@pytest.mark.integration
@pytest.mark.asyncio
async def test_login_rejects_wrong_password(
integration_client: AsyncClient,
) -> None:
await _seed_password("correct horse")
resp = await _login(integration_client, "wrong horse")
assert resp.status_code == 401
# --- password change -------------------------------------------------------
@pytest.mark.integration
@pytest.mark.asyncio
async def test_update_password_rehashes_so_only_new_works(
integration_client: AsyncClient,
) -> None:
await _seed_password("old password")
login = await _login(integration_client, "old password")
token = login.json()["token"]
integration_client.headers["Authorization"] = f"Bearer {token}"
resp = await integration_client.patch(
"/admin/api/password",
json={"current_password": "old password", "new_password": "new password"},
)
assert resp.status_code == 200, resp.text
# Drop admin auth so the login calls aren't treated as authenticated noise.
integration_client.headers.pop("Authorization", None)
assert (await _login(integration_client, "old password")).status_code == 401
assert (await _login(integration_client, "new password")).status_code == 200
@pytest.mark.integration
@pytest.mark.asyncio
async def test_update_password_rejects_wrong_current(
integration_client: AsyncClient,
) -> None:
await _seed_password("old password")
login = await _login(integration_client, "old password")
token = login.json()["token"]
integration_client.headers["Authorization"] = f"Bearer {token}"
resp = await integration_client.patch(
"/admin/api/password",
json={"current_password": "not the password", "new_password": "new password"},
)
assert resp.status_code == 401
@pytest.mark.integration
@pytest.mark.asyncio
async def test_update_password_rejects_short_new(
integration_client: AsyncClient,
) -> None:
await _seed_password("old password")
login = await _login(integration_client, "old password")
token = login.json()["token"]
integration_client.headers["Authorization"] = f"Bearer {token}"
resp = await integration_client.patch(
"/admin/api/password",
json={"current_password": "old password", "new_password": "x"},
)
assert resp.status_code == 400
@@ -0,0 +1,113 @@
"""Tests for the admin nsec rotation endpoint (issue #553).
The Nostr identity is a secret: it lives encrypted in the Secret store, never in
the settings blob, so it cannot be set through the general settings PATCH. This
dedicated endpoint is the supported way to set/rotate/clear it — it encrypts the
key at rest, updates the live runtime identity (so signing picks it up without a
restart), and derives the npub. Invalid keys are rejected.
"""
from __future__ import annotations
import secrets
import time
from collections.abc import AsyncGenerator
import pytest
import pytest_asyncio
from httpx import AsyncClient
from routstr.core import vault
from routstr.core.admin import admin_sessions
from routstr.core.db import AsyncSession, get_secret
from routstr.core.settings import derive_npub_from_nsec, settings
# A valid 64-char hex private key (accepted by nsec_to_keypair, as in bootstrap).
NSEC_HEX = "1" * 64
@pytest_asyncio.fixture
async def admin_client(
integration_client: AsyncClient,
) -> AsyncGenerator[AsyncClient, None]:
"""An integration_client pre-authenticated with an admin session token."""
token = secrets.token_urlsafe(24)
admin_sessions[token] = int(time.time()) + 3600
integration_client.headers["Authorization"] = f"Bearer {token}"
yield integration_client
admin_sessions.pop(token, None)
@pytest.mark.integration
@pytest.mark.asyncio
async def test_update_nsec_stores_encrypted_and_derives_npub(
admin_client: AsyncClient,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings, "nsec", "")
monkeypatch.setattr(settings, "npub", "")
resp = await admin_client.patch("/admin/api/nsec", json={"nsec": NSEC_HEX})
assert resp.status_code == 200
expected_npub = derive_npub_from_nsec(NSEC_HEX)
assert resp.json() == {"ok": True, "npub": expected_npub}
# Stored encrypted at rest, decryptable back to the original key.
integration_session.expunge_all()
secret = await get_secret(integration_session)
assert secret.encrypted_nsec is not None
assert vault.is_encrypted(secret.encrypted_nsec)
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
# Live runtime identity updated so Nostr signing reflects it without restart.
assert settings.nsec == NSEC_HEX
assert settings.npub == expected_npub
@pytest.mark.integration
@pytest.mark.asyncio
async def test_update_nsec_rejects_invalid_key(
admin_client: AsyncClient,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings, "nsec", "")
monkeypatch.setattr(settings, "npub", "")
resp = await admin_client.patch(
"/admin/api/nsec", json={"nsec": "not-a-real-nsec"}
)
assert resp.status_code == 400
# Nothing stored, live identity untouched.
integration_session.expunge_all()
secret = await get_secret(integration_session)
assert secret.encrypted_nsec is None
assert settings.nsec == ""
@pytest.mark.integration
@pytest.mark.asyncio
async def test_update_nsec_clears_identity_with_empty_value(
admin_client: AsyncClient,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Start from a node that has an identity...
monkeypatch.setattr(settings, "nsec", "")
monkeypatch.setattr(settings, "npub", "")
set_resp = await admin_client.patch("/admin/api/nsec", json={"nsec": NSEC_HEX})
assert set_resp.status_code == 200
# ...then clear it.
clear_resp = await admin_client.patch("/admin/api/nsec", json={"nsec": ""})
assert clear_resp.status_code == 200
assert clear_resp.json() == {"ok": True, "npub": ""}
integration_session.expunge_all()
secret = await get_secret(integration_session)
assert secret.encrypted_nsec is None
assert settings.nsec == ""
assert settings.npub == ""
@@ -0,0 +1,82 @@
"""Tests for the admin settings endpoint's handling of secrets (issue #553).
``admin_password`` is no longer a settings field (it lives only as a one-way
hash in the Secret store), so it must never appear in the GET/PATCH payloads.
``nsec`` (in-memory at runtime) and ``upstream_api_key`` (still in the settings
blob) are both redacted on read and ignored on write — they cannot be set
through the general settings endpoint, only through their dedicated paths.
"""
from __future__ import annotations
import secrets
import time
from collections.abc import AsyncGenerator
import pytest
import pytest_asyncio
from httpx import AsyncClient
from routstr.core.admin import admin_sessions
from routstr.core.db import AsyncSession
from routstr.core.settings import SettingsService, settings
@pytest_asyncio.fixture
async def admin_client(
integration_client: AsyncClient,
) -> AsyncGenerator[AsyncClient, None]:
"""An integration_client pre-authenticated with an admin session token."""
token = secrets.token_urlsafe(24)
admin_sessions[token] = int(time.time()) + 3600
integration_client.headers["Authorization"] = f"Bearer {token}"
yield integration_client
admin_sessions.pop(token, None)
@pytest.mark.integration
@pytest.mark.asyncio
async def test_get_settings_omits_admin_password_and_redacts_secrets(
admin_client: AsyncClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(settings, "nsec", "nsec-secret")
monkeypatch.setattr(settings, "upstream_api_key", "sk-secret")
resp = await admin_client.get("/admin/api/settings")
assert resp.status_code == 200
data = resp.json()
assert "admin_password" not in data
assert data["nsec"] == "[REDACTED]"
assert data["upstream_api_key"] == "[REDACTED]"
@pytest.mark.integration
@pytest.mark.asyncio
async def test_patch_settings_ignores_secret_fields(
admin_client: AsyncClient,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# The PATCH path persists through SettingsService, which needs an
# initialized current snapshot and a settings row in the shared test DB.
await SettingsService.initialize(integration_session)
monkeypatch.setattr(settings, "nsec", "original-nsec")
resp = await admin_client.patch(
"/admin/api/settings",
json={
"name": "Renamed",
"nsec": "attacker-nsec",
"upstream_api_key": "attacker-key",
"admin_password": "attacker-pw",
},
)
assert resp.status_code == 200
data = resp.json()
assert data["name"] == "Renamed"
assert "admin_password" not in data
assert data["nsec"] == "[REDACTED]"
# The live secret was not overwritten through the general settings endpoint.
assert settings.nsec == "original-nsec"
@@ -15,6 +15,7 @@ from unittest.mock import patch
import pytest
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.payment.cost_calculation import CostData
@@ -23,7 +24,7 @@ def _make_key(balance: int, reserved: int) -> ApiKey:
return ApiKey(
hashed_key=f"test_{uuid.uuid4().hex}",
balance=balance,
reserved_balance=reserved,
reserved_balance=0,
total_spent=0,
total_requests=1,
)
@@ -75,6 +76,8 @@ async def test_balance_never_negative_when_cost_exceeds_reservation(
key = _make_key(balance=deducted_max_cost, reserved=deducted_max_cost)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
@@ -111,6 +114,8 @@ async def test_balance_floor_at_zero_on_overrun(
key = _make_key(balance=500, reserved=500)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 50}}
@@ -152,6 +157,8 @@ async def test_full_cost_charged_when_balance_sufficient_for_overrun(
key = _make_key(balance=2000, reserved=990)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
@@ -190,7 +197,11 @@ async def test_concurrent_cost_overruns_never_negative(
"""Concurrent finalization with cost overruns must never produce negative balance."""
import asyncio
from routstr.auth import adjust_payment_for_tokens, pay_for_request
from routstr.auth import (
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
)
from routstr.core.db import create_session
deducted_max_cost = 990
@@ -216,12 +227,14 @@ async def test_concurrent_cost_overruns_never_negative(
async with create_session() as session:
key_to_reserve = await session.get(ApiKey, key_hash)
assert key_to_reserve is not None
reservations = []
for _ in range(n_requests):
await pay_for_request(key_to_reserve, deducted_max_cost, session)
reservations.append(await get_reservation_snapshot(key_to_reserve, session))
await session.refresh(key_to_reserve)
# Now finalize all concurrently with cost overrun
async def finalize() -> None:
async def finalize(reservation: ReservationSnapshot) -> None:
response_data = {
"model": "test-model",
"usage": {"prompt_tokens": 100, "completion_tokens": 100},
@@ -230,7 +243,11 @@ async def test_concurrent_cost_overruns_never_negative(
fresh_key = await session.get(ApiKey, key_hash)
assert fresh_key is not None
await adjust_payment_for_tokens(
fresh_key, response_data, session, deducted_max_cost, None, None
fresh_key,
response_data,
session,
deducted_max_cost,
reservation_snapshot=reservation,
)
# Patch once around the gather: entering the same patch target from
@@ -240,7 +257,7 @@ async def test_concurrent_cost_overruns_never_negative(
"routstr.auth.calculate_cost",
return_value=_cost_data(actual_token_cost),
):
await asyncio.gather(*[finalize() for _ in range(n_requests)])
await asyncio.gather(*(finalize(r) for r in reservations))
async with create_session() as session:
final_key = await session.get(ApiKey, key_hash)
@@ -281,6 +298,8 @@ async def test_zero_free_balance_overrun_is_safe(
key = _make_key(balance=1000, reserved=1000)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 100}}
@@ -319,7 +338,11 @@ async def test_parallel_requests_no_free_inference(
"""Second parallel finalization must be charged even when first depleted free balance."""
import asyncio
from routstr.auth import adjust_payment_for_tokens
from routstr.auth import (
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
)
from routstr.core.db import create_session
deducted_max_cost = 100
@@ -340,14 +363,18 @@ async def test_parallel_requests_no_free_inference(
key = ApiKey(
hashed_key=key_hash,
balance=starting_balance,
reserved_balance=deducted_max_cost * 2, # both slots pre-reserved
reserved_balance=0,
total_spent=0,
total_requests=2,
)
session.add(key)
await session.commit()
reservations = []
for _ in range(2):
await pay_for_request(key, deducted_max_cost, session)
reservations.append(await get_reservation_snapshot(key, session))
async def finalize() -> None:
async def finalize(reservation: ReservationSnapshot) -> None:
response_data = {
"model": "test-model",
"usage": {"prompt_tokens": 50, "completion_tokens": 100},
@@ -356,7 +383,11 @@ async def test_parallel_requests_no_free_inference(
fresh_key = await session.get(ApiKey, key_hash)
assert fresh_key is not None
await adjust_payment_for_tokens(
fresh_key, response_data, session, deducted_max_cost, None, None
fresh_key,
response_data,
session,
deducted_max_cost,
reservation_snapshot=reservation,
)
# Patch once around the gather: entering the same patch target from two
@@ -366,7 +397,7 @@ async def test_parallel_requests_no_free_inference(
"routstr.auth.calculate_cost",
return_value=_cost_data(actual_token_cost),
):
await asyncio.gather(finalize(), finalize())
await asyncio.gather(*(finalize(r) for r in reservations))
async with create_session() as session:
final_key = await session.get(ApiKey, key_hash)
+127 -7
View File
@@ -14,13 +14,17 @@ from unittest.mock import patch
import httpx
import pytest
from httpx import AsyncClient
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey, ReservationRelease
from routstr.payment.models import Architecture, Model, Pricing
from routstr.proxy import refresh_model_maps
from routstr.upstream.base import BaseUpstreamProvider
CHEAP_BASE_URL = "https://cheap.example.com/v1"
EXPENSIVE_BASE_URL = "https://expensive.example.com/v1"
THIRD_BASE_URL = "https://third.example.com/v1"
def _make_model(
@@ -55,9 +59,7 @@ def _make_model(
class _StaticProvider(BaseUpstreamProvider):
"""Upstream provider with a fixed model catalog and no remote refresh."""
def __init__(
self, base_url: str, api_key: str, fee: float, model: Model
) -> None:
def __init__(self, base_url: str, api_key: str, fee: float, model: Model) -> None:
super().__init__(base_url, api_key, fee)
self.provider_type = "custom"
self._static_model = model
@@ -98,7 +100,7 @@ async def dual_provider_maps(
EXPENSIVE_BASE_URL,
"key-expensive",
1.0,
_make_model("provb/dual-model", 0.005, 0.010),
_make_model("provb/dual-model", 0.005, 0.010, max_cost=100.0),
)
async for _ in _install_providers([cheap, expensive]):
yield cheap, expensive
@@ -142,6 +144,7 @@ def _upstream_response(request: httpx.Request) -> httpx.Response:
async def test_failover_serve_billed_at_serving_providers_rate(
authenticated_client: AsyncClient,
dual_provider_maps: tuple[_StaticProvider, _StaticProvider],
integration_session: AsyncSession,
) -> None:
"""A fallback serve is billed at the fallback's price, not the winner's.
@@ -199,6 +202,17 @@ async def test_failover_serve_billed_at_serving_providers_rate(
# Billed at the serving provider's rate: 1000/1000*5000 + 500/1000*10000.
assert payload["cost"]["total_msats"] == 10_000
# The fallback's larger max-cost envelope requires a replacement
# reservation. The failed candidate is released, the serving candidate is
# charged, and no request-owned reservation remains active.
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
records = (
await integration_session.exec(
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
)
).all()
assert sorted(record.status for record in records) == ["charged", "released"]
@pytest.fixture
async def same_id_provider_maps(
@@ -349,9 +363,7 @@ async def test_usd_cost_serve_carries_serving_providers_fee(
if request.url.host == "cheap.example.com":
return httpx.Response(
502,
content=json.dumps(
{"error": {"message": "bad gateway"}}
).encode(),
content=json.dumps({"error": {"message": "bad gateway"}}).encode(),
headers={"content-type": "application/json"},
)
body = {
@@ -478,6 +490,101 @@ async def test_failover_beyond_balance_envelope_is_rejected(
assert [r.url.host for r in sent_requests] == ["cheap.example.com"]
@pytest.fixture
async def three_candidate_child_maps(
patched_db_engine: None,
) -> AsyncGenerator[None, None]:
"""Second candidate cannot fit the child limit; third restores and serves."""
first = _StaticProvider(
CHEAP_BASE_URL,
"key-first",
1.0,
_make_model("dual-model", 0.001, 0.002, max_cost=50.0),
)
too_large = _StaticProvider(
EXPENSIVE_BASE_URL,
"key-too-large",
1.0,
_make_model("dual-model", 0.002, 0.003, max_cost=100.0),
)
third = _StaticProvider(
THIRD_BASE_URL,
"key-third",
1.0,
_make_model("dual-model", 0.003, 0.004, max_cost=50.0),
)
async for _ in _install_providers([first, too_large, third]):
yield
@pytest.mark.integration
@pytest.mark.asyncio
async def test_child_failover_rolls_back_failed_larger_reserve_before_restoring(
authenticated_client: AsyncClient,
three_candidate_child_maps: None,
integration_session: AsyncSession,
) -> None:
"""A failed child guard cannot leak its parent update into restoration."""
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
child = await integration_session.get(ApiKey, key_hash)
assert child is not None
parent = ApiKey(hashed_key="failover-parent", balance=10_000_000)
child.parent_key_hash = parent.hashed_key
child.balance_limit = 75_000
integration_session.add(parent)
integration_session.add(child)
await integration_session.commit()
sent_requests: list[httpx.Request] = []
async def fake_transport(
request: httpx.Request, *args: Any, **kwargs: Any
) -> httpx.Response:
sent_requests.append(request)
return _upstream_response(request)
with (
patch(
"httpx.AsyncHTTPTransport.handle_async_request",
side_effect=fake_transport,
),
patch(
"routstr.payment.cost_calculation.sats_usd_price",
return_value=0.0005,
),
):
response = await authenticated_client.post(
"/v1/chat/completions",
json={
"model": "dual-model",
"messages": [{"role": "user", "content": "hello"}],
},
)
assert response.status_code == 200
# The 100-sat candidate is rejected before forwarding; the third serves.
assert [request.url.host for request in sent_requests] == [
"cheap.example.com",
"third.example.com",
]
await integration_session.refresh(parent)
await integration_session.refresh(child)
assert parent.reserved_balance == 0
assert child.reserved_balance == 0
assert parent.total_spent == response.json()["cost"]["total_msats"]
records = (
await integration_session.exec(
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
)
).all()
assert len(records) == 2
assert sorted(record.status for record in records) == ["charged", "released"]
assert len({record.reserved_msats for record in records}) == 1
assert all(record.status != "active" for record in records)
@pytest.fixture
async def raised_envelope_provider_maps(
patched_db_engine: None,
@@ -504,6 +611,7 @@ async def raised_envelope_provider_maps(
async def test_failover_reserves_serving_candidates_envelope(
authenticated_client: AsyncClient,
raised_envelope_provider_maps: None,
integration_session: AsyncSession,
) -> None:
"""An affordable pricier fallback is re-reserved, served, and billed.
@@ -544,3 +652,15 @@ async def test_failover_reserves_serving_candidates_envelope(
"expensive.example.com",
]
assert response.json()["cost"]["total_msats"] == 10_000
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
records = (
await integration_session.exec(
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
)
).all()
assert len(records) == 2
released = next(record for record in records if record.status == "released")
charged = next(record for record in records if record.status == "charged")
assert charged.reserved_msats > released.reserved_msats
assert all(record.status != "active" for record in records)
@@ -38,7 +38,7 @@ async def test_overrun_charges_after_reservation_swept(
integration_session: AsyncSession,
) -> None:
"""Overrun finalize must charge even when the reservation was already released."""
from routstr.auth import adjust_payment_for_tokens
from routstr.auth import adjust_payment_for_tokens, pay_for_request
deducted_max_cost = 990 # discounted reservation
actual_token_cost = 1000 # actual cost overruns the reservation
@@ -47,6 +47,10 @@ async def test_overrun_charges_after_reservation_swept(
key = _make_key(balance=1000, reserved=0)
integration_session.add(key)
await integration_session.commit()
await pay_for_request(key, deducted_max_cost, integration_session)
key.reserved_balance = 0
integration_session.add(key)
await integration_session.commit()
response_data = {
"model": "test-model",
@@ -79,8 +83,16 @@ async def test_free_response_path_closed_end_to_end(
patched_db_engine: None,
) -> None:
"""A reservation released by the real sweeper must not yield a free response."""
from routstr.auth import adjust_payment_for_tokens, pay_for_request
from routstr.core.db import create_session, release_stale_reservations
from routstr.auth import (
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
)
from routstr.core.db import (
ReservationRelease,
create_session,
release_stale_reservations,
)
deducted_max_cost = 990
actual_token_cost = 1000
@@ -104,10 +116,15 @@ async def test_free_response_path_closed_end_to_end(
key = await session.get(ApiKey, key_hash)
assert key is not None
await pay_for_request(key, deducted_max_cost, session)
snapshot = await get_reservation_snapshot(key, session)
await session.refresh(key)
assert key.reserved_balance == deducted_max_cost
key.reserved_at = int(time.time()) - 10_000
record = await session.get(ReservationRelease, snapshot.release_id)
assert record is not None
record.created_at = int(time.time()) - 10_000
session.add(key)
session.add(record)
await session.commit()
# Sweeper releases the stale reservation without charging.
@@ -129,18 +146,20 @@ async def test_free_response_path_closed_end_to_end(
return_value=_cost_data(actual_token_cost),
):
await adjust_payment_for_tokens(
key, response_data, session, deducted_max_cost, None, None
key,
response_data,
session,
deducted_max_cost,
reservation_snapshot=snapshot,
)
async with create_session() as session:
final = await session.get(ApiKey, key_hash)
assert final is not None
assert final.total_spent == actual_token_cost, (
f"Free response: total_spent={final.total_spent}, expected {actual_token_cost}"
)
assert final.balance == 1000 - actual_token_cost, (
f"Balance not charged after sweep: {final.balance}"
)
# Stale release is terminal for this reservation. A late finalizer must not
# charge aggregate balance that may now belong to a newer request.
assert final.total_spent == 0
assert final.balance == 1000
assert final.balance >= 0
assert final.reserved_balance == 0
@@ -141,7 +141,7 @@ async def test_revert_with_zero_reserved_balance_is_noop(
Previously this would drive reserved_balance negative. With the floor guard,
it should return False and leave reserved_balance at 0.
"""
from routstr.auth import revert_pay_for_request
from routstr.auth import pay_for_request, revert_pay_for_request
unique_key = f"test_revert_key_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
@@ -151,8 +151,12 @@ async def test_revert_with_zero_reserved_balance_is_noop(
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 100, integration_session)
test_key.reserved_balance = 0
integration_session.add(test_key)
await integration_session.commit()
# Try to revert more than available — should be a no-op
# A stale cleanup already released the aggregate reservation.
result = await revert_pay_for_request(test_key, integration_session, 100)
await integration_session.refresh(test_key)
@@ -161,8 +165,8 @@ async def test_revert_with_zero_reserved_balance_is_noop(
assert test_key.reserved_balance == 0, (
f"Reserved balance should remain 0, got: {test_key.reserved_balance}"
)
assert test_key.total_requests == 0, (
f"Total requests should remain 0, got: {test_key.total_requests}"
assert test_key.total_requests == 1, (
f"Total requests should remain 1, got: {test_key.total_requests}"
)
@@ -171,17 +175,18 @@ async def test_revert_with_sufficient_reserved_balance_succeeds(
integration_session: AsyncSession,
) -> None:
"""Test that revert_pay_for_request works correctly when there is enough reserved balance."""
from routstr.auth import revert_pay_for_request
from routstr.auth import pay_for_request, revert_pay_for_request
unique_key = f"test_revert_ok_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=5000,
reserved_balance=500,
total_requests=3,
reserved_balance=0,
total_requests=2,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
result = await revert_pay_for_request(test_key, integration_session, 500)
@@ -202,17 +207,21 @@ async def test_revert_partial_reserved_balance_is_noop(
integration_session: AsyncSession,
) -> None:
"""Test that reverting more than the current reserved_balance is a no-op."""
from routstr.auth import revert_pay_for_request
from routstr.auth import pay_for_request, revert_pay_for_request
unique_key = f"test_revert_partial_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=5000,
reserved_balance=50,
total_requests=1,
reserved_balance=0,
total_requests=0,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
test_key.reserved_balance = 50
integration_session.add(test_key)
await integration_session.commit()
# Try to revert 500 when only 50 is reserved — should be no-op
result = await revert_pay_for_request(test_key, integration_session, 500)
@@ -237,20 +246,28 @@ async def test_double_revert_prevented(
This simulates the double-revert scenario where both upstream/base.py
and proxy.py attempt to revert the same reservation.
"""
from routstr.auth import revert_pay_for_request
from routstr.auth import (
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
)
unique_key = f"test_double_revert_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=10000,
reserved_balance=500,
total_requests=5,
reserved_balance=0,
total_requests=4,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
snapshot = await get_reservation_snapshot(test_key, integration_session)
# First revert — should succeed
result1 = await revert_pay_for_request(test_key, integration_session, 500)
result1 = await revert_pay_for_request(
test_key, integration_session, 500, snapshot
)
await integration_session.refresh(test_key)
assert result1 is True
@@ -258,7 +275,9 @@ async def test_double_revert_prevented(
assert test_key.total_requests == 4
# Second revert of the same amount — should be no-op
result2 = await revert_pay_for_request(test_key, integration_session, 500)
result2 = await revert_pay_for_request(
test_key, integration_session, 500, snapshot
)
await integration_session.refresh(test_key)
assert result2 is False, "Second revert should be a no-op"
@@ -279,22 +298,30 @@ async def test_sequential_reverts_never_go_negative(
Simulates the double-revert scenario where multiple code paths
attempt to revert the same reservation.
"""
from routstr.auth import revert_pay_for_request
from routstr.auth import (
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
)
unique_key = f"test_multi_revert_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=10000,
reserved_balance=500,
total_requests=5,
reserved_balance=0,
total_requests=4,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
snapshot = await get_reservation_snapshot(test_key, integration_session)
# Run 5 sequential reverts for the same 500 reservation
results = []
for _ in range(5):
r = await revert_pay_for_request(test_key, integration_session, 500)
r = await revert_pay_for_request(
test_key, integration_session, 500, snapshot
)
results.append(r)
await integration_session.refresh(test_key)
@@ -317,7 +344,11 @@ async def test_child_key_revert_floor_guard(
integration_session: AsyncSession,
) -> None:
"""Test that child key reserved_balance also has floor guard on revert."""
from routstr.auth import revert_pay_for_request
from routstr.auth import (
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
)
parent_key_hash = f"test_parent_{uuid.uuid4().hex[:8]}"
child_key_hash = f"test_child_{uuid.uuid4().hex[:8]}"
@@ -325,22 +356,26 @@ async def test_child_key_revert_floor_guard(
parent_key = ApiKey(
hashed_key=parent_key_hash,
balance=10000,
reserved_balance=500,
total_requests=3,
reserved_balance=0,
total_requests=2,
)
child_key = ApiKey(
hashed_key=child_key_hash,
balance=0,
reserved_balance=500,
total_requests=3,
reserved_balance=0,
total_requests=2,
parent_key_hash=parent_key_hash,
)
integration_session.add(parent_key)
integration_session.add(child_key)
await integration_session.commit()
await pay_for_request(child_key, 500, integration_session)
snapshot = await get_reservation_snapshot(child_key, integration_session)
# First revert succeeds
result1 = await revert_pay_for_request(child_key, integration_session, 500)
result1 = await revert_pay_for_request(
child_key, integration_session, 500, snapshot
)
await integration_session.refresh(parent_key)
await integration_session.refresh(child_key)
@@ -349,7 +384,9 @@ async def test_child_key_revert_floor_guard(
assert child_key.reserved_balance == 0
# Second revert is a no-op for both parent and child
result2 = await revert_pay_for_request(child_key, integration_session, 500)
result2 = await revert_pay_for_request(
child_key, integration_session, 500, snapshot
)
await integration_session.refresh(parent_key)
await integration_session.refresh(child_key)
@@ -0,0 +1,76 @@
"""Tests for the ``reset_admin_password`` recovery script (issue #553).
The script is the lockout escape hatch: it works without ``ROUTSTR_SECRET_KEY``
(scrypt hashing is key-independent). Two explicit, mutually exclusive actions
``--password`` sets a new hash now, ``--regenerate`` clears the hash so the next
boot generates and logs a fresh one. A bare invocation is informational only and
must never touch the database (so nobody resets their password by accident).
"""
import pytest
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core import vault
from routstr.core.db import get_secret, set_admin_password
from scripts.reset_admin_password import apply_reset, build_parser, main
@pytest.mark.asyncio
async def test_password_sets_a_verifiable_hash(
integration_session: AsyncSession,
) -> None:
await apply_reset(integration_session, password="recover-me-123")
secret = await get_secret(integration_session)
assert secret.admin_password_hash is not None
assert vault.verify_password("recover-me-123", secret.admin_password_hash) is True
assert secret.updated_at is not None
@pytest.mark.asyncio
async def test_regenerate_clears_the_hash(
integration_session: AsyncSession,
) -> None:
# Start from a node that already has an admin password set.
await set_admin_password(integration_session, "old-password-9")
assert (await get_secret(integration_session)).admin_password_hash is not None
await apply_reset(integration_session, regenerate=True)
secret = await get_secret(integration_session)
# Cleared -> the next boot's bootstrap_secrets generates and logs a new one.
assert secret.admin_password_hash is None
assert secret.updated_at is not None
@pytest.mark.asyncio
async def test_password_below_min_length_is_rejected(
integration_session: AsyncSession,
) -> None:
await set_admin_password(integration_session, "old-password-9")
with pytest.raises(ValueError, match="8 characters"):
await apply_reset(integration_session, password="short")
# The existing password is untouched by the rejected reset.
secret = await get_secret(integration_session)
assert vault.verify_password("old-password-9", secret.admin_password_hash or "")
def test_password_and_regenerate_are_mutually_exclusive() -> None:
parser = build_parser()
with pytest.raises(SystemExit):
parser.parse_args(["--password", "abcd1234", "--regenerate"])
def test_no_args_prints_help_and_never_opens_a_session(
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch,
) -> None:
def _fail() -> None:
raise AssertionError("a bare invocation must not touch the database")
monkeypatch.setattr("scripts.reset_admin_password.create_session", _fail)
assert main([]) == 0
assert "usage" in capsys.readouterr().out.lower()
+484
View File
@@ -0,0 +1,484 @@
"""Tests for ``bootstrap_secrets`` — moving node secrets into the Secret store.
Specifies the per-secret bootstrap that runs at startup (issue #553). For both
the admin password and the nsec it follows the same three branches: use the
column if already set, otherwise migrate any legacy plaintext (env first, then
the old settings blob), otherwise admin password only generate and log one.
A column written under a different ROUTSTR_SECRET_KEY fails fast rather than
silently corrupting state.
"""
import json
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any, AsyncGenerator
import pytest
from sqlmodel import text
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core import vault
from routstr.core.db import NsecState, get_secret, set_nsec
from routstr.core.settings import (
SettingsService,
bootstrap_secrets,
derive_npub_from_nsec,
settings,
)
# Valid Fernet keys; must match the suite default in tests/conftest.py.
TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU="
TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ="
NSEC_HEX = "1" * 64
# A different key, standing in for a stale value left behind in env/blob after
# the vault has taken ownership of the real one.
STALE_NSEC_HEX = "2" * 64
@pytest.fixture
def clean_secret_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""No ambient legacy secrets, and a known in-memory settings baseline."""
monkeypatch.delenv("ADMIN_PASSWORD", raising=False)
monkeypatch.delenv("NSEC", raising=False)
monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY)
monkeypatch.setattr(settings, "nsec", "")
monkeypatch.setattr(settings, "npub", "")
monkeypatch.setattr(settings, "http_url", "")
async def _create_settings_blob(session: AsyncSession, data: dict) -> None:
await session.exec( # type: ignore
text(
"CREATE TABLE IF NOT EXISTS settings "
"(id INTEGER PRIMARY KEY, data TEXT NOT NULL, "
"updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)"
)
)
await session.exec( # type: ignore
text("INSERT INTO settings (id, data) VALUES (1, :data)").bindparams(
data=json.dumps(data)
)
)
await session.commit()
# --- admin password --------------------------------------------------------
@pytest.mark.asyncio
async def test_generates_admin_password_when_none(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
await bootstrap_secrets(integration_session)
secret = await get_secret(integration_session)
assert secret.admin_password_hash is not None
assert secret.admin_password_hash.startswith("scrypt:")
@pytest.mark.asyncio
async def test_admin_password_generation_is_idempotent(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
await bootstrap_secrets(integration_session)
first = (await get_secret(integration_session)).admin_password_hash
await bootstrap_secrets(integration_session)
second = (await get_secret(integration_session)).admin_password_hash
assert first is not None and first == second
@pytest.mark.asyncio
async def test_hashes_legacy_admin_password_from_env(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("ADMIN_PASSWORD", "hunter2")
await bootstrap_secrets(integration_session)
secret = await get_secret(integration_session)
assert secret.admin_password_hash is not None
assert vault.verify_password("hunter2", secret.admin_password_hash) is True
@pytest.mark.asyncio
async def test_hashes_legacy_admin_password_from_blob(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
# No ADMIN_PASSWORD in env, but the old settings blob carries one.
await _create_settings_blob(integration_session, {"admin_password": "blobpw"})
await bootstrap_secrets(integration_session)
secret = await get_secret(integration_session)
assert vault.verify_password("blobpw", secret.admin_password_hash or "") is True
@pytest.mark.asyncio
async def test_admin_password_race_adopts_winner_without_clobber(
clean_secret_env: None,
integration_engine: Any,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
# Two workers boot against one shared DB and both read a null admin password.
# The first to commit "wins" and shows the operator its generated password. A
# worker that read null but lost the race must NOT overwrite the winner's hash
# (which the operator may already be logging in with) and must NOT print a
# second password that will never work.
#
# The race window is forced deterministically: a hook fires inside bootstrap's
# generate branch (so it only runs once this worker has committed to
# generating) and commits the winner's password on a separate connection
# before this worker writes its own.
import sqlite3
from routstr.core import settings as settings_mod
db_file = integration_engine.url.database
winner_hash = vault.hash_password("winner-password-123")
real_token = settings_mod.secrets.token_urlsafe
def commit_winner_then_generate(nbytes: int) -> str:
conn = sqlite3.connect(db_file)
conn.execute(
"UPDATE secrets SET admin_password_hash = ? WHERE id = 1", (winner_hash,)
)
conn.commit()
conn.close()
return real_token(nbytes)
monkeypatch.setattr(
settings_mod.secrets, "token_urlsafe", commit_winner_then_generate
)
await get_secret(integration_session) # row exists, password still null
capsys.readouterr() # drop anything emitted before the race resolves
await bootstrap_secrets(integration_session)
secret = await get_secret(integration_session)
assert secret.admin_password_hash is not None
# The winner's password survives and still verifies — no clobber.
assert vault.verify_password("winner-password-123", secret.admin_password_hash)
# The losing worker stayed silent — no second generated password was leaked.
assert "generated a temporary" not in capsys.readouterr().out
# --- nsec ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_encrypts_legacy_nsec_from_env_and_derives_npub(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("NSEC", NSEC_HEX)
await bootstrap_secrets(integration_session)
secret = await get_secret(integration_session)
assert secret.encrypted_nsec is not None
assert vault.is_encrypted(secret.encrypted_nsec) is True
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
# In-memory runtime value is the decrypted nsec, and npub is derived from it.
assert settings.nsec == NSEC_HEX
assert settings.npub == derive_npub_from_nsec(NSEC_HEX)
@pytest.mark.asyncio
async def test_decrypts_existing_nsec_column(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
secret = await get_secret(integration_session)
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
secret.nsec_state = NsecState.encrypted
integration_session.add(secret)
await integration_session.commit()
stored = secret.encrypted_nsec
await bootstrap_secrets(integration_session)
reloaded = await get_secret(integration_session)
assert settings.nsec == NSEC_HEX
# The column is reused, not re-encrypted.
assert reloaded.encrypted_nsec == stored
@pytest.mark.asyncio
async def test_fail_fast_when_nsec_encrypted_with_different_key(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Encrypt the column under the alternate key, then bootstrap under the
# suite key -> the value cannot be decrypted -> clear startup failure.
monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY_ALT)
secret = await get_secret(integration_session)
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
secret.nsec_state = NsecState.encrypted
integration_session.add(secret)
await integration_session.commit()
monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY)
with pytest.raises(RuntimeError, match="ROUTSTR_SECRET_KEY"):
await bootstrap_secrets(integration_session)
# --- encryption is mandatory, key custody is not: upgrade without a key --------
@pytest.mark.asyncio
async def test_legacy_nsec_without_secret_key_generates_and_encrypts(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
) -> None:
# A node upgrading with a legacy plaintext NSEC but no ROUTSTR_SECRET_KEY must
# NOT break. Encryption at rest stays mandatory (the nsec is never persisted
# in plaintext), but the key custody is flexible: bootstrap generates a master
# key, persists it to the key file, warns loudly, and encrypts the identity —
# so the node keeps running instead of refusing to boot.
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
key_file = tmp_path / "routstr_secret.key"
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
monkeypatch.setenv("NSEC", NSEC_HEX)
await bootstrap_secrets(integration_session)
# A master key was generated and persisted...
assert key_file.exists()
# ...the nsec is encrypted at rest under it, never stored in plaintext...
secret = await get_secret(integration_session)
assert secret.encrypted_nsec is not None
assert vault.is_encrypted(secret.encrypted_nsec) is True
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
assert secret.nsec_state == NsecState.encrypted
# ...the node holds the live identity (npub derived from it)...
assert settings.nsec == NSEC_HEX
assert settings.npub == derive_npub_from_nsec(NSEC_HEX)
# ...and the operator is loudly told a key was generated and must be backed up
# (path shown, but never the key value) so an upgrade cannot silently create
# an unbacked key nor leak the key into captured stdout / aggregated logs.
out = capsys.readouterr().out
assert str(key_file) in out
assert key_file.read_text().strip() not in out
assert "BACK UP" in out.upper()
# --- boot ordering: rescue legacy blob secrets before they are stripped ----
@pytest.mark.asyncio
async def test_blob_only_nsec_is_migrated_before_blob_is_stripped(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
# Legacy node whose nsec lives ONLY in the settings blob (never in env).
# bootstrap_secrets must run *before* SettingsService.initialize strips the
# blob, or the only copy of the secret would be lost.
await _create_settings_blob(
integration_session, {"nsec": NSEC_HEX, "name": "LegacyNode"}
)
await bootstrap_secrets(integration_session)
await SettingsService.initialize(integration_session)
secret = await get_secret(integration_session)
# The plaintext nsec has been moved into the encrypted Secret store...
assert secret.encrypted_nsec is not None
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
assert settings.nsec == NSEC_HEX
# ...and stripped from the persisted settings blob.
row = await integration_session.exec( # type: ignore
text("SELECT data FROM settings WHERE id = 1")
)
blob = json.loads(row.first()[0])
assert "nsec" not in blob
assert blob["name"] == "LegacyNode"
@pytest.mark.asyncio
async def test_initialize_does_not_clobber_store_only_nsec(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
# Steady state after migration: the nsec lives ONLY in the encrypted Secret
# store (NSEC removed from env, blob already stripped on a previous boot).
# bootstrap decrypts it into memory; initialize then re-derives settings from
# the secret-free blob and must NOT wipe the live nsec back to empty (or the
# node would silently stop signing Nostr announcements).
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
secret = await get_secret(integration_session)
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
secret.nsec_state = NsecState.encrypted
integration_session.add(secret)
await integration_session.commit()
await bootstrap_secrets(integration_session)
assert settings.nsec == NSEC_HEX # bootstrap decrypted it into memory
await SettingsService.initialize(integration_session)
# The live secret survives initialize even though no env/blob carries it...
assert settings.nsec == NSEC_HEX
# ...and is still never written back to the persisted blob.
row = await integration_session.exec( # type: ignore
text("SELECT data FROM settings WHERE id = 1")
)
assert "nsec" not in json.loads(row.first()[0])
@pytest.mark.asyncio
async def test_stale_env_nsec_does_not_override_vault_nsec(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# The vault owns the nsec, but a stale NSEC (e.g. the operator rotated the
# key in the UI yet left the old value in .env) is still in the environment.
# bootstrap decrypts the store value; initialize must NOT let the stale env
# value clobber it, or a restart silently reverts to the old identity.
await set_nsec(integration_session, NSEC_HEX)
monkeypatch.setenv("NSEC", STALE_NSEC_HEX)
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
await bootstrap_secrets(integration_session)
await SettingsService.initialize(integration_session)
# The vault value wins; the stale env value is ignored.
assert settings.nsec == NSEC_HEX
@pytest.mark.asyncio
async def test_stale_env_nsec_does_not_split_npub_from_vault_nsec(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# As above, the vault owns the nsec while a stale NSEC lingers in env. The
# private key correctly comes from the vault, but the npub must too: if
# initialize derives the public key from the stale env nsec, the node ends up
# with a private key from the vault and a public key from the old env value,
# and anything reading settings.npub announces the wrong Nostr identity.
expected_npub = derive_npub_from_nsec(NSEC_HEX)
stale_npub = derive_npub_from_nsec(STALE_NSEC_HEX)
assert expected_npub and stale_npub and expected_npub != stale_npub # guard
await set_nsec(integration_session, NSEC_HEX)
monkeypatch.setenv("NSEC", STALE_NSEC_HEX)
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
await bootstrap_secrets(integration_session)
await SettingsService.initialize(integration_session)
assert settings.nsec == NSEC_HEX
assert settings.npub == expected_npub
@pytest.mark.asyncio
async def test_cleared_nsec_stays_cleared_across_reboot(
clean_secret_env: None,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# An identity was imported from env, then the operator cleared it via the
# admin API. The old NSEC is still in env. On the NEXT PROCESS the cleared
# identity must stay cleared, not get resurrected from the stale env value.
monkeypatch.setenv("NSEC", NSEC_HEX)
await bootstrap_secrets(integration_session)
assert settings.nsec == NSEC_HEX
# Clear via the admin path (store empty, vault owns it).
await set_nsec(integration_session, "")
# Simulate a fresh process rather than pre-clearing the live singleton: the
# pydantic settings global reloads the (still-stale) NSEC from env and derives
# its npub, which is exactly the in-memory state a new boot starts from before
# bootstrap runs. The cleared store must win over this stale live value.
monkeypatch.setattr(settings, "nsec", NSEC_HEX)
monkeypatch.setattr(settings, "npub", derive_npub_from_nsec(NSEC_HEX))
await bootstrap_secrets(integration_session)
reloaded = await get_secret(integration_session)
assert reloaded.nsec_state == NsecState.cleared
assert reloaded.encrypted_nsec is None # not re-imported
assert settings.nsec == "" # stays cleared
assert settings.npub == "" # and no derived public identity survives
@pytest.mark.asyncio
async def test_initialize_keeps_npub_matching_store_only_nsec(
clean_secret_env: None, integration_session: AsyncSession
) -> None:
# Steady state with mandatory encryption: the nsec lives ONLY in the
# encrypted Secret store (env carries no NSEC) and the blob has no npub.
# bootstrap decrypts the nsec and derives the npub into memory; initialize
# then re-derives settings from the npub-less blob and must NOT wipe the npub
# back to empty, or the node holds a private key with no matching public key
# and silently stops announcing a usable Nostr identity.
expected_npub = derive_npub_from_nsec(NSEC_HEX)
assert expected_npub # guard: the test key must yield a real npub
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
secret = await get_secret(integration_session)
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
secret.nsec_state = NsecState.encrypted
integration_session.add(secret)
await integration_session.commit()
await bootstrap_secrets(integration_session)
assert settings.npub == expected_npub # bootstrap derived it
await SettingsService.initialize(integration_session)
# The npub still matches the live nsec...
assert settings.nsec == NSEC_HEX
assert settings.npub == expected_npub
# ...and is persisted to the blob (it is public, not a stripped secret).
row = await integration_session.exec( # type: ignore
text("SELECT data FROM settings WHERE id = 1")
)
assert json.loads(row.first()[0])["npub"] == expected_npub
@pytest.mark.asyncio
async def test_startup_runs_bootstrap_before_settings_initialize(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# The two tests above prove the migration outcome *given* the call order;
# they hardcode that order themselves. This one guards the order at its real
# call site — the application lifespan — so a reorder in main.py (which would
# strip a blob-only secret before bootstrap could rescue it) is caught.
import routstr.core.main as main
order: list[str] = []
class _Abort(Exception):
pass
@asynccontextmanager
async def fake_create_session() -> AsyncGenerator[None, None]:
yield None
async def fake_bootstrap(session: Any) -> None:
order.append("bootstrap")
async def fake_initialize(session: Any) -> None:
order.append("initialize")
# Stop startup here, before the background-task fan-out (prices, nostr,
# upstreams) that we don't want to run in a unit test.
raise _Abort()
async def noop_init_db() -> None:
return None
monkeypatch.setattr(main, "configure_litellm", lambda: None)
monkeypatch.setattr(main, "register_deepseek_v4_pricing", lambda: None)
monkeypatch.setattr(main, "run_migrations", lambda: None)
monkeypatch.setattr(main, "init_db", noop_init_db)
monkeypatch.setattr(main, "create_session", fake_create_session)
monkeypatch.setattr(main, "bootstrap_secrets", fake_bootstrap)
monkeypatch.setattr(main.SettingsService, "initialize", fake_initialize)
with pytest.raises(_Abort):
async with main.lifespan(main.app):
pass
assert order == ["bootstrap", "initialize"]
+93
View File
@@ -0,0 +1,93 @@
"""Tests for the ``Secret`` singleton model (issue #553).
Specifies the node-level secret store: a single row (``id=1``, like
``RoutstrFee``) holding the one-way admin-password hash and the encrypted nsec.
``get_secret`` is get-or-create, so callers always get the singleton without
worrying whether it has been initialised yet. Encoding of the values themselves
lives in ``routstr.core.vault``; here we only assert the row persists and stays
a singleton.
"""
import time
from typing import Any
import pytest
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import Secret, get_secret
@pytest.mark.asyncio
async def test_get_secret_creates_singleton(
integration_session: AsyncSession,
) -> None:
secret = await get_secret(integration_session)
assert secret.id == 1
# Fresh row carries no secret material yet.
assert secret.admin_password_hash is None
assert secret.encrypted_nsec is None
assert secret.updated_at is None
@pytest.mark.asyncio
async def test_get_secret_is_idempotent(
integration_session: AsyncSession,
) -> None:
first = await get_secret(integration_session)
second = await get_secret(integration_session)
assert first.id == second.id == 1
rows = (await integration_session.exec(select(Secret))).all()
assert len(rows) == 1
@pytest.mark.asyncio
async def test_secret_fields_round_trip(
integration_session: AsyncSession,
) -> None:
secret = await get_secret(integration_session)
secret.admin_password_hash = "scrypt:16384:8:1:c2FsdA==:aGFzaA=="
secret.encrypted_nsec = "fernet:v1:gAAAAA"
secret.updated_at = int(time.time())
integration_session.add(secret)
await integration_session.commit()
integration_session.expunge_all()
reloaded = await get_secret(integration_session)
assert reloaded.admin_password_hash == "scrypt:16384:8:1:c2FsdA==:aGFzaA=="
assert reloaded.encrypted_nsec == "fernet:v1:gAAAAA"
assert reloaded.updated_at is not None
@pytest.mark.asyncio
async def test_get_secret_tolerates_concurrent_first_insert(
integration_engine: Any,
integration_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
# A second worker wins the race and commits the singleton row first.
async with AsyncSession(integration_engine, expire_on_commit=False) as other:
other.add(Secret(id=1, admin_password_hash="scrypt:from-other-worker"))
await other.commit()
# Reproduce the race window: our session's first read still sees no row, so
# it attempts to INSERT a duplicate id=1. The real IntegrityError that follows
# must be recovered (roll back, re-read) rather than crashing startup.
real_get = integration_session.get
calls = {"n": 0}
async def stale_first_read(model: Any, pk: Any) -> Any:
calls["n"] += 1
if calls["n"] == 1:
return None
return await real_get(model, pk)
monkeypatch.setattr(integration_session, "get", stale_first_read)
secret = await get_secret(integration_session)
# Recovered the other worker's row; no crash, still a single row.
assert secret.id == 1
assert secret.admin_password_hash == "scrypt:from-other-worker"
rows = (await integration_session.exec(select(Secret))).all()
assert len(rows) == 1
+40
View File
@@ -0,0 +1,40 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from fastapi import FastAPI
from httpx import ASGITransport, AsyncClient
from routstr import balance as balance_module
from routstr.core.db import get_session
@pytest.mark.asyncio
async def test_create_balance_accepts_large_cashu_token_in_post_body(
monkeypatch: pytest.MonkeyPatch,
) -> None:
token = "cashuA" + "x" * 20_000
key = SimpleNamespace(hashed_key="hashed", balance=123_000)
validate_bearer_key = AsyncMock(return_value=key)
session = AsyncMock()
monkeypatch.setattr(balance_module, "validate_bearer_key", validate_bearer_key)
async def override_get_session(): # type: ignore[no-untyped-def]
yield session
app = FastAPI()
app.include_router(balance_module.balance_router)
app.dependency_overrides[get_session] = override_get_session
async with AsyncClient(
transport=ASGITransport(app=app), # type: ignore[arg-type]
base_url="http://test",
) as client:
response = await client.post(
"/v1/balance/create",
json={"initial_balance_token": token},
)
assert response.status_code == 200
assert response.json() == {"api_key": "sk-hashed", "balance": 123_000}
validate_bearer_key.assert_awaited_once_with(token, session)
+63 -40
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from typing import AsyncGenerator
from typing import Any, AsyncGenerator
from unittest.mock import AsyncMock, MagicMock
import pytest
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
@@ -8,7 +9,8 @@ from sqlalchemy.pool import StaticPool
from sqlmodel import SQLModel, select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey
from routstr.auth import get_reservation_snapshot, pay_for_request
from routstr.core.db import ApiKey, ReservationRelease
from routstr.upstream.ehbp import (
finalize_ehbp_actual_cost_payment,
finalize_ehbp_max_cost_payment,
@@ -43,18 +45,38 @@ async def _api_key(session: AsyncSession, hashed_key: str) -> ApiKey | None:
).one_or_none()
def _fail_nth_api_key_update(
session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
target_update: int,
) -> None:
"""Return rowcount=0 for one API-key UPDATE without mutating the database."""
original_exec = session.exec
api_key_updates = 0
async def exec_with_failure(
statement: Any, *args: Any, **kwargs: Any
) -> Any:
nonlocal api_key_updates
table = getattr(statement, "table", None)
if getattr(table, "name", None) == "api_keys":
api_key_updates += 1
if api_key_updates == target_update:
return MagicMock(rowcount=0)
return await original_exec(statement, *args, **kwargs)
monkeypatch.setattr(session, "exec", exec_with_failure)
@pytest.mark.asyncio
async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve(
session: AsyncSession,
) -> None:
key = ApiKey(
hashed_key="ehbp-actual",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
key = ApiKey(hashed_key="ehbp-actual", balance=10_000)
session.add(key)
await session.commit()
await pay_for_request(key, 3_000, session)
reservation = await get_reservation_snapshot(key, session)
await finalize_ehbp_actual_cost_payment(
key,
@@ -68,6 +90,7 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
"input_msats": 500,
"output_msats": 700,
},
reservation_snapshot=reservation,
)
updated = await _api_key(session, "ehbp-actual")
@@ -82,28 +105,22 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
session: AsyncSession,
) -> None:
parent = ApiKey(
hashed_key="ehbp-parent",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
parent = ApiKey(hashed_key="ehbp-parent", balance=10_000)
child = ApiKey(
hashed_key="ehbp-child",
balance=0,
reserved_balance=3_000,
reserved_at=123,
parent_key_hash="ehbp-parent",
hashed_key="ehbp-child", balance=0, parent_key_hash="ehbp-parent"
)
session.add(parent)
session.add(child)
await session.commit()
await pay_for_request(child, 3_000, session)
reservation = await get_reservation_snapshot(child, session)
await finalize_ehbp_max_cost_payment(
child,
session,
max_cost_for_model=3_000,
model_id="tinfoil/model",
reservation_snapshot=reservation,
)
updated_parent = await _api_key(session, "ehbp-parent")
@@ -123,17 +140,16 @@ async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
@pytest.mark.asyncio
async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matches_no_rows(
session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
key = ApiKey(
hashed_key="ehbp-missing-parent",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
key = ApiKey(hashed_key="ehbp-missing-parent", balance=10_000)
session.add(key)
await session.commit()
await session.delete(key)
await session.commit()
await pay_for_request(key, 3_000, session)
reservation = await get_reservation_snapshot(key, session)
_fail_nth_api_key_update(session, monkeypatch, target_update=1)
rollback_spy = AsyncMock(wraps=session.rollback)
monkeypatch.setattr(session, "rollback", rollback_spy)
await finalize_ehbp_actual_cost_payment(
key,
@@ -141,45 +157,52 @@ async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matche
reserved_cost_for_model=3_000,
model_id="tinfoil/model",
cost_info={"total_msats": 1_200},
reservation_snapshot=reservation,
)
assert await _api_key(session, "ehbp-missing-parent") is None
rollback_spy.assert_awaited_once()
updated = await _api_key(session, "ehbp-missing-parent")
assert updated is not None
assert updated.balance == 10_000
assert updated.reserved_balance == 3_000
assert updated.total_spent == 0
release = await session.get(ReservationRelease, reservation.release_id)
assert release is not None
assert release.status == "active"
@pytest.mark.asyncio
async def test_finalize_max_cost_payment_rolls_back_parent_when_child_update_matches_no_rows(
session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
parent = ApiKey(
hashed_key="ehbp-rollback-parent",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
parent = ApiKey(hashed_key="ehbp-rollback-parent", balance=10_000)
child = ApiKey(
hashed_key="ehbp-missing-child",
balance=0,
reserved_balance=3_000,
reserved_at=123,
parent_key_hash="ehbp-rollback-parent",
)
session.add(parent)
session.add(child)
await session.commit()
await session.delete(child)
await session.commit()
await pay_for_request(child, 3_000, session)
reservation = await get_reservation_snapshot(child, session)
_fail_nth_api_key_update(session, monkeypatch, target_update=2)
await finalize_ehbp_max_cost_payment(
child,
session,
max_cost_for_model=3_000,
model_id="tinfoil/model",
reservation_snapshot=reservation,
)
updated_parent = await _api_key(session, "ehbp-rollback-parent")
assert updated_parent is not None
assert updated_parent.balance == 10_000
assert updated_parent.reserved_balance == 3_000
assert updated_parent.reserved_at == 123
assert updated_parent.total_spent == 0
assert await _api_key(session, "ehbp-missing-child") is None
updated_child = await _api_key(session, "ehbp-missing-child")
assert updated_child is not None
assert updated_child.reserved_balance == 3_000
assert updated_child.total_spent == 0
+64
View File
@@ -18,6 +18,37 @@ def _run_alembic(root: Path, database_url: str, revision: str) -> None:
)
def test_fresh_node_migrates_fee_payout_schema_to_head(tmp_path: Path) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "fresh-node.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
_run_alembic(root, database_url, "head")
with sqlite3.connect(database_path) as connection:
version = connection.execute(
"SELECT version_num FROM alembic_version"
).fetchone()
columns = {
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
}
fee = connection.execute(
"SELECT id, accumulated_msats, total_paid_msats, last_paid_at, "
"payout_in_progress_msats, payout_started_at FROM routstr_fees"
).fetchone()
assert version == ("c7d5f8638599",)
assert {
"id",
"accumulated_msats",
"total_paid_msats",
"last_paid_at",
"payout_in_progress_msats",
"payout_started_at",
} <= columns
assert fee == (1, 0, 0, None, 0, None)
def test_fee_payout_checkpoint_migration_preserves_existing_row(
tmp_path: Path,
) -> None:
@@ -44,3 +75,36 @@ def test_fee_payout_checkpoint_migration_preserves_existing_row(
).fetchone()
assert row == (5000, 1000, 123, 0, None)
def test_fee_payout_checkpoint_repair_restores_columns_missing_at_old_head(
tmp_path: Path,
) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "migration.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
old_head = "7f2843d3f4e4"
_run_alembic(root, database_url, old_head)
# Reproduce a database that was stamped to head after a duplicate-column or
# unknown-revision recovery skipped part of the migration chain.
with sqlite3.connect(database_path) as connection:
connection.execute("ALTER TABLE routstr_fees DROP COLUMN payout_started_at")
connection.execute(
"ALTER TABLE routstr_fees DROP COLUMN payout_in_progress_msats"
)
connection.commit()
_run_alembic(root, database_url, "head")
with sqlite3.connect(database_path) as connection:
columns = {
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
}
row = connection.execute(
"SELECT payout_in_progress_msats, payout_started_at "
"FROM routstr_fees WHERE id = 1"
).fetchone()
assert {"payout_in_progress_msats", "payout_started_at"} <= columns
assert row == (0, None)
+29 -2
View File
@@ -28,7 +28,9 @@ async def _fake_session(): # type: ignore[no-untyped-def]
yield MagicMock()
def _patches(proof_amount: int = 1000): # type: ignore[no-untyped-def]
def _patches( # type: ignore[no-untyped-def]
proof_amount: int = 1000, user_balance_msats: int = 0
):
proof = MagicMock(amount=proof_amount)
return [
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
@@ -42,7 +44,7 @@ def _patches(proof_amount: int = 1000): # type: ignore[no-untyped-def]
),
patch(
"routstr.wallet.db.balances_for_mint_and_unit",
AsyncMock(return_value=0),
AsyncMock(return_value=user_balance_msats),
),
patch("routstr.wallet.db.create_session", _fake_session),
]
@@ -225,6 +227,31 @@ async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
assert details[1]["error_code"] == "unreachable"
async def test_fetch_all_balances_reports_liability_when_wallet_is_empty() -> None:
"""An empty wallet must not hide outstanding user liabilities."""
from routstr.core.settings import settings
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
):
for p in _patches(proof_amount=0, user_balance_msats=5000):
p.start()
try:
details, total_wallet, total_user, owner = await fetch_all_balances(
units=["sat"]
)
finally:
patch.stopall()
assert details[0]["wallet_balance"] == 0
assert details[0]["user_balance"] == 5
assert details[0]["owner_balance"] == -5
assert total_wallet == 0
assert total_user == 5
assert owner == -5
@pytest.mark.asyncio
async def test_fetch_all_balances_no_duplicate_primary_mint() -> None:
"""primary_mint already in cashu_mints is not inspected twice."""
@@ -18,6 +18,7 @@ from fastapi.responses import Response, StreamingResponse
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
os.environ.setdefault("UPSTREAM_API_KEY", "test")
from routstr.auth import ReservationSnapshot # noqa: E402
from routstr.core.db import ApiKey # noqa: E402
from routstr.payment.cost_calculation import CostData # noqa: E402
from routstr.payment.models import Architecture, Model, Pricing # noqa: E402
@@ -498,6 +499,12 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
yield {"type": "message_stop"}
fake_cost = {"total_msats": 4321, "total_usd": 0.00015}
reservation = ReservationSnapshot(
release_id="messages-stream",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=10_000,
)
captured_cost_call: dict[str, Any] = {}
@@ -508,9 +515,11 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
max_cost: int,
model_obj: Any = None,
provider_fee: Any = None,
reservation_snapshot: Any = None,
) -> dict:
captured_cost_call["combined_data"] = combined_data
captured_cost_call["max_cost"] = max_cost
captured_cost_call["reservation_snapshot"] = reservation_snapshot
return fake_cost
fake_session = MagicMock()
@@ -544,6 +553,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
session=session,
max_cost_for_model=10_000,
model_obj=model,
reservation_snapshot=reservation,
)
assert isinstance(result, StreamingResponse)
@@ -566,6 +576,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
assert combined["usage"]["input_tokens"] == 5
assert combined["usage"]["output_tokens"] == 7
assert combined["model"] == "openai/gpt-4o-mini"
assert captured_cost_call["reservation_snapshot"] is reservation
@pytest.mark.asyncio
@@ -593,6 +604,12 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
fake_cost = {"total_msats": 999, "total_usd": 0.0001}
reservation = ReservationSnapshot(
release_id="messages-byte-stream",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=10_000,
)
captured: dict[str, Any] = {}
async def fake_adjust(
@@ -602,8 +619,10 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
max_cost: int,
model_obj: Any = None,
provider_fee: Any = None,
reservation_snapshot: Any = None,
) -> dict:
captured["combined_data"] = combined_data
captured["reservation_snapshot"] = reservation_snapshot
return fake_cost
fake_session = MagicMock()
@@ -636,6 +655,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
session=session,
max_cost_for_model=10_000,
model_obj=model,
reservation_snapshot=reservation,
)
assert isinstance(result, StreamingResponse)
@@ -659,6 +679,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
assert combined["usage"]["input_tokens"] == 3
assert combined["usage"]["output_tokens"] == 4
assert combined["model"] == "openai/gpt-4o-mini"
assert captured["reservation_snapshot"] is reservation
# ---------------------------------------------------------------------------
+47
View File
@@ -0,0 +1,47 @@
import os
import sqlite3
import subprocess
import sys
from pathlib import Path
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
env = os.environ.copy()
env["DATABASE_URL"] = database_url
subprocess.run(
[sys.executable, "-m", "alembic", command, revision],
cwd=root,
env=env,
check=True,
capture_output=True,
text=True,
)
def _lightning_invoice_columns(database_path: Path) -> set[str]:
with sqlite3.connect(database_path) as connection:
return {
row[1]
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
}
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
tmp_path: Path,
) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "mint-url-migration.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
previous_head = "9c4d8e2f1a6b"
_run_alembic(root, database_url, "upgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "c7d5f8638599")
assert "mint_url" in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "downgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "head")
assert "mint_url" in _lightning_invoice_columns(database_path)
+132 -1
View File
@@ -1,3 +1,4 @@
import json
import os
import pytest
@@ -6,7 +7,15 @@ from sqlalchemy.ext.asyncio import create_async_engine
from sqlmodel import text
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.settings import Settings, SettingsService
from routstr.core.settings import Settings, SettingsService, settings
NSEC_HEX = "1" * 64
async def _read_settings_blob(session: AsyncSession) -> dict:
"""Return the raw persisted settings JSON (id=1) as a dict."""
row = await session.exec(text("SELECT data FROM settings WHERE id = 1")) # type: ignore
return json.loads(row.first()[0])
@pytest.mark.asyncio
@@ -120,3 +129,125 @@ async def test_settings_initialize_discards_unknown_keys() -> None:
assert '"enable_analytics_sharing": true' in stored_data
assert "nostr_analytics_enabled" not in stored_data
assert "unknown_key" not in stored_data
# ── Secret fields are never written to the settings blob (issue #553) ────────
def test_settings_model_drops_admin_password_field() -> None:
# admin_password now lives only as a one-way hash in the Secret store; it is
# no longer a settings field at all.
assert "admin_password" not in Settings.__fields__
# nsec remains a runtime value held in memory; upstream_api_key is ordinary
# config that still lives in the persisted blob.
assert "nsec" in Settings.__fields__
assert "upstream_api_key" in Settings.__fields__
@pytest.mark.asyncio
async def test_secret_fields_kept_in_memory_but_not_persisted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# bootstrap_secrets (which runs first at boot) owns the nsec and has already
# decrypted it into memory; simulate that live value. initialize must keep it
# in memory for runtime consumers yet never write it to the settings blob.
monkeypatch.setattr(settings, "nsec", NSEC_HEX)
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with AsyncSession(engine, expire_on_commit=False) as session:
s = await SettingsService.initialize(session)
# Runtime consumers still see the live secret value.
assert s.nsec == NSEC_HEX
# ...but it is never written to the settings blob.
blob = await _read_settings_blob(session)
assert "nsec" not in blob
assert "admin_password" not in blob
# Non-secret derived/public values are still persisted.
assert blob["npub"] == s.npub
@pytest.mark.asyncio
async def test_upstream_api_key_survives_persistence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# upstream_api_key is provider-scoped config, not a vault secret: it has no
# encrypted home yet, so it must stay in the settings blob. Stripping it
# would load it once, rewrite the blob without it, and lose it on the next
# restart. Guard the on-disk survival path: blob-only value, no env.
monkeypatch.delenv("UPSTREAM_API_KEY", raising=False)
monkeypatch.setattr(settings, "upstream_api_key", "")
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with AsyncSession(engine, expire_on_commit=False) as session:
await SettingsService.initialize(session)
await session.exec( # type: ignore
text("UPDATE settings SET data = :d WHERE id = 1").bindparams(
d=json.dumps({"name": "LegacyNode", "upstream_api_key": "sk-only-in-db"})
)
)
await session.commit()
# A reload must not drop the key from the blob...
await SettingsService.initialize(session)
blob = await _read_settings_blob(session)
assert blob["upstream_api_key"] == "sk-only-in-db"
# ...and it stays live for the proxy hot path.
assert settings.upstream_api_key == "sk-only-in-db"
@pytest.mark.asyncio
async def test_existing_blob_secrets_are_stripped_on_initialize(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("NSEC", raising=False)
monkeypatch.delenv("UPSTREAM_API_KEY", raising=False)
monkeypatch.delenv("ADMIN_PASSWORD", raising=False)
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with AsyncSession(engine, expire_on_commit=False) as session:
await SettingsService.initialize(session)
# Simulate a legacy row that still carries plaintext secrets in the blob.
await session.exec( # type: ignore
text("UPDATE settings SET data = :d WHERE id = 1").bindparams(
d=json.dumps(
{
"name": "LegacyNode",
"admin_password": "pw",
"nsec": NSEC_HEX,
"upstream_api_key": "sk-legacy",
}
)
)
)
await session.commit()
await SettingsService.initialize(session)
blob = await _read_settings_blob(session)
assert "admin_password" not in blob
assert "nsec" not in blob
# Non-secret values survive the migration, including upstream_api_key,
# which is not vaulted yet and so must stay in the blob.
assert blob["name"] == "LegacyNode"
assert blob["upstream_api_key"] == "sk-legacy"
@pytest.mark.asyncio
async def test_update_does_not_persist_secret_fields(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("NSEC", NSEC_HEX)
monkeypatch.setattr(settings, "nsec", "")
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with AsyncSession(engine, expire_on_commit=False) as session:
await SettingsService.initialize(session)
await SettingsService.update({"name": "Updated"}, session)
blob = await _read_settings_blob(session)
assert blob["name"] == "Updated"
assert "nsec" not in blob
assert "admin_password" not in blob
+52 -8
View File
@@ -16,13 +16,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlalchemy.pool import StaticPool
from sqlmodel import SQLModel
from sqlmodel import SQLModel, select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.auth import pay_for_request
from routstr.balance import refund_wallet_endpoint
from routstr.core.db import (
ApiKey,
ReservationRelease,
release_stale_reservations,
reset_all_reserved_balances,
)
@@ -70,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
@pytest.mark.asyncio
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
async def test_pay_for_request_sets_reserved_at_on_child_key(
session: AsyncSession,
) -> None:
parent = ApiKey(hashed_key="parentkey", balance=10_000)
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
session.add(parent)
@@ -150,6 +153,39 @@ async def test_release_stale_reservations_releases_old(session: AsyncSession) ->
assert key.reserved_at is None
@pytest.mark.asyncio
async def test_targeted_parent_cleanup_releases_child_owned_reservation(
session: AsyncSession,
) -> None:
parent = ApiKey(hashed_key="stale-parent", balance=5_000)
child = ApiKey(
hashed_key="stale-child", parent_key_hash=parent.hashed_key, balance=0
)
session.add_all([parent, child])
await session.commit()
await pay_for_request(child, 1_000, session)
reservation = (
await session.exec(
select(ReservationRelease).where(
ReservationRelease.key_hash == child.hashed_key
)
)
).one()
reservation.created_at = int(time.time()) - 1_000
session.add(reservation)
await session.commit()
released = await release_stale_reservations(
session, max_age_seconds=300, key_hash=parent.hashed_key
)
assert released == 1
await session.refresh(parent)
await session.refresh(child)
assert parent.reserved_balance == 0
assert child.reserved_balance == 0
@pytest.mark.asyncio
async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) -> None:
key = ApiKey(
@@ -170,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
@pytest.mark.asyncio
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
async def test_release_stale_reservations_skips_null_reserved_at(
session: AsyncSession,
) -> None:
# Reservations without a timestamp may belong to instances running older
# code (rolling deploy) — the background sweeper must not touch them.
key = ApiKey(
@@ -190,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
@pytest.mark.asyncio
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
async def test_reset_all_reserved_balances_clears_reserved_at(
session: AsyncSession,
) -> None:
key = ApiKey(
hashed_key="resetkey",
balance=5_000,
@@ -352,6 +392,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
upstream.forward_request = AsyncMock(side_effect=asyncio.CancelledError())
session = MagicMock()
reservation_snapshot = MagicMock()
revert_mock = AsyncMock(return_value=True)
with (
@@ -369,13 +410,16 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
"get_reservation_snapshot",
AsyncMock(return_value=reservation_snapshot),
),
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
):
with pytest.raises(asyncio.CancelledError):
await proxy_module.proxy(request, "v1/chat/completions", session=session)
revert_mock.assert_awaited_once_with(key, session, 900)
revert_mock.assert_awaited_once_with(key, session, 900, reservation_snapshot)
+7
View File
@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.upstream.base import BaseUpstreamProvider
@@ -67,6 +68,12 @@ async def test_stream_with_id_injection() -> None:
max_cost_for_model=100,
background_tasks=background_tasks,
requested_model="test-model",
reservation_snapshot=ReservationSnapshot(
release_id="test-release",
key_hash="test_hash",
billing_key_hash="test_hash",
reserved_msats=100,
),
)
results = []
@@ -0,0 +1,448 @@
import asyncio
from collections.abc import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlmodel import SQLModel
from sqlmodel.ext.asyncio.session import AsyncSession
import routstr.auth as auth_module
from routstr.auth import (
ReservationSnapshot,
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
release_reservation,
)
from routstr.core.db import ApiKey, ReservationRelease
from routstr.payment.cost_calculation import MaxCostData
from routstr.upstream.base import BaseUpstreamProvider
async def _engine() -> AsyncEngine:
engine = create_async_engine("sqlite+aiosqlite://")
async with engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
return engine
@pytest.mark.asyncio
async def test_release_reservation_is_durable_and_idempotent() -> None:
engine = await _engine()
key = ApiKey(hashed_key="key", balance=1_000)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 500, session)
snapshot = await get_reservation_snapshot(key, session)
record = await session.get(ReservationRelease, snapshot.release_id)
assert record is not None and record.status == "active"
assert await release_reservation(snapshot, session, 500) is True
assert await release_reservation(snapshot, session, 500) is True
await session.refresh(key)
await session.refresh(record)
assert key.reserved_balance == 0
assert key.reserved_at is None
assert record.status == "released"
await engine.dispose()
@pytest.mark.asyncio
async def test_release_only_owns_its_concurrent_reservation() -> None:
engine = await _engine()
key = ApiKey(hashed_key="key", balance=1_000)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 400, session)
first = await get_reservation_snapshot(key, session)
await pay_for_request(key, 400, session)
second = await get_reservation_snapshot(key, session)
assert first.release_id != second.release_id
assert await release_reservation(first, session, 400) is True
assert await release_reservation(first, session, 400) is True
await session.refresh(key)
assert key.reserved_balance == 400
assert await release_reservation(second, session, 400) is True
await session.refresh(key)
assert key.reserved_balance == 0
await engine.dispose()
@pytest.mark.asyncio
async def test_release_updates_parent_and_child_atomically() -> None:
engine = await _engine()
parent = ApiKey(hashed_key="parent", balance=1_000)
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add_all([parent, child])
await session.commit()
await pay_for_request(child, 500, session)
snapshot = await get_reservation_snapshot(child, session)
assert await release_reservation(snapshot, session, 500) is True
await session.refresh(parent)
await session.refresh(child)
assert (parent.reserved_balance, child.reserved_balance) == (0, 0)
assert (parent.reserved_at, child.reserved_at) == (None, None)
await engine.dispose()
@pytest.mark.asyncio
async def test_release_rolls_back_partial_parent_child_update() -> None:
engine = await _engine()
parent = ApiKey(hashed_key="parent", balance=1_000)
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add_all([parent, child])
await session.commit()
await pay_for_request(child, 500, session)
snapshot = await get_reservation_snapshot(child, session)
child.reserved_balance = 100
session.add(child)
await session.commit()
assert await release_reservation(snapshot, session, 500) is False
await session.refresh(parent)
await session.refresh(child)
record = await session.get(ReservationRelease, snapshot.release_id)
assert (parent.reserved_balance, child.reserved_balance) == (500, 100)
assert record is not None and record.status == "active"
await engine.dispose()
@pytest.mark.asyncio
async def test_post_commit_failure_cannot_release_charged_reservation() -> None:
engine = await _engine()
key = ApiKey(hashed_key="key", balance=1_000)
cost = MaxCostData(
base_msats=500,
input_msats=0,
output_msats=0,
total_msats=500,
)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 500, session)
snapshot = await get_reservation_snapshot(key, session)
with (
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
patch.object(
session,
"refresh",
AsyncMock(side_effect=SQLAlchemyError("post-commit refresh failed")),
),
):
with pytest.raises(SQLAlchemyError, match="post-commit refresh failed"):
await adjust_payment_for_tokens(key, {}, session, 500)
await session.rollback()
assert await release_reservation(snapshot, session, 500) is False
charged_key = await session.get(ApiKey, "key")
record = await session.get(ReservationRelease, snapshot.release_id)
assert charged_key is not None
assert (charged_key.balance, charged_key.reserved_balance) == (500, 0)
assert record is not None and record.status == "charged"
await engine.dispose()
@pytest.mark.asyncio
async def test_generic_background_settlement_uses_explicit_reservation() -> None:
engine = await _engine()
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key", provider_fee=1.0
)
key = ApiKey(hashed_key="generic-key", balance=1_000)
cost = MaxCostData(
base_msats=500,
input_msats=0,
output_msats=0,
total_msats=500,
)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 500, session)
snapshot = await get_reservation_snapshot(key, session)
context_token = auth_module._current_reservation.set(None)
try:
with (
patch(
"routstr.upstream.base.create_session",
side_effect=lambda: AsyncSession(engine, expire_on_commit=False),
),
patch(
"routstr.upstream.base.adjust_payment_for_tokens",
auth_module.adjust_payment_for_tokens,
),
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
):
await provider._finalize_generic_streaming_payment(
key.hashed_key,
500,
"audio/speech",
model_obj=None,
provider_fee=provider.provider_fee,
reservation_snapshot=snapshot,
)
finally:
auth_module._current_reservation.reset(context_token)
async with AsyncSession(engine, expire_on_commit=False) as session:
settled_key = await session.get(ApiKey, key.hashed_key)
record = await session.get(ReservationRelease, snapshot.release_id)
assert settled_key is not None
assert (settled_key.balance, settled_key.reserved_balance) == (500, 0)
assert record is not None and record.status == "charged"
await engine.dispose()
@pytest.mark.asyncio
async def test_streaming_release_is_terminal_and_suppresses_background_charge() -> None:
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key"
)
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
yield b"data: [DONE]\n\n"
upstream_response = MagicMock()
upstream_response.status_code = 200
upstream_response.headers = {"content-type": "text/event-stream"}
upstream_response.aiter_bytes = aiter_bytes
key = MagicMock(spec=ApiKey)
key.hashed_key = "test-key-hash"
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.rollback = AsyncMock()
session_context = MagicMock()
session_context.__aenter__ = AsyncMock(return_value=session)
session_context.__aexit__ = AsyncMock(return_value=None)
release = AsyncMock(return_value=True)
reservation_snapshot = MagicMock()
reservation_snapshot.reserved_msats = 500
background_tasks = MagicMock()
with (
patch(
"routstr.upstream.base.adjust_payment_for_tokens",
AsyncMock(side_effect=SQLAlchemyError("database unavailable")),
),
patch(
"routstr.upstream.base.get_reservation_snapshot",
AsyncMock(return_value=reservation_snapshot),
),
patch("routstr.upstream.base.release_reservation", release),
patch("routstr.upstream.base.create_session", return_value=session_context),
):
response = await provider.handle_streaming_chat_completion(
response=upstream_response,
key=key,
max_cost_for_model=500,
background_tasks=background_tasks,
)
with pytest.raises(SQLAlchemyError, match="database unavailable"):
async for _ in response.body_iterator:
pass
session.rollback.assert_awaited_once()
release.assert_awaited_once_with(reservation_snapshot, session, 500)
background_tasks.add_task.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"release_outcome",
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
)
async def test_responses_streaming_releases_and_raises_on_billing_failure(
release_outcome: bool | BaseException,
) -> None:
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key"
)
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
yield (
b'data: {"type":"response.completed","response":{"model":"test",'
b'"usage":{"input_tokens":1,"output_tokens":1}}}\n\n'
)
yield b"data: [DONE]\n\n"
upstream_response = MagicMock(
status_code=200,
headers={"content-type": "text/event-stream"},
)
upstream_response.aiter_bytes = aiter_bytes
key = MagicMock(spec=ApiKey)
key.hashed_key = "responses-key"
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.rollback = AsyncMock()
session_context = MagicMock()
session_context.__aenter__ = AsyncMock(return_value=session)
session_context.__aexit__ = AsyncMock(return_value=None)
snapshot = ReservationSnapshot(
release_id="responses-release",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=500,
)
release = (
AsyncMock(side_effect=release_outcome)
if isinstance(release_outcome, BaseException)
else AsyncMock(return_value=release_outcome)
)
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
with (
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
patch("routstr.upstream.base.release_reservation", release),
patch("routstr.upstream.base.create_session", return_value=session_context),
):
response = await provider.handle_streaming_responses_completion(
response=upstream_response,
key=key,
max_cost_for_model=500,
reservation_snapshot=snapshot,
)
emitted = bytearray()
with pytest.raises(SQLAlchemyError, match="database unavailable"):
async for chunk in response.body_iterator:
if isinstance(chunk, str):
emitted.extend(chunk.encode())
else:
emitted.extend(bytes(chunk))
assert b'"total_msats": 0' not in emitted
adjust.assert_awaited_once()
session.rollback.assert_awaited_once()
release.assert_awaited_once_with(snapshot, session, 500)
@pytest.mark.asyncio
@pytest.mark.parametrize("via_litellm", [False, True])
@pytest.mark.parametrize(
"release_outcome",
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
)
async def test_messages_streaming_releases_and_raises_on_billing_failure(
via_litellm: bool,
release_outcome: bool | BaseException,
) -> None:
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key"
)
key = MagicMock(spec=ApiKey)
key.hashed_key = "messages-key"
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.rollback = AsyncMock()
session_context = MagicMock()
session_context.__aenter__ = AsyncMock(return_value=session)
session_context.__aexit__ = AsyncMock(return_value=None)
snapshot = ReservationSnapshot(
release_id=f"messages-{'litellm' if via_litellm else 'native'}",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=500,
)
release = (
AsyncMock(side_effect=release_outcome)
if isinstance(release_outcome, BaseException)
else AsyncMock(return_value=release_outcome)
)
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
async def native_chunks() -> AsyncGenerator[bytes, None]:
yield (
b'event: message_start\ndata: {"type":"message_start","message":'
b'{"model":"test","usage":{"input_tokens":1,"output_tokens":0}}}\n\n'
)
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
async def litellm_chunks() -> AsyncGenerator[dict, None]:
yield {
"type": "message_start",
"message": {
"model": "test",
"usage": {"input_tokens": 1, "output_tokens": 0},
},
}
yield {"type": "message_stop"}
with (
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
patch("routstr.upstream.base.release_reservation", release),
patch("routstr.upstream.base.create_session", return_value=session_context),
):
if via_litellm:
response = provider._stream_litellm_messages(
iterator=litellm_chunks(),
key=key,
max_cost_for_model=500,
requested_model=None,
reservation_snapshot=snapshot,
)
else:
upstream_response = MagicMock(
status_code=200,
headers={"content-type": "text/event-stream"},
)
upstream_response.aiter_bytes = native_chunks
response = await provider.handle_streaming_messages_completion(
response=upstream_response,
key=key,
max_cost_for_model=500,
reservation_snapshot=snapshot,
)
with pytest.raises(SQLAlchemyError, match="database unavailable"):
async for _ in response.body_iterator:
pass
adjust.assert_awaited_once()
session.rollback.assert_awaited_once()
release.assert_awaited_once_with(snapshot, session, 500)
@pytest.mark.asyncio
async def test_cross_key_reservation_snapshot_is_rejected_without_mutation() -> None:
engine = await _engine()
first = ApiKey(hashed_key="first", balance=1_000)
second = ApiKey(hashed_key="second", balance=1_000)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(first)
session.add(second)
await session.commit()
await pay_for_request(first, 500, session)
snapshot = await get_reservation_snapshot(first, session)
with pytest.raises(RuntimeError, match="does not belong"):
await adjust_payment_for_tokens(
second,
{"model": "test", "usage": None},
session,
500,
reservation_snapshot=snapshot,
)
await session.refresh(first)
await session.refresh(second)
assert first.reserved_balance == 500
assert second.reserved_balance == 0
await engine.dispose()
@@ -24,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.upstream import base
from routstr.upstream.base import BaseUpstreamProvider
@@ -67,6 +68,12 @@ async def _drive(chunks: list[bytes], requested_model: str | None = None) -> lis
max_cost_for_model=100,
background_tasks=MagicMock(),
requested_model=requested_model,
reservation_snapshot=ReservationSnapshot(
release_id="test-release",
key_hash="test_hash",
billing_key_hash="test_hash",
reserved_msats=100,
),
)
out: list[bytes] = []
+14 -4
View File
@@ -331,6 +331,7 @@ async def test_5xx_wrapped_rate_limit_is_classified(
@pytest.mark.asyncio
async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
from routstr import proxy as proxy_module
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.core.exceptions import UpstreamError
@@ -359,6 +360,12 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
)
session = MagicMock()
reservation = ReservationSnapshot(
release_id="rate-limit-release",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=1_000,
)
revert_mock = AsyncMock(return_value=True)
with (
@@ -376,10 +383,13 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
"get_reservation_snapshot",
AsyncMock(return_value=reservation),
),
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
):
response = await proxy_module.proxy(
@@ -396,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
assert RAW_ORG_ID not in serialized
assert "org-[REDACTED]" in serialized
# Single upstream failed -> reservation reverted exactly once (no double-charge).
revert_mock.assert_awaited_once_with(key, session, 900)
revert_mock.assert_awaited_once_with(key, session, 900, reservation)
+339
View File
@@ -0,0 +1,339 @@
"""Tests for ``routstr.core.vault`` — the secret encrypt/hash/fingerprint helpers.
Specifies the primitives that the rest of the secret-storage work (issue #553)
builds on, independent of any database or app wiring:
- ``encrypt``/``decrypt`` Fernet symmetric encryption emitting self-describing
``fernet:v1:`` ciphertext, so a value can be told apart from legacy plaintext
and from ciphertext written under a different ``ROUTSTR_SECRET_KEY`` (which
surfaces as a hard ``InvalidToken`` rather than silent corruption).
- ``hash_password``/``verify_password`` salted scrypt hashing that is
*key-independent* (does not depend on ``ROUTSTR_SECRET_KEY``), so password
login and the recovery script keep working even if the key is lost.
- a missing/malformed ``ROUTSTR_SECRET_KEY`` fails fast with the generation
command in the message.
"""
from pathlib import Path
import pytest
from cryptography.fernet import InvalidToken
from routstr.core import vault
# Two distinct, valid Fernet keys held fixed so ciphertext/fingerprints are
# reproducible across runs and we can exercise the wrong-key path.
KEY_A = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU="
KEY_B = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ="
def _use_key(monkeypatch: pytest.MonkeyPatch, key: str) -> None:
monkeypatch.setenv("ROUTSTR_SECRET_KEY", key)
# --- encrypt / decrypt -----------------------------------------------------
def test_encrypt_decrypt_round_trips(monkeypatch: pytest.MonkeyPatch) -> None:
_use_key(monkeypatch, KEY_A)
assert vault.decrypt(vault.encrypt("nsec1secret")) == "nsec1secret"
def test_encrypt_emits_self_describing_prefix(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_use_key(monkeypatch, KEY_A)
assert vault.encrypt("x").startswith("fernet:v1:")
def test_encrypt_is_non_deterministic(monkeypatch: pytest.MonkeyPatch) -> None:
# Fernet embeds a random IV/timestamp: equal plaintext -> different
# ciphertext. This is exactly why upstream-key equality needs a blind index.
_use_key(monkeypatch, KEY_A)
assert vault.encrypt("same") != vault.encrypt("same")
def test_is_encrypted_distinguishes_ciphertext_from_plaintext(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_use_key(monkeypatch, KEY_A)
assert vault.is_encrypted(vault.encrypt("x")) is True
assert vault.is_encrypted("sk-plaintext-api-key") is False
assert vault.is_encrypted("") is False
def test_decrypt_rejects_unprefixed_value(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Guards the migration paths: a legacy plaintext value must never be
# mistaken for ciphertext and "decrypted".
_use_key(monkeypatch, KEY_A)
with pytest.raises(ValueError):
vault.decrypt("not-encrypted")
def test_decrypt_with_wrong_key_raises(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# The fail-fast signal: ciphertext written under KEY_A cannot be read under
# KEY_B -> InvalidToken (bootstrap turns this into a clear startup error).
_use_key(monkeypatch, KEY_A)
token = vault.encrypt("secret")
_use_key(monkeypatch, KEY_B)
with pytest.raises(InvalidToken):
vault.decrypt(token)
# --- password hashing (key-independent) ------------------------------------
def test_hash_and_verify_password(monkeypatch: pytest.MonkeyPatch) -> None:
_use_key(monkeypatch, KEY_A)
stored = vault.hash_password("correct horse")
assert vault.verify_password("correct horse", stored) is True
assert vault.verify_password("wrong", stored) is False
def test_password_hash_is_salted(monkeypatch: pytest.MonkeyPatch) -> None:
_use_key(monkeypatch, KEY_A)
a = vault.hash_password("pw")
b = vault.hash_password("pw")
assert a != b
assert vault.verify_password("pw", a) is True
assert vault.verify_password("pw", b) is True
def test_verify_password_rejects_malformed_stored_value(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# A garbage or non-scrypt stored value must verify to False, never raise.
_use_key(monkeypatch, KEY_A)
assert vault.verify_password("pw", "") is False
assert vault.verify_password("pw", "not-a-hash") is False
assert vault.verify_password("pw", "bcrypt:1:2:3:x:y") is False
def test_password_hashing_is_key_independent(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# scrypt does not use ROUTSTR_SECRET_KEY, so login and the recovery script
# work even when the key is missing.
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
stored = vault.hash_password("pw")
assert vault.verify_password("pw", stored) is True
# --- fail-fast on missing/malformed key ------------------------------------
def test_decrypt_without_any_key_fails_fast_with_generation_command(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# Reading secrets is strict: with no key in env AND no key file, decrypt
# fails fast with the generation command rather than silently minting a new
# key (a fresh key could never match already-encrypted ciphertext). Only the
# encrypt path auto-provisions; the read path never does.
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(tmp_path / "absent.key"))
with pytest.raises(RuntimeError) as exc:
vault.decrypt("fernet:v1:not-real-ciphertext")
msg = str(exc.value)
assert "ROUTSTR_SECRET_KEY" in msg
assert "Fernet.generate_key" in msg
def test_malformed_key_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key")
with pytest.raises(RuntimeError):
vault.encrypt("x")
def test_malformed_env_key_does_not_self_provision(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# A malformed env key is an operator mistake, not an unset key: it must fail
# loudly, never silently generate a different key to a file (which would hide
# the mistake and could brick secrets the operator meant to key differently).
monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key")
key_file = tmp_path / "routstr_secret.key"
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
with pytest.raises(RuntimeError):
vault.encrypt("x")
assert not key_file.exists()
# --- auto-provisioned key file (non-breaking upgrade path) -----------------
@pytest.fixture
def generated_key_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""No env key; the key file points at a fresh, empty tmp location.
Exercises what an existing node hits when it upgrades without setting
ROUTSTR_SECRET_KEY: the master key is auto-generated and persisted here so
boot does not break, while secrets are still never written in plaintext.
"""
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
key_file = tmp_path / "routstr_secret.key"
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
return key_file
def test_encrypt_without_key_generates_and_persists_key_file(
generated_key_file: Path,
) -> None:
# Encryption at rest stays mandatory, but a missing key is provisioned rather
# than fatal: encrypt generates a key, writes it to the key file (owner-only),
# and the value round-trips — decrypt, still with no env key, reads the same
# file key back.
key_file = generated_key_file
assert not key_file.exists()
token = vault.encrypt("nsec1secret")
assert token.startswith("fernet:v1:")
assert key_file.exists()
assert key_file.stat().st_mode & 0o077 == 0 # not group/other-accessible
assert vault.decrypt(token) == "nsec1secret"
def test_generated_key_warns_operator_with_path_not_value(
generated_key_file: Path, capsys: pytest.CaptureFixture[str]
) -> None:
# The notice names the file to back up and shouts the back-up imperative so an
# upgrading operator cannot miss it, but it MUST NOT echo the key value: the
# secret lives in the 0600 file, and printing it would leak it into captured
# stdout / aggregated container logs.
key_file = generated_key_file
vault.encrypt("x")
out = capsys.readouterr().out
assert "ROUTSTR_SECRET_KEY" in out
assert str(key_file) in out
assert key_file.read_text().strip() not in out
assert "BACK UP" in out.upper()
def test_existing_key_file_is_reused_and_warns_only_once(
generated_key_file: Path, capsys: pytest.CaptureFixture[str]
) -> None:
# Once the key exists, later encrypts reuse it (never rotate a key that
# secrets were already encrypted under) and stay silent (no repeated notice).
key_file = generated_key_file
vault.encrypt("a")
first_key = key_file.read_text()
capsys.readouterr() # drain the one-time notice
vault.encrypt("b")
assert key_file.read_text() == first_key
assert capsys.readouterr().out == ""
def test_generated_key_is_published_atomically(
generated_key_file: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# The key must appear at its final path only as a complete file: it is written
# to a temp file and atomically linked into place. If the publish (link) step
# fails — e.g. the process crashes — the destination must be absent, never a
# half-written or empty file that a later boot would read as a corrupt key and
# then refuse to decrypt every secret. No temp debris is left behind.
key_file = generated_key_file
def boom(src: object, dst: object) -> None:
raise OSError("crash during atomic publish")
monkeypatch.setattr(vault.os, "link", boom)
with pytest.raises(OSError):
vault.encrypt("x")
assert not key_file.exists()
assert list(key_file.parent.iterdir()) == []
def test_racing_worker_adopts_winners_key_without_clobber(
generated_key_file: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
# Two workers auto-generate a key concurrently. The first to link "wins" and
# its key is the one on disk; a worker that loses the link race must adopt the
# winner's key — secrets may already be encrypted under it — rather than
# clobber it or crash. Force the race deterministically: the winner publishes
# its key at the final path just as this worker tries to link, so os.link
# raises FileExistsError and the loser reads the winner's key back.
key_file = generated_key_file
def winner_links_first(src: object, dst: object) -> None:
key_file.write_text(KEY_A) # the winner's already-published key
raise FileExistsError
monkeypatch.setattr(vault.os, "link", winner_links_first)
token = vault.encrypt("secret")
# The winner's key stays put and the loser encrypted under it, so the value
# round-trips under KEY_A even though this worker had generated its own key.
assert key_file.read_text().strip() == KEY_A
assert vault.decrypt(token) == "secret"
# The losing worker left no temp debris behind.
assert [p.name for p in key_file.parent.iterdir()] == [key_file.name]
def test_loose_key_file_perms_are_tightened_on_read(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# A key file left group/other-readable (e.g. written under a loose umask
# before this hardening, or by a careless operator) is a leaked-secret risk.
# Reading it repairs the permissions to owner-only rather than trusting a
# world-readable master key, while still using the key so boot is not broken.
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
key_file = tmp_path / "routstr_secret.key"
key_file.write_text(KEY_A)
key_file.chmod(0o644)
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
token = vault.encrypt("secret") # reads the loose file, repairs its perms
assert key_file.stat().st_mode & 0o077 == 0
assert vault.decrypt(token) == "secret"
def test_env_key_takes_precedence_over_key_file(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# An explicit env key wins over a persisted file key (an operator-supplied
# key from a secrets manager overrides the auto-generated one), and the file
# is left untouched.
key_file = tmp_path / "routstr_secret.key"
key_file.write_text(KEY_B)
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
monkeypatch.setenv("ROUTSTR_SECRET_KEY", KEY_A)
token = vault.encrypt("x")
assert vault.decrypt(token) == "x" # env key (A) decrypts it
monkeypatch.setenv("ROUTSTR_SECRET_KEY", KEY_B)
with pytest.raises(InvalidToken):
vault.decrypt(token) # the file key (B) does not
assert key_file.read_text() == KEY_B # file key never used or overwritten
def test_generated_key_defaults_beside_the_database(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
# With no env key and no explicit key-file path, the key is provisioned next
# to the SQLite database, so it rides whatever volume already persists the
# data instead of landing in the working directory (where a container
# recreate would lose it and brick decryption).
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
monkeypatch.delenv("ROUTSTR_SECRET_KEY_FILE", raising=False)
monkeypatch.chdir(tmp_path) # isolate the working-dir fallback from the repo
db_dir = tmp_path / "data"
db_dir.mkdir()
monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{db_dir}/routstr.db")
token = vault.encrypt("beside-the-db")
assert (db_dir / "routstr_secret.key").exists()
assert not (tmp_path / "routstr_secret.key").exists() # not the working dir
assert vault.decrypt(token) == "beside-the-db"
+97 -10
View File
@@ -18,6 +18,7 @@ from routstr.wallet import (
get_balance,
is_mint_connection_error,
recieve_token,
send,
send_token,
)
@@ -49,7 +50,12 @@ async def test_get_balance() -> None:
mock_wallet.load_mint = AsyncMock()
mock_wallet.load_proofs = AsyncMock()
with patch("routstr.wallet.Wallet.with_db", return_value=mock_wallet):
# Reset the module-level wallet cache so a real wallet cached by an earlier
# test (e.g. an unmocked admin-withdraw path) can't shadow the mock here.
with (
patch("routstr.wallet._wallets", {}),
patch("routstr.wallet.Wallet.with_db", return_value=mock_wallet),
):
balance = await get_balance("sat")
assert balance == 50000
@@ -263,6 +269,93 @@ async def test_refund_mint_falls_back_to_trusted_mint_with_funds() -> None:
assert mint == secondary
@pytest.mark.asyncio
async def test_send_falls_back_when_preferred_mint_has_only_reserved_balance() -> None:
from routstr.core.settings import settings
preferred = "http://preferred:3338"
primary = "http://primary:3338"
preferred_wallet = Mock(keysets={}, proofs=[])
preferred_wallet.select_to_send = AsyncMock()
primary_wallet = Mock(keysets={}, proofs=[])
primary_wallet.select_to_send = AsyncMock()
primary_wallet.serialize_proofs = AsyncMock(return_value="primary-token")
primary_wallet.set_reserved_for_send = AsyncMock()
preferred_liquid = Mock(amount=500, reserved=False)
preferred_reserved = Mock(amount=600, reserved=True)
primary_liquid = Mock(amount=1000, reserved=False)
primary_wallet.select_to_send.return_value = ([primary_liquid], None)
async def get_wallet(mint_url: str, unit: str, **_: object) -> Mock:
assert unit == "sat"
return primary_wallet if mint_url == primary else preferred_wallet
def get_proofs(
wallet: Mock,
mint_url: str,
unit: str,
*,
not_reserved: bool = False,
) -> list[Mock]:
assert unit == "sat"
if wallet is primary_wallet:
assert mint_url == primary
proofs = [primary_liquid]
else:
assert mint_url == preferred
proofs = [preferred_liquid, preferred_reserved]
return (
[proof for proof in proofs if not proof.reserved]
if not_reserved
else proofs
)
with (
patch.object(settings, "primary_mint", primary),
patch.object(settings, "cashu_mints", [primary, preferred]),
patch("routstr.wallet.get_wallet", side_effect=get_wallet),
patch("routstr.wallet.get_proofs_per_mint_and_unit", side_effect=get_proofs),
):
amount, token = await send(1000, "sat", preferred)
assert (amount, token) == (1000, "primary-token")
preferred_wallet.select_to_send.assert_not_awaited()
primary_wallet.select_to_send.assert_awaited_once_with(
[primary_liquid], 1000, set_reserved=False, include_fees=False
)
@pytest.mark.asyncio
async def test_send_primary_with_only_reserved_proofs_still_raises() -> None:
from routstr.core.settings import settings
primary = "http://primary:3338"
wallet = Mock(keysets={}, proofs=[])
wallet.select_to_send = AsyncMock()
reserved = Mock(amount=1000, reserved=True)
def get_proofs(
_wallet: Mock,
_mint_url: str,
_unit: str,
*,
not_reserved: bool = False,
) -> list[Mock]:
return [] if not_reserved else [reserved]
with (
patch.object(settings, "primary_mint", primary),
patch.object(settings, "cashu_mints", [primary]),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.wallet.get_proofs_per_mint_and_unit", side_effect=get_proofs),
pytest.raises(ValueError, match="No trusted mint has"),
):
await send(1000, "sat", primary)
wallet.select_to_send.assert_not_awaited()
@pytest.mark.asyncio
async def test_credit_balance() -> None:
token_data = {
@@ -2182,9 +2275,7 @@ def _http_500_error(message: str = "") -> httpx.HTTPStatusError:
),
],
)
def test_is_mint_rate_limited_strictness(
error: BaseException, expected: bool
) -> None:
def test_is_mint_rate_limited_strictness(error: BaseException, expected: bool) -> None:
assert _is_mint_rate_limited(error) is expected
@@ -2231,9 +2322,7 @@ def test_classify_rate_limit_takes_priority_over_connection_error() -> None:
def test_classify_connection_error_still_returns_mint_unreachable() -> None:
"""Transport failures without a 429 in the chain are still
classified as mint_unreachable."""
classified = classify_redemption_error(
httpx.ConnectError("connection refused")
)
classified = classify_redemption_error(httpx.ConnectError("connection refused"))
assert classified is not None
type_, status, _msg, code = classified
assert type_ == "mint_unreachable"
@@ -2276,9 +2365,7 @@ async def test_probe_does_not_escalate_consecutive_rate_limits() -> None:
# Simulate probe failure: _run_probe uses apply_cooldown, NOT
# apply_rate_limit_cooldown, so the counter stays at 1.
guard.apply_cooldown(
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, reason="rate_limited"
)
guard.apply_cooldown(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, reason="rate_limited")
assert guard._consecutive_rate_limits == 1 # unchanged!
assert guard._needs_probe is True
@@ -91,25 +91,27 @@ export function CashuPaymentWorkflow({
setIsCreatingKey(true);
try {
const params = new URLSearchParams({
const requestPayload: {
initial_balance_token: string;
balance_limit?: number;
balance_limit_reset?: string;
validity_date?: number;
} = {
initial_balance_token: initialToken.trim(),
});
if (balanceLimit) params.append('balance_limit', balanceLimit);
};
if (balanceLimit) requestPayload.balance_limit = Number(balanceLimit);
if (balanceLimitReset)
params.append('balance_limit_reset', balanceLimitReset);
requestPayload.balance_limit_reset = balanceLimitReset;
if (validityDate) {
const timestamp = Math.floor(
requestPayload.validity_date = Math.floor(
new Date(validityDate + 'T23:59:59').getTime() / 1000
);
params.append('validity_date', timestamp.toString());
}
const response = await fetch(
`${baseUrl}/v1/balance/create?${params.toString()}`,
{
method: 'GET',
headers: { 'Content-Type': 'application/json' },
}
);
const response = await fetch(`${baseUrl}/v1/balance/create`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(requestPayload),
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(errorText || 'Failed to create API key');
@@ -161,13 +161,11 @@ export function RoutstrCreateKeySection({
setIsCreatingCashu(true);
try {
const params = new URLSearchParams({
initial_balance_token: cashuToken.trim(),
const resp = await fetch(`${cleanUrl}/v1/balance/create`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ initial_balance_token: cashuToken.trim() }),
});
const resp = await fetch(
`${cleanUrl}/v1/balance/create?${params.toString()}`,
{ method: 'GET', headers: { 'Content-Type': 'application/json' } }
);
if (!resp.ok) {
const errorText = await resp.text();
+21 -6
View File
@@ -116,9 +116,24 @@ export function AdminSettings() {
setSaving(true);
setError('');
const updatedData = await AdminService.updateSettings(settings);
setSettings(updatedData as SettingsData);
setInitialSettings(updatedData as SettingsData);
// The nsec is a secret with its own endpoint (the general settings PATCH
// strips it); only send it when the operator actually changed it, so an
// untouched redacted value is never written back. The npub is derived
// server-side from the new nsec — fold it into the settings payload so the
// persisted blob stays consistent with the stored key.
let settingsPayload = settings;
if (hasFieldChanged('nsec')) {
const result = await AdminService.updateNsec(
(settings.nsec as string) || ''
);
settingsPayload = { ...settings, npub: result.npub };
}
const updatedData = (await AdminService.updateSettings(
settingsPayload
)) as SettingsData;
setSettings(updatedData);
setInitialSettings(updatedData);
toast.success('Settings saved successfully');
} catch (err) {
const message =
@@ -140,8 +155,8 @@ export function AdminSettings() {
return;
}
if (passwordData.new_password.length < 6) {
setPasswordError('New password must be at least 6 characters');
if (passwordData.new_password.length < 8) {
setPasswordError('New password must be at least 8 characters');
return;
}
@@ -944,7 +959,7 @@ export function AdminSettings() {
new_password: e.target.value,
}))
}
placeholder='Enter new password (min 6 characters)'
placeholder='Enter new password (min 8 characters)'
/>
</div>
+9
View File
@@ -801,6 +801,15 @@ export class AdminService {
);
}
static async updateNsec(
nsec: string
): Promise<{ ok: boolean; npub: string }> {
return await apiClient.patch<{ ok: boolean; npub: string }>(
'/admin/api/nsec',
{ nsec }
);
}
static async login(password: string): Promise<{
ok: boolean;
token: string;