Compare commits

...
Author SHA1 Message Date
9qeklajc c7e5fba910 better fallback 2026-08-05 00:32:06 +02:00
9qeklajcandGitHub 93df935446 Merge pull request #647 from Routstr/fix/invoice-quote-not-found-v2
fix: treat 'quote not found' as definitive unpaid for invoice expiry
2026-08-04 23:19:44 +02:00
9qeklajc 27f53948ca clean up 2026-08-04 22:59:50 +02:00
thefux 48c69857ed fix: treat 'quote not found' as definitive unpaid for invoice expiry
When the mint no longer has a Lightning quote (e.g. after TTL purge or
restart), check_invoice_payment() was logging an ERROR and returning False.
This caused the periodic_invoice_watcher to keep polling the same dead
quote every 10s forever, producing infinite log spam.

Now _is_quote_not_found() detects 'Mint Error: quote not found (Code: 0)'
and returns True, allowing _expire_invoice_if_authoritatively_unpaid()
to mark the invoice as expired so the watcher stops polling it.

The check is case-insensitive and requires code 0 to avoid false positives
from other quote-related errors.
2026-08-04 19:39:43 +00:00
9qeklajcandGitHub c971862ac6 Merge pull request #636 from Routstr/ppq-auto-topup
ppq-auto-topup
2026-08-04 01:50:59 +02:00
9qeklajc 22b35ff93d increase daily topup 2026-08-04 01:47:21 +02:00
9qeklajc 2f2820eb33 fix test 2026-08-04 01:44:01 +02:00
9qeklajc a6c129c02d harden autotopup 2026-08-04 01:32:25 +02:00
9qeklajc a2e2a5c662 clean up 2026-08-04 01:13:20 +02:00
9qeklajc 8e8a9a46b6 Merge remote-tracking branch 'origin/main' into ppq-auto-topup
# Conflicts:
#	routstr/upstream/auto_topup.py
#	routstr/wallet.py
#	tests/unit/test_wallet.py
2026-08-04 00:24:27 +02:00
9qeklajcandGitHub 47d0d87a88 Merge pull request #631 from Routstr/fix/xcashu-cost-breakdown-port
fix: expose cost breakdown in paid responses
2026-08-04 00:20:20 +02:00
9qeklajc 3b2c5a0671 Merge remote-tracking branch 'origin/main' into fix/xcashu-cost-breakdown-port 2026-08-04 00:12:37 +02:00
9qeklajcandGitHub 0609c5ed77 Merge pull request #597 from Routstr/fix/mint-rate-limit-and-fallback
fix: per-mint rate limiting, trusted-mint fallback, and retry factory…
2026-08-04 00:11:08 +02:00
9qeklajc 694bc04623 harden melting 2026-08-04 00:06:53 +02:00
9qeklajc da859f2f84 clean up 2026-08-03 23:32:06 +02:00
9qeklajc dd8c4a9a8a update migartion 2026-08-03 00:05:36 +02:00
9qeklajc 2b4e4c2430 Merge remote-tracking branch 'origin/main' into fix/mint-rate-limit-and-fallback
# Conflicts:
#	tests/unit/test_fee_payout_migration.py
2026-08-02 23:29:16 +02:00
9qeklajcandGitHub 8c3d8f52ba Merge pull request #588 from Routstr/model-paths
add model path endpoint
2026-08-02 23:24:31 +02:00
9qeklajc e903aa3a9f clean up 2026-08-02 23:16:01 +02:00
9qeklajc f3eefc2638 Merge remote-tracking branch 'origin/main' into model-paths
# Conflicts:
#	routstr/core/settings.py
#	tests/unit/test_fee_payout_migration.py
2026-08-02 22:27:39 +02:00
9qeklajcandGitHub 3463149b38 Merge pull request #645 from Routstr/fix/increase-default-db-pool
fix(db): increase default connection pool capacity
2026-07-31 02:53:50 +02:00
9qeklajc dc13cde00c chore: remove unrelated PPQ PR changes 2026-07-31 02:42:14 +02:00
9qeklajc 55dba5136c docs: document PPQ auto top-up 2026-07-31 02:24:28 +02:00
9qeklajc c0aad3b3ab test: isolate BOLT11 mint selection liabilities 2026-07-31 02:16:03 +02:00
9qeklajc 19236ecc9d fix(db): increase default connection pool capacity 2026-07-31 02:14:50 +02:00
9qeklajc 7a2b485af6 Merge main into ppq-auto-topup 2026-07-31 02:10:43 +02:00
9qeklajc 2ec6b27200 fix: resolve mint fallback review comments 2026-07-31 02:10:00 +02:00
9qeklajc f9980e5c66 fix: harden PPQ auto top-up reconciliation 2026-07-31 02:06:50 +02:00
9qeklajcandGitHub 98fe5a37cd Merge pull request #644 from Routstr/fix/api-key-refund-token-retrieval-main
fix: return persisted API-key refund token
2026-07-31 01:25:18 +02:00
9qeklajc 60566313dc fix: return persisted API-key refund token 2026-07-31 01:19:18 +02:00
9qeklajc 443c910b9e Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-30 08:38:27 +02:00
9qeklajcandGitHub 88d301398b Merge pull request #642 from Routstr/fix/payout-safety-pool-lifecycle
Fix payout safety and proxy session lifecycle
2026-07-30 03:17:37 +02:00
9qeklajc 4cc9aef61f fix: make payouts and proxy sessions safe 2026-07-30 02:57:49 +02:00
9qeklajc 3befe063f4 fix: annotate lightning settlement test session 2026-07-30 01:19:35 +02:00
9qeklajc f8adaee362 revert: restore default compose port 2026-07-30 01:14:36 +02:00
9qeklajc 895ea90bfa Merge branch 'main' into fix/mint-rate-limit-and-fallback
# Conflicts:
#	routstr/core/settings.py
#	routstr/lightning.py
#	routstr/wallet.py
#	tests/integration/test_lightning_invoice_constraints.py
#	tests/unit/test_fee_payout_migration.py
#	tests/unit/test_fetch_all_balances.py
2026-07-30 01:09:32 +02:00
9qeklajcandGitHub c4d27ba02a Merge pull request #634 from Routstr/fix/combined-db-pool-exhaustion
fix: combine DB pool-exhaustion and session-lifecycle fixes
2026-07-30 00:45:00 +02:00
9qeklajc 5ea5024608 resolve review comments 2026-07-29 22:50:33 +02:00
9qeklajc bb2a05b67c add url and specific model infos to paht 2026-07-29 00:08:34 +02:00
thefux c75dee147a fix: set routstr-core port to 8011 to avoid Portainer conflict on 8000 2026-07-28 00:08:47 +00:00
9qeklajc e2f89a2645 fix: address follow-up model path review 2026-07-27 23:17:13 +02:00
9qeklajc 48c11eb7bc fix Lightning settlement test typing 2026-07-27 00:07:36 +02:00
9qeklajc c829685f80 fix Cashu fallback and Lightning settlement 2026-07-26 23:23:32 +02:00
9qeklajc 16fc548b48 fix: make model path identity selectable 2026-07-26 20:12:55 +02:00
9qeklajc c5da73f1e9 revert: remove unrelated repository formatting 2026-07-26 20:12:12 +02:00
9qeklajc 06dba681c5 fix: satisfy strict mypy in model-paths tests
Replace untyped lambdas with typed handler/provider functions; CI runs
mypy over tests as well.
2026-07-26 13:26:49 +02:00
9qeklajc f96acbb99c fix: address model-paths review findings
Provider scoping (items 1/2/6):
- Key visibility maps on (model_id.lower(), upstream_provider_id), matching
  refresh_model_maps, so a disable/override row on one provider never leaks
  onto another provider's model, and matching is case-insensitive.

Data safety (items 3/5):
- Degraded OpenRouter fetches (network error, 429, non-200, bad payload)
  return None (unknown) instead of []; a provider whose path set is unknown
  keeps its previously persisted rows instead of being wiped.
- Endpoint payload parsing moved fully inside try, with a list guard, so
  endpoints:null or non-list shapes are swallowed as documented.
- refresh with an empty live upstream list is a no-op; the unfiltered
  DELETE in the prune path is gone (prune now keys off enabled DB rows).

Hot path (items 4/12/14):
- Persist uses chunked bulk INSERTs (one statement per 500 rows) instead of
  per-row ORM adds; redundant ix_model_paths_model_id index dropped.
- Read routes filter in SQL instead of materializing the whole table, and
  output ordering is deterministic (public id + path), independent of rowid.
- Visibility no longer rebuilds fully priced Model objects per override row;
  it reads id/forwarded_model_id/canonical_slug straight off ModelRow.

Path/id contract (items 7/8/9/11):
- discovery_path_for_subprovider/discovery_base_paths hooks on
  BaseUpstreamProvider, overridden by OpenRouterUpstreamProvider, mirror
  _apply_provider_field so discovery and response stamping cannot drift
  (openrouter:OpenRouter now correctly maps to unknown).
- openrouter_author_slug falls back to a slash-containing forwarded_model_id,
  so admin-created alias rows are discoverable.
- public_model_id splits on the first slash, same as get_base_model_id, so
  discovery ids can be sent to chat completions verbatim.

Lifecycle (items 10/13):
- ENABLE_MODEL_PATHS_REFRESH kill switch; interval and flag re-read every
  loop iteration, and the task idles (not exits) while disabled.
- First 429 latches and aborts the remaining fan-out for the cycle; a
  per-cycle cache dedupes fetches across providers sharing a base URL.
- refresh_model_maps prunes paths of disabled/deleted providers so admin
  mutations take effect immediately; rows carry updated_at and both
  endpoints expose it.

Tests (item 15) rewritten through the public refresh entry point with
transport-level httpx.MockTransport fakes, FK enforcement on, and coverage
for the periodic loop. Migration re-chained onto 9c4d8e2f1a6b.
2026-07-26 13:23:31 +02:00
9qeklajc 0a00527626 chore: apply ruff format repo-wide
CI only runs ruff check, so format drift accumulated. Committed separately
so the reformat noise stays out of functional commits.
2026-07-26 13:23:11 +02:00
9qeklajc 73a3f12469 Merge remote-tracking branch 'origin/main' into model-paths 2026-07-26 13:09:07 +02:00
9qeklajc 39f801561b fix: recreate refund sweep migration on latest head 2026-07-26 12:52:24 +02:00
9qeklajc ff55788e2d fix: address PR 634 review feedback 2026-07-26 12:44:37 +02:00
9qeklajc e38cd32fa3 add tests 2026-07-26 02:40:35 +02:00
9qeklajc 423e2cba73 ppq-auto-topup 2026-07-26 02:38:07 +02:00
9qeklajc 1138cdd4ef Merge main and recreate mint URL migration 2026-07-26 00:16:10 +02:00
9qeklajcandGitHub f15eab9f10 Merge pull request #635 from Routstr/re-apply-migration
reapply fee migration
2026-07-25 23:54:53 +02:00
9qeklajc 344c3c5f21 reapply fee migration 2026-07-25 23:42:50 +02:00
9qeklajc 4c6bc49e07 unify payment path 2026-07-25 00:36:32 +02:00
9qeklajcandGitHub 1d4b8d7cb2 Merge pull request #633 from Routstr/fix/cashu-token-create-post
fix: avoid 414 errors when creating keys from Cashu tokens
2026-07-24 23:43:56 +02:00
9qeklajc 1131c2d583 test: keep dynamic settings validation mypy-safe 2026-07-24 23:41:01 +02:00
9qeklajc 7108d554c8 merge: combine PR #632 with broader pool-exhaustion fixes 2026-07-24 23:22:01 +02:00
9qeklajc 1eddf89d52 merge: preserve PR #630 history 2026-07-24 23:06:51 +02:00
9qeklajc b0c70ecddc fix: send Cashu token creation payload in request body 2026-07-24 23:01:10 +02:00
Jeroen UbbinkandClaude Opus 4.8 a2cedd6769 feat: make the DB connection pool env-configurable
Add DATABASE_POOL_SIZE / DATABASE_MAX_OVERFLOW / DATABASE_POOL_TIMEOUT,
consumed like every other typed env var through the pydantic Settings
(constrained Fields, defaults 5/10/30 matching SQLAlchemy's own baseline
so leaving them unset is behaviour-neutral). An out-of-range or
non-integer value fails validation and refuses to boot — the same
fail-loud behaviour a malformed DATABASE_URL already has — rather than
silently starting up misconfigured. create_db_engine sizes the pool from
these and logs the effective values at startup so they can be confirmed
from the boot output during an incident. In-memory SQLite (StaticPool,
which rejects the pool kwargs) is detected and built without them.

pool_pre_ping is deliberately not exposed: the default backend is a local
SQLite file with no network peer to drop idle connections, so it would add
a SELECT 1 per checkout for no benefit — and it detects dead connections,
not the live-but-wedged ones behind the exhaustion this series addresses.

These knobs are infrastructure the node needs before it can open a DB
session, so they can never be sourced from the DB (chicken-and-egg). A new
ENV_ONLY_FIELDS set keeps them out of the persisted settings blob and
stops a DB value from shadowing env in both SettingsService.initialize and
.update, so env stays authoritative.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-24 22:26:57 +02:00
Jeroen UbbinkandClaude Opus 4.8 7b0ade3987 fix: stop AsyncSession pool exhaustion in balance reads
fetch_all_balances shared a single AsyncSession across the tasks it ran
with asyncio.gather. AsyncSession is not safe for concurrent use: the
overlapping queries raised "concurrent operations are not permitted" and
left connections wedged until the QueuePool was exhausted, after which
admin/API endpoints returned 500/502 until a container restart.

Read all outstanding user liabilities up front in one short-lived session
with a single grouped query (balances_by_mint_and_unit), then run the
per-mint balance checks concurrently with no session in scope. A failure
reading liabilities now degrades gracefully — the page still reports the
known wallet custody and blanks only the unknowable user/owner split,
tagging each mint with the error — instead of 500-ing the whole page.

periodic_payout reads each liability fresh, immediately before the payout
decision, rather than from a single pre-loop snapshot: the per-mint round
trip is slow, and a user top-up during the cycle would otherwise let a
later mint/unit act on a stale-low liability and over-send funds owed to
users (related to the payout-safety concern in issue #611).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-24 22:26:25 +02:00
9qeklajc 81c0ff57e9 fix: expose cost breakdown in paid responses 2026-07-24 22:24:24 +02:00
9qeklajc 2410a4a6ce fix: address review findings on payout liability staleness, sweep races, and cancellation safety
- periodic_payout: fetch liability per mint/unit right before computing
  available balance, so a concurrent top-up can only shrink the payout
- refund sweep: atomically claim each refund before redeeming; release the
  claim on failure so retries still happen and concurrent sweeps cannot
  misreport a sweep as client-collected
- check_invoice_payment: catch BaseException so task cancellation after a
  successful mint still emits the reconciliation alert
- tests: DB-guard race test where both mints succeed (exactly one credit);
  pool_size=1 test proving the fee payout releases its connection during
  the external send
2026-07-24 22:14:57 +02:00
9qeklajc 1b09639265 docs: document DB pool and mint concurrency env vars in .env.example 2026-07-24 21:44:11 +02:00
9qeklajc 7394b10e75 fix-concurent-issue 2026-07-24 21:29:21 +02:00
9qeklajc 4c292580e8 rebase model paths migration onto latest head 2026-07-24 21:15:55 +02:00
9qeklajc 04879cad56 Merge remote-tracking branch 'origin/main' into model-paths-pr588 2026-07-24 21:14:08 +02:00
9qeklajc ab80657507 recreate model paths migration 2026-07-24 20:56:34 +02:00
9qeklajcandGitHub b94d95fc2b Merge pull request #623 from Routstr/fix/streaming-billing-finalization
fix: fail safely on streaming billing errors
2026-07-24 20:19:56 +02:00
9qeklajc 27f81dbf42 update migration 2026-07-24 02:36:38 +02:00
9qeklajc 66ba31d0df Merge remote-tracking branch 'origin/main' into fix/streaming-billing-finalization 2026-07-24 02:05:58 +02:00
9qeklajc 16679c1f4e Merge branch 'main' into model-paths 2026-07-24 02:02:48 +02:00
9qeklajc 2ed20b1b85 resolve reviews 2026-07-24 00:24:24 +02:00
9qeklajc 92246b78d0 add import 2026-07-23 01:08:34 +02:00
9qeklajc 6023c03959 Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-23 00:22:38 +02:00
9qeklajc dbe7a53afd fix format 2026-07-23 00:09:03 +02:00
9qeklajc e0c74e3a46 Merge branch 'main' into fix/streaming-billing-finalization
# Conflicts:
#	routstr/auth.py
#	routstr/proxy.py
#	routstr/upstream/base.py
#	tests/integration/test_balance_negative_on_cost_overrun.py
#	tests/integration/test_free_response_stale_reservation.py
2026-07-23 00:03:01 +02:00
9qeklajc 97dc10a8ad harden impl. 2026-07-22 23:10:27 +02:00
9qeklajc a9a6381614 fix: recreate mint URL migration from latest head 2026-07-22 22:27:50 +02:00
9qeklajc 040799a4d7 Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-22 21:27:46 +02:00
9qeklajc 87850c97b9 Merge branch 'main' into fix/streaming-billing-finalization 2026-07-22 21:21:06 +02:00
thefux 586af15a1b chore: fix ruff lint errors (E402, F401, I001) 2026-07-18 14:41:32 +00:00
thefux 3e906605a0 fix: strict rate-limit detection, probe non-escalation, distinct error codes
Fixes three issues that caused per-mint rate-limit state to never recover:

1. _is_mint_rate_limited: remove substring matching on 'rate limit' /
   'too many requests' in exception messages.  Only HTTP 429
   (httpx.HTTPStatusError) is now classified as a rate limit, preventing
   false positives (e.g. a 503 with 'database rate exceeded' in its body).

2. _run_probe: use apply_cooldown() instead of apply_rate_limit_cooldown()
   when a probe fails due to a rate limit.  The probe is a recovery check,
   not a new request, so it should not escalate the exponential backoff
   counter (_consecutive_rate_limits).  This prevents the cooldown from
   ratcheting 60s → 120s → 240s → ... → 7h on repeated probe failures.

3. classify_redemption_error: split the combined _is_mint_rate_limited ||
   is_mint_connection_error check into two separate classifications:
   - mint_rate_limited / cashu_mint_rate_limited (503, retryable)
   - mint_unreachable / cashu_mint_unreachable (503, retryable)
   Callers (routstrd) can now distinguish temporary rate limits from
   permanent connection failures when deciding fallback strategy.

Tests: 20 new tests covering strict 429 detection, classification
priority, probe non-escalation, and cooldown reset behaviour.
2026-07-18 14:34:42 +00:00
9qeklajc 4defe4f227 fix: identify reservation releases 2026-07-18 14:59:57 +02:00
9qeklajc f8125a8a2d Merge branch 'fix/fee-payout-crash-guard' into fix/streaming-billing-finalization 2026-07-18 14:57:15 +02:00
9qeklajc 999a5634fa fix: snapshot reservation cleanup state 2026-07-18 14:54:06 +02:00
9qeklajc 2c218cce49 fix: make reservation cleanup atomic 2026-07-18 14:48:02 +02:00
9qeklajc fa0b366f9a fix: fail safely on streaming billing errors 2026-07-18 14:42:03 +02:00
9qeklajc 1957e716a3 clean up keyset unit recog. 2026-07-15 01:45:19 +02:00
9qeklajc 69f19ff991 defensive cooldown 2026-07-15 01:43:00 +02:00
9qeklajc 8b942f3c14 show mint status correclty 2026-07-15 01:00:32 +02:00
9qeklajc 09e1c7bf2d better cooldown 2026-07-15 00:10:17 +02:00
9qeklajc cc2a96e2ef make trusted mint available for lightning topup 2026-07-14 21:59:43 +02:00
9qeklajc 93ab1d927b mint cooldown 2026-07-14 01:55:35 +02:00
9qeklajc 39970d8bee clean up 2026-07-14 01:23:14 +02:00
9qeklajc d7c401d204 primary mint fallback 2026-07-14 01:10:23 +02:00
9qeklajc d44b98fd0d fix fallback 2026-07-14 00:54:44 +02:00
9qeklajc 6fa3610423 fix: account for mint fees in balance checks 2026-07-14 00:11:27 +02:00
9qeklajc 65702171e4 fix: harden mint fallback and refund recovery 2026-07-13 23:37:43 +02:00
9qeklajc 65abcbce92 fix: harden mint fallback and refund recovery 2026-07-13 23:33:56 +02:00
9qeklajc 40153d4c36 fix: report cashu transaction persistence 2026-07-12 15:07:31 +02:00
9qeklajc 40bf976fbc fix: recreate mint URL migration 2026-07-12 15:04:43 +02:00
9qeklajc eae20f04a7 Merge branch 'main' into fix/mint-rate-limit-and-fallback 2026-07-12 15:02:07 +02:00
9qeklajc acb630f6cf refactor: adapt mint throttling to 429 responses 2026-07-10 23:54:05 +02:00
9qeklajc 1230d528de fix: avoid rate limiting balance proof checks 2026-07-10 23:46:09 +02:00
9qeklajc d23c90b939 fix: type wallet test fixture 2026-07-10 21:50:50 +02:00
9qeklajc d8db2a3051 fix: harden mint rate limiting and fallback 2026-07-10 21:46:56 +02:00
9qeklajc 0bbbf902cd Merge origin/main into fix/mint-rate-limit-and-fallback 2026-07-10 21:12:54 +02:00
9qeklajc 7ed18a9d02 fix: per-mint rate limiting, trusted-mint fallback, and retry factory fix 2026-07-10 20:43:07 +02:00
9qeklajc dc25659cff only activ model should be visible 2026-07-07 11:11:46 +02:00
9qeklajc 349d8dd009 add model path endpoint 2026-07-06 23:49:56 +02:00
88 changed files with 16531 additions and 1258 deletions
+18
View File
@@ -22,6 +22,19 @@ ROUTSTR_SECRET_KEY=
# Database
# DATABASE_URL=sqlite+aiosqlite:///keys.db
# Pool controls are validated at boot, sourced only from the environment, and
# logged at startup. Keep total capacity across all workers below the database
# connection limit. Pre-ping is automatic for networked backends; SQLite may
# explicitly opt in if desired.
# DATABASE_POOL_SIZE=10
# DATABASE_MAX_OVERFLOW=20
# DATABASE_POOL_TIMEOUT=15
# DATABASE_POOL_RECYCLE=1800
# DATABASE_POOL_PRE_PING=false
# Warn when a checkout is held this many seconds.
# DATABASE_POOL_HOLD_WARN_SECONDS=10
# SQLite serialises writes; increasing its pool can trade pool timeouts for
# "database is locked" errors rather than increasing write throughput.
# Node Information
# NAME=My Routstr Node
@@ -31,7 +44,12 @@ ROUTSTR_SECRET_KEY=
# RELAYS="wss://relay.damus.io,wss://relay.nostr.band,wss://eden.nostr.land,wss://relay.routstr.com"
# ENABLE_ANALYTICS_SHARING=true
# CASHU_MINTS="https://mint.minibits.cash/Bitcoin,https://mint.cubabitcoin.org,https://ecashmint.otrta.me"
# MINT_OPERATION_CONCURRENCY=4
# MINT_OPERATION_TIMEOUT_SECONDS=30
# MINT_MAX_CONCURRENCY=4
# MINT_RETRY_MAX_ATTEMPTS=3
# RECEIVE_LN_ADDRESS=
# REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS=900
# Custom Pricing Configuration
# MODEL_BASED_PRICING=true
+56
View File
@@ -327,6 +327,62 @@ GET /v1/models
}
```
### List Model Paths
Get the selectable upstream routes for each advertised model. This endpoint is
discovery-only; request-side selection will be added separately.
```http
GET /v1/models/paths
```
**Response:**
```json
{
"data": [
{
"id": "anthropic/claude-sonnet-4",
"paths": [
{
"path": "url=https%3A%2F%2Fapi.anthropic.com%2Fv1&provider-id=12&model-id=anthropic%2Fclaude-sonnet-4",
"provider": {"id": 12, "slug": "anthropic-primary", "type": "anthropic"},
"endpoint": null
},
{
"path": "url=https%3A%2F%2Fopenrouter.ai%2Fapi%2Fv1&provider-id=42&model-id=anthropic%2Fclaude-sonnet-4&endpoint=google-vertex%2Fus",
"provider": {"id": 42, "slug": "openrouter-main", "type": "openrouter"},
"endpoint": {"tag": "google-vertex/us", "name": "Google"}
}
]
}
],
"updated_at": 1753500000
}
```
`path` is an opaque, percent-encoded selector. Clients must store and return it
unchanged rather than parsing or reconstructing it. It identifies the exact
configured route with `url`, `provider-id`, and `model-id`. To avoid exposing
private network details, a configured private IP address or any URL with an
explicit port is advertised as `http://localhost`. OpenRouter routes additionally
preserve the exact machine-readable endpoint `tag`. Provider slugs/types and
endpoint names remain display data. When request-side selection is implemented,
an endpoint tag must not silently fall back to another backend.
### List Paths for One Model
Use the exact model ID advertised by `/v1/models`. The query parameter safely
supports IDs containing `/`.
```http
GET /v1/models/paths/model?model_id=anthropic/claude-sonnet-4
```
The response uses the same path objects and `updated_at` field as the collection
endpoint. An unknown model returns `404 Model not found`. A known model whose
paths have not been discovered yet returns `200` with an empty `data` array.
## Wallet Management
### Create Wallet (Coming Soon)
+2 -1
View File
@@ -100,6 +100,7 @@ All errors follow a consistent format:
Standard OpenAI-compatible endpoints:
- **Models**: `/v1/models`
- **Model paths**: `/v1/models/paths`, `/v1/models/paths/model?model_id=...`
- **Responses**: `/v1/responses`
- **Chat Completions**: `/v1/chat/completions`
- **Embeddings**: `/v1/embeddings`
@@ -302,7 +303,7 @@ Get node metadata:
GET /v1/info
```
Supported models and pricing are available at `/v1/models`.
Supported models and pricing are available at `/v1/models`. Upstream provider path discovery is available at `/v1/models/paths` and `/v1/models/paths/model?model_id=...`.
## Next Steps
+78
View File
@@ -48,6 +48,68 @@ Connect to your AI provider(s):
| **Upstream URL** | API endpoint (e.g., `https://api.openai.com/v1`) |
| **API Key** | Your provider's API key |
### PPQ Auto Top-up
PPQ providers can automatically purchase more credits when their USD balance
falls below a configured threshold. Configure this per provider in the Admin
Dashboard by editing a **PPQ.AI** provider and opening **PPQ Auto Top-up**.
There are no environment variables for this feature.
#### Requirements
Before enabling auto top-up, make sure that:
- the PPQ provider has a valid API key;
- at least one trusted Cashu mint is configured;
- the node wallet has enough **node-owned** funds at one mint to pay the
Lightning invoice; client balances are never used; and
- the node has a current BTC/USD price for validating the invoice amount.
| Setting | Description |
| ------- | ----------- |
| **Enable Auto Top-up** | Enables automatic PPQ credit purchases for this provider. |
| **When credits are below (USD)** | Starts a top-up when the reported PPQ balance is below this positive USD value. |
| **Purchase this amount (USD)** | Amount of PPQ credit to buy per top-up. Must be a whole number from **1 to 500 USD**. |
For example, a threshold of `5` and purchase amount of `20` buys 20 USD of
credit when the PPQ balance drops below 5 USD.
#### How it works
The worker checks eligible providers approximately once per minute. When the
balance is below the threshold, it:
1. verifies the node has enough owner funds before creating an invoice;
2. requests a USD-denominated Lightning top-up invoice from PPQ;
3. rejects expired, mismatched, or unexpectedly expensive invoices (more than
10% above the local BTC/USD estimate);
4. pays from the configured Cashu mint with sufficient owner funds; and
5. waits for PPQ to confirm that the credit settled.
Only one attempt can be active for a provider. An attempt that was active at
the start of a cycle suppresses another top-up for that entire cycle, even if
PPQ reports it settled immediately. This prevents a temporarily stale PPQ
balance from causing a duplicate purchase.
Completed PPQ payments appear in the dashboard transaction history with source
`ppq_auto_topup`. The payment record is separate from the internal claim used
to prevent concurrent attempts.
#### Payment recovery
If the Cashu mint paid the invoice but PPQ settlement cannot be confirmed, the
provider card shows **Auto top-up needs review**. A payment still owned by a
running worker is shown as **Paying invoice** and cannot be released.
Before choosing **Release top-up**, manually verify both PPQ and the Cashu mint.
Release the claim only when the previous Lightning payment is definitively
unable to settle. Releasing an ambiguous payment allows the next cycle to try
again and can therefore cause a duplicate top-up.
Disabling auto top-up prevents new purchases, but the node continues to
reconcile an already active payment until it reaches a safe terminal state or
requires operator review.
### Node Identity
How your node appears to clients:
@@ -136,12 +198,23 @@ Use environment variables for:
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
| `MINT_OPERATION_CONCURRENCY` | Concurrent mint/unit balance reads | `4` |
| `MINT_OPERATION_TIMEOUT_SECONDS` | Per-attempt timeout for mint network calls | `30` |
| `MINT_MAX_CONCURRENCY` | Concurrent operations allowed per mint (`0` disables the limit) | `4` |
| `MINT_RETRY_MAX_ATTEMPTS` | Retries after a timeout or HTTP 429 (`0` disables retries) | `3` |
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
| `MIN_PAYOUT_SAT` | Min payout balance in sats (applies to all mints) | `210` |
| `PAYOUT_INTERVAL_SECONDS` | Payout loop interval (seconds) | `900` |
| `TOR_PROXY_URL` | SOCKS5 proxy for Tor | `socks5://127.0.0.1:9050` |
| `CORS_ORIGINS` | Allowed CORS origins | `*` |
| `RELAYS` | Nostr relays (comma-separated) | (default set) |
| `MODEL_PATHS_REFRESH_INTERVAL_SECONDS` | How often to refresh `/v1/models/paths` discovery data; set `0` to pause the refresh (previously discovered paths keep being served) | `600` |
| `ENABLE_MODEL_PATHS_REFRESH` | Kill switch for the background model-path refresh (OpenRouter endpoint fan-out) | `true` |
Mint HTTP 429 responses create a per-mint cooldown. Operations that already hold
Routstr's wallet mutation lock fail fast during that cooldown instead of waiting
while blocking every other wallet mutation. Callers receive an error and may retry
later; the current response does not include the cooldown duration.
### Priority
@@ -175,3 +248,8 @@ Manage which AI models you offer:
- **Create aliases** — friendly names for models
See [Pricing](pricing.md) for per-model pricing strategies.
Model path discovery is refreshed in the background and exposed through
`/v1/models/paths`. The response groups each client-visible model ID with the
provider paths that may appear in chat-completion response metadata. Tune the
refresh cadence with `MODEL_PATHS_REFRESH_INTERVAL_SECONDS`.
@@ -0,0 +1,52 @@
"""add model paths table
Revision ID: 64ed5594df1f
Revises: aa50fde387a2
Create Date: 2026-08-02 22:26:33.280409
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
# revision identifiers, used by Alembic.
revision = "64ed5594df1f"
down_revision = "aa50fde387a2"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"model_paths",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("model_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column("path", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column("provider_slug", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column("provider_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column("endpoint_tag", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column("endpoint_name", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column("upstream_provider_id", sa.Integer(), nullable=False),
sa.Column("updated_at", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(
["upstream_provider_id"], ["upstream_providers.id"], ondelete="CASCADE"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"model_id",
"path",
"upstream_provider_id",
name="uq_model_paths_model_path_provider",
),
)
op.create_index(
op.f("ix_model_paths_upstream_provider_id"),
"model_paths",
["upstream_provider_id"],
unique=False,
)
def downgrade() -> None:
op.drop_index(op.f("ix_model_paths_upstream_provider_id"), table_name="model_paths")
op.drop_table("model_paths")
@@ -0,0 +1,62 @@
"""add reservation release idempotency records
Revision ID: 7f2843d3f4e4
Revises: fc4fa29630d2
Create Date: 2026-07-24 02:06:06.066726
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "7f2843d3f4e4"
down_revision = "fc4fa29630d2"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"reservation_releases",
sa.Column("id", sa.String(), nullable=False),
sa.Column("key_hash", sa.String(), nullable=False),
sa.Column("billing_key_hash", sa.String(), nullable=False),
sa.Column("reserved_msats", sa.Integer(), nullable=False),
sa.Column(
"status", sa.String(), nullable=False, server_default="active"
),
sa.Column("created_at", sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"ix_reservation_releases_key_hash",
"reservation_releases",
["key_hash"],
)
op.create_index(
"ix_reservation_releases_billing_key_hash",
"reservation_releases",
["billing_key_hash"],
)
op.create_index(
"ix_reservation_releases_status_created_at",
"reservation_releases",
["status", "created_at"],
)
def downgrade() -> None:
op.drop_index(
"ix_reservation_releases_status_created_at",
table_name="reservation_releases",
)
op.drop_index(
"ix_reservation_releases_billing_key_hash",
table_name="reservation_releases",
)
op.drop_index(
"ix_reservation_releases_key_hash",
table_name="reservation_releases",
)
op.drop_table("reservation_releases")
@@ -0,0 +1,47 @@
"""repair missing fee payout checkpoint columns
Revision ID: 9c4d8e2f1a6b
Revises: 7f2843d3f4e4
Create Date: 2026-07-25 00:00:00.000000
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "9c4d8e2f1a6b"
down_revision = "7f2843d3f4e4"
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Repair databases stamped past the original checkpoint migration."""
conn = op.get_bind()
columns = {
column["name"] for column in sa.inspect(conn).get_columns("routstr_fees")
}
if "payout_in_progress_msats" not in columns:
op.add_column(
"routstr_fees",
sa.Column(
"payout_in_progress_msats",
sa.Integer(),
nullable=False,
server_default="0",
),
)
if "payout_started_at" not in columns:
op.add_column(
"routstr_fees",
sa.Column("payout_started_at", sa.Integer(), nullable=True),
)
def downgrade() -> None:
# The preceding revision already expects both columns. This migration only
# repairs schema drift, so downgrading it must preserve the expected schema.
pass
@@ -0,0 +1,39 @@
"""add refund sweep claim lease
Revision ID: aa50fde387a2
Revises: 9c4d8e2f1a6b
Create Date: 2026-07-26 12:50:10.509217
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "aa50fde387a2"
down_revision = "9c4d8e2f1a6b"
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
columns = {
column["name"]
for column in sa.inspect(conn).get_columns("cashu_transactions")
}
if "sweep_started_at" not in columns:
op.add_column(
"cashu_transactions",
sa.Column("sweep_started_at", sa.Integer(), nullable=True),
)
def downgrade() -> None:
conn = op.get_bind()
columns = {
column["name"]
for column in sa.inspect(conn).get_columns("cashu_transactions")
}
if "sweep_started_at" in columns:
op.drop_column("cashu_transactions", "sweep_started_at")
@@ -0,0 +1,67 @@
"""add mint url to lightning invoices
Revision ID: ecfa0d6e2a36
Revises: 64ed5594df1f
Create Date: 2026-08-02 23:53:00.037456
"""
import json
import os
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "ecfa0d6e2a36"
down_revision = "64ed5594df1f"
branch_labels = None
depends_on = None
def _resolve_backfill_mint_url(bind: sa.engine.Connection) -> str | None:
"""Best-effort resolution of the mint that issued pre-existing invoices.
Order: persisted settings JSON -> PRIMARY_MINT_URL env -> first CASHU_MINTS entry.
"""
try:
row = bind.execute(
sa.text("SELECT data FROM settings ORDER BY id LIMIT 1")
).fetchone()
if row and row[0]:
data = json.loads(row[0])
mint = data.get("primary_mint") or next(
iter(data.get("cashu_mints") or []), None
)
if mint:
return str(mint)
except Exception:
pass
env_mint = os.environ.get("PRIMARY_MINT_URL", "").strip()
if env_mint:
return env_mint
cashu_mints = os.environ.get("CASHU_MINTS", "").strip()
if cashu_mints:
return cashu_mints.split(",")[0].strip() or None
return None
def upgrade() -> None:
op.add_column(
"lightning_invoices", sa.Column("mint_url", sa.String(), nullable=True)
)
bind = op.get_bind()
backfill_mint = _resolve_backfill_mint_url(bind)
if backfill_mint:
bind.execute(
sa.text(
"UPDATE lightning_invoices SET mint_url = :mint WHERE mint_url IS NULL"
),
{"mint": backfill_mint},
)
def downgrade() -> None:
op.drop_column("lightning_invoices", "mint_url")
+429 -169
View File
@@ -3,16 +3,25 @@ import hashlib
import math
import random
import time
import uuid
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING, Optional
from fastapi import HTTPException
from sqlalchemy import case
from sqlalchemy import case, inspect
from sqlalchemy.exc import IntegrityError
from sqlmodel import col, select, update
from .core import get_logger
from .core.db import ApiKey, AsyncSession, accumulate_routstr_fee
from .core.db import (
ApiKey,
AsyncSession,
ReservationRelease,
accumulate_routstr_fee,
create_session,
)
from .core.settings import settings
from .payment.cost_calculation import (
CostData,
@@ -24,6 +33,7 @@ from .wallet import (
classify_redemption_error,
credit_balance,
deserialize_token_from_string,
wallet_operation_guard,
)
if TYPE_CHECKING:
@@ -34,10 +44,50 @@ payments_logger = get_logger("routstr.payments")
# Routstr platform fee constants
ROUTSTR_FEE_PERCENT: float = 2.1
ROUTSTR_LN_ADDRESS: str = "npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
ROUTSTR_LN_ADDRESS: str = (
"npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
)
ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
def _format_msat_amount(amount: int) -> str:
sats = f"{amount / 1000:.3f}".rstrip("0").rstrip(".")
return f"{sats} sats ({amount} msats)"
def _model_balance_error(required: int, available: int) -> dict[str, dict[str, str]]:
return {
"error": {
"message": (
f"Insufficient balance: {_format_msat_amount(required)} required "
f"for this model; {_format_msat_amount(available)} available."
),
"type": "insufficient_quota",
"code": "insufficient_balance",
}
}
@dataclass(frozen=True)
class ReservationSnapshot:
release_id: str
key_hash: str
billing_key_hash: str
reserved_msats: int
_current_reservation: ContextVar[ReservationSnapshot | None] = ContextVar(
"current_billing_reservation", default=None
)
def _clear_current_reservation(snapshot: ReservationSnapshot) -> None:
current = _current_reservation.get()
if current is not None and current.release_id == snapshot.release_id:
_current_reservation.set(None)
# TODO: implement prepaid api key (not like it was before)
# PREPAID_API_KEY = os.environ.get("PREPAID_API_KEY", None)
# PREPAID_BALANCE = int(os.environ.get("PREPAID_BALANCE", "0")) * 1000 # Convert to msats
@@ -122,6 +172,29 @@ async def validate_bearer_key(
refund_address: Optional[str] = None,
key_expiry_time: Optional[int] = None,
min_cost: int = 0,
) -> ApiKey:
if bearer_key.startswith("cashu"):
# Acquire before the first lookup/flush so concurrent token creation
# cannot hold SQLite write transactions while waiting to mutate proofs.
async with wallet_operation_guard():
return await _validate_bearer_key_locked(
bearer_key,
session,
refund_address,
key_expiry_time,
min_cost,
)
return await _validate_bearer_key_locked(
bearer_key, session, refund_address, key_expiry_time, min_cost
)
async def _validate_bearer_key_locked(
bearer_key: str,
session: AsyncSession,
refund_address: Optional[str] = None,
key_expiry_time: Optional[int] = None,
min_cost: int = 0,
) -> ApiKey:
"""
Validates the provided API key using SQLModel.
@@ -209,13 +282,7 @@ async def validate_bearer_key(
)
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Insufficient balance: {min_cost} mSats required for this model. {billing_key.total_balance} available.",
"type": "insufficient_quota",
"code": "insufficient_balance",
}
},
detail=_model_balance_error(min_cost, billing_key.total_balance),
)
# Early check: Spending limit check (Child key limit)
@@ -305,13 +372,9 @@ async def validate_bearer_key(
if min_cost > 0 and existing_key.total_balance < min_cost:
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Insufficient balance: {min_cost} mSats required for this model. {existing_key.total_balance} available.",
"type": "insufficient_quota",
"code": "insufficient_balance",
}
},
detail=_model_balance_error(
min_cost, existing_key.total_balance
),
)
return existing_key
@@ -324,11 +387,23 @@ async def validate_bearer_key(
"has_expiry_time": bool(key_expiry_time),
},
)
if token_obj.mint in settings.cashu_mints:
if token_obj.mint == settings.primary_mint:
if token_obj.unit != settings.primary_mint_unit:
raise redemption_error_to_http_exception(
ValueError(
"Cashu token unit does not match the configured primary "
f"mint unit: expected {settings.primary_mint_unit}, "
f"got {token_obj.unit}"
)
)
refund_currency = token_obj.unit
refund_mint_url = settings.primary_mint
elif token_obj.mint in settings.cashu_mints:
refund_currency = token_obj.unit
refund_mint_url = token_obj.mint
else:
refund_currency = "sat"
# Foreign tokens are swapped into the configured primary mint.
refund_currency = settings.primary_mint_unit
refund_mint_url = settings.primary_mint
new_key = ApiKey(
@@ -595,6 +670,16 @@ async def pay_for_request(
},
)
# Create the durable reservation identity before changing aggregate balances.
# The row and balance updates commit together, so every reserved amount has one
# owner that can reach exactly one terminal state.
reservation = ReservationSnapshot(
release_id=uuid.uuid4().hex,
key_hash=key.hashed_key,
billing_key_hash=billing_key.hashed_key,
reserved_msats=cost_per_request,
)
# Charge the base cost for the request atomically to avoid race conditions
reserved_at_now = int(time.time())
stmt = (
@@ -656,22 +741,83 @@ async def pay_for_request(
child_result = await session.exec(child_stmt) # type: ignore[call-overload]
if child_result.rowcount == 0:
# Build the error before rollback expires ORM attributes.
limit_message = (
f"Balance limit exceeded: {key.balance_limit} mSats limit. "
f"{key.total_spent} already spent ({key.reserved_balance} reserved), "
f"{cost_per_request} required for this request."
)
# The parent reservation update already ran in this transaction.
# Roll it back before failover code attempts to restore the previous
# reservation; otherwise that later commit can persist both updates.
await session.rollback()
raise HTTPException(
status_code=402,
detail={
"error": {
"message": f"Balance limit exceeded: {key.balance_limit} mSats limit. {key.total_spent} already spent ({key.reserved_balance} reserved), {cost_per_request} required for this request.",
"message": limit_message,
"type": "insufficient_quota",
"code": "balance_limit_exceeded",
}
},
)
await session.commit()
session.add(
ReservationRelease(
id=reservation.release_id,
key_hash=reservation.key_hash,
billing_key_hash=reservation.billing_key_hash,
reserved_msats=reservation.reserved_msats,
status="active",
)
)
# Publish the identity before commit. If the commit succeeds but its
# acknowledgement is interrupted, exact cleanup can still recover the
# durable row. A definitely failed commit is harmless because every
# terminal transition validates that row before touching balances.
_current_reservation.set(reservation)
try:
await session.commit()
except BaseException:
# The database may have committed even if acknowledgement was cancelled
# or the connection failed. Reconcile using a fresh transaction and the
# exact durable identity; no upstream request has started yet.
try:
await session.rollback()
except Exception:
pass
try:
async with create_session() as cleanup_session:
record = await cleanup_session.get(
ReservationRelease, reservation.release_id
)
if record is not None and record.status == "active":
await _transition_reservation_to_released(
reservation,
cleanup_session,
decrement_requests=True,
idempotent_success=True,
)
except Exception:
logger.exception(
"Failed to reconcile ambiguous reservation commit",
extra={"reservation_id": reservation.release_id},
)
finally:
_clear_current_reservation(reservation)
raise
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
try:
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
except Exception:
# The reservation transaction is already committed and durable. Logging
# refresh failures must not make the caller treat it as unreserved.
logger.exception(
"Reservation committed but post-commit refresh failed",
extra={"reservation_id": reservation.release_id},
)
logger.info(
"Payment processed successfully",
@@ -701,81 +847,185 @@ async def pay_for_request(
async def revert_pay_for_request(
key: ApiKey, session: AsyncSession, cost_per_request: int
key: ApiKey,
session: AsyncSession,
cost_per_request: int,
reservation_snapshot: ReservationSnapshot | None = None,
) -> bool:
"""Revert a previously reserved payment. Returns True if revert succeeded,
False if the reservation was already released (prevents negative reserved_balance)."""
billing_key = await get_billing_key(key, session)
# Keep reserved_at while other reservations remain
cleared_reserved_at = case(
(col(ApiKey.reserved_balance) - cost_per_request > 0, col(ApiKey.reserved_at)),
else_=None,
"""Revert the current request's durable reservation exactly once."""
snapshot = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(key, snapshot, session, require_active=False)
if cost_per_request != snapshot.reserved_msats:
return False
return await _transition_reservation_to_released(
snapshot,
session,
decrement_requests=True,
idempotent_success=False,
)
stmt = (
async def _validate_reservation_snapshot(
key: ApiKey,
snapshot: ReservationSnapshot,
session: AsyncSession,
*,
require_active: bool = True,
) -> None:
"""Reject cross-request or forged reservation handles before any mutation."""
state = inspect(key)
identity = state.identity if state is not None else None
key_hash = str(identity[0]) if identity else key.__dict__.get("hashed_key")
if snapshot.key_hash != key_hash:
raise RuntimeError("Billing reservation does not belong to this key")
persisted_key = await session.get(ApiKey, snapshot.key_hash)
if persisted_key is None:
raise RuntimeError("Billing reservation key no longer exists")
expected_billing_hash = persisted_key.parent_key_hash or persisted_key.hashed_key
if snapshot.billing_key_hash != expected_billing_hash:
raise RuntimeError("Billing reservation does not belong to this billing key")
record = await session.get(ReservationRelease, snapshot.release_id)
if (
record is None
or (require_active and record.status != "active")
or record.key_hash != snapshot.key_hash
or record.billing_key_hash != snapshot.billing_key_hash
or record.reserved_msats != snapshot.reserved_msats
):
raise RuntimeError("Billing reservation record does not match the request")
async def get_reservation_snapshot(
key: ApiKey, session: AsyncSession
) -> ReservationSnapshot:
"""Return the durable reservation created for the current request."""
snapshot = _current_reservation.get()
if snapshot is None:
raise RuntimeError("No billing reservation is associated with this request")
await _validate_reservation_snapshot(key, snapshot, session)
return snapshot
async def _transition_reservation_to_released(
snapshot: ReservationSnapshot,
session: AsyncSession,
*,
decrement_requests: bool,
idempotent_success: bool,
) -> bool:
transition = (
update(ReservationRelease)
.where(col(ReservationRelease.id) == snapshot.release_id)
.where(col(ReservationRelease.status) == "active")
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
.values(status="released")
)
transition_result = await session.exec(transition) # type: ignore[call-overload]
if transition_result.rowcount != 1:
await session.rollback()
existing = await session.get(ReservationRelease, snapshot.release_id)
return bool(
idempotent_success
and existing is not None
and existing.status == "released"
and existing.key_hash == snapshot.key_hash
and existing.billing_key_hash == snapshot.billing_key_hash
and existing.reserved_msats == snapshot.reserved_msats
)
values: dict[str, object] = {
"reserved_balance": col(ApiKey.reserved_balance) - snapshot.reserved_msats,
"reserved_at": case(
(
col(ApiKey.reserved_balance) - snapshot.reserved_msats > 0,
col(ApiKey.reserved_at),
),
else_=None,
),
}
if decrement_requests:
values["total_requests"] = col(ApiKey.total_requests) - 1
release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.where(col(ApiKey.reserved_balance) >= cost_per_request)
.values(
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
reserved_at=cleared_reserved_at,
total_requests=col(ApiKey.total_requests) - 1,
)
.where(col(ApiKey.hashed_key) == snapshot.billing_key_hash)
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
.values(**values)
)
result = await session.exec(release_stmt) # type: ignore[call-overload]
if result.rowcount != 1:
await session.rollback()
return False
result = await session.exec(stmt) # type: ignore[call-overload]
# Also decrement total_requests and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_stmt = (
if snapshot.billing_key_hash != snapshot.key_hash:
child_release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.where(col(ApiKey.reserved_balance) >= cost_per_request)
.values(
total_requests=col(ApiKey.total_requests) - 1,
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
reserved_at=cleared_reserved_at,
)
.where(col(ApiKey.hashed_key) == snapshot.key_hash)
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
.values(**values)
)
await session.exec(child_stmt) # type: ignore[call-overload]
child_result = await session.exec( # type: ignore[call-overload]
child_release_stmt
)
if child_result.rowcount != 1:
await session.rollback()
return False
await session.commit()
if result.rowcount == 0:
logger.warning(
"Revert skipped - reservation already released (no-op to prevent negative reserved_balance)",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"cost_to_revert": cost_per_request,
"current_reserved_balance": billing_key.reserved_balance,
},
)
return False
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
payments_logger.info(
"REVERT",
extra={
"event": "revert",
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"cost_reverted": cost_per_request,
"balance": billing_key.balance,
"reserved_balance": billing_key.reserved_balance,
},
)
_clear_current_reservation(snapshot)
return True
async def release_reservation(
snapshot: ReservationSnapshot,
session: AsyncSession,
reserved_msats: int,
) -> bool:
"""Release one durable reservation exactly once without charging."""
if reserved_msats <= 0 or reserved_msats != snapshot.reserved_msats:
return False
return await _transition_reservation_to_released(
snapshot,
session,
decrement_requests=False,
idempotent_success=True,
)
async def _claim_reservation_for_charge(
snapshot: ReservationSnapshot, session: AsyncSession
) -> bool:
"""Claim an active reservation in the caller's charge transaction."""
statement = (
update(ReservationRelease)
.where(col(ReservationRelease.id) == snapshot.release_id)
.where(col(ReservationRelease.status) == "active")
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
.values(status="charged")
)
result = await session.exec(statement) # type: ignore[call-overload]
if result.rowcount == 1:
_clear_current_reservation(snapshot)
return True
await session.rollback()
return False
async def adjust_payment_for_tokens(
key: ApiKey,
response_data: dict,
session: AsyncSession,
deducted_max_cost: int,
model_obj: "Model | None",
provider_fee: float | None,
model_obj: "Model | None" = None,
provider_fee: float | None = None,
reservation_snapshot: ReservationSnapshot | None = None,
) -> dict:
"""
Adjusts the payment based on token usage in the response.
@@ -790,6 +1040,13 @@ async def adjust_payment_for_tokens(
``calculate_cost``.
"""
billing_key = await get_billing_key(key, session)
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(
key, reservation, session, require_active=False
)
# The persisted amount is authoritative if request-level minimum pricing
# changed the caller's original estimate.
deducted_max_cost = reservation.reserved_msats
model = response_data.get("model", "unknown")
logger.debug(
@@ -805,50 +1062,21 @@ async def adjust_payment_for_tokens(
)
async def release_reservation_only() -> None:
"""Fallback to release reservation without charging when main update fails."""
"""Fallback to release this request's reservation without charging."""
try:
release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
.values(
reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost
)
released = await release_reservation(
reservation, session, reservation.reserved_msats
)
logger.warning(
"Released reservation without charging (fallback)"
if released
else "Reservation was already finalized; fallback skipped",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"deducted_max_cost": deducted_max_cost,
},
)
result = await session.exec(release_stmt) # type: ignore[call-overload]
# Also release on child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
.values(
reserved_balance=col(ApiKey.reserved_balance)
- deducted_max_cost
)
)
await session.exec(child_release_stmt) # type: ignore[call-overload]
await session.commit()
if result.rowcount == 0: # type: ignore[union-attr]
logger.warning(
"Release reservation skipped - already released (no-op to prevent negative reserved_balance)",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"deducted_max_cost": deducted_max_cost,
},
)
else:
logger.warning(
"Released reservation without charging (fallback)",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"deducted_max_cost": deducted_max_cost,
},
)
except Exception as e:
logger.error(
"Failed to release reservation in fallback",
@@ -870,9 +1098,17 @@ async def adjust_payment_for_tokens(
extra={"error": str(e), "fee_msats": fee_msats},
)
match await calculate_cost(
calculated_cost = await calculate_cost(
response_data, deducted_max_cost, model_obj, provider_fee
):
)
if not isinstance(calculated_cost, CostDataError):
if not await _claim_reservation_for_charge(reservation, session):
# A prior charge or release already owns this reservation. Returning
# the calculated metadata is safe; the aggregate balances must not
# be modified a second time.
return calculated_cost.dict()
match calculated_cost:
case MaxCostData() as cost:
logger.debug(
"Using max cost data (no token adjustment)",
@@ -900,8 +1136,10 @@ async def adjust_payment_for_tokens(
)
safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
@@ -919,8 +1157,10 @@ async def adjust_payment_for_tokens(
# Also update total_spent and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
child_stmt = (
@@ -1030,8 +1270,10 @@ async def adjust_payment_for_tokens(
)
exact_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
@@ -1049,8 +1291,10 @@ async def adjust_payment_for_tokens(
# Also update total_spent and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_exact_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
child_stmt = (
@@ -1089,31 +1333,45 @@ async def adjust_payment_for_tokens(
# actual cost exceeded discounted reservation (due to tolerance_percentage)
if cost_difference > 0:
# Always release the reservation and charge min(actual_cost, balance).
# CASE expressions keep this atomic and safe even when the
# stale-reservation sweeper has already released the reservation.
chargeable = case(
(col(ApiKey.balance) >= total_cost_msats, total_cost_msats),
else_=col(ApiKey.balance),
)
overrun_safe_reserved = case(
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
finalize_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.values(
reserved_balance=overrun_safe_reserved,
balance=col(ApiKey.balance) - chargeable,
total_spent=col(ApiKey.total_spent) + chargeable,
# Lock the billing row so the parent and child record the same
# database-determined charge under concurrent finalizations.
actual_charge_msats = 0
for attempt in range(5):
locked_billing_key = (
await session.exec(
select(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.with_for_update()
.execution_options(populate_existing=True)
)
).one()
observed_balance = locked_billing_key.balance
actual_charge_msats = min(observed_balance, total_cost_msats)
overrun_safe_reserved = case(
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
)
await session.exec(finalize_stmt) # type: ignore[call-overload]
finalize_result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
.where(col(ApiKey.balance) == observed_balance)
.values(
reserved_balance=overrun_safe_reserved,
balance=col(ApiKey.balance) - actual_charge_msats,
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
)
)
if finalize_result.rowcount == 1:
break
await session.rollback()
if not await _claim_reservation_for_charge(reservation, session):
return cost.dict()
else:
await session.rollback()
raise RuntimeError("Could not atomically finalize cost overrun")
if billing_key.hashed_key != key.hashed_key:
child_stmt = (
@@ -1121,7 +1379,7 @@ async def adjust_payment_for_tokens(
.where(col(ApiKey.hashed_key) == key.hashed_key)
.values(
reserved_balance=overrun_safe_reserved,
total_spent=col(ApiKey.total_spent) + min(billing_key.balance, total_cost_msats),
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
)
)
await session.exec(child_stmt) # type: ignore[call-overload]
@@ -1131,18 +1389,18 @@ async def adjust_payment_for_tokens(
await session.refresh(billing_key)
if billing_key.hashed_key != key.hashed_key:
await session.refresh(key)
cost.total_msats = total_cost_msats
cost.total_msats = actual_charge_msats
logger.info(
"Finalized payment with additional charge",
extra={
"key_hash": key.hashed_key[:8] + "...",
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"charged_amount": total_cost_msats,
"charged_amount": actual_charge_msats,
"new_balance": billing_key.balance,
"model": model,
},
)
await _accumulate_fee(total_cost_msats)
await _accumulate_fee(actual_charge_msats)
payments_logger.info(
"FINALIZE",
extra={
@@ -1151,7 +1409,7 @@ async def adjust_payment_for_tokens(
"billing_key_hash": billing_key.hashed_key[:8] + "...",
"model": model,
"cost_reserved": deducted_max_cost,
"cost_charged": total_cost_msats,
"cost_charged": actual_charge_msats,
"input_tokens": cost.input_tokens,
"output_tokens": cost.output_tokens,
"balance": billing_key.balance,
@@ -1191,8 +1449,10 @@ async def adjust_payment_for_tokens(
)
refund_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
@@ -1210,8 +1470,10 @@ async def adjust_payment_for_tokens(
# Also update total_spent and reserved_balance on the child key if it's different
if billing_key.hashed_key != key.hashed_key:
child_refund_safe_reserved = case(
(col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost),
(
col(ApiKey.reserved_balance) >= deducted_max_cost,
col(ApiKey.reserved_balance) - deducted_max_cost,
),
else_=0,
)
child_stmt = (
@@ -1386,9 +1648,7 @@ async def periodic_dead_key_prune() -> None:
try:
async with create_session() as session:
await prune_dead_api_keys(
session, settings.dead_key_min_age_seconds
)
await prune_dead_api_keys(session, settings.dead_key_min_age_seconds)
except asyncio.CancelledError:
break
except Exception as e:
+192 -39
View File
@@ -7,7 +7,7 @@ from typing import Annotated, NoReturn
from fastapi import APIRouter, Depends, Header, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from sqlmodel import col, or_, select, update
from sqlmodel import col, select, update
from .auth import get_billing_key, validate_bearer_key
from .core.db import (
@@ -15,6 +15,7 @@ from .core.db import (
AsyncSession,
CashuTransaction,
get_session,
release_stale_reservations,
)
from .core.db import (
store_cashu_transaction_with_retry as store_cashu_transaction,
@@ -22,6 +23,7 @@ from .core.db import (
from .core.logging import get_logger
from .core.settings import settings
from .lightning import lightning_router
from .payment.lnurl import MeltOutcomeAmbiguousError
from .wallet import (
classify_redemption_error,
credit_balance,
@@ -29,6 +31,7 @@ from .wallet import (
recieve_token,
send_to_lnurl,
send_token,
token_mint_url,
)
router = APIRouter()
@@ -108,13 +111,19 @@ async def account_info(
# Note: validate_bearer_key already supports refund_address and key_expiry_time params
@router.get("/create")
async def create_balance(
class BalanceCreateRequest(BaseModel):
initial_balance_token: str
balance_limit: int | None = None
balance_limit_reset: str | None = None
validity_date: int | None = None
async def _create_balance(
initial_balance_token: str,
balance_limit: int | None = None,
balance_limit_reset: str | None = None,
validity_date: int | None = None,
session: AsyncSession = Depends(get_session),
balance_limit: int | None,
balance_limit_reset: str | None,
validity_date: int | None,
session: AsyncSession,
) -> dict:
key = await validate_bearer_key(initial_balance_token, session)
@@ -134,6 +143,37 @@ async def create_balance(
}
@router.post("/create")
async def create_balance_from_body(
payload: BalanceCreateRequest,
session: AsyncSession = Depends(get_session),
) -> dict:
return await _create_balance(
payload.initial_balance_token,
payload.balance_limit,
payload.balance_limit_reset,
payload.validity_date,
session,
)
@router.get("/create")
async def create_balance(
initial_balance_token: str,
balance_limit: int | None = None,
balance_limit_reset: str | None = None,
validity_date: int | None = None,
session: AsyncSession = Depends(get_session),
) -> dict:
return await _create_balance(
initial_balance_token,
balance_limit,
balance_limit_reset,
validity_date,
session,
)
@router.get("/info")
async def wallet_info(
key: ApiKey = Depends(get_key_from_header),
@@ -146,6 +186,17 @@ class TopupRequest(BaseModel):
cashu_token: str
def _error_chain(error: BaseException) -> list[dict[str, str]]:
chain: list[dict[str, str]] = []
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
chain.append({"type": type(current).__name__, "message": str(current)})
current = current.__cause__ or current.__context__
return chain
@router.post("/topup")
async def topup_wallet_endpoint(
cashu_token: str | None = None,
@@ -163,6 +214,18 @@ async def topup_wallet_endpoint(
cashu_token = cashu_token.replace("\n", "").replace("\r", "").replace("\t", "")
if len(cashu_token) < 10 or "cashu" not in cashu_token:
raise HTTPException(status_code=400, detail="Invalid token format")
source_mint = token_mint_url(cashu_token, "unknown")
logger.info(
"Cashu wallet top-up started",
extra={
"event": "cashu_topup_started",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"key_hash": billing_key.hashed_key[:8],
},
)
try:
amount_msats = await credit_balance(cashu_token, billing_key, session)
except Exception as e:
@@ -171,12 +234,41 @@ async def topup_wallet_endpoint(
classified = classify_redemption_error(e)
if classified is None:
logger.error(
"topup_wallet_endpoint: unhandled error",
extra={"error": str(e), "error_type": type(e).__name__},
"Cashu wallet top-up failed with an unhandled error",
extra={
"event": "cashu_topup_failed",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"error_chain": _error_chain(e),
},
)
raise HTTPException(status_code=500, detail="Internal server error")
_type, status_code, message, _code = classified
error_type, status_code, message, error_code = classified
logger.warning(
"Cashu wallet top-up failed",
extra={
"event": "cashu_topup_failed",
"source_mint": source_mint,
"primary_mint": settings.primary_mint,
"trusted_mints": settings.cashu_mints,
"status_code": status_code,
"error_type": error_type,
"error_code": error_code,
"error_chain": _error_chain(e),
},
)
raise HTTPException(status_code=status_code, detail=message)
logger.info(
"Cashu wallet top-up completed",
extra={
"event": "cashu_topup_completed",
"source_mint": source_mint,
"credited_msats": amount_msats,
"key_hash": billing_key.hashed_key[:8],
},
)
return {"msats": amount_msats}
@@ -221,8 +313,42 @@ async def _lookup_key_no_create(
return None
async def _get_persisted_api_key_refund(
key: ApiKey, session: AsyncSession
) -> dict[str, str] | None:
result = await session.exec(
select(CashuTransaction)
.where(
CashuTransaction.api_key_hashed_key == key.hashed_key,
CashuTransaction.type == "out",
CashuTransaction.source == "apikey",
)
.order_by(col(CashuTransaction.created_at).desc())
)
refund = result.first()
if refund is None:
return None
if refund.swept:
raise HTTPException(status_code=410, detail="Refund has been swept")
refund.collected = True
session.add(refund)
await session.commit()
persisted = {"token": refund.token}
if refund.unit == "sat":
persisted["sats"] = str(refund.amount)
else:
persisted["msats"] = str(refund.amount)
return persisted
async def _restore_balance(
session: AsyncSession, hashed_key: str, balance: int, reserved_balance: int, mint_url: str
session: AsyncSession,
hashed_key: str,
balance: int,
reserved_balance: int,
mint_url: str,
) -> None:
"""Restore balance after a failed refund mint attempt."""
restore_stmt = (
@@ -237,7 +363,11 @@ async def _restore_balance(
await session.commit()
logger.info(
"refund_wallet_endpoint: balance restored after mint failure",
extra={"hashed_key": hashed_key, "restored_balance": balance, "mint_url": mint_url},
extra={
"hashed_key": hashed_key,
"restored_balance": balance,
"mint_url": mint_url,
},
)
@@ -315,6 +445,8 @@ async def refund_wallet_endpoint(
if key.total_balance <= 0:
if cached := await _refund_cache_get(bearer_value):
return cached
if persisted := await _get_persisted_api_key_refund(key, session):
return persisted
if key.parent_key_hash:
raise HTTPException(
@@ -323,30 +455,19 @@ async def refund_wallet_endpoint(
)
if key.reserved_balance > 0:
# Release the reservation if it is stale
cutoff = int(time.time()) - settings.stale_reservation_timeout_seconds
stale_release_stmt = (
update(ApiKey)
.where(col(ApiKey.hashed_key) == key.hashed_key)
.where(col(ApiKey.reserved_balance) > 0)
.where(
or_(
col(ApiKey.reserved_at).is_(None),
col(ApiKey.reserved_at) < cutoff,
)
)
.values(reserved_balance=0, reserved_at=None)
# Release only durable reservations old enough to be stale. A newer
# request on the same aggregate balance must remain reserved.
await release_stale_reservations(
session,
settings.stale_reservation_timeout_seconds,
key_hash=key.hashed_key,
)
stale_result = await session.exec(stale_release_stmt) # type: ignore[call-overload]
await session.commit()
if stale_result.rowcount == 0:
await session.refresh(key)
if key.reserved_balance > 0:
raise HTTPException(
status_code=400,
detail="Cannot refund key. There are ongoing requests for this api key.",
)
await session.refresh(key)
logger.warning(
"refund_wallet_endpoint: released stale reservation before refund",
extra={
@@ -391,15 +512,14 @@ async def refund_wallet_endpoint(
detail="Balance changed concurrently. Please retry the refund.",
)
# --- MINT: balance is locked at zero, safe to create the refund token ---
# Proofs from untrusted mints are swapped to primary_mint on receive.
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
# The balance is locked at zero, so it is safe to create the refund token.
effective_refund_mint = (
key.refund_mint_url
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
else settings.primary_mint
)
try:
refund_currency = key.refund_currency or "sat"
if key.refund_address:
await send_to_lnurl(
remaining_balance,
@@ -409,10 +529,10 @@ async def refund_wallet_endpoint(
)
result = {"recipient": key.refund_address}
else:
refund_currency = key.refund_currency or "sat"
token = await send_token(
remaining_balance, refund_currency, effective_refund_mint
)
effective_refund_mint = token_mint_url(token, effective_refund_mint)
result = {"token": token}
if key.refund_currency == "sat":
@@ -431,13 +551,47 @@ async def refund_wallet_endpoint(
},
)
except MeltOutcomeAmbiguousError as e:
# The melt was dispatched and may still settle. Restoring the balance
# here would let the same debit be paid out twice; keep the debit and
# leave the outcome to reconciliation.
logger.error(
"refund_wallet_endpoint: melt outcome ambiguous; balance withheld "
"pending reconciliation",
extra={
"error": str(e),
"hashed_key": key.hashed_key,
"remaining_balance": remaining_balance,
"refund_currency": key.refund_currency,
"refund_mint_url": key.refund_mint_url,
},
)
raise HTTPException(
status_code=502,
detail=(
"Refund was dispatched but its outcome is unconfirmed; the "
"balance is withheld until reconciliation completes"
),
)
except HTTPException:
# Minting failed — restore the debited balance
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
await _restore_balance(
session,
key.hashed_key,
pre_debit_balance,
pre_debit_reserved,
key.refund_mint_url or "",
)
raise
except Exception as e:
# Minting failed — restore the debited balance
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved, key.refund_mint_url or "")
await _restore_balance(
session,
key.hashed_key,
pre_debit_balance,
pre_debit_reserved,
key.refund_mint_url or "",
)
error_msg = str(e)
logger.error(
"refund_wallet_endpoint: mint/send failed",
@@ -464,7 +618,7 @@ async def refund_wallet_endpoint(
token=result["token"],
amount=remaining_balance,
unit=key.refund_currency or "sat",
mint_url=key.refund_mint_url,
mint_url=effective_refund_mint,
typ="out",
collected=False,
source="apikey",
@@ -658,7 +812,6 @@ async def reset_child_key_spent(
return {"success": True, "message": "Child key balance reset successfully."}
@router.api_route(
"/{path:path}",
methods=["GET", "POST", "PUT", "DELETE"],
+222 -30
View File
@@ -13,13 +13,7 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from ..payment.models import _row_to_model, list_models
from ..proxy import refresh_model_maps, reinitialize_upstreams
from ..wallet import (
fetch_all_balances,
get_proofs_per_mint_and_unit,
get_wallet,
send_token,
slow_filter_spend_proofs,
)
from ..wallet import fetch_all_balances, send_token, token_mint_url
from . import vault
from .db import (
ApiKey,
@@ -51,6 +45,13 @@ ADMIN_SESSION_DURATION = 3600
MAX_USAGE_ANALYTICS_HOURS = 365 * 24
async def _refresh_provider_model_paths(upstream_provider_id: int) -> None:
"""Queue discovery sync without blocking the committed admin mutation."""
from ..upstream.model_paths import schedule_model_paths_refresh_for_provider
await schedule_model_paths_refresh_for_provider(upstream_provider_id)
async def require_admin_api(request: Request) -> None:
auth_header = request.headers.get("Authorization")
if not auth_header or not auth_header.startswith("Bearer "):
@@ -435,37 +436,31 @@ class WithdrawRequest(BaseModel):
async def withdraw(
request: Request, withdraw_request: WithdrawRequest
) -> dict[str, str]:
# Get wallet and check balance
from .settings import settings as global_settings
effective_mint = withdraw_request.mint_url or global_settings.primary_mint
wallet = await get_wallet(effective_mint, withdraw_request.unit)
proofs = get_proofs_per_mint_and_unit(
wallet,
effective_mint,
withdraw_request.unit,
not_reserved=True,
)
proofs = await slow_filter_spend_proofs(proofs, wallet)
current_balance = sum(proof.amount for proof in proofs)
if withdraw_request.amount <= 0:
raise HTTPException(
status_code=400, detail="Withdrawal amount must be positive"
)
if withdraw_request.amount > current_balance:
raise HTTPException(status_code=400, detail="Insufficient wallet balance")
token = await send_token(
withdraw_request.amount, withdraw_request.unit, effective_mint
)
try:
token = await send_token(
withdraw_request.amount, withdraw_request.unit, effective_mint
)
except ValueError as error:
if not str(error).startswith("No trusted mint has "):
raise
raise HTTPException(
status_code=400, detail="Insufficient wallet balance"
) from error
actual_mint = token_mint_url(token, effective_mint)
try:
await store_cashu_transaction(
token=token,
amount=withdraw_request.amount,
unit=withdraw_request.unit,
mint_url=effective_mint,
mint_url=actual_mint,
typ="out",
collected=False,
source="admin",
@@ -476,10 +471,10 @@ async def withdraw(
extra={
"amount": withdraw_request.amount,
"unit": withdraw_request.unit,
"mint_url": effective_mint,
"mint_url": actual_mint,
},
)
return {"token": token}
return {"token": token, "mint_url": actual_mint}
class ModelCreate(BaseModel):
@@ -579,6 +574,7 @@ async def upsert_provider_model(
await session.refresh(row)
await refresh_model_maps()
await _refresh_provider_model_paths(provider_pk)
return _row_to_model(
row, apply_provider_fee=True, provider_fee=provider.provider_fee
).dict() # type: ignore
@@ -633,6 +629,7 @@ async def delete_provider_model(provider_id: str, model_id: str) -> dict[str, ob
await session.delete(row)
await session.commit()
await refresh_model_maps()
await _refresh_provider_model_paths(provider_pk)
return {"ok": True, "deleted_id": model_id}
@@ -652,6 +649,7 @@ async def delete_all_provider_models(provider_id: str) -> dict[str, object]:
await session.delete(row) # type: ignore
await session.commit()
await refresh_model_maps()
await _refresh_provider_model_paths(provider_pk)
return {"ok": True, "deleted": len(rows)}
@@ -743,6 +741,7 @@ async def batch_override_provider_models(
await session.commit()
await refresh_model_maps()
await _refresh_provider_model_paths(provider_pk)
return {
"ok": True,
"count": overridden_count,
@@ -863,6 +862,33 @@ class UpstreamProviderUpdateBySlug(BaseModel):
provider_settings: dict | None = None
async def _active_ppq_claim_in_session(session: AsyncSession, provider_id: int) -> bool:
"""Check for an active claim inside the caller's transaction.
Must share the transaction of whatever destructive write it is guarding —
a check in its own session leaves a window for a worker to create the
claim between the check and the commit.
"""
from ..upstream.auto_topup import _ppq_state_id_for_provider
claim = await session.get(CashuTransaction, _ppq_state_id_for_provider(provider_id))
return claim is not None and not claim.collected and not claim.swept
def _require_valid_ppq_auto_topup(
provider_type: str, settings: dict | None
) -> None:
"""Reject PPQ auto top-up settings the worker would later refuse."""
if provider_type != "ppqai":
return
from ..upstream.auto_topup import validate_ppq_auto_topup_settings
problem = validate_ppq_auto_topup_settings(settings)
if problem is not None:
raise HTTPException(status_code=400, detail=problem)
async def _apply_provider_update(
session: AsyncSession,
provider: UpstreamProviderRow,
@@ -874,6 +900,29 @@ async def _apply_provider_update(
await _ensure_unique_slug(session, validated, exclude_id=provider.id)
provider.slug = validated
provider_type_changed = (
payload.provider_type is not None
and payload.provider_type != provider.provider_type
)
ppq_type_changed = provider_type_changed and (
provider.provider_type == "ppqai" or payload.provider_type == "ppqai"
)
if (
provider_type_changed
and provider.provider_type == "ppqai"
and provider.id is not None
and await _active_ppq_claim_in_session(session, provider.id)
):
# Changing the type would orphan the claim: the PPQ endpoints refuse
# non-ppqai providers, so nobody could ever inspect or release it.
raise HTTPException(
status_code=409,
detail=(
"This provider has an active PPQ auto top-up claim. Release "
"it before changing the provider type"
),
)
if payload.provider_type is not None:
provider.provider_type = payload.provider_type
if payload.base_url is not None:
@@ -886,6 +935,41 @@ async def _apply_provider_update(
provider.enabled = payload.enabled
if payload.provider_fee is not None:
provider.provider_fee = payload.provider_fee
# Auto-top-up fields have provider-specific units and meaning. Reusing
# enabled Routstr settings for PPQ (or vice versa) can silently reinterpret
# sats as USD, so a type change must provide settings for the new type.
if (
ppq_type_changed
and payload.provider_settings is None
and provider.provider_settings
):
try:
stored_settings = json.loads(provider.provider_settings)
except (json.JSONDecodeError, TypeError):
stored_settings = None
if isinstance(stored_settings, dict) and stored_settings.get("auto_topup"):
raise HTTPException(
status_code=400,
detail=(
"Changing provider type requires explicit auto-top-up "
"settings because the units are provider-specific"
),
)
# Validate against the effective type and effective settings.
effective_settings = payload.provider_settings
if effective_settings is None and payload.provider_type is not None:
try:
effective_settings = (
json.loads(provider.provider_settings)
if provider.provider_settings
else None
)
except (json.JSONDecodeError, TypeError):
effective_settings = None
if effective_settings is not None:
_require_valid_ppq_auto_topup(provider.provider_type, effective_settings)
if payload.provider_settings is not None:
provider.provider_settings = json.dumps(payload.provider_settings)
@@ -925,6 +1009,10 @@ async def create_upstream_provider(
else:
slug = await allocate_unique_provider_slug(session, payload.provider_type)
_require_valid_ppq_auto_topup(
payload.provider_type, payload.provider_settings
)
provider = UpstreamProviderRow(
slug=slug,
provider_type=payload.provider_type,
@@ -943,6 +1031,7 @@ async def create_upstream_provider(
await reinitialize_upstreams()
await refresh_model_maps()
await _refresh_provider_model_paths(_provider_pk(provider))
return _serialize_provider(provider)
@@ -968,6 +1057,7 @@ async def update_upstream_provider(
await reinitialize_upstreams()
await refresh_model_maps()
await _refresh_provider_model_paths(_provider_pk(provider))
return _serialize_provider(provider)
@@ -1003,6 +1093,7 @@ async def update_upstream_provider_by_slug(
await reinitialize_upstreams()
await refresh_model_maps()
await _refresh_provider_model_paths(_provider_pk(provider))
return _serialize_provider(provider)
@@ -1013,6 +1104,25 @@ async def delete_upstream_provider(provider_id: str) -> dict[str, object]:
async with create_session() as session:
provider = await _get_upstream_provider_by_ref(session, provider_id)
deleted_id = _provider_pk(provider)
# Checked inside the delete transaction: the worker's claim creation
# re-reads the provider inside its own transaction, so these two
# writes serialise — either the claim lands first and this 409s, or
# the delete lands first and the worker refuses to claim.
if provider.provider_type == "ppqai" and await _active_ppq_claim_in_session(
session, deleted_id
):
# Deleting now would orphan the claim and any funds it tracks:
# the PPQ endpoints 404 without the provider row, so the claim
# could never again be inspected or released.
raise HTTPException(
status_code=409,
detail=(
"This provider has an active PPQ auto top-up claim. "
"Resolve and release it before deleting the provider"
),
)
await session.delete(provider)
await session.commit()
await reinitialize_upstreams()
@@ -1621,6 +1731,78 @@ async def get_log_dates_api(request: Request) -> dict[str, object]:
return {"dates": dates}
_PPQ_RELEASE_ERRORS = {
"no_active_claim": "No active PPQ claim to release",
"stale_state": ("The claim changed since it was reviewed; reload and check again"),
"payment_in_flight": (
"A Lightning payment is still in flight for this claim. Wait for it to "
"finish or expire before releasing"
),
"claim_changed": (
"The claim changed while the release was being applied; reload and check again"
),
}
class ReleasePPQAutoTopupRequest(BaseModel):
confirmed_safe_to_retry: bool
# Echoes the state_token the admin reviewed — the claim's full versioned
# state, not just its operation id. Any change since the review (a new
# attempt, a phase change, a renewed lease) fails the match, so the
# release cannot land on a state the admin never saw.
state_token: str | None = None
async def _require_ppq_provider(provider_id: int) -> UpstreamProviderRow:
async with create_session() as session:
provider = await session.get(UpstreamProviderRow, provider_id)
if provider is None:
raise HTTPException(status_code=404, detail="Provider not found")
if provider.provider_type != "ppqai":
raise HTTPException(status_code=400, detail="Provider is not PPQ")
return provider
@admin_router.get(
"/api/upstream-providers/{provider_id}/ppq-auto-topup",
dependencies=[Depends(require_admin_api)],
)
async def get_ppq_auto_topup_api(provider_id: int) -> dict[str, object]:
await _require_ppq_provider(provider_id)
from ..upstream.auto_topup import get_ppq_auto_topup_state
return {"ok": True, **await get_ppq_auto_topup_state(provider_id)}
@admin_router.post(
"/api/upstream-providers/{provider_id}/ppq-auto-topup/release",
dependencies=[Depends(require_admin_api)],
)
async def release_ppq_auto_topup_api(
provider_id: int, payload: ReleasePPQAutoTopupRequest
) -> dict[str, object]:
await _require_ppq_provider(provider_id)
if not payload.confirmed_safe_to_retry:
raise HTTPException(
status_code=400,
detail="Confirm the Lightning payment outcome is safe before releasing",
)
from ..upstream.auto_topup import release_ppq_auto_topup_state
outcome = await release_ppq_auto_topup_state(
provider_id, state_token=payload.state_token
)
if not outcome.released:
raise HTTPException(status_code=409, detail=_PPQ_RELEASE_ERRORS[outcome.reason])
logger.warning(
"Admin released PPQ auto top-up claim after manual reconciliation",
extra={"provider_id": provider_id, "state_token": payload.state_token},
)
return {"ok": True, "released": True}
@admin_router.get("/api/transactions", dependencies=[Depends(require_admin_api)])
async def get_transactions_api(
type: str | None = None,
@@ -1633,7 +1815,11 @@ async def get_transactions_api(
async with create_session() as session:
from sqlmodel import col, func
base = select(CashuTransaction)
# Hide only the deterministic PPQ claim-lock rows. Append-only PPQ
# payment rows remain visible as the audit trail for irreversible melts.
base = select(CashuTransaction).where(
~col(CashuTransaction.id).like("ppq-auto-topup-%")
)
if type:
base = base.where(CashuTransaction.type == type)
if source:
@@ -1669,12 +1855,18 @@ async def get_transactions_api(
)
total = count_result.one()
stmt = base.order_by(col(CashuTransaction.created_at).desc()).offset(offset).limit(limit)
stmt = (
base.order_by(col(CashuTransaction.created_at).desc())
.offset(offset)
.limit(limit)
)
results = await session.exec(stmt)
transactions = results.all()
return {
"transactions": [tx.dict() for tx in transactions],
"transactions": [
tx.dict(exclude={"sweep_started_at"}) for tx in transactions
],
"total": total,
}
+303 -33
View File
@@ -12,21 +12,80 @@ from typing import AsyncGenerator
from alembic import command
from alembic.config import Config
from alembic.util.exc import CommandError
from sqlalchemy import UniqueConstraint, delete
from sqlalchemy import Index, UniqueConstraint, case, delete, event, or_
from sqlalchemy.engine import make_url
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.ext.asyncio.engine import create_async_engine
from sqlalchemy.orm import aliased
from sqlmodel import Field, Relationship, SQLModel, col, func, select, update
from sqlmodel.ext.asyncio.session import AsyncSession
from .logging import get_logger
from .settings import settings
logger = get_logger(__name__)
DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite+aiosqlite:///keys.db")
engine = create_async_engine(DATABASE_URL, echo=False) # echo=True for debugging SQL
def create_db_engine(database_url: str = DATABASE_URL) -> AsyncEngine:
"""Build and instrument an async engine from environment-only settings."""
url = make_url(database_url)
backend = url.get_backend_name()
is_sqlite = backend == "sqlite"
is_memory_sqlite = is_sqlite and url.database in {None, "", ":memory:"}
pool_pre_ping = settings.database_pool_pre_ping or not is_sqlite
options: dict[str, int | float | bool] = {"pool_pre_ping": pool_pre_ping}
if not is_memory_sqlite:
options.update(
pool_size=settings.database_pool_size,
max_overflow=settings.database_max_overflow,
pool_timeout=settings.database_pool_timeout,
pool_recycle=settings.database_pool_recycle,
)
logger.info(
"Database pool configured",
extra={
"database_url_backend": backend,
"in_memory_sqlite": is_memory_sqlite,
**options,
},
)
created_engine = create_async_engine(database_url, echo=False, **options)
hold_warn_seconds = settings.database_pool_hold_warn_seconds
def record_pool_checkout(
dbapi_connection: object, connection_record: object, proxy: object
) -> None:
connection_record.info["routstr_checked_out_at"] = time.monotonic() # type: ignore[attr-defined]
def record_pool_checkin(
dbapi_connection: object, connection_record: object
) -> None:
checked_out_at = connection_record.info.pop( # type: ignore[attr-defined]
"routstr_checked_out_at", None
)
if checked_out_at is None:
return
held_seconds = time.monotonic() - checked_out_at
if held_seconds >= hold_warn_seconds:
logger.warning(
"Database connection held longer than threshold",
extra={
"held_seconds": round(held_seconds, 3),
"threshold_seconds": hold_warn_seconds,
"pool_status": created_engine.pool.status(),
},
)
event.listen(created_engine.sync_engine, "checkout", record_pool_checkout)
event.listen(created_engine.sync_engine, "checkin", record_pool_checkin)
return created_engine
engine = create_db_engine()
class ApiKey(SQLModel, table=True): # type: ignore
@@ -99,32 +158,133 @@ class ApiKey(SQLModel, table=True): # type: ignore
async def reset_all_reserved_balances(session: AsyncSession) -> None:
stmt = update(ApiKey).values(reserved_balance=0, reserved_at=None)
await session.exec(stmt) # type: ignore[call-overload]
"""Release every active durable reservation during explicit startup reset."""
await session.exec( # type: ignore[call-overload]
update(ReservationRelease)
.where(col(ReservationRelease.status) == "active")
.values(status="released")
)
await session.exec( # type: ignore[call-overload]
update(ApiKey).values(reserved_balance=0, reserved_at=None)
)
await session.commit()
logger.info("Reset reserved balances on startup")
async def release_stale_reservations(
session: AsyncSession, max_age_seconds: int
session: AsyncSession,
max_age_seconds: int,
*,
key_hash: str | None = None,
) -> int:
"""Release reservations whose last reserve is older than max_age_seconds.
"""
"""Release stale durable reservations without touching newer reservations."""
cutoff = int(time.time()) - max_age_seconds
stmt = (
update(ApiKey)
.where(col(ApiKey.reserved_balance) > 0)
.where(col(ApiKey.reserved_at).is_not(None))
.where(col(ApiKey.reserved_at) < cutoff)
.values(reserved_balance=0, reserved_at=None)
query = (
select(ReservationRelease)
.where(col(ReservationRelease.status) == "active")
.where(col(ReservationRelease.created_at) < cutoff)
)
result = await session.exec(stmt) # type: ignore[call-overload]
if key_hash is not None:
query = query.where(
or_(
col(ReservationRelease.key_hash) == key_hash,
col(ReservationRelease.billing_key_hash) == key_hash,
)
)
reservations = (await session.exec(query)).all()
released = 0
for reservation in reservations:
transition = await session.exec( # type: ignore[call-overload]
update(ReservationRelease)
.where(col(ReservationRelease.id) == reservation.id)
.where(col(ReservationRelease.status) == "active")
.values(status="released")
)
if transition.rowcount != 1:
continue
values = {
"reserved_balance": col(ApiKey.reserved_balance)
- reservation.reserved_msats,
"reserved_at": case(
(
col(ApiKey.reserved_balance) - reservation.reserved_msats > 0,
col(ApiKey.reserved_at),
),
else_=None,
),
}
parent_result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == reservation.billing_key_hash)
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
.values(**values)
)
if parent_result.rowcount != 1:
await session.rollback()
return 0
if reservation.billing_key_hash != reservation.key_hash:
child_result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == reservation.key_hash)
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
.values(**values)
)
if child_result.rowcount != 1:
await session.rollback()
return 0
released += 1
# Rolling upgrades can leave aggregate reservations created before durable
# reservation rows existed. Release only stale aggregates that have no active
# durable owner; targeted refund cleanup also heals legacy NULL timestamps.
legacy_query = select(ApiKey).where(col(ApiKey.reserved_balance) > 0)
if key_hash is None:
legacy_query = legacy_query.where(col(ApiKey.reserved_at).is_not(None)).where(
col(ApiKey.reserved_at) < cutoff
)
else:
legacy_query = legacy_query.where(
or_(
col(ApiKey.hashed_key) == key_hash,
col(ApiKey.parent_key_hash) == key_hash,
)
).where(
or_(col(ApiKey.reserved_at).is_(None), col(ApiKey.reserved_at) < cutoff)
)
for legacy_key in (await session.exec(legacy_query)).all():
active_owner = (
await session.exec(
select(ReservationRelease.id)
.where(col(ReservationRelease.status) == "active")
.where(
or_(
col(ReservationRelease.key_hash) == legacy_key.hashed_key,
col(ReservationRelease.billing_key_hash)
== legacy_key.hashed_key,
)
)
.limit(1)
)
).first()
if active_owner is not None:
continue
legacy_key.reserved_balance = 0
legacy_key.reserved_at = None
session.add(legacy_key)
released += 1
await session.commit()
released = int(result.rowcount or 0)
if released:
logger.warning(
"Released stale balance reservations",
extra={"released_keys": released, "max_age_seconds": max_age_seconds},
"Released stale reservations",
extra={
"released_reservations": released,
"max_age_seconds": max_age_seconds,
},
)
return released
@@ -133,7 +293,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
"""Delete dead parentless API keys; return the count removed.
Dead = 0 balance/reservation/spend/requests, older than the grace period,
no parent, no children, no pending invoice. Cashu rows are unlinked (not
no parent, no children, no retryable invoice. Cashu rows are unlinked (not
deleted) first to keep the audit trail.
"""
cutoff = int(time.time()) - min_age_seconds
@@ -147,7 +307,9 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
pending_invoice = (
select(LightningInvoice.id)
.where(col(LightningInvoice.api_key_hash) == col(ApiKey.hashed_key))
.where(col(LightningInvoice.status) == "pending")
.where(
col(LightningInvoice.status).in_(("pending", "settlement_pending"))
)
).exists()
eligible_hashes = (
@@ -157,9 +319,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
.where(col(ApiKey.total_spent) == 0)
.where(col(ApiKey.total_requests) == 0)
.where(col(ApiKey.parent_key_hash).is_(None))
.where(
(col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff)
)
.where((col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff))
.where(~pending_invoice)
.where(~has_children)
)
@@ -213,6 +373,60 @@ class ModelRow(SQLModel, table=True): # type: ignore
upstream_provider: "UpstreamProviderRow" = Relationship(back_populates="models")
class ModelPathRow(SQLModel, table=True): # type: ignore
"""Upstream provider path a model is reachable through.
Discovery/visibility data only. ``model_id`` is intentionally NOT globally
unique: it is the client-visible ``/v1/models`` id (``forwarded_model_id or
id``) grouped across every provider that exposes the model. A single model
can therefore have several rows — one per direct provider path plus one per
OpenRouter sub-provider endpoint.
"""
__tablename__ = "model_paths"
__table_args__ = (
UniqueConstraint(
"model_id",
"path",
"upstream_provider_id",
name="uq_model_paths_model_path_provider",
),
)
id: int | None = Field(default=None, primary_key=True)
# No standalone index on model_id: the unique constraint's autoindex already
# leads on model_id, so a second index only adds write amplification.
model_id: str = Field(
description="Client-visible /v1/models id (forwarded_model_id or id)"
)
path: str = Field(
description=(
"Opaque selector containing upstream URL, provider ID, model ID, "
"and optional endpoint tag"
)
)
provider_slug: str = Field(
description="Public slug of the configured upstream provider"
)
provider_type: str = Field(description="Configured upstream provider type")
endpoint_tag: str | None = Field(
default=None,
description="Exact OpenRouter endpoint tag used for request-side selection",
)
endpoint_name: str | None = Field(
default=None, description="Human-readable endpoint display name"
)
upstream_provider_id: int = Field(
index=True,
foreign_key="upstream_providers.id",
ondelete="CASCADE",
description="upstream_providers.id this path was discovered from",
)
updated_at: int = Field(
default=0,
description="Unix timestamp of the refresh cycle that wrote this row",
)
class LightningInvoice(SQLModel, table=True): # type: ignore
__tablename__ = "lightning_invoices"
@@ -222,12 +436,19 @@ class LightningInvoice(SQLModel, table=True): # type: ignore
description: str = Field(description="Invoice description")
payment_hash: str = Field(description="Payment hash for tracking", unique=True)
status: str = Field(
default="pending", description="pending, paid, expired, cancelled"
default="pending",
description=(
"pending, settlement_pending, paid, expired, cancelled, "
"reconciliation_required"
),
)
api_key_hash: str | None = Field(
default=None, description="Associated API key hash for topup operations"
)
purpose: str = Field(description="create or topup")
mint_url: str | None = Field(
default=None, description="Mint URL where the quote was created (fallback tracking)"
)
created_at: int = Field(
default_factory=lambda: int(time.time()), description="Unix timestamp"
)
@@ -267,6 +488,10 @@ class CashuTransaction(SQLModel, table=True): # type: ignore
)
collected: bool = Field(default=False)
swept: bool = Field(default=False)
sweep_started_at: int | None = Field(
default=None,
description="Unix timestamp for a recoverable refund-sweep claim",
)
source: str = Field(
default="x-cashu",
description="Payment source: x-cashu or apikey",
@@ -434,6 +659,20 @@ class UpstreamProviderRow(SQLModel, table=True): # type: ignore
)
class ReservationRelease(SQLModel, table=True): # type: ignore
__tablename__ = "reservation_releases"
__table_args__ = (
Index("ix_reservation_releases_status_created_at", "status", "created_at"),
)
id: str = Field(primary_key=True)
key_hash: str = Field(index=True)
billing_key_hash: str = Field(index=True)
reserved_msats: int
status: str = Field(default="active")
created_at: int = Field(default_factory=lambda: int(time.time()))
class RoutstrFee(SQLModel, table=True): # type: ignore
__tablename__ = "routstr_fees"
id: int = Field(default=1, primary_key=True)
@@ -484,9 +723,7 @@ class CliToken(SQLModel, table=True): # type: ignore
"""Long-lived authorization token for CLI/agent use against admin endpoints."""
__tablename__ = "cli_tokens"
id: str = Field(
primary_key=True, default_factory=lambda: uuid.uuid4().hex
)
id: str = Field(primary_key=True, default_factory=lambda: uuid.uuid4().hex)
token: str = Field(unique=True, index=True, description="Bearer token value")
name: str = Field(description="Human-readable label for this token")
created_at: int = Field(default_factory=lambda: int(time.time()))
@@ -585,9 +822,7 @@ async def reset_routstr_fee(session: AsyncSession, paid_msats: int) -> bool:
return result.rowcount == 1
async def complete_routstr_fee_payout(
session: AsyncSession, paid_msats: int
) -> bool:
async def complete_routstr_fee_payout(session: AsyncSession, paid_msats: int) -> bool:
"""Mark a checkpointed payout complete after the external payment succeeds."""
stmt = (
update(RoutstrFee)
@@ -605,14 +840,49 @@ async def complete_routstr_fee_payout(
return result.rowcount == 1
async def balances_for_mint_and_unit(
async def total_user_liability(db_session: AsyncSession) -> int:
"""Return all outstanding API-key balances in millisatoshis."""
result = await db_session.exec(select(func.sum(ApiKey.balance)))
return int(result.one() or 0)
async def balance_for_mint_and_unit(
db_session: AsyncSession, mint_url: str, unit: str
) -> int:
query = select(func.sum(ApiKey.balance)).where(
ApiKey.refund_mint_url == mint_url, ApiKey.refund_currency == unit
"""Return the user liability for one mint and unit in millisatoshis."""
result = await db_session.exec(
select(func.sum(ApiKey.balance)).where(
col(ApiKey.refund_mint_url) == mint_url,
col(ApiKey.refund_currency) == unit,
)
)
return int(result.one() or 0)
async def balances_by_mint_and_unit(
db_session: AsyncSession, mint_urls: list[str], units: list[str]
) -> dict[tuple[str, str], int]:
"""Return requested user liabilities grouped by mint and unit."""
if not mint_urls or not units:
return {}
query = (
select(
col(ApiKey.refund_mint_url),
col(ApiKey.refund_currency),
func.sum(ApiKey.balance),
)
.where(
col(ApiKey.refund_mint_url).in_(mint_urls),
col(ApiKey.refund_currency).in_(units),
)
.group_by(col(ApiKey.refund_mint_url), col(ApiKey.refund_currency))
)
result = await db_session.exec(query)
return result.one() or 0
return {
(mint_url, unit): int(balance or 0)
for mint_url, unit, balance in result.all()
if mint_url is not None and unit is not None
}
async def init_db() -> None:
+15 -9
View File
@@ -58,6 +58,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
providers_task = None
models_refresh_task = None
model_maps_refresh_task = None
model_paths_refresh_task = None
key_reset_task = None
stale_reservation_task = None
dead_key_prune_task = None
@@ -130,6 +131,13 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
refresh_upstreams_models_periodically(get_upstreams)
)
model_maps_refresh_task = asyncio.create_task(refresh_model_maps_periodically())
# Always started: the loop re-reads the enable flag and interval every
# iteration, so 0 -> N (or re-enabling) takes effect without a restart.
from ..upstream.model_paths import refresh_model_paths_periodically
model_paths_refresh_task = asyncio.create_task(
refresh_model_paths_periodically(get_upstreams)
)
payout_task = asyncio.create_task(periodic_payout())
if global_settings.nsec:
nip91_task = asyncio.create_task(announce_provider())
@@ -137,9 +145,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
if global_settings.providers_refresh_interval_seconds > 0:
providers_task = asyncio.create_task(providers_cache_refresher())
key_reset_task = asyncio.create_task(periodic_key_reset())
stale_reservation_task = asyncio.create_task(
periodic_stale_reservation_sweep()
)
stale_reservation_task = asyncio.create_task(periodic_stale_reservation_sweep())
dead_key_prune_task = asyncio.create_task(periodic_dead_key_prune())
auto_topup_task = asyncio.create_task(periodic_auto_topup())
refund_sweep_task = asyncio.create_task(periodic_refund_sweep())
@@ -176,6 +182,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
models_refresh_task.cancel()
if model_maps_refresh_task is not None:
model_maps_refresh_task.cancel()
if model_paths_refresh_task is not None:
model_paths_refresh_task.cancel()
if key_reset_task is not None:
key_reset_task.cancel()
if stale_reservation_task is not None:
@@ -209,6 +217,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
tasks_to_wait.append(models_refresh_task)
if model_maps_refresh_task is not None:
tasks_to_wait.append(model_maps_refresh_task)
if model_paths_refresh_task is not None:
tasks_to_wait.append(model_paths_refresh_task)
if key_reset_task is not None:
tasks_to_wait.append(key_reset_task)
if stale_reservation_task is not None:
@@ -245,9 +255,7 @@ class _ImmutableStaticFiles(StaticFiles):
async def get_response(self, path: str, scope: Scope) -> StarletteResponse:
response = await super().get_response(path, scope)
if response.status_code == 200:
response.headers["Cache-Control"] = (
"public, max-age=31536000, immutable"
)
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
return response
@@ -321,9 +329,7 @@ if UI_DIST_PATH.exists() and UI_DIST_PATH.is_dir():
# Serve the App Router RSC payload for the home page.
@app.get("/index.txt", include_in_schema=False)
async def serve_root_rsc() -> FileResponse:
return FileResponse(
UI_DIST_PATH / "index.txt", media_type="text/x-component"
)
return FileResponse(UI_DIST_PATH / "index.txt", media_type="text/x-component")
# Next.js is built with `trailingSlash: true`, so all UI page URLs end
# with a slash (e.g. `/login/`). The proxy router catches `/{path:path}`
+81 -8
View File
@@ -41,6 +41,9 @@ class Settings(BaseSettings):
receive_ln_address: str = Field(default="", env="RECEIVE_LN_ADDRESS")
primary_mint: str = Field(default="", env="PRIMARY_MINT_URL")
primary_mint_unit: str = Field(default="sat", env="PRIMARY_MINT_UNIT")
mint_operation_concurrency: int = Field(
default=4, ge=1, env="MINT_OPERATION_CONCURRENCY"
)
# Lightning payout configuration
# Minimum available balance (in satoshis) before profit is paid out over
@@ -50,6 +53,18 @@ class Settings(BaseSettings):
payout_interval_seconds: int = Field(
default=900, gt=0, env="PAYOUT_INTERVAL_SECONDS"
)
# Timeout (seconds) for individual mint API operations (melt, mint, swap,
# checkstate). When a mint is slow or rate-limiting, operations are
# cancelled after this delay instead of hanging indefinitely.
mint_operation_timeout_seconds: int = Field(
default=30, gt=0, env="MINT_OPERATION_TIMEOUT_SECONDS"
)
# Maximum concurrent API operations per mint. Actual mint quotas vary by
# endpoint, so 429 responses drive adaptive cooldown instead of fixed RPM
# pacing. 0 = unlimited concurrency.
mint_max_concurrency: int = Field(default=4, ge=0, env="MINT_MAX_CONCURRENCY")
# Max retries when a mint returns 429 or times out (exponential backoff).
mint_retry_max_attempts: int = Field(default=3, ge=0, env="MINT_RETRY_MAX_ATTEMPTS")
# Pricing
# Default behavior: derive pricing from MODELS
@@ -95,10 +110,36 @@ class Settings(BaseSettings):
models_refresh_interval_seconds: int = Field(
default=360, env="MODELS_REFRESH_INTERVAL_SECONDS"
)
model_paths_refresh_interval_seconds: int = Field(
default=600, env="MODEL_PATHS_REFRESH_INTERVAL_SECONDS"
)
enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
enable_model_paths_refresh: bool = Field(
default=True, env="ENABLE_MODEL_PATHS_REFRESH"
)
refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
refund_sweep_ttl_seconds: int = Field(default=604800, env="REFUND_SWEEP_TTL_SECONDS")
refund_sweep_ttl_seconds: int = Field(
default=604800, env="REFUND_SWEEP_TTL_SECONDS"
)
refund_sweep_claim_timeout_seconds: int = Field(
default=900, gt=0, env="REFUND_SWEEP_CLAIM_TIMEOUT_SECONDS"
)
# Database connection-pool controls (advanced). Capacity defaults provide
# headroom for Routstr's concurrent request and background-payment workload.
# Pre-ping is enabled by the engine factory for networked backends; SQLite
# can explicitly opt in. These fields are env-only below.
database_pool_size: int = Field(default=10, ge=1, env="DATABASE_POOL_SIZE")
database_max_overflow: int = Field(default=20, ge=0, env="DATABASE_MAX_OVERFLOW")
database_pool_timeout: float = Field(
default=15.0, gt=0, env="DATABASE_POOL_TIMEOUT"
)
database_pool_recycle: int = Field(default=1800, ge=0, env="DATABASE_POOL_RECYCLE")
database_pool_pre_ping: bool = Field(default=False, env="DATABASE_POOL_PRE_PING")
database_pool_hold_warn_seconds: float = Field(
default=10.0, gt=0, env="DATABASE_POOL_HOLD_WARN_SECONDS"
)
# Logging
log_level: str = Field(default="INFO", env="LOG_LEVEL")
@@ -117,9 +158,8 @@ class Settings(BaseSettings):
# Discovery
relays: list[str] = Field(default_factory=list, env="RELAYS")
enable_analytics_sharing: bool = Field(
default=True, env="ENABLE_ANALYTICS_SHARING"
)
enable_analytics_sharing: bool = Field(default=True, env="ENABLE_ANALYTICS_SHARING")
def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
"""Discard unknown keys from persisted settings."""
@@ -144,10 +184,32 @@ def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
# ``routstr.core.vault``.
SECRET_FIELDS = frozenset({"admin_password", "nsec"})
# Infrastructure the node needs *before* it can open a DB session — so it can
# never be configured from the DB (chicken-and-egg) and stays env-only. Unlike
# secrets (owned by bootstrap), these are excluded so the DB settings blob can
# neither store nor shadow them; env is always authoritative.
ENV_ONLY_FIELDS = frozenset(
{
"database_pool_size",
"database_max_overflow",
"database_pool_timeout",
"database_pool_recycle",
"database_pool_pre_ping",
"database_pool_hold_warn_seconds",
}
)
_NON_PERSISTED_FIELDS = SECRET_FIELDS | ENV_ONLY_FIELDS
def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]:
"""Return a copy of ``data`` without any secret fields (for persistence)."""
return {k: v for k, v in data.items() if k not in SECRET_FIELDS}
"""Return a copy of ``data`` without secret or env-only fields.
Both are kept out of the persisted settings blob: secrets for confidentiality,
env-only fields (e.g. DB pool sizing) because they must never be sourced from
the database.
"""
return {k: v for k, v in data.items() if k not in _NON_PERSISTED_FIELDS}
def _apply_to_live_settings(data: dict[str, Any]) -> None:
@@ -327,7 +389,13 @@ class SettingsService:
valid_fields = set(env_resolved.dict().keys())
merged_dict: dict[str, Any] = dict(env_resolved.dict())
merged_dict.update(
{k: v for k, v in db_json.items() if v not in (None, "", [], {}) and k in valid_fields}
{
k: v
for k, v in db_json.items()
if v not in (None, "", [], {})
and k in valid_fields
and k not in ENV_ONLY_FIELDS
}
)
merged_dict = Settings(**merged_dict).dict()
@@ -394,8 +462,13 @@ class SettingsService:
)
)
await db_session.commit()
# Update in-place
# Update in-place. Env-only fields (e.g. DB pool sizing) are never
# applied here: the engine pool is already built at boot from env,
# so letting an update mutate the live value would only make it
# diverge from the running pool.
for k, v in candidate.dict().items():
if k in ENV_ONLY_FIELDS:
continue
setattr(settings, k, v)
cls._current = settings
return settings
+571 -86
View File
@@ -1,22 +1,101 @@
import asyncio
import hashlib
import re
import secrets
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass
from typing import Any, AsyncGenerator
from cashu.core.base import MintQuoteState
from fastapi import APIRouter, Depends, Header, HTTPException
from pydantic import BaseModel, Field
from sqlmodel import col, select
from sqlalchemy.orm.attributes import set_committed_value
from sqlmodel import col, select, update
from sqlmodel.ext.asyncio.session import AsyncSession
from .core.db import ApiKey, LightningInvoice, create_session, get_session
from .core.logging import get_logger
from .core.settings import settings
from .wallet import get_wallet
from .mint import (
is_mint_rate_limited,
mint_cooldown_remaining,
run_mint_operation,
)
from .wallet import (
MintConnectionError,
get_wallet,
is_mint_connection_error,
wallet_operation_guard,
)
logger = get_logger(__name__)
lightning_router = APIRouter(prefix="/lightning")
# Avoid duplicate work within one process. Cross-process settlement is fenced
# by claiming a paid quote before minting and by the final conditional update.
@dataclass
class _InvoiceLockEntry:
lock: asyncio.Lock
users: int = 0
_invoice_settlement_locks: dict[str, _InvoiceLockEntry] = {}
@asynccontextmanager
async def _invoice_settlement_lock(invoice_id: str) -> AsyncGenerator[None, None]:
"""Serialize one invoice and remove its lock after the last waiter leaves."""
entry = _invoice_settlement_locks.get(invoice_id)
if entry is None:
entry = _InvoiceLockEntry(asyncio.Lock())
_invoice_settlement_locks[invoice_id] = entry
entry.users += 1
try:
async with entry.lock:
yield
finally:
entry.users -= 1
if entry.users == 0 and _invoice_settlement_locks.get(invoice_id) is entry:
del _invoice_settlement_locks[invoice_id]
@dataclass(frozen=True)
class _InvoiceSettlement:
id: str
payment_hash: str
amount_sats: int
purpose: str
api_key_hash: str | None
mint_url: str | None
balance_limit: int | None
balance_limit_reset: str | None
validity_date: int | None
@classmethod
def from_invoice(cls, invoice: LightningInvoice) -> "_InvoiceSettlement":
return cls(
id=invoice.id,
payment_hash=invoice.payment_hash,
amount_sats=invoice.amount_sats,
purpose=invoice.purpose,
api_key_hash=invoice.api_key_hash,
mint_url=invoice.mint_url,
balance_limit=invoice.balance_limit,
balance_limit_reset=invoice.balance_limit_reset,
validity_date=invoice.validity_date,
)
def _publish_invoice_value(invoice: LightningInvoice, key: str, value: Any) -> None:
"""Update a caller view without marking a mapped object dirty."""
try:
set_committed_value(invoice, key, value)
except AttributeError:
setattr(invoice, key, value)
class InvoiceCreateRequest(BaseModel):
amount_sats: int = Field(gt=0, le=1_000_000, description="Amount in satoshis")
@@ -60,16 +139,102 @@ class InvoiceStatusResponse(BaseModel):
expires_at: int
_RETRYABLE_INVOICE_STATUSES = ("pending", "settlement_pending")
class InvoiceRecoverRequest(BaseModel):
bolt11: str = Field(description="BOLT11 invoice string")
def _trusted_mint_candidates() -> list[str]:
return [
mint
for mint in dict.fromkeys([settings.primary_mint, *settings.cashu_mints])
if mint
]
async def _request_mint_with_fallback(
amount_sats: int,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
"""Request a quote, falling back only among the allowed trusted mints.
Guards against amount_sats <= 0: the cashu library's PostMintQuoteRequest
enforces ``amount > 0`` (Pydantic Field(gt=0)), so passing 0 raises a
cryptic validation error deep in the stack. Fail fast with context.
"""
if amount_sats <= 0:
raise ValueError(
f"generate_lightning_invoice: amount_sats must be > 0, got {amount_sats}."
)
tried: list[str] = []
trusted = _trusted_mint_candidates()
if allowed_mints:
# Persisted mint preferences (e.g. an API key's refund_mint_url) must
# not outlive the operator's trusted-mint configuration.
candidates = [m for m in dict.fromkeys(allowed_mints) if m in trusted]
if not candidates:
logger.warning(
"Requested mints are no longer trusted; falling back to "
"configured mints",
extra={
"requested_mints": list(dict.fromkeys(allowed_mints)),
"op_name": "request_mint_invoice",
},
)
candidates = trusted
else:
candidates = trusted
for mint_url in candidates:
cooldown = mint_cooldown_remaining(mint_url)
if cooldown > 0:
tried.append(f"{mint_url}: cooling down")
logger.info(
"Skipping mint during cooldown",
extra={
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"op_name": "request_mint_invoice",
},
)
continue
try:
wallet = await get_wallet(mint_url, "sat", retry_on_rate_limit=False)
quote = await run_mint_operation(
lambda: wallet.request_mint(amount_sats),
op_name="request_mint_invoice",
mint_url=mint_url,
retry_on_rate_limit=False,
)
return quote.request, quote.quote, mint_url
except Exception as e:
tried.append(f"{mint_url}: {type(e).__name__}")
if not is_mint_connection_error(e) and not is_mint_rate_limited(e):
raise
logger.warning(
"request_mint failed, trying fallback mint",
extra={
"failed_mint": mint_url,
"error": str(e),
"tried": tried,
},
)
continue
raise MintConnectionError(f"All mints failed for request_mint: {tried}")
async def generate_lightning_invoice(
amount_sats: int, description: str
) -> tuple[str, str]:
wallet = await get_wallet(settings.primary_mint, "sat")
quote = await wallet.request_mint(amount_sats)
return quote.request, quote.quote
amount_sats: int,
description: str,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
bolt11, payment_hash, mint_url = await _request_mint_with_fallback(
amount_sats, allowed_mints=allowed_mints
)
return bolt11, payment_hash, mint_url
def generate_invoice_id() -> str:
@@ -83,6 +248,7 @@ async def create_invoice(
session: AsyncSession = Depends(get_session),
) -> InvoiceCreateResponse:
api_key_token = _extract_bearer_api_key(authorization) or request.api_key
topup_api_key: ApiKey | None = None
if request.purpose == "topup":
if not api_key_token:
@@ -93,14 +259,23 @@ async def create_invoice(
if not api_key_token.startswith("sk-"):
raise HTTPException(status_code=400, detail="Invalid API key format")
api_key = await session.get(ApiKey, api_key_token[3:])
if not api_key:
topup_api_key = await session.get(ApiKey, api_key_token[3:])
if not topup_api_key:
raise HTTPException(status_code=404, detail="API key not found")
try:
description = f"Routstr {request.purpose} {request.amount_sats} sats"
bolt11, payment_hash = await generate_lightning_invoice(
request.amount_sats, description
allowed_mints = None
if request.purpose == "topup":
assert topup_api_key is not None
# A key's liabilities are attributed to a single refund mint. Keep
# top-up collateral on that same mint so balances and payouts cannot
# misclassify funds held by another mint as owner profit.
allowed_mints = [
topup_api_key.refund_mint_url or settings.primary_mint
]
bolt11, payment_hash, mint_url = await generate_lightning_invoice(
request.amount_sats, description, allowed_mints=allowed_mints
)
invoice_id = generate_invoice_id()
@@ -115,6 +290,7 @@ async def create_invoice(
status="pending",
api_key_hash=api_key_token[3:] if api_key_token else None,
purpose=request.purpose,
mint_url=mint_url,
balance_limit=request.balance_limit,
balance_limit_reset=request.balance_limit_reset,
validity_date=request.validity_date,
@@ -160,12 +336,12 @@ async def get_invoice_status(
if not invoice:
raise HTTPException(status_code=404, detail="Invoice not found")
if invoice.status == "pending":
await check_invoice_payment(invoice, session)
if invoice.status == "pending" and int(time.time()) > invoice.expires_at:
invoice.status = "expired"
await session.commit()
definitively_unpaid = False
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
definitively_unpaid = await check_invoice_payment(invoice, session)
await _expire_invoice_if_authoritatively_unpaid(
invoice, session, definitively_unpaid
)
api_key = None
if invoice.status == "paid" and invoice.purpose == "create":
@@ -199,8 +375,12 @@ async def recover_invoice(
if not invoice:
raise HTTPException(status_code=404, detail="Invoice not found")
if invoice.status == "pending":
await check_invoice_payment(invoice, session)
definitively_unpaid = False
if invoice.status in _RETRYABLE_INVOICE_STATUSES:
definitively_unpaid = await check_invoice_payment(invoice, session)
await _expire_invoice_if_authoritatively_unpaid(
invoice, session, definitively_unpaid
)
api_key = None
if invoice.status == "paid":
@@ -219,114 +399,419 @@ async def recover_invoice(
)
async def _claim_paid_invoice_for_settlement(
invoice: LightningInvoice,
caller_session: AsyncSession,
observed_status: str,
) -> bool:
"""Claim an authoritative paid quote before consuming it at the mint."""
if observed_status == "settlement_pending":
return True
if observed_status != "pending":
await _reload_invoice_view(invoice, caller_session)
return False
async with create_session() as claim_session:
claim = await claim_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice.id,
col(LightningInvoice.status) == "pending",
)
.values(status="settlement_pending")
.execution_options(synchronize_session=False)
)
await claim_session.commit()
if claim.rowcount != 1:
await _reload_invoice_view(invoice, caller_session)
return False
_publish_invoice_value(invoice, "status", "settlement_pending")
return True
async def check_invoice_payment(
invoice: LightningInvoice, session: AsyncSession
) -> None:
try:
wallet = await get_wallet(settings.primary_mint, "sat")
mint_status = await wallet.get_mint_quote(invoice.payment_hash)
if mint_status.paid:
invoice.status = "paid"
invoice.paid_at = int(time.time())
if invoice.purpose == "create":
api_key = await create_api_key_from_invoice(invoice, session)
invoice.api_key_hash = api_key.hashed_key
elif invoice.purpose == "topup" and invoice.api_key_hash:
await topup_api_key_from_invoice(invoice, session)
) -> bool:
"""Settle an invoice and report whether its quote is definitively unpaid.
False covers paid, pending, and ambiguous transport/DB outcomes so callers
never expire a quote merely because reconciliation could not complete.
"""
async with _invoice_settlement_lock(invoice.id), wallet_operation_guard():
minted = False
payment_confirmed = False
try:
# Snapshot the row and end the caller's read transaction before any
# potentially slow mint I/O. All final DB mutations use owned,
# short-lived sessions below.
await session.refresh(invoice)
if invoice.status not in _RETRYABLE_INVOICE_STATUSES:
await session.commit()
return False
observed_status = invoice.status
settlement = _InvoiceSettlement.from_invoice(invoice)
await session.commit()
mint_url = settlement.mint_url or settings.primary_mint
wallet = await get_wallet(mint_url, "sat")
try:
mint_status = await run_mint_operation(
lambda: wallet.get_mint_quote(settlement.payment_hash),
op_name="get_mint_quote",
mint_url=mint_url,
)
except Exception as error:
if not _is_quote_not_found(error):
raise
logger.info(
"Invoice quote no longer exists at mint, marking expired",
extra={"invoice_id": invoice.id, "error": str(error)},
)
return True
if not mint_status.paid:
return getattr(mint_status, "state", None) == MintQuoteState.unpaid
payment_confirmed = True
# Fence expiry and other workers before consuming the paid quote.
# If a concurrent expiry/finalization won, this worker must not mint.
if not await _claim_paid_invoice_for_settlement(
invoice, session, observed_status
):
return False
# Reject a paid top-up whose target was pruned before redeeming its
# single-use quote. The validation session is closed before mint I/O.
if settlement.purpose == "topup":
if not settlement.api_key_hash:
raise ValueError("No API key associated with topup invoice")
async with create_session() as validation_session:
target = await validation_session.get(
ApiKey, settlement.api_key_hash
)
if target is None:
terminal = await validation_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == settlement.id,
col(LightningInvoice.status).in_(
_RETRYABLE_INVOICE_STATUSES
),
)
.values(status="reconciliation_required")
)
await validation_session.commit()
if terminal.rowcount == 1:
_publish_invoice_value(
invoice, "status", "reconciliation_required"
)
else:
await _reload_invoice_view(invoice, session)
logger.critical(
"Paid topup invoice target API key was not found; reconciliation required",
extra={"invoice_id": settlement.id},
)
return False
# Quote-linked proof verification makes an ambiguous mint response
# retryable without crediting unrelated wallet balance growth.
await _mint_invoice_quote(wallet, settlement)
minted = True
paid_at = int(time.time())
async with create_session() as finalization_session:
settled, api_key_hash = await _finalize_invoice_settlement(
settlement, finalization_session, paid_at
)
if not settled:
await _reload_invoice_view(invoice, session)
return False
_publish_invoice_value(invoice, "status", "paid")
_publish_invoice_value(invoice, "paid_at", paid_at)
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
logger.info(
"Lightning invoice paid",
extra={
"invoice_id": invoice.id,
"amount_sats": invoice.amount_sats,
"purpose": invoice.purpose,
"api_key_hash": invoice.api_key_hash[:8] + "..."
if invoice.api_key_hash
"invoice_id": settlement.id,
"amount_sats": settlement.amount_sats,
"purpose": settlement.purpose,
"api_key_hash": api_key_hash[:8] + "..."
if api_key_hash
else None,
},
)
except Exception as e:
logger.error(f"Failed to check invoice payment: {e}")
return False
except BaseException as error:
# Never roll back the caller-owned session: doing so expires invoice
# and sibling ORM objects. Owned sessions roll themselves back.
if payment_confirmed and invoice.status != "settlement_pending":
try:
async with create_session() as state_session:
pending = await state_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice.id,
col(LightningInvoice.status).in_(
_RETRYABLE_INVOICE_STATUSES
),
)
.values(status="settlement_pending")
)
await state_session.commit()
if pending.rowcount == 1:
_publish_invoice_value(
invoice, "status", "settlement_pending"
)
except Exception as state_error:
logger.critical(
"Paid invoice reconciliation state could not be persisted",
extra={"invoice_id": invoice.id, "error": str(state_error)},
)
if minted:
logger.critical(
"Invoice mint succeeded but DB finalization failed; reconciliation required",
extra={"invoice_id": invoice.id, "purpose": invoice.purpose},
)
try:
await _reload_invoice_view(invoice, session)
except Exception:
pass
if not isinstance(error, Exception):
raise
logger.error(f"Failed to check invoice payment: {error}")
return False
async def create_api_key_from_invoice(
invoice: LightningInvoice, session: AsyncSession
) -> ApiKey:
wallet = await get_wallet(settings.primary_mint, "sat")
await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash)
def _is_quote_not_found(error: BaseException) -> bool:
"""Check if the error indicates the mint no longer has this quote."""
message = str(error)
return bool(
re.search(r"\bquote\s+not\s+found\b", message, re.IGNORECASE)
and re.search(r"\bcode\s*:?\s*0\b", message, re.IGNORECASE)
)
def _is_outputs_already_signed(error: BaseException) -> bool:
message = str(error)
return bool(
re.search(
r"\boutputs?\s+(?:have\s+)?already\s+(?:been\s+)?signed(?:\s+before)?\b",
message,
re.IGNORECASE,
)
and re.search(r"\bcode\s*:\s*11003\b", message, re.IGNORECASE)
)
def _invoice_quote_proof_amount(wallet: Any, quote_id: str) -> int:
"""Return spendable wallet value minted by one Lightning quote."""
return sum(
proof.amount
for proof in wallet.proofs
if proof.mint_id == quote_id and not proof.reserved
)
async def _mint_invoice_quote(
wallet: Any, invoice: LightningInvoice | _InvoiceSettlement
) -> None:
"""Mint a paid quote, proving quote-linked outputs before DB credit."""
mint_url = invoice.mint_url or settings.primary_mint
await wallet.load_proofs(reload=True)
if _invoice_quote_proof_amount(wallet, invoice.payment_hash) >= invoice.amount_sats:
return
try:
await run_mint_operation(
lambda: wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash),
op_name=f"invoice_mint_{invoice.purpose}",
mint_url=mint_url,
retry_timeouts=False,
)
except Exception as error:
if not _is_outputs_already_signed(error):
raise
for keyset_id in wallet.keysets:
await wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
await wallet.load_proofs(reload=True)
recovered = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
if recovered < invoice.amount_sats:
raise RuntimeError(
"Invoice outputs were already signed but quote-linked recovery returned "
f"{recovered} sats; expected at least {invoice.amount_sats}"
) from error
else:
await wallet.load_proofs(reload=True)
minted_amount = _invoice_quote_proof_amount(wallet, invoice.payment_hash)
if minted_amount < invoice.amount_sats:
raise RuntimeError(
"Invoice mint succeeded but quote-linked proofs total "
f"{minted_amount} sats; expected at least {invoice.amount_sats}"
)
def _invoice_api_key_hash(invoice: LightningInvoice | _InvoiceSettlement) -> str:
dummy_token = f"invoice-{invoice.id}-{invoice.payment_hash}"
hashed_key = hashlib.sha256(dummy_token.encode()).hexdigest()
return hashlib.sha256(dummy_token.encode()).hexdigest()
async def _create_api_key_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> ApiKey:
mint_url = invoice.mint_url or settings.primary_mint
api_key = ApiKey(
hashed_key=hashed_key,
balance=invoice.amount_sats * 1000, # Convert to msats
hashed_key=_invoice_api_key_hash(invoice),
balance=invoice.amount_sats * 1000,
refund_currency="sat",
refund_mint_url=settings.primary_mint,
refund_mint_url=mint_url,
balance_limit=invoice.balance_limit,
balance_limit_reset=invoice.balance_limit_reset,
validity_date=invoice.validity_date,
)
session.add(api_key)
await session.flush()
return api_key
async def topup_api_key_from_invoice(
invoice: LightningInvoice, session: AsyncSession
async def _topup_api_key_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> None:
wallet = await get_wallet(settings.primary_mint, "sat")
await wallet.mint(invoice.amount_sats, quote_id=invoice.payment_hash)
if not invoice.api_key_hash:
raise ValueError("No API key associated with topup invoice")
api_key = await session.get(ApiKey, invoice.api_key_hash)
if not api_key:
result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == invoice.api_key_hash)
.values(balance=col(ApiKey.balance) + invoice.amount_sats * 1000)
.execution_options(synchronize_session=False)
)
if result.rowcount != 1:
raise ValueError("Associated API key not found")
api_key.balance += invoice.amount_sats * 1000 # Convert to msats
await session.flush()
async def _finalize_invoice_settlement(
invoice: _InvoiceSettlement, session: AsyncSession, paid_at: int
) -> tuple[bool, str | None]:
"""Atomically fence and apply one invoice credit in the provided owned session."""
api_key_hash = (
_invoice_api_key_hash(invoice)
if invoice.purpose == "create"
else invoice.api_key_hash
)
claim = await session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(col(LightningInvoice.id) == invoice.id)
.where(
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
)
.values(status="paid", paid_at=paid_at, api_key_hash=api_key_hash)
.execution_options(synchronize_session=False)
)
if claim.rowcount != 1:
await session.rollback()
return False, None
if invoice.purpose == "create":
await _create_api_key_record(invoice, session)
elif invoice.purpose == "topup":
await _topup_api_key_record(invoice, session)
else:
raise ValueError(f"Unsupported invoice purpose: {invoice.purpose}")
await session.commit()
return True, api_key_hash
INVOICE_WATCH_INTERVAL_SECONDS = 5
async def _reload_invoice_view(
invoice: LightningInvoice, _caller_session: AsyncSession
) -> None:
"""Publish committed invoice state without touching the caller transaction."""
async with create_session() as reload_session:
stored = await reload_session.get(LightningInvoice, invoice.id)
if stored is None:
return
status = stored.status
paid_at = stored.paid_at
api_key_hash = stored.api_key_hash
await reload_session.commit()
_publish_invoice_value(invoice, "status", status)
_publish_invoice_value(invoice, "paid_at", paid_at)
_publish_invoice_value(invoice, "api_key_hash", api_key_hash)
async def _expire_invoice_if_authoritatively_unpaid(
invoice: LightningInvoice,
caller_session: AsyncSession,
definitively_unpaid: bool,
) -> bool:
"""Expire one overdue unpaid invoice without overwriting concurrent settlement."""
if (
not definitively_unpaid
or invoice.status != "pending"
or int(time.time()) <= invoice.expires_at
):
return False
async with create_session() as expiry_session:
expired = await expiry_session.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(
col(LightningInvoice.id) == invoice.id,
col(LightningInvoice.status) == "pending",
)
.values(status="expired")
.execution_options(synchronize_session=False)
)
await expiry_session.commit()
if expired.rowcount == 1:
_publish_invoice_value(invoice, "status", "expired")
return True
await _reload_invoice_view(invoice, caller_session)
return False
async def _credit_topup_record(
invoice: LightningInvoice | _InvoiceSettlement, session: AsyncSession
) -> None:
await _topup_api_key_record(invoice, session)
# Nutshell mints throttle Lightning backend lookups to once per 10s per
# quote, so polling faster just burns the global request budget for nothing.
INVOICE_WATCH_INTERVAL_SECONDS = 10
INVOICE_WATCH_BATCH_LIMIT = 100
async def periodic_invoice_watcher() -> None:
"""Background task: detect paid Lightning invoices and credit balances.
async def _process_invoice_watch_batch(session: AsyncSession) -> None:
result = await session.exec(
select(LightningInvoice)
.where(
col(LightningInvoice.status).in_(_RETRYABLE_INVOICE_STATUSES)
)
.limit(INVOICE_WATCH_BATCH_LIMIT)
)
for invoice in result.all():
try:
definitively_unpaid = await check_invoice_payment(invoice, session)
await _expire_invoice_if_authoritatively_unpaid(
invoice, session, definitively_unpaid
)
except Exception as e:
logger.error(
"Invoice watcher failed for invoice",
extra={"invoice_id": invoice.id, "error": str(e)},
)
Removes the need for clients to poll the status endpoint after paying.
"""
async def periodic_invoice_watcher() -> None:
"""Background task: detect paid Lightning invoices and credit balances."""
while True:
try:
async with create_session() as session:
now = int(time.time())
result = await session.exec(
select(LightningInvoice)
.where(
LightningInvoice.status == "pending",
col(LightningInvoice.expires_at) > now,
)
.limit(INVOICE_WATCH_BATCH_LIMIT)
)
pending = result.all()
for invoice in pending:
try:
await check_invoice_payment(invoice, session)
except Exception as e:
logger.error(
"Invoice watcher failed for invoice",
extra={"invoice_id": invoice.id, "error": str(e)},
)
await _process_invoice_watch_batch(session)
except asyncio.CancelledError:
raise
except Exception as e:
+343
View File
@@ -0,0 +1,343 @@
"""Shared policy for bounded, rate-aware Cashu mint API operations."""
from __future__ import annotations
import asyncio
import socket
import time
from contextlib import asynccontextmanager
from contextvars import ContextVar
from typing import Any, AsyncGenerator, Awaitable, Callable
import httpx
from .core.logging import get_logger
from .core.settings import settings
logger = get_logger(__name__)
MINT_TRANSPORT_EXCEPTIONS: tuple[type[BaseException], ...] = (
httpx.NetworkError,
httpx.TimeoutException,
ConnectionError,
socket.gaierror,
asyncio.TimeoutError,
)
MINT_TRANSPORT_COOLDOWN_SECONDS = 30.0
_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS = 60.0
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS = 7 * 60 * 60
_fail_fast_depth: ContextVar[int] = ContextVar("mint_fail_fast_depth", default=0)
class MintRateLimitedError(httpx.HTTPStatusError):
"""Typed boundary error preserving a Cashu mint's HTTP 429 response."""
class MintCooldownError(Exception):
"""A mint is cooling down and this operation must not wait."""
def __init__(self, mint_url: str, retry_after_seconds: float):
self.mint_url = mint_url
self.retry_after_seconds = max(0.0, retry_after_seconds)
super().__init__(
f"Mint {mint_url} is cooling down; retry after "
f"{self.retry_after_seconds:.2f}s"
)
@asynccontextmanager
async def fail_fast_mint_operations() -> AsyncGenerator[None, None]:
"""Make mint cooldown/probe waits fail fast in the current task.
Wallet mutation code holds a process-wide file lock. It enters this scope so
an existing mint cooldown can never turn that lock into a multi-hour wait.
"""
token = _fail_fast_depth.set(_fail_fast_depth.get() + 1)
try:
yield
finally:
_fail_fast_depth.reset(token)
class MintRateGuard:
"""Limit concurrency and remember per-mint cooldown/probe state."""
_guards: dict[str, "MintRateGuard"] = {}
@classmethod
def get(cls, mint_url: str) -> "MintRateGuard":
concurrency = settings.mint_max_concurrency
guard = cls._guards.get(mint_url)
if guard is None or guard._max_concurrency != concurrency:
previous = guard
guard = cls(mint_url, concurrency)
if previous is not None:
# Concurrency changed at runtime: keep the live cooldown/backoff
# state so an active 429 cooldown is not silently discarded.
guard._cooldown_until = previous._cooldown_until
guard._cooldown_reason = previous._cooldown_reason
guard._consecutive_rate_limits = previous._consecutive_rate_limits
guard._needs_probe = previous._needs_probe
cls._guards[mint_url] = guard
return guard
def __init__(self, mint_url: str, max_concurrency: int):
self._mint_url = mint_url
self._max_concurrency = max_concurrency
self._semaphore = (
asyncio.Semaphore(max_concurrency) if max_concurrency > 0 else None
)
self._cooldown_until = 0.0
self._cooldown_reason: str | None = None
self._consecutive_rate_limits = 0
self._needs_probe = False
self._probe_lock = asyncio.Lock()
def apply_cooldown(self, delay: float, *, reason: str | None = None) -> None:
deadline = time.monotonic() + max(0.0, delay)
if deadline >= self._cooldown_until:
self._cooldown_until = deadline
if reason is not None:
self._cooldown_reason = reason
elif self._cooldown_reason is None and reason is not None:
self._cooldown_reason = reason
self._needs_probe = True
def apply_rate_limit_cooldown(self, retry_after: float | None = None) -> float:
remaining = self.cooldown_remaining()
if remaining > 0 and self._cooldown_reason == "rate_limited":
minimum = min(
_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS,
max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0),
)
if minimum > remaining:
self.apply_cooldown(minimum, reason="rate_limited")
return minimum
return remaining
self._consecutive_rate_limits += 1
base = max(_MINT_RATE_LIMIT_BASE_COOLDOWN_SECONDS, retry_after or 0.0)
multiplier = 2 ** min(self._consecutive_rate_limits - 1, 10)
delay = min(_MINT_RATE_LIMIT_MAX_COOLDOWN_SECONDS, base * multiplier)
self.apply_cooldown(delay, reason="rate_limited")
return delay
def cooldown_remaining(self) -> float:
return max(0.0, self._cooldown_until - time.monotonic())
def cooldown_reason(self) -> str | None:
return self._cooldown_reason if self.cooldown_remaining() > 0 else None
def _raise_if_wait_forbidden(self) -> None:
remaining = self.cooldown_remaining()
if _fail_fast_depth.get() and remaining > 0:
raise MintCooldownError(self._mint_url, remaining)
async def _wait_for_cooldown(self) -> None:
while True:
self._raise_if_wait_forbidden()
deadline = self._cooldown_until
wait = max(0.0, deadline - time.monotonic())
if wait <= 0:
return
logger.debug(
"Mint rate guard: cooling down",
extra={"mint_url": self._mint_url, "wait_seconds": round(wait, 2)},
)
await asyncio.sleep(wait)
if self._cooldown_until <= deadline:
return
async def _run_probe(self, factory: Callable[[], Awaitable[Any]]) -> Any:
await self._wait_for_cooldown()
logger.info(
"Mint cooldown ended; sending one probe request",
extra={"event": "mint_cooldown_probe_started", "mint_url": self._mint_url},
)
try:
result = await factory()
except Exception as error:
if is_mint_rate_limited(error):
retry_after = None
if isinstance(error, httpx.HTTPStatusError):
retry_after = parse_retry_after(error.response.headers)
self.apply_rate_limit_cooldown(retry_after)
else:
self.apply_cooldown(1.0)
logger.warning(
"Mint cooldown probe failed",
extra={
"event": "mint_cooldown_probe_failed",
"mint_url": self._mint_url,
"error": str(error),
"error_type": type(error).__name__,
"cooldown_seconds": round(self.cooldown_remaining(), 2),
"consecutive_rate_limits": self._consecutive_rate_limits,
},
)
raise
self._needs_probe = False
self._cooldown_until = 0.0
self._cooldown_reason = None
self._consecutive_rate_limits = 0
logger.info(
"Mint cooldown probe succeeded; restoring normal concurrency",
extra={
"event": "mint_cooldown_probe_succeeded",
"mint_url": self._mint_url,
},
)
return result
async def run(self, factory: Callable[[], Awaitable[Any]]) -> Any:
while True:
self._raise_if_wait_forbidden()
if self._needs_probe or self.cooldown_remaining() > 0:
if _fail_fast_depth.get() and self._probe_lock.locked():
raise MintCooldownError(self._mint_url, self.cooldown_remaining())
async with self._probe_lock:
self._raise_if_wait_forbidden()
if self.cooldown_remaining() > 0:
self._needs_probe = True
if self._needs_probe:
return await self._run_probe(factory)
continue
if self._semaphore is None:
return await factory()
async with self._semaphore:
self._raise_if_wait_forbidden()
if self._needs_probe:
continue
return await factory()
def mint_cooldown_remaining(mint_url: str) -> float:
return MintRateGuard.get(mint_url).cooldown_remaining()
def mint_cooldown_reason(mint_url: str) -> str | None:
return MintRateGuard.get(mint_url).cooldown_reason()
def is_mint_rate_limited(error: BaseException) -> bool:
"""Return whether an exception chain represents HTTP 429/cooldown."""
current: BaseException | None = error
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
if isinstance(current, MintCooldownError):
return True
if isinstance(current, httpx.HTTPStatusError):
if current.response.status_code == 429:
return True
current = current.__cause__ or current.__context__
return False
def parse_retry_after(headers: Any) -> float | None:
raw = headers.get("retry-after") or headers.get("Retry-After")
if raw is None:
return None
try:
return float(str(raw).strip())
except (TypeError, ValueError):
return None
async def run_mint_operation(
factory: Callable[[], Awaitable[Any]],
*,
op_name: str = "mint_operation",
mint_url: str = "",
retry_timeouts: bool = True,
retry_on_rate_limit: bool = True,
) -> Any:
"""Run one mint operation with bounded concurrency and adaptive cooldown."""
guard = MintRateGuard.get(mint_url) if mint_url else None
timeout = settings.mint_operation_timeout_seconds
max_attempts = settings.mint_retry_max_attempts + 1
async def timed_factory() -> Any:
if timeout > 0:
return await asyncio.wait_for(factory(), timeout=timeout)
return await factory()
async def invoke() -> Any:
if guard is not None:
return await guard.run(timed_factory)
return await timed_factory()
for attempt in range(max_attempts):
try:
return await invoke()
except MintCooldownError:
raise
except (asyncio.TimeoutError, httpx.TimeoutException) as exc:
if retry_timeouts and attempt < max_attempts - 1:
backoff = (2**attempt) + (time.monotonic() % 1.0)
logger.warning(
"Mint operation timed out, retrying",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"backoff_seconds": round(backoff, 2),
},
)
await asyncio.sleep(backoff)
continue
raise httpx.TimeoutException(
f"{op_name} timed out (attempts: {attempt + 1})"
) from exc
except Exception as exc:
if not is_mint_rate_limited(exc):
raise
backoff = (2**attempt) + (time.monotonic() % 1.0)
if isinstance(exc, httpx.HTTPStatusError):
retry_after = parse_retry_after(exc.response.headers)
if retry_after is not None:
backoff = max(retry_after, backoff)
cooldown = backoff
if guard is not None:
cooldown = guard.apply_rate_limit_cooldown(backoff)
if not retry_on_rate_limit:
logger.warning(
"Mint rate-limited, skipping retries for fallback",
extra={
"op_name": op_name,
"mint_url": mint_url,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
raise
if attempt >= max_attempts - 1:
raise
logger.warning(
"Mint rate-limited, applying cooldown",
extra={
"op_name": op_name,
"mint_url": mint_url,
"attempt": attempt + 1,
"cooldown_seconds": round(cooldown, 2),
"consecutive_rate_limits": guard._consecutive_rate_limits
if guard is not None
else attempt + 1,
},
)
if guard is None:
await asyncio.sleep(cooldown)
raise RuntimeError(f"{op_name}: exhausted retries unexpectedly")
+68 -5
View File
@@ -183,6 +183,28 @@ async def calculate_cost(
cost_details.get("output_cost")
or cost_details.get("upstream_inference_completions_cost")
)
cache_pricing_rates: tuple[float, float, float, float] | None = None
if cache_read_tokens > 0 or cache_creation_tokens > 0:
try:
cache_pricing_rates = _get_pricing_rates(
response_data, model_obj, provider_fee
)
except ValueError:
logger.warning(
"Cache pricing unavailable for USD cost breakdown; "
"leaving cache cost components unknown",
extra={"model": response_data.get("model", "unknown")},
)
if cache_pricing_rates is None and settings.fixed_pricing:
fixed_input_rate = (
float(settings.fixed_per_1k_input_tokens) * 1000.0
)
cache_pricing_rates = (
fixed_input_rate,
float(settings.fixed_per_1k_output_tokens) * 1000.0,
fixed_input_rate,
fixed_input_rate,
)
return _calculate_from_usd_cost(
usd_cost,
input_usd,
@@ -193,6 +215,7 @@ async def calculate_cost(
output_tokens,
response_data,
provider_fee,
cache_pricing_rates,
)
except Exception as e:
logger.warning(
@@ -451,6 +474,7 @@ def _calculate_from_usd_cost(
output_tokens: int,
response_data: dict,
provider_fee: float | None,
pricing_rates: tuple[float, float, float, float] | None = None,
) -> CostData:
"""Calculate cost from USD figures, deriving input/output split from tokens."""
if provider_fee is None:
@@ -460,15 +484,20 @@ def _calculate_from_usd_cost(
output_usd = output_usd * provider_fee
sats_per_usd = 1.0 / sats_usd_price()
cost_in_sats = usd_cost * sats_per_usd
cost_in_msats = math.ceil(cost_in_sats * 1000)
raw_cost_msats = cost_in_sats * 1000
cost_in_msats = math.ceil(raw_cost_msats)
raw_input_msats = 0.0
if input_usd > 0 or output_usd > 0:
# The total is the authoritative billed amount. Allocating that integer
# total proportionally avoids losing sub-millisatoshi remainders when
# input and output components are each truncated independently.
component_usd = input_usd + output_usd
input_msats = math.floor(cost_in_msats * input_usd / component_usd)
output_msats = cost_in_msats - input_msats
# Match the token-priced path: truncate the visible output component
# and assign the authoritative total's rounding remainder to input.
output_msats = math.floor(cost_in_msats * output_usd / component_usd)
input_msats = cost_in_msats - output_msats
raw_input_msats = raw_cost_msats * input_usd / component_usd
else:
effective_input_tokens = (
input_tokens + cache_read_tokens + cache_creation_tokens
@@ -480,6 +509,38 @@ def _calculate_from_usd_cost(
else 0
)
output_msats = cost_in_msats - input_msats
raw_input_msats = (
raw_cost_msats * effective_input_tokens / total_tokens
if total_tokens > 0
else 0.0
)
# Preserve the same cache-rate ratios as the token-priced path while the
# upstream USD total remains authoritative. Cache values are informational
# subcomponents of the inclusive input cost.
cache_read_msats = 0
cache_creation_msats = 0
if pricing_rates is not None:
input_rate, _, cache_read_rate, cache_creation_rate = pricing_rates
regular_weight = input_tokens * input_rate
cache_read_weight = cache_read_tokens * cache_read_rate
cache_creation_weight = cache_creation_tokens * cache_creation_rate
total_input_weight = (
regular_weight + cache_read_weight + cache_creation_weight
)
if total_input_weight > 0:
cache_read_msats = int(
round(
raw_input_msats * cache_read_weight / total_input_weight,
3,
)
)
cache_creation_msats = int(
round(
raw_input_msats * cache_creation_weight / total_input_weight,
3,
)
)
logger.info(
"Using cost from usage data/details",
@@ -487,6 +548,8 @@ def _calculate_from_usd_cost(
"usd_cost": usd_cost,
"cost_in_sats": cost_in_sats,
"cost_in_msats": cost_in_msats,
"cache_read_msats": cache_read_msats,
"cache_creation_msats": cache_creation_msats,
"model": response_data.get("model", "unknown"),
},
)
@@ -501,8 +564,8 @@ def _calculate_from_usd_cost(
output_tokens=output_tokens,
cache_read_input_tokens=cache_read_tokens,
cache_creation_input_tokens=cache_creation_tokens,
cache_read_msats=0,
cache_creation_msats=0,
cache_read_msats=cache_read_msats,
cache_creation_msats=cache_creation_msats,
)
+1 -2
View File
@@ -18,7 +18,6 @@ from ..wallet import deserialize_token_from_string
logger = get_logger(__name__)
def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> None:
if x_cashu := headers.get("x-cashu", None):
cashu_token = x_cashu
@@ -243,7 +242,7 @@ async def calculate_discounted_max_cost(
},
)
return max(0, adjusted)
return max(settings.min_request_msat, adjusted)
def estimate_tokens(messages: list) -> int:
+63 -14
View File
@@ -1,16 +1,17 @@
from __future__ import annotations
import asyncio
import math
from typing import TypedDict
import httpx
from cashu.core.base import MeltQuoteState
from cashu.wallet.wallet import Proof, Wallet
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
# very slow mint can block a melt (and any caller, e.g. the payout loop)
# indefinitely. Bound it here so callers fail instead of hanging forever.
MELT_TIMEOUT_SECONDS = 60
from ..mint import (
MINT_TRANSPORT_EXCEPTIONS,
is_mint_rate_limited,
run_mint_operation,
)
try:
from bech32 import bech32_decode, convertbits # type: ignore
@@ -31,6 +32,15 @@ class LNURLError(Exception):
"""LNURL related errors."""
class MeltOutcomeAmbiguousError(LNURLError):
"""A melt was dispatched but its final outcome could not be confirmed.
Callers must NOT treat this as a clean failure: the payment may still
settle, so debits backing it must be kept until reconciliation confirms
the true outcome.
"""
async def decode_lnurl(lnurl: str) -> str:
"""Decode LNURL to get the actual URL.
@@ -221,23 +231,62 @@ async def raw_send_to_lnurl(
lnurl_data["callback_url"], final_amount
)
melt_quote_resp = await wallet.melt_quote(invoice=bolt11_invoice)
melt_quote_resp = await run_mint_operation(
lambda: wallet.melt_quote(invoice=bolt11_invoice),
op_name="lnurl_melt_quote",
mint_url=str(wallet.url),
)
if amount:
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
try:
_ = await asyncio.wait_for(
wallet.melt(
melt_response = await run_mint_operation(
lambda: wallet.melt(
proofs=proofs,
invoice=bolt11_invoice,
fee_reserve_sat=melt_quote_resp.fee_reserve,
quote_id=melt_quote_resp.quote,
),
timeout=MELT_TIMEOUT_SECONDS,
op_name="lnurl_melt",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except asyncio.TimeoutError as e:
raise LNURLError(
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
) from e
return final_amount
except Exception as error:
if is_mint_rate_limited(error):
# Cooldown failures happen before dispatch, and HTTP 429 means the
# mint rejected the request. Neither outcome may keep proofs
# reserved as though a Lightning payment could still settle.
await wallet.set_reserved_for_send(proofs, reserved=False)
raise
if not isinstance(error, MINT_TRANSPORT_EXCEPTIONS):
raise
melt_response = None
melt_error: BaseException | None = error
else:
melt_error = None
if getattr(melt_response, "state", None) == MeltQuoteState.paid:
return final_amount
try:
quote = await run_mint_operation(
lambda: wallet.get_melt_quote(melt_quote_resp.quote),
op_name="reconcile_lnurl_melt_quote",
mint_url=str(wallet.url),
retry_timeouts=False,
)
except Exception as reconciliation_error:
raise MeltOutcomeAmbiguousError(
"Melt outcome is ambiguous; quote reconciliation failed and proofs "
"must not be retried"
) from reconciliation_error
if quote is not None and quote.state == MeltQuoteState.paid:
return final_amount
state = getattr(getattr(quote, "state", None), "value", "unknown")
raise MeltOutcomeAmbiguousError(
"Melt outcome is ambiguous; proofs must not be retried "
f"(quote_state={state})"
) from melt_error
+35 -4
View File
@@ -455,7 +455,9 @@ async def _update_sats_pricing_once() -> None:
for m in upstream.get_cached_models()
]
upstream._models_cache = updated_models
upstream._models_by_id = {m.forwarded_model_id or m.id: m for m in updated_models}
upstream._models_by_id = {
m.forwarded_model_id or m.id: m for m in updated_models
}
updated_count += len(updated_models)
if updated_count > 0:
@@ -510,9 +512,7 @@ class ModelTestRequest(V2BaseModel):
request_data: dict
@models_router.post(
"/api/models/test", dependencies=[Depends(_require_admin_api)]
)
@models_router.post("/api/models/test", dependencies=[Depends(_require_admin_api)])
async def test_model(
payload: ModelTestRequest,
session: AsyncSession = Depends(get_session),
@@ -595,6 +595,37 @@ async def test_model(
}
@models_router.get("/v1/models/paths")
@models_router.get("/v1/models/paths/", include_in_schema=False)
async def model_paths() -> dict:
"""All models with every upstream provider path they are reachable through."""
from ..upstream.model_paths import get_all_model_paths
return await get_all_model_paths()
@models_router.get("/v1/models/paths/model")
@models_router.get("/v1/models/paths/model/", include_in_schema=False)
async def model_paths_for_model(model_id: str) -> dict:
"""Paths for a single model.
Uses a query parameter (``?model_id=...``) under a fully static route so
model ids containing ``/`` (e.g. ``anthropic/claude-opus-4.6``) need no URL
encoding and there is no dynamic-route ambiguity.
"""
from ..proxy import get_unique_models
from ..upstream.model_paths import get_paths_for_model
result = await get_paths_for_model(model_id)
if not result["data"]:
advertised_ids = {
model.forwarded_model_id or model.id for model in get_unique_models()
}
if model_id not in advertised_ids:
raise HTTPException(status_code=404, detail="Model not found")
return result
@models_router.get("/v1/models")
@models_router.get("/v1/models/", include_in_schema=False)
@models_router.get("/models")
+90 -19
View File
@@ -1,4 +1,5 @@
import asyncio
import inspect
import json
from typing import Any
@@ -7,7 +8,13 @@ from fastapi.responses import Response, StreamingResponse
from sqlmodel import select
from .algorithm import create_model_mappings
from .auth import pay_for_request, revert_pay_for_request, validate_bearer_key
from .auth import (
ReservationSnapshot,
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
validate_bearer_key,
)
from .core import get_logger
from .core.db import (
ApiKey,
@@ -19,7 +26,6 @@ from .core.db import (
)
from .core.exceptions import UpstreamError
from .core.not_found import build_not_found_response
from .core.settings import settings
from .payment.helpers import (
calculate_discounted_max_cost,
check_token_balance,
@@ -43,6 +49,13 @@ _provider_map: dict[
_unique_models: dict[str, Model] = {} # Unique model.id -> Model (no duplicates)
async def _finish_read_transaction(session: AsyncSession) -> None:
"""Release a read transaction without assuming a particular session mock."""
commit_result = session.commit()
if inspect.isawaitable(commit_result):
await commit_result
async def initialize_upstreams() -> None:
"""Initialize upstream providers from database during application startup."""
global _upstreams
@@ -177,6 +190,19 @@ async def refresh_model_maps() -> None:
disabled_model_keys=disabled_model_keys,
)
# Keep model-path discovery in sync with admin mutations: disabling or
# deleting a provider must stop advertising its paths immediately rather
# than after the next timed refresh.
from .upstream.model_paths import prune_model_paths_for_inactive_providers
try:
await prune_model_paths_for_inactive_providers()
except Exception as e: # noqa: BLE001 - discovery sync must not break routing
logger.warning(
"Failed to prune model paths for inactive providers",
extra={"error": str(e), "error_type": type(e).__name__},
)
async def refresh_model_maps_periodically() -> None:
"""Background task to refresh model maps every minute."""
@@ -214,6 +240,20 @@ _API_PATH_PREFIXES = (
@proxy_router.api_route("/{path:path}", methods=["GET", "POST"], response_model=None)
async def proxy(
request: Request, path: str, session: AsyncSession = Depends(get_session)
) -> Response | StreamingResponse:
"""Run proxy setup in a short request session, never across response streaming."""
try:
return await _proxy(request, path, session)
finally:
# FastAPI yield dependencies normally close after the response body is
# sent. Close explicitly so a long stream cannot retain DB resources.
close_result = session.close()
if inspect.isawaitable(close_result):
await close_result
async def _proxy(
request: Request, path: str, session: AsyncSession
) -> Response | StreamingResponse:
# GET requests must hit a known API prefix; otherwise return a 404 (HTML
# for browsers, JSON for API clients). POST requests are always forwarded
@@ -326,8 +366,6 @@ async def proxy(
max_cost_for_model = await calculate_discounted_max_cost(
_max_cost_for_model, request_body_dict, model_obj=model_obj
)
# Ensure max_cost_for_model is at least the minimum allowed request cost
max_cost_for_model = max(max_cost_for_model, settings.min_request_msat)
check_token_balance(headers, request_body_dict, max_cost_for_model)
@@ -440,8 +478,13 @@ async def proxy(
"upstream_error", "All upstreams failed", 502, request=request
)
reservation_snapshot: ReservationSnapshot | None = None
if is_ehbp or request_body_dict:
await pay_for_request(key, max_cost_for_model, session)
reservation_snapshot = await get_reservation_snapshot(key, session)
# Snapshot validation performs SELECTs after pay_for_request commits.
# End that read transaction before waiting on upstream response headers.
await _finish_read_transaction(session)
# Tracks request params already removed in response to upstream rejections,
# shared across providers so a stripped param stays stripped on failover and
@@ -461,16 +504,21 @@ async def proxy(
candidate_max = await calculate_discounted_max_cost(
candidate_max, request_body_dict, model_obj=model_obj
)
candidate_max = max(candidate_max, settings.min_request_msat)
if candidate_max > max_cost_for_model:
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
try:
await pay_for_request(key, candidate_max, session)
except HTTPException:
if i == len(candidates) - 1:
raise
await pay_for_request(key, max_cost_for_model, session)
reservation_snapshot = await get_reservation_snapshot(key, session)
await _finish_read_transaction(session)
continue
reservation_snapshot = await get_reservation_snapshot(key, session)
await _finish_read_transaction(session)
max_cost_for_model = candidate_max
headers = upstream.prepare_headers(dict(request.headers))
@@ -499,6 +547,7 @@ async def proxy(
max_cost_for_model=max_cost_for_model,
session=session,
model_obj=model_obj,
reservation_snapshot=reservation_snapshot,
)
elif is_responses_api:
response = await upstream.forward_responses_request(
@@ -510,6 +559,7 @@ async def proxy(
max_cost_for_model,
session,
model_obj,
reservation_snapshot,
)
else:
response = await upstream.forward_request(
@@ -521,6 +571,7 @@ async def proxy(
max_cost_for_model,
session,
model_obj,
reservation_snapshot,
)
except UpstreamError:
# Let the outer UpstreamError handler manage retry/revert
@@ -537,7 +588,9 @@ async def proxy(
"max_cost_for_model": max_cost_for_model,
},
)
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
raise
# Reactive recovery: some models reject one specific request
@@ -606,7 +659,9 @@ async def proxy(
continue
# 4xx error (user error), or other non-retryable error, or last provider failed
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
logger.warning(
"Upstream request failed, revert payment "
"(provider=%s model=%s status=%s path=%s)",
@@ -638,8 +693,10 @@ async def proxy(
"max_cost_for_model": max_cost_for_model,
},
)
await asyncio.shield(
revert_pay_for_request(key, session, max_cost_for_model)
# The cancellation has been caught, so complete exact cleanup in
# this task before the request-scoped session can be torn down.
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
raise
@@ -659,7 +716,9 @@ async def proxy(
# If this was the last provider
if i == len(candidates) - 1:
await revert_pay_for_request(key, session, max_cost_for_model)
await revert_pay_for_request(
key, session, max_cost_for_model, reservation_snapshot
)
return create_upstream_error_response(e, request)
# Otherwise loop continues to next provider
@@ -744,17 +803,29 @@ async def get_bearer_token_key(
},
)
return key
except Exception as e:
key_preview = bearer_key[:20] + "..." if len(bearer_key) > 20 else bearer_key
logger.error(
f"Bearer token validation failed: {type(e).__name__}: {e} path={path} model={model_id!r} min_cost={min_cost} key={key_preview!r}",
except HTTPException as error:
detail: dict[str, Any] = error.detail if isinstance(error.detail, dict) else {}
raw_error = detail.get("error")
error_info = raw_error if isinstance(raw_error, dict) else {}
logger.warning(
"Bearer token rejected",
extra={
"error": str(e),
"error_type": type(e).__name__,
"status_code": error.status_code,
"error_code": error_info.get("code"),
"path": path,
"model_id": model_id,
"min_cost_msat": min_cost,
"bearer_key_preview": key_preview,
"required_msat": min_cost,
},
)
raise
except Exception as error:
logger.exception(
"Bearer token validation failed",
extra={
"error_type": type(error).__name__,
"path": path,
"model_id": model_id,
"required_msat": min_cost,
},
)
raise
File diff suppressed because it is too large Load Diff
+400 -139
View File
File diff suppressed because it is too large Load Diff
+28 -2
View File
@@ -15,7 +15,11 @@ from sqlmodel import col, update
from ..auth import (
ROUTSTR_FEE_PERCENT,
ReservationSnapshot,
_claim_reservation_for_charge,
_validate_reservation_snapshot,
get_billing_key,
get_reservation_snapshot,
payments_logger,
)
from ..core import get_logger
@@ -502,8 +506,14 @@ async def finalize_ehbp_actual_cost_payment(
reserved_cost_for_model: int,
model_id: str,
cost_info: dict,
reservation_snapshot: ReservationSnapshot | None = None,
) -> None:
"""Finalize an EHBP bearer request using clamped provider usage metrics."""
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(key, reservation, session)
if not await _claim_reservation_for_charge(reservation, session):
return
reserved_cost_for_model = reservation.reserved_msats
billing_key = await get_billing_key(key, session)
key_hash = key.hashed_key
billing_key_hash = billing_key.hashed_key
@@ -606,6 +616,7 @@ async def finalize_ehbp_max_cost_payment(
session: AsyncSession,
max_cost_for_model: int,
model_id: str,
reservation_snapshot: ReservationSnapshot | None = None,
) -> None:
"""Finalize an EHBP bearer request by charging the reserved max cost.
@@ -613,6 +624,11 @@ async def finalize_ehbp_max_cost_payment(
normal completion handlers, this intentionally charges the pre-reserved max
cost and releases the reservation.
"""
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
await _validate_reservation_snapshot(key, reservation, session)
if not await _claim_reservation_for_charge(reservation, session):
return
max_cost_for_model = reservation.reserved_msats
billing_key = await get_billing_key(key, session)
key_hash = key.hashed_key
billing_key_hash = billing_key.hashed_key
@@ -766,6 +782,7 @@ async def forward_ehbp_request(
max_cost_for_model: int,
session: AsyncSession,
model_obj: Model,
reservation_snapshot: ReservationSnapshot | None = None,
) -> Response | StreamingResponse:
"""Forward an EHBP bearer-auth request and finalize billing.
@@ -883,7 +900,12 @@ async def forward_ehbp_request(
# the requested model.
billing_model = cost_info.pop("actual_model", None) or model_obj.id
await finalize_ehbp_actual_cost_payment(
key, session, max_cost_for_model, billing_model, cost_info
key,
session,
max_cost_for_model,
billing_model,
cost_info,
reservation_snapshot,
)
cost_data = {**cost_info, "total_usd": 0.0}
else:
@@ -897,7 +919,11 @@ async def forward_ehbp_request(
},
)
await finalize_ehbp_max_cost_payment(
key, session, max_cost_for_model, model_obj.id
key,
session,
max_cost_for_model,
model_obj.id,
reservation_snapshot,
)
cost_data = {
"total_msats": max_cost_for_model,
+819
View File
@@ -0,0 +1,819 @@
"""Model-path discovery service.
Exposes every selectable upstream route a Routstr model is reachable through.
This PR remains discovery-only: request-side routing will consume the opaque
selectors in a follow-up.
A path is a standard percent-encoded query string containing the configured
upstream URL, provider ID, client-visible model ID and, for an exact OpenRouter
endpoint, its machine-readable tag::
url=https%3A%2F%2Fapi.anthropic.com%2Fv1&provider-id=12&model-id=claude-sonnet-4
url=https%3A%2F%2Fopenrouter.ai%2Fapi%2Fv1&provider-id=42&model-id=claude-sonnet-4&endpoint=google-vertex%2Fus
"""
from __future__ import annotations
import asyncio
import ipaddress
import random
import time
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Callable
from urllib.parse import urlencode, urlsplit
import httpx
from sqlalchemy.dialects.sqlite import insert
from sqlalchemy.orm import selectinload
from sqlmodel import col, delete, select
from ..core.db import ModelPathRow, ModelRow, UpstreamProviderRow, create_session
from ..core.logging import get_logger
if TYPE_CHECKING:
from sqlmodel.ext.asyncio.session import AsyncSession
from .base import BaseUpstreamProvider
logger = get_logger(__name__)
# Bound the per-model OpenRouter /endpoints fan-out so a provider with hundreds
# of models does not open hundreds of concurrent requests every refresh.
_OPENROUTER_CONCURRENCY = 5
_OPENROUTER_TIMEOUT_SECONDS = 10.0
# Rows inserted per statement during persist. Keeps each INSERT bounded while
# avoiding per-row round-trips that hold SQLite's write lock for ~1s per cycle.
_PERSIST_CHUNK_SIZE = 500
# Admin mutations enqueue provider IDs here instead of running OpenRouter's
# per-model endpoint fan-out inside the request. One worker serializes refreshes
# and coalesces repeated mutations for the same provider.
_scheduled_provider_refresh_ids: set[int] = set()
_scheduled_provider_refresh_task: asyncio.Task[None] | None = None
# Visibility key used across this module: routing carries the provider
# dimension everywhere (ModelRow's primary key is (id, upstream_provider_id)),
# so all model-id keyed maps here do too, lowercased like proxy.refresh_model_maps.
ModelKey = tuple[str, int]
@dataclass(frozen=True)
class EndpointIdentity:
"""Exact OpenRouter endpoint identity returned by ``/endpoints``."""
tag: str
provider_name: str | None
@dataclass(frozen=True)
class ConfiguredProviderIdentity:
"""Public identity of one configured upstream provider."""
id: int
slug: str
provider_type: str
base_url: str
@dataclass(frozen=True)
class DiscoveredPath:
"""One model route ready for persistence and API serialization."""
model_id: str
path: str
provider: ConfiguredProviderIdentity
endpoint_tag: str | None = None
endpoint_name: str | None = None
@dataclass(frozen=True)
class ProviderPathSnapshot:
"""Refresh result plus model IDs whose prior rows must survive degradation."""
paths: tuple[DiscoveredPath, ...]
preserve_model_ids: frozenset[str] = frozenset()
def public_provider_url(base_url: str) -> str:
"""Mask private IP addresses and URLs with explicit ports."""
parsed = urlsplit(base_url)
try:
if parsed.port is not None:
return "http://localhost"
except ValueError:
# An invalid explicit port must not accidentally leak through.
return "http://localhost"
hostname = parsed.hostname
if hostname is None:
return base_url
try:
address = ipaddress.ip_address(hostname)
except ValueError:
return base_url
return "http://localhost" if address.is_private else base_url
def encode_model_path(
base_url: str,
provider_id: int,
model_id: str,
endpoint_tag: str | None = None,
) -> str:
"""Encode the complete upstream route selector advertised to clients."""
components: list[tuple[str, str | int]] = [
("url", base_url),
("provider-id", provider_id),
("model-id", model_id),
]
if endpoint_tag:
components.append(("endpoint", endpoint_tag))
return urlencode(components)
def _make_http_client() -> httpx.AsyncClient:
"""Client factory, separated so tests can substitute a mock transport."""
return httpx.AsyncClient()
def is_openrouter_base_url(base_url: str | None) -> bool:
"""True when ``base_url`` points at OpenRouter.
Deliberately separate from ``BaseUpstreamProvider._upstream_accepts_cache_control``:
that predicate also returns True for native Anthropic (correct for
cache-control, wrong for OpenRouter endpoint discovery). This one keys only
on the URL so a ``GenericUpstreamProvider`` aimed at OpenRouter is matched
while native Anthropic is not.
"""
return "openrouter.ai" in (base_url or "")
def exposed_model_id(model: object) -> str:
"""Return exactly the ID advertised by ``/v1/models``.
A forwarded ID is already a public routable alias and must remain intact,
including any slash. Without one, ``/v1/models`` exposes the base ID.
"""
forwarded = getattr(model, "forwarded_model_id", None)
if forwarded:
return forwarded
return public_model_id(getattr(model, "id"))
def public_model_id(model_id: str) -> str:
"""Model id exposed by model-path API responses.
Uses the same rule as ``create_model_mappings.get_base_model_id`` and
``resolve_model_alias`` — strip everything before the *first* slash — so
the id shown here can be sent back to ``/v1/chat/completions`` verbatim.
"""
return model_id.split("/", 1)[1] if "/" in model_id else model_id
def openrouter_author_slug(model: object) -> str | None:
"""Return a canonical ``author/slug`` for the OpenRouter endpoints API.
Prefer ``canonical_slug``, then a slash-containing ``id``, then a
slash-containing ``forwarded_model_id``. The forwarded id is exactly what
the proxy sends upstream for admin-created alias rows (``base.py`` forwards
``forwarded_model_id or id``), so it is a valid OpenRouter id when the
bare ``id`` is a local alias with no slash.
"""
canonical = getattr(model, "canonical_slug", None)
if canonical and "/" in canonical:
return canonical
model_id = getattr(model, "id", None)
if model_id and "/" in model_id:
return model_id
forwarded = getattr(model, "forwarded_model_id", None)
if forwarded and "/" in forwarded:
return forwarded
return None
class _RefreshCycleState:
"""Per-refresh shared state: fetch dedupe cache and rate-limit latch.
``endpoint_cache`` dedupes byte-identical ``/endpoints`` fetches when two
providers point at the same OpenRouter base URL. ``rate_limited`` latches
on the first 429 so the rest of the cycle stops hammering a throttled API;
the whole provider result then degrades to "unknown" instead of an empty
list, which preserves previously persisted rows.
"""
def __init__(self) -> None:
self.endpoint_cache: dict[tuple[str, str], list[EndpointIdentity] | None] = {}
self.rate_limited = False
async def _fetch_openrouter_endpoint_subproviders(
client: httpx.AsyncClient,
base_url: str,
api_key: str,
author_slug: str,
semaphore: asyncio.Semaphore,
cycle: _RefreshCycleState,
) -> list[EndpointIdentity] | None:
"""Return exact endpoint identities for one model, or ``None`` when unknown.
``None`` (not ``[]``) signals a degraded fetch — network failure, rate
limit, non-200, or an unparseable payload — so callers can distinguish
"this model has no endpoints" from "we could not find out". Failures are
logged and swallowed so one model never breaks the whole refresh.
"""
cache_key = (base_url, author_slug)
if cache_key in cycle.endpoint_cache:
return cycle.endpoint_cache[cache_key]
if cycle.rate_limited:
return None
url = f"{base_url.rstrip('/')}/models/{author_slug}/endpoints"
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
result: list[EndpointIdentity] | None
async with semaphore:
try:
resp = await client.get(
url, headers=headers, timeout=_OPENROUTER_TIMEOUT_SECONDS
)
except Exception as e: # noqa: BLE001 - isolate per-model failures
logger.warning(
"OpenRouter endpoint discovery request failed",
extra={"author_slug": author_slug, "error": str(e)},
)
cycle.endpoint_cache[cache_key] = None
return None
if resp.status_code == 429:
logger.warning(
"OpenRouter endpoint discovery rate-limited; aborting cycle",
extra={"author_slug": author_slug},
)
cycle.rate_limited = True
cycle.endpoint_cache[cache_key] = None
return None
if resp.status_code != 200:
logger.warning(
"OpenRouter endpoint discovery non-200",
extra={"author_slug": author_slug, "status_code": resp.status_code},
)
cycle.endpoint_cache[cache_key] = None
return None
try:
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
endpoints = data.get("endpoints") if isinstance(data, dict) else None
if not isinstance(endpoints, list):
raise ValueError("endpoints must be a list")
identities: dict[str, EndpointIdentity] = {}
for endpoint in endpoints:
if not isinstance(endpoint, dict):
continue
tag = endpoint.get("tag")
if not isinstance(tag, str) or not tag.strip():
continue
provider_name = endpoint.get("provider_name")
identities.setdefault(
tag,
EndpointIdentity(
tag=tag,
provider_name=provider_name
if isinstance(provider_name, str) and provider_name
else None,
),
)
if endpoints and not identities:
raise ValueError("endpoints contain no usable tags")
result = list(identities.values())
except Exception as e: # noqa: BLE001
logger.warning(
"OpenRouter endpoint discovery bad payload",
extra={"author_slug": author_slug, "error": str(e)},
)
result = None
cycle.endpoint_cache[cache_key] = result
return result
async def _load_model_visibility() -> tuple[
dict[ModelKey, ModelRow],
set[ModelKey],
dict[int, ConfiguredProviderIdentity],
]:
"""Load the same DB model visibility inputs used by routing.
``refresh_model_maps`` builds routing from enabled providers, enabled DB
override rows, and disabled model keys — all keyed on
``(model_id.lower(), upstream_provider_id)`` because ``ModelRow``'s primary
key is composite and the same id legitimately exists on several providers.
Model-path discovery uses the same keying so disabling a model on one
provider never hides it on another, and one provider's
``forwarded_model_id`` alias is never applied to a different provider.
"""
async with create_session() as session:
query = select(UpstreamProviderRow).options(
selectinload(UpstreamProviderRow.models) # type: ignore[arg-type]
)
provider_rows = (await session.exec(query)).all()
overrides_by_key: dict[ModelKey, ModelRow] = {}
disabled_model_keys: set[ModelKey] = set()
provider_identities: dict[int, ConfiguredProviderIdentity] = {}
for provider in provider_rows:
if not provider.enabled or provider.id is None:
continue
provider_identities[provider.id] = ConfiguredProviderIdentity(
id=provider.id,
slug=provider.slug or f"provider-{provider.id}",
provider_type=provider.provider_type,
base_url=public_provider_url(provider.base_url),
)
for model in provider.models:
key = (model.id.lower(), provider.id)
if model.enabled:
overrides_by_key[key] = model
else:
disabled_model_keys.add(key)
return overrides_by_key, disabled_model_keys, provider_identities
def _apply_model_visibility(
upstream: BaseUpstreamProvider,
overrides_by_key: dict[ModelKey, ModelRow] | None,
disabled_model_keys: set[ModelKey] | None,
) -> list[object]:
"""Return provider models after DB disabled/override state is applied.
Only the identity fields (``id``, ``forwarded_model_id``,
``canonical_slug``) matter for path discovery, so DB override rows are used
directly rather than rebuilt into fully priced ``Model`` objects — the
pricing pipeline costs ~0.7ms of event-loop CPU per row for data this
module immediately discards.
"""
overrides_by_key = overrides_by_key or {}
disabled_model_keys = disabled_model_keys or set()
upstream_provider_id = getattr(upstream, "db_id", None)
if not isinstance(upstream_provider_id, int):
return [
model
for model in upstream.get_cached_models()
if getattr(model, "enabled", True)
]
visible_models: list[object] = []
seen_model_ids: set[str] = set()
for model in upstream.get_cached_models():
model_id = getattr(model, "id", "")
key = (model_id.lower(), upstream_provider_id)
if not getattr(model, "enabled", True) or key in disabled_model_keys:
continue
# Apply overrides only for this provider's own model row.
override_row = overrides_by_key.get(key)
visible: object = model if override_row is None else override_row
visible_models.append(visible)
seen_model_ids.add(model_id.lower())
# DB-only override rows for this provider with no cached counterpart.
for (model_id_lower, provider_id), override_row in overrides_by_key.items():
if provider_id != upstream_provider_id:
continue
if model_id_lower in seen_model_ids:
continue
visible_models.append(override_row)
seen_model_ids.add(model_id_lower)
return visible_models
async def _collect_provider_paths(
upstream: BaseUpstreamProvider,
provider_identity: ConfiguredProviderIdentity,
overrides_by_key: dict[ModelKey, ModelRow] | None = None,
disabled_model_keys: set[ModelKey] | None = None,
cycle: _RefreshCycleState | None = None,
) -> ProviderPathSnapshot:
"""Collect selectable routes while marking model-level degraded fetches.
A failed OpenRouter lookup preserves only that model's prior rows. Other
models in the same provider still refresh, so a partial outage cannot erase
valid discovery data or freeze the entire provider snapshot.
"""
cycle = cycle or _RefreshCycleState()
models = _apply_model_visibility(upstream, overrides_by_key, disabled_model_keys)
def _base_path(model: object) -> DiscoveredPath:
model_id = exposed_model_id(model)
return DiscoveredPath(
model_id=model_id,
path=encode_model_path(
provider_identity.base_url, provider_identity.id, model_id
),
provider=provider_identity,
)
if not is_openrouter_base_url(upstream.base_url):
return ProviderPathSnapshot(paths=tuple(_base_path(model) for model in models))
if not (upstream.provider_type or "").strip():
return ProviderPathSnapshot(paths=())
semaphore = asyncio.Semaphore(_OPENROUTER_CONCURRENCY)
async with _make_http_client() as client:
async def _for_model(
model: object,
) -> tuple[list[DiscoveredPath], str | None]:
model_id = exposed_model_id(model)
author_slug = openrouter_author_slug(model)
if not author_slug:
return [_base_path(model)], None
endpoints = await _fetch_openrouter_endpoint_subproviders(
client,
upstream.base_url,
upstream.api_key,
author_slug,
semaphore,
cycle,
)
if endpoints is None:
return [], model_id
paths = [_base_path(model)]
paths.extend(
DiscoveredPath(
model_id=model_id,
path=encode_model_path(
provider_identity.base_url,
provider_identity.id,
model_id,
endpoint.tag,
),
provider=provider_identity,
endpoint_tag=endpoint.tag,
endpoint_name=endpoint.provider_name,
)
for endpoint in endpoints
)
return paths, None
results = await asyncio.gather(
*(_for_model(model) for model in models), return_exceptions=True
)
paths: list[DiscoveredPath] = []
preserve_model_ids: set[str] = set()
for model, result in zip(models, results):
if isinstance(result, BaseException):
model_id = exposed_model_id(model)
preserve_model_ids.add(model_id)
logger.warning(
"OpenRouter endpoint discovery task errored",
extra={"provider": upstream.provider_type, "error": str(result)},
)
continue
model_paths, preserved_model_id = result
paths.extend(model_paths)
if preserved_model_id:
preserve_model_ids.add(preserved_model_id)
return ProviderPathSnapshot(
paths=tuple(paths), preserve_model_ids=frozenset(preserve_model_ids)
)
async def _persist_provider_paths(
upstream_provider_id: int, snapshot: ProviderPathSnapshot
) -> None:
"""Replace refreshed rows while retaining model-level degraded snapshots."""
unique_paths = list(
{(path.model_id, path.path): path for path in snapshot.paths}.values()
)
now = int(time.time())
async with create_session() as session:
delete_stmt = delete(ModelPathRow).where(
col(ModelPathRow.upstream_provider_id) == upstream_provider_id
)
if snapshot.preserve_model_ids:
delete_stmt = delete_stmt.where(
col(ModelPathRow.model_id).not_in(sorted(snapshot.preserve_model_ids))
)
await session.exec(delete_stmt) # type: ignore[call-overload]
for start in range(0, len(unique_paths), _PERSIST_CHUNK_SIZE):
chunk = unique_paths[start : start + _PERSIST_CHUNK_SIZE]
values = [
{
"model_id": discovered.model_id,
"path": discovered.path,
"provider_slug": discovered.provider.slug,
"provider_type": discovered.provider.provider_type,
"endpoint_tag": discovered.endpoint_tag,
"endpoint_name": discovered.endpoint_name,
"upstream_provider_id": upstream_provider_id,
"updated_at": now,
}
for discovered in chunk
]
insert_stmt = insert(ModelPathRow).values(values)
await session.execute(
insert_stmt.on_conflict_do_update(
index_elements=["model_id", "path", "upstream_provider_id"],
set_={
"provider_slug": insert_stmt.excluded.provider_slug,
"provider_type": insert_stmt.excluded.provider_type,
"endpoint_tag": insert_stmt.excluded.endpoint_tag,
"endpoint_name": insert_stmt.excluded.endpoint_name,
"updated_at": insert_stmt.excluded.updated_at,
},
)
)
await session.commit()
async def prune_model_paths_for_inactive_providers() -> None:
"""Delete paths whose provider is no longer enabled in the database.
Called from ``refresh_model_maps`` so admin mutations (disable/delete
provider) stop advertising a provider's paths immediately instead of
waiting for the next timed refresh. Uses the DB as the source of truth, so
it is safe at boot even before upstreams initialize.
"""
async with create_session() as session:
enabled_ids = (
await session.exec(
select(UpstreamProviderRow.id).where(
col(UpstreamProviderRow.enabled).is_(True)
)
)
).all()
stmt = delete(ModelPathRow)
if enabled_ids:
stmt = stmt.where(
col(ModelPathRow.upstream_provider_id).not_in(
[pid for pid in enabled_ids if pid is not None]
)
)
await session.exec(stmt) # type: ignore[call-overload]
await session.commit()
async def refresh_model_paths(
upstreams: list[BaseUpstreamProvider],
) -> None:
"""Recompute and persist model paths for every enabled provider.
One provider's failure is logged and isolated; it must not break the rest.
A provider whose paths could not be determined this cycle keeps its
previously persisted rows. An empty ``upstreams`` list (e.g. a failed
``initialize_upstreams`` at boot) is treated as "unknown" and touches
nothing.
"""
if not upstreams:
logger.warning("Skipping model paths refresh: no live upstreams")
return
(
overrides_by_key,
disabled_model_keys,
provider_identities,
) = await _load_model_visibility()
await prune_model_paths_for_inactive_providers()
cycle = _RefreshCycleState()
for upstream in upstreams:
if upstream.db_id is None or upstream.db_id not in provider_identities:
continue
try:
snapshot = await _collect_provider_paths(
upstream,
provider_identity=provider_identities[upstream.db_id],
overrides_by_key=overrides_by_key,
disabled_model_keys=disabled_model_keys,
cycle=cycle,
)
if snapshot.preserve_model_ids:
logger.warning(
"Some model paths are unknown; keeping their previous rows",
extra={
"provider": upstream.provider_type or upstream.base_url,
"db_id": upstream.db_id,
"preserved_models": len(snapshot.preserve_model_ids),
},
)
await _persist_provider_paths(upstream.db_id, snapshot)
except Exception as e: # noqa: BLE001 - isolate per-provider failures
logger.error(
"Failed to refresh model paths for provider",
extra={
"provider": upstream.provider_type or upstream.base_url,
"db_id": upstream.db_id,
"error": str(e),
"error_type": type(e).__name__,
},
)
async def refresh_model_paths_for_provider(upstream_provider_id: int) -> None:
"""Synchronize one provider when model-path discovery is enabled."""
if _refresh_interval_seconds() <= 0:
return
from ..proxy import get_upstreams
matching = [
upstream
for upstream in get_upstreams()
if upstream.db_id == upstream_provider_id
]
if matching:
await refresh_model_paths(matching)
else:
await prune_model_paths_for_inactive_providers()
async def _drain_scheduled_provider_refreshes() -> None:
"""Serialize and coalesce model-path refreshes scheduled by admin writes."""
global _scheduled_provider_refresh_task
try:
# Let mutations in the same event-loop turn collapse into one refresh.
await asyncio.sleep(0)
while _scheduled_provider_refresh_ids:
if _refresh_interval_seconds() <= 0:
_scheduled_provider_refresh_ids.clear()
return
provider_id = min(_scheduled_provider_refresh_ids)
_scheduled_provider_refresh_ids.remove(provider_id)
try:
await refresh_model_paths_for_provider(provider_id)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - background best effort
logger.warning(
"Failed to refresh model paths after admin mutation",
extra={
"upstream_provider_id": provider_id,
"error": str(exc),
"error_type": type(exc).__name__,
},
)
finally:
_scheduled_provider_refresh_task = None
async def schedule_model_paths_refresh_for_provider(
upstream_provider_id: int,
) -> None:
"""Queue a non-blocking, coalesced refresh after an admin mutation."""
global _scheduled_provider_refresh_task
if _refresh_interval_seconds() <= 0:
return
_scheduled_provider_refresh_ids.add(upstream_provider_id)
if (
_scheduled_provider_refresh_task is None
or _scheduled_provider_refresh_task.done()
):
_scheduled_provider_refresh_task = asyncio.create_task(
_drain_scheduled_provider_refreshes(),
name="model-path-admin-refresh",
)
def _refresh_interval_seconds() -> int:
"""Current interval, re-read every loop so runtime setting changes apply."""
from ..core.settings import settings
if not getattr(settings, "enable_model_paths_refresh", True):
return 0
return int(getattr(settings, "model_paths_refresh_interval_seconds", 0) or 0)
async def refresh_model_paths_periodically(
upstreams_provider: (
Callable[[], list[BaseUpstreamProvider]] | list[BaseUpstreamProvider]
),
) -> None:
"""Background task mirroring ``refresh_upstreams_models_periodically``.
The interval and enable flag are re-read every iteration, so the refresh
can be turned off (or on) and retuned without a restart. While disabled the
task idles instead of exiting, so re-enabling takes effect.
"""
_DISABLED_POLL_SECONDS = 60.0
def _resolve_upstreams() -> list[BaseUpstreamProvider]:
if callable(upstreams_provider):
return upstreams_provider()
return upstreams_provider
while True:
interval = _refresh_interval_seconds()
if interval <= 0:
try:
await asyncio.sleep(_DISABLED_POLL_SECONDS)
except asyncio.CancelledError:
break
continue
try:
await refresh_model_paths(_resolve_upstreams())
except asyncio.CancelledError:
break
except Exception as e: # noqa: BLE001
logger.error(
"Error in model paths refresh loop",
extra={"error": str(e), "error_type": type(e).__name__},
)
try:
jitter = max(0.0, float(interval) * 0.1)
await asyncio.sleep(interval + random.uniform(0, jitter))
except asyncio.CancelledError:
break
def _serialize_path(row: ModelPathRow) -> dict[str, Any]:
endpoint = None
if row.endpoint_tag or row.endpoint_name:
endpoint = {"tag": row.endpoint_tag, "name": row.endpoint_name}
return {
"path": row.path,
"provider": {
"id": row.upstream_provider_id,
"slug": row.provider_slug,
"type": row.provider_type,
},
"endpoint": endpoint,
}
async def get_all_model_paths() -> dict:
"""All models with their exact selectable routes."""
async with create_session() as session:
rows = (
await session.exec(
select(ModelPathRow).order_by(
col(ModelPathRow.model_id),
col(ModelPathRow.path),
col(ModelPathRow.upstream_provider_id),
)
)
).all()
grouped: dict[str, list[dict[str, Any]]] = {}
seen_paths: dict[str, set[str]] = {}
updated_at = 0
for row in rows:
updated_at = max(updated_at, row.updated_at)
if row.path in seen_paths.setdefault(row.model_id, set()):
continue
seen_paths[row.model_id].add(row.path)
grouped.setdefault(row.model_id, []).append(_serialize_path(row))
data = [
{
"id": grouped_model_id,
"paths": grouped[grouped_model_id],
}
for grouped_model_id in sorted(grouped)
]
return {"data": data, "updated_at": updated_at or None}
async def get_paths_for_model(model_id: str) -> dict:
"""Return paths for an advertised ID or its provider-prefixed alias."""
async def load_rows(session: AsyncSession, lookup_id: str) -> list[ModelPathRow]:
return list(
(
await session.exec(
select(ModelPathRow)
.where(col(ModelPathRow.model_id) == lookup_id)
.order_by(
col(ModelPathRow.path),
col(ModelPathRow.upstream_provider_id),
)
)
).all()
)
async with create_session() as session:
rows = await load_rows(session, model_id)
if not rows:
unprefixed_id = public_model_id(model_id)
if unprefixed_id != model_id:
rows = await load_rows(session, unprefixed_id)
seen: set[str] = set()
paths: list[dict] = []
updated_at = 0
for row in rows:
updated_at = max(updated_at, row.updated_at)
if row.path in seen:
continue
seen.add(row.path)
paths.append(_serialize_path(row))
return {"data": paths, "updated_at": updated_at or None}
+1 -1
View File
@@ -441,7 +441,7 @@ class PPQAIUpstreamProvider(BaseUpstreamProvider):
"""
data = await self.check_balance()
balance = data.get("balance")
if isinstance(balance, (int, float)):
if isinstance(balance, (int, float)) and not isinstance(balance, bool):
return float(balance)
return None
+1640 -320
View File
File diff suppressed because it is too large Load Diff
+7 -2
View File
@@ -203,8 +203,13 @@ class TestmintWallet:
token_base64 = base64.urlsafe_b64encode(token_json.encode()).decode()
return f"cashuA{token_base64}"
async def redeem_token(self, token: str) -> Tuple[int, str, str]:
"""Redeem a Cashu token - compatible with wallet.recieve_token"""
async def redeem_token(
self,
token: str,
destination_mint: str | None = None,
destination_unit: str | None = None,
) -> Tuple[int, str, str]:
"""Redeem a Cashu token - compatible with wallet.recieve_token."""
if not self.wallet:
await self.init()
@@ -15,6 +15,7 @@ from unittest.mock import patch
import pytest
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.payment.cost_calculation import CostData
@@ -23,7 +24,7 @@ def _make_key(balance: int, reserved: int) -> ApiKey:
return ApiKey(
hashed_key=f"test_{uuid.uuid4().hex}",
balance=balance,
reserved_balance=reserved,
reserved_balance=0,
total_spent=0,
total_requests=1,
)
@@ -75,6 +76,8 @@ async def test_balance_never_negative_when_cost_exceeds_reservation(
key = _make_key(balance=deducted_max_cost, reserved=deducted_max_cost)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
@@ -111,6 +114,8 @@ async def test_balance_floor_at_zero_on_overrun(
key = _make_key(balance=500, reserved=500)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 50}}
@@ -152,6 +157,8 @@ async def test_full_cost_charged_when_balance_sufficient_for_overrun(
key = _make_key(balance=2000, reserved=990)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
@@ -190,7 +197,11 @@ async def test_concurrent_cost_overruns_never_negative(
"""Concurrent finalization with cost overruns must never produce negative balance."""
import asyncio
from routstr.auth import adjust_payment_for_tokens, pay_for_request
from routstr.auth import (
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
)
from routstr.core.db import create_session
deducted_max_cost = 990
@@ -216,12 +227,14 @@ async def test_concurrent_cost_overruns_never_negative(
async with create_session() as session:
key_to_reserve = await session.get(ApiKey, key_hash)
assert key_to_reserve is not None
reservations = []
for _ in range(n_requests):
await pay_for_request(key_to_reserve, deducted_max_cost, session)
reservations.append(await get_reservation_snapshot(key_to_reserve, session))
await session.refresh(key_to_reserve)
# Now finalize all concurrently with cost overrun
async def finalize() -> None:
async def finalize(reservation: ReservationSnapshot) -> None:
response_data = {
"model": "test-model",
"usage": {"prompt_tokens": 100, "completion_tokens": 100},
@@ -230,7 +243,11 @@ async def test_concurrent_cost_overruns_never_negative(
fresh_key = await session.get(ApiKey, key_hash)
assert fresh_key is not None
await adjust_payment_for_tokens(
fresh_key, response_data, session, deducted_max_cost, None, None
fresh_key,
response_data,
session,
deducted_max_cost,
reservation_snapshot=reservation,
)
# Patch once around the gather: entering the same patch target from
@@ -240,7 +257,7 @@ async def test_concurrent_cost_overruns_never_negative(
"routstr.auth.calculate_cost",
return_value=_cost_data(actual_token_cost),
):
await asyncio.gather(*[finalize() for _ in range(n_requests)])
await asyncio.gather(*(finalize(r) for r in reservations))
async with create_session() as session:
final_key = await session.get(ApiKey, key_hash)
@@ -281,6 +298,8 @@ async def test_zero_free_balance_overrun_is_safe(
key = _make_key(balance=1000, reserved=1000)
integration_session.add(key)
await integration_session.commit()
from routstr.auth import pay_for_request
await pay_for_request(key, deducted_max_cost, integration_session)
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 100}}
@@ -319,7 +338,11 @@ async def test_parallel_requests_no_free_inference(
"""Second parallel finalization must be charged even when first depleted free balance."""
import asyncio
from routstr.auth import adjust_payment_for_tokens
from routstr.auth import (
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
)
from routstr.core.db import create_session
deducted_max_cost = 100
@@ -340,14 +363,18 @@ async def test_parallel_requests_no_free_inference(
key = ApiKey(
hashed_key=key_hash,
balance=starting_balance,
reserved_balance=deducted_max_cost * 2, # both slots pre-reserved
reserved_balance=0,
total_spent=0,
total_requests=2,
)
session.add(key)
await session.commit()
reservations = []
for _ in range(2):
await pay_for_request(key, deducted_max_cost, session)
reservations.append(await get_reservation_snapshot(key, session))
async def finalize() -> None:
async def finalize(reservation: ReservationSnapshot) -> None:
response_data = {
"model": "test-model",
"usage": {"prompt_tokens": 50, "completion_tokens": 100},
@@ -356,7 +383,11 @@ async def test_parallel_requests_no_free_inference(
fresh_key = await session.get(ApiKey, key_hash)
assert fresh_key is not None
await adjust_payment_for_tokens(
fresh_key, response_data, session, deducted_max_cost, None, None
fresh_key,
response_data,
session,
deducted_max_cost,
reservation_snapshot=reservation,
)
# Patch once around the gather: entering the same patch target from two
@@ -366,7 +397,7 @@ async def test_parallel_requests_no_free_inference(
"routstr.auth.calculate_cost",
return_value=_cost_data(actual_token_cost),
):
await asyncio.gather(finalize(), finalize())
await asyncio.gather(*(finalize(r) for r in reservations))
async with create_session() as session:
final_key = await session.get(ApiKey, key_hash)
+127 -7
View File
@@ -14,13 +14,17 @@ from unittest.mock import patch
import httpx
import pytest
from httpx import AsyncClient
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey, ReservationRelease
from routstr.payment.models import Architecture, Model, Pricing
from routstr.proxy import refresh_model_maps
from routstr.upstream.base import BaseUpstreamProvider
CHEAP_BASE_URL = "https://cheap.example.com/v1"
EXPENSIVE_BASE_URL = "https://expensive.example.com/v1"
THIRD_BASE_URL = "https://third.example.com/v1"
def _make_model(
@@ -55,9 +59,7 @@ def _make_model(
class _StaticProvider(BaseUpstreamProvider):
"""Upstream provider with a fixed model catalog and no remote refresh."""
def __init__(
self, base_url: str, api_key: str, fee: float, model: Model
) -> None:
def __init__(self, base_url: str, api_key: str, fee: float, model: Model) -> None:
super().__init__(base_url, api_key, fee)
self.provider_type = "custom"
self._static_model = model
@@ -98,7 +100,7 @@ async def dual_provider_maps(
EXPENSIVE_BASE_URL,
"key-expensive",
1.0,
_make_model("provb/dual-model", 0.005, 0.010),
_make_model("provb/dual-model", 0.005, 0.010, max_cost=100.0),
)
async for _ in _install_providers([cheap, expensive]):
yield cheap, expensive
@@ -142,6 +144,7 @@ def _upstream_response(request: httpx.Request) -> httpx.Response:
async def test_failover_serve_billed_at_serving_providers_rate(
authenticated_client: AsyncClient,
dual_provider_maps: tuple[_StaticProvider, _StaticProvider],
integration_session: AsyncSession,
) -> None:
"""A fallback serve is billed at the fallback's price, not the winner's.
@@ -199,6 +202,17 @@ async def test_failover_serve_billed_at_serving_providers_rate(
# Billed at the serving provider's rate: 1000/1000*5000 + 500/1000*10000.
assert payload["cost"]["total_msats"] == 10_000
# The fallback's larger max-cost envelope requires a replacement
# reservation. The failed candidate is released, the serving candidate is
# charged, and no request-owned reservation remains active.
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
records = (
await integration_session.exec(
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
)
).all()
assert sorted(record.status for record in records) == ["charged", "released"]
@pytest.fixture
async def same_id_provider_maps(
@@ -349,9 +363,7 @@ async def test_usd_cost_serve_carries_serving_providers_fee(
if request.url.host == "cheap.example.com":
return httpx.Response(
502,
content=json.dumps(
{"error": {"message": "bad gateway"}}
).encode(),
content=json.dumps({"error": {"message": "bad gateway"}}).encode(),
headers={"content-type": "application/json"},
)
body = {
@@ -478,6 +490,101 @@ async def test_failover_beyond_balance_envelope_is_rejected(
assert [r.url.host for r in sent_requests] == ["cheap.example.com"]
@pytest.fixture
async def three_candidate_child_maps(
patched_db_engine: None,
) -> AsyncGenerator[None, None]:
"""Second candidate cannot fit the child limit; third restores and serves."""
first = _StaticProvider(
CHEAP_BASE_URL,
"key-first",
1.0,
_make_model("dual-model", 0.001, 0.002, max_cost=50.0),
)
too_large = _StaticProvider(
EXPENSIVE_BASE_URL,
"key-too-large",
1.0,
_make_model("dual-model", 0.002, 0.003, max_cost=100.0),
)
third = _StaticProvider(
THIRD_BASE_URL,
"key-third",
1.0,
_make_model("dual-model", 0.003, 0.004, max_cost=50.0),
)
async for _ in _install_providers([first, too_large, third]):
yield
@pytest.mark.integration
@pytest.mark.asyncio
async def test_child_failover_rolls_back_failed_larger_reserve_before_restoring(
authenticated_client: AsyncClient,
three_candidate_child_maps: None,
integration_session: AsyncSession,
) -> None:
"""A failed child guard cannot leak its parent update into restoration."""
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
child = await integration_session.get(ApiKey, key_hash)
assert child is not None
parent = ApiKey(hashed_key="failover-parent", balance=10_000_000)
child.parent_key_hash = parent.hashed_key
child.balance_limit = 75_000
integration_session.add(parent)
integration_session.add(child)
await integration_session.commit()
sent_requests: list[httpx.Request] = []
async def fake_transport(
request: httpx.Request, *args: Any, **kwargs: Any
) -> httpx.Response:
sent_requests.append(request)
return _upstream_response(request)
with (
patch(
"httpx.AsyncHTTPTransport.handle_async_request",
side_effect=fake_transport,
),
patch(
"routstr.payment.cost_calculation.sats_usd_price",
return_value=0.0005,
),
):
response = await authenticated_client.post(
"/v1/chat/completions",
json={
"model": "dual-model",
"messages": [{"role": "user", "content": "hello"}],
},
)
assert response.status_code == 200
# The 100-sat candidate is rejected before forwarding; the third serves.
assert [request.url.host for request in sent_requests] == [
"cheap.example.com",
"third.example.com",
]
await integration_session.refresh(parent)
await integration_session.refresh(child)
assert parent.reserved_balance == 0
assert child.reserved_balance == 0
assert parent.total_spent == response.json()["cost"]["total_msats"]
records = (
await integration_session.exec(
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
)
).all()
assert len(records) == 2
assert sorted(record.status for record in records) == ["charged", "released"]
assert len({record.reserved_msats for record in records}) == 1
assert all(record.status != "active" for record in records)
@pytest.fixture
async def raised_envelope_provider_maps(
patched_db_engine: None,
@@ -504,6 +611,7 @@ async def raised_envelope_provider_maps(
async def test_failover_reserves_serving_candidates_envelope(
authenticated_client: AsyncClient,
raised_envelope_provider_maps: None,
integration_session: AsyncSession,
) -> None:
"""An affordable pricier fallback is re-reserved, served, and billed.
@@ -544,3 +652,15 @@ async def test_failover_reserves_serving_candidates_envelope(
"expensive.example.com",
]
assert response.json()["cost"]["total_msats"] == 10_000
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
records = (
await integration_session.exec(
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
)
).all()
assert len(records) == 2
released = next(record for record in records if record.status == "released")
charged = next(record for record in records if record.status == "charged")
assert charged.reserved_msats > released.reserved_msats
assert all(record.status != "active" for record in records)
@@ -38,7 +38,7 @@ async def test_overrun_charges_after_reservation_swept(
integration_session: AsyncSession,
) -> None:
"""Overrun finalize must charge even when the reservation was already released."""
from routstr.auth import adjust_payment_for_tokens
from routstr.auth import adjust_payment_for_tokens, pay_for_request
deducted_max_cost = 990 # discounted reservation
actual_token_cost = 1000 # actual cost overruns the reservation
@@ -47,6 +47,10 @@ async def test_overrun_charges_after_reservation_swept(
key = _make_key(balance=1000, reserved=0)
integration_session.add(key)
await integration_session.commit()
await pay_for_request(key, deducted_max_cost, integration_session)
key.reserved_balance = 0
integration_session.add(key)
await integration_session.commit()
response_data = {
"model": "test-model",
@@ -79,8 +83,16 @@ async def test_free_response_path_closed_end_to_end(
patched_db_engine: None,
) -> None:
"""A reservation released by the real sweeper must not yield a free response."""
from routstr.auth import adjust_payment_for_tokens, pay_for_request
from routstr.core.db import create_session, release_stale_reservations
from routstr.auth import (
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
)
from routstr.core.db import (
ReservationRelease,
create_session,
release_stale_reservations,
)
deducted_max_cost = 990
actual_token_cost = 1000
@@ -104,10 +116,15 @@ async def test_free_response_path_closed_end_to_end(
key = await session.get(ApiKey, key_hash)
assert key is not None
await pay_for_request(key, deducted_max_cost, session)
snapshot = await get_reservation_snapshot(key, session)
await session.refresh(key)
assert key.reserved_balance == deducted_max_cost
key.reserved_at = int(time.time()) - 10_000
record = await session.get(ReservationRelease, snapshot.release_id)
assert record is not None
record.created_at = int(time.time()) - 10_000
session.add(key)
session.add(record)
await session.commit()
# Sweeper releases the stale reservation without charging.
@@ -129,18 +146,20 @@ async def test_free_response_path_closed_end_to_end(
return_value=_cost_data(actual_token_cost),
):
await adjust_payment_for_tokens(
key, response_data, session, deducted_max_cost, None, None
key,
response_data,
session,
deducted_max_cost,
reservation_snapshot=snapshot,
)
async with create_session() as session:
final = await session.get(ApiKey, key_hash)
assert final is not None
assert final.total_spent == actual_token_cost, (
f"Free response: total_spent={final.total_spent}, expected {actual_token_cost}"
)
assert final.balance == 1000 - actual_token_cost, (
f"Balance not charged after sweep: {final.balance}"
)
# Stale release is terminal for this reservation. A late finalizer must not
# charge aggregate balance that may now belong to a newer request.
assert final.total_spent == 0
assert final.balance == 1000
assert final.balance >= 0
assert final.reserved_balance == 0
+30 -3
View File
@@ -207,8 +207,35 @@ async def test_pay_for_request_succeeds_when_balance_equals_cost(
assert key.balance == model_cost # balance unchanged, only reserved goes up
@pytest.mark.asyncio
async def test_full_model_maximum_is_required_and_reserved(
integration_session: AsyncSession,
) -> None:
from routstr.auth import pay_for_request, validate_bearer_key
short_key = _key(balance=95_000)
exact_key = _key(balance=100_000)
integration_session.add(short_key)
integration_session.add(exact_key)
await integration_session.commit()
with pytest.raises(HTTPException) as insufficient:
await validate_bearer_key(
f"sk-{short_key.hashed_key}", integration_session, min_cost=100_000
)
assert insufficient.value.status_code == 402
validated = await validate_bearer_key(
f"sk-{exact_key.hashed_key}", integration_session, min_cost=100_000
)
await pay_for_request(validated, 100_000, integration_session)
await integration_session.refresh(exact_key)
assert exact_key.reserved_balance == 100_000
# ---------------------------------------------------------------------------
# Test 6 — HTTP layer returns 402 JSON with the right shape
# HTTP layer returns 402 JSON with the right shape
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@@ -266,8 +293,8 @@ async def test_http_402_response_shape_on_insufficient_balance(
error = body["detail"]["error"]
assert error["code"] == "insufficient_balance"
assert error["type"] == "insufficient_quota"
assert str(model_cost) in error["message"]
assert str(user_balance) in error["message"]
assert "622.888 sats (622888 msats) required" in error["message"]
assert "20.32 sats (20320 msats) available" in error["message"]
# Balance must be completely untouched
await integration_session.refresh(key)
@@ -3,20 +3,30 @@
Covers two things:
- The three constraint fields (balance_limit, balance_limit_reset, validity_date)
are persisted on LightningInvoice and survive a DB round-trip.
- create_api_key_from_invoice propagates those fields to the created ApiKey,
so the constraints are actually enforced when the key is used.
- The production-path API-key record helper propagates those fields to the
created ApiKey, so the constraints are actually enforced when the key is used.
"""
from __future__ import annotations
import asyncio
import time
from unittest.mock import AsyncMock, patch
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from cashu.core.base import Proof
from sqlalchemy import inspect
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey, LightningInvoice
from routstr.lightning import create_api_key_from_invoice
from routstr.lightning import _create_api_key_record
def _configure_quote_proof_wallet(wallet: MagicMock) -> None:
wallet.proofs = []
wallet.keysets = {}
wallet.load_proofs = AsyncMock()
def _make_invoice(**kwargs: object) -> LightningInvoice:
@@ -39,7 +49,15 @@ def _make_invoice(**kwargs: object) -> LightningInvoice:
def mock_wallet_mint() -> object:
with patch("routstr.lightning.get_wallet") as mock_get_wallet:
wallet = AsyncMock()
wallet.mint = AsyncMock(return_value=[])
wallet.proofs = []
wallet.load_proofs = AsyncMock()
async def mint(amount: int, quote_id: str) -> list[Proof]:
proofs = [Proof(amount=amount, mint_id=quote_id)]
wallet.proofs.extend(proofs)
return proofs
wallet.mint = AsyncMock(side_effect=mint)
mock_get_wallet.return_value = wallet
yield mock_get_wallet
@@ -48,6 +66,7 @@ def mock_wallet_mint() -> object:
# Persistence
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_invoice_persists_balance_limit(
integration_session: AsyncSession,
@@ -92,6 +111,7 @@ async def test_invoice_persists_validity_date(
# Propagation to ApiKey
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_created_key_receives_balance_limit(
integration_session: AsyncSession,
@@ -100,7 +120,7 @@ async def test_created_key_receives_balance_limit(
integration_session.add(invoice)
await integration_session.flush()
api_key = await create_api_key_from_invoice(invoice, integration_session)
api_key = await _create_api_key_record(invoice, integration_session)
await integration_session.commit()
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
@@ -116,7 +136,7 @@ async def test_created_key_receives_balance_limit_reset(
integration_session.add(invoice)
await integration_session.flush()
api_key = await create_api_key_from_invoice(invoice, integration_session)
api_key = await _create_api_key_record(invoice, integration_session)
await integration_session.commit()
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
@@ -133,7 +153,7 @@ async def test_created_key_receives_validity_date(
integration_session.add(invoice)
await integration_session.flush()
api_key = await create_api_key_from_invoice(invoice, integration_session)
api_key = await _create_api_key_record(invoice, integration_session)
await integration_session.commit()
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
@@ -141,6 +161,254 @@ async def test_created_key_receives_validity_date(
assert stored_key.validity_date == expiry
@pytest.mark.asyncio
async def test_payment_check_releases_connection_during_mint_quote(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _make_invoice(id="inv_slow_quote", status="pending", paid_at=None)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(invoice)
await setup.commit()
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
assert stored is not None
async def quote_status(*args: object, **kwargs: object) -> MagicMock:
assert integration_engine.pool.checkedout() == 0 # type: ignore[attr-defined]
return MagicMock(paid=False)
wallet = MagicMock()
wallet.get_mint_quote = AsyncMock(side_effect=quote_status)
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
from routstr.lightning import check_invoice_payment
await check_invoice_payment(stored, session)
@pytest.mark.asyncio
async def test_concurrent_payment_checks_mint_and_credit_invoice_once(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _make_invoice(id="inv_concurrent", status="pending", paid_at=None)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(invoice)
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
mint_calls = 0
async def single_use_mint(*args: object, **kwargs: object) -> list[object]:
# Real mints enforce single-use quotes: the second concurrent minter
# gets rejected at the mint, mirroring cashu quote semantics.
nonlocal mint_calls
mint_calls += 1
call_number = mint_calls
await asyncio.sleep(0.05)
if call_number > 1:
raise Exception("quote already issued")
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=single_use_mint)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as first,
AsyncSession(integration_engine, expire_on_commit=False) as second,
):
first_invoice = await first.get(LightningInvoice, invoice.id)
second_invoice = await second.get(LightningInvoice, invoice.id)
assert first_invoice is not None
assert second_invoice is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
from routstr.lightning import check_invoice_payment
await asyncio.gather(
check_invoice_payment(first_invoice, first),
check_invoice_payment(second_invoice, second),
)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
assert stored_invoice is not None
assert stored_invoice.status == "paid"
assert stored_invoice.api_key_hash is not None
stored_key = await verify.get(ApiKey, stored_invoice.api_key_hash)
assert stored_key is not None
assert stored_key.balance == invoice.amount_sats * 1000
@pytest.mark.asyncio
async def test_failed_mint_marks_invoice_for_settlement_retry(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _make_invoice(id="inv_mint_failure", status="pending", paid_at=None)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(invoice)
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
wallet.mint = AsyncMock(side_effect=TimeoutError("mint unavailable"))
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
assert stored is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
from routstr.lightning import check_invoice_payment
await check_invoice_payment(stored, session)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "settlement_pending"
@pytest.mark.asyncio
async def test_unpaid_topup_does_not_query_target_key(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _make_invoice(
id="inv_unpaid_topup",
status="pending",
paid_at=None,
purpose="topup",
api_key_hash="target-key",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(invoice)
await setup.commit()
wallet = MagicMock()
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=False))
create_session = MagicMock(side_effect=RuntimeError("target lookup should not run"))
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
assert stored is not None
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", create_session),
):
from routstr.lightning import check_invoice_payment
await check_invoice_payment(stored, session)
wallet.get_mint_quote.assert_awaited_once_with(invoice.payment_hash)
create_session.assert_not_called()
@pytest.mark.asyncio
async def test_missing_topup_target_is_rejected_before_mint(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _make_invoice(
id="inv_missing_topup_target",
status="pending",
paid_at=None,
purpose="topup",
api_key_hash="pruned-key",
expires_at=int(time.time()) - 1,
)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(invoice)
await setup.commit()
wallet = MagicMock()
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
wallet.mint = AsyncMock()
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
assert stored is not None
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.logger.critical") as critical,
):
from routstr.lightning import get_invoice_status
response = await get_invoice_status(invoice.id, session)
assert response.status == "reconciliation_required"
assert stored.status == "reconciliation_required"
assert stored not in session.dirty
critical.assert_called_once()
wallet.mint.assert_not_awaited()
async with AsyncSession(integration_engine) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "reconciliation_required"
@pytest.mark.asyncio
async def test_post_mint_db_failure_keeps_invoice_pending_for_reconciliation(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _make_invoice(id="inv_finalize_failure", status="pending", paid_at=None)
sibling = _make_invoice(
id="inv_finalize_failure_sibling",
bolt11="lnbc1000n1sibling",
payment_hash="cafebabe" * 8,
status="pending",
paid_at=None,
)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add_all([invoice, sibling])
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
async def successful_mint(*args: object, **kwargs: object) -> list[Proof]:
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=successful_mint)
async with AsyncSession(integration_engine, expire_on_commit=False) as session:
stored = await session.get(LightningInvoice, invoice.id)
stored_sibling = await session.get(LightningInvoice, sibling.id)
assert stored is not None
assert stored_sibling is not None
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._create_api_key_record",
AsyncMock(side_effect=RuntimeError("database unavailable")),
),
):
from routstr.lightning import check_invoice_payment
await check_invoice_payment(stored, session)
stored_state = inspect(stored)
sibling_state = inspect(stored_sibling)
assert stored_state is not None
assert sibling_state is not None
assert stored_state.expired is False
assert sibling_state.expired is False
assert stored.status == "settlement_pending"
assert stored_sibling.id == sibling.id
assert wallet.mint.await_count == 1
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "settlement_pending"
@pytest.mark.asyncio
async def test_created_key_without_constraints_has_none_fields(
integration_session: AsyncSession,
@@ -149,7 +417,7 @@ async def test_created_key_without_constraints_has_none_fields(
integration_session.add(invoice)
await integration_session.flush()
api_key = await create_api_key_from_invoice(invoice, integration_session)
api_key = await _create_api_key_record(invoice, integration_session)
await integration_session.commit()
stored_key = await integration_session.get(ApiKey, api_key.hashed_key)
@@ -157,3 +425,87 @@ async def test_created_key_without_constraints_has_none_fields(
assert stored_key.balance_limit is None
assert stored_key.balance_limit_reset is None
assert stored_key.validity_date is None
@pytest.mark.asyncio
async def test_db_guard_credits_once_when_both_mints_succeed(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
"""Even if the mint fails to enforce single-use quotes and both racers
mint successfully, the conditional status update must credit exactly once."""
key = ApiKey(hashed_key="race-key", balance=1_000)
invoice = _make_invoice(
id="inv_db_guard",
status="pending",
paid_at=None,
purpose="topup",
api_key_hash="race-key",
)
sibling = _make_invoice(
id="inv_db_guard_sibling",
bolt11="lnbc1000n1race-sibling",
payment_hash="01234567" * 8,
status="pending",
paid_at=None,
)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add_all([key, invoice, sibling])
await setup.commit()
wallet = MagicMock()
_configure_quote_proof_wallet(wallet)
wallet.get_mint_quote = AsyncMock(return_value=MagicMock(paid=True))
async def always_succeeding_mint(*args: object, **kwargs: object) -> list[Proof]:
await asyncio.sleep(0.05)
proof = Proof(amount=invoice.amount_sats, mint_id=invoice.payment_hash)
wallet.proofs.append(proof)
return [proof]
wallet.mint = AsyncMock(side_effect=always_succeeding_mint)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as first,
AsyncSession(integration_engine, expire_on_commit=False) as second,
):
first_invoice = await first.get(LightningInvoice, invoice.id)
first_sibling = await first.get(LightningInvoice, sibling.id)
second_invoice = await second.get(LightningInvoice, invoice.id)
assert first_invoice is not None
assert first_sibling is not None
assert second_invoice is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
from routstr.lightning import check_invoice_payment
await asyncio.gather(
check_invoice_payment(first_invoice, first),
check_invoice_payment(second_invoice, second),
)
first_state = inspect(first_invoice)
sibling_state = inspect(first_sibling)
second_state = inspect(second_invoice)
assert first_state is not None
assert sibling_state is not None
assert second_state is not None
assert first_state.expired is False
assert sibling_state.expired is False
assert second_state.expired is False
assert first_invoice.id == invoice.id
assert first_sibling.id == sibling.id
assert second_invoice.id == invoice.id
assert first_invoice.status == "paid"
assert second_invoice.status == "paid"
assert first_invoice not in first.dirty
assert second_invoice not in second.dirty
assert wallet.mint.await_count == 1
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
assert stored_invoice is not None
assert stored_invoice.status == "paid"
stored_key = await verify.get(ApiKey, "race-key")
assert stored_key is not None
assert stored_key.balance == 1_000 + invoice.amount_sats * 1000
@@ -26,11 +26,17 @@ async def patch_invoice_generation() -> Any:
"""Stub out `generate_lightning_invoice` so no mint round-trip is needed."""
counter = {"n": 0}
async def fake_generate(amount_sats: int, description: str) -> tuple[str, str]:
async def fake_generate(
amount_sats: int,
description: str,
*,
allowed_mints: list[str] | None = None,
) -> tuple[str, str, str]:
counter["n"] += 1
return (
f"lnbc{amount_sats}n1pfakeinvoice{counter['n']}",
f"payment_hash_{counter['n']}",
"http://localhost:3338",
)
with patch(
@@ -95,6 +101,8 @@ async def test_topup_with_authorization_header(
body = resp.json()
assert body["amount_sats"] == 500
assert body["bolt11"].startswith("lnbc")
allowed_mints = patch_invoice_generation.call_args.kwargs["allowed_mints"]
assert allowed_mints == ["http://localhost:3338"]
@pytest.mark.integration
@@ -0,0 +1,365 @@
import asyncio
import time
import uuid
from unittest.mock import AsyncMock, Mock, patch
import pytest
from cashu.core.base import Proof
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel import col, update
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey, LightningInvoice
from routstr.lightning import (
_expire_invoice_if_authoritatively_unpaid,
_finalize_invoice_settlement,
_InvoiceSettlement,
check_invoice_payment,
)
def _lightning_invoice(**overrides: object) -> LightningInvoice:
suffix = uuid.uuid4().hex
values = {
"id": f"invoice-{suffix}",
"bolt11": f"lnbc-{suffix}",
"amount_sats": 100,
"description": "settlement test",
"payment_hash": f"quote-{suffix}",
"status": "pending",
"purpose": "create",
"mint_url": "http://mint:3338",
"expires_at": int(time.time()) + 3600,
}
values.update(overrides)
return LightningInvoice(**values) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_invoice_read_transaction_closes_before_external_mint_io(
integration_session: AsyncSession,
) -> None:
invoice = _lightning_invoice()
integration_session.add(invoice)
await integration_session.commit()
stored = await integration_session.get(LightningInvoice, invoice.id)
assert stored is not None
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=False)))
async def get_wallet_without_open_db_transaction(
*args: object, **kwargs: object
) -> Mock:
assert not integration_session.in_transaction()
return wallet
with patch(
"routstr.lightning.get_wallet", side_effect=get_wallet_without_open_db_transaction
):
await check_invoice_payment(stored, integration_session)
assert not integration_session.in_transaction()
@pytest.mark.asyncio
async def test_separate_sessions_cas_topup_credit_exactly_once(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup",
api_key_hash=key_hash,
amount_sats=100,
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
snapshot_a = _InvoiceSettlement.from_invoice(invoice)
snapshot_b = _InvoiceSettlement.from_invoice(invoice)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as session_a,
AsyncSession(integration_engine, expire_on_commit=False) as session_b,
):
results = await asyncio.gather(
_finalize_invoice_settlement(snapshot_a, session_a, 1_700_000_000),
_finalize_invoice_settlement(snapshot_b, session_b, 1_700_000_001),
)
assert sorted(settled for settled, _ in results) == [False, True]
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_invoice = await verify.get(LightningInvoice, invoice.id)
stored_key = await verify.get(ApiKey, key_hash)
assert stored_invoice is not None
assert stored_invoice.status == "paid"
assert stored_key is not None
assert stored_key.balance == 200_000
@pytest.mark.asyncio
async def test_topup_atomic_increment_preserves_concurrent_balance_mutation(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup", api_key_hash=key_hash, amount_sats=100
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
async def debit_balance(session: AsyncSession) -> None:
result = await session.exec( # type: ignore[call-overload]
update(ApiKey)
.where(col(ApiKey.hashed_key) == key_hash)
.values(balance=col(ApiKey.balance) - 10_000)
.execution_options(synchronize_session=False)
)
assert result.rowcount == 1
await session.commit()
snapshot = _InvoiceSettlement.from_invoice(invoice)
async with (
AsyncSession(integration_engine, expire_on_commit=False) as settlement,
AsyncSession(integration_engine, expire_on_commit=False) as debit,
):
settlement_result, _ = await asyncio.gather(
_finalize_invoice_settlement(snapshot, settlement, 1_700_000_000),
debit_balance(debit),
)
assert settlement_result[0]
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored_key = await verify.get(ApiKey, key_hash)
assert stored_key is not None
assert stored_key.balance == 190_000
@pytest.mark.asyncio
async def test_failed_final_commit_rolls_back_claim_and_credit_for_retry(
integration_engine: AsyncEngine,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup",
api_key_hash=key_hash,
amount_sats=100,
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
snapshot = _InvoiceSettlement.from_invoice(invoice)
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
with patch.object(
failed, "commit", AsyncMock(side_effect=Exception("db unavailable"))
):
with pytest.raises(Exception, match="db unavailable"):
await _finalize_invoice_settlement(snapshot, failed, 1_700_000_000)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
pending = await verify.get(LightningInvoice, invoice.id)
unchanged = await verify.get(ApiKey, key_hash)
assert pending is not None
assert pending.status == "pending"
assert unchanged is not None
assert unchanged.balance == 100_000
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
settled, _ = await _finalize_invoice_settlement(
snapshot, retry, 1_700_000_001
)
assert settled
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
paid = await verify.get(LightningInvoice, invoice.id)
credited = await verify.get(ApiKey, key_hash)
assert paid is not None
assert paid.status == "paid"
assert credited is not None
assert credited.balance == 200_000
@pytest.mark.asyncio
async def test_check_invoice_payment_retries_after_mint_success_and_db_failure(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
key_hash = uuid.uuid4().hex
invoice = _lightning_invoice(
purpose="topup", api_key_hash=key_hash, amount_sats=100
)
key = ApiKey(
hashed_key=key_hash,
balance=100_000,
refund_currency="sat",
refund_mint_url="http://mint:3338",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(key)
seed.add(invoice)
await seed.commit()
wallet = Mock(
proofs=[],
keysets={"keyset-1": Mock()},
load_proofs=AsyncMock(),
get_mint_quote=AsyncMock(return_value=Mock(paid=True)),
restore_tokens_for_keyset=AsyncMock(),
)
async def mint(amount: int, quote_id: str) -> list[Proof]:
proofs = [Proof(amount=amount, mint_id=quote_id)]
wallet.proofs.extend(proofs)
return proofs
wallet.mint = AsyncMock(side_effect=mint)
async with AsyncSession(integration_engine, expire_on_commit=False) as failed:
stored = await failed.get(LightningInvoice, invoice.id)
assert stored is not None
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._finalize_invoice_settlement",
AsyncMock(side_effect=Exception("db unavailable")),
),
):
await check_invoice_payment(stored, failed)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
pending = await verify.get(LightningInvoice, invoice.id)
unchanged = await verify.get(ApiKey, key_hash)
assert pending is not None
assert pending.status == "settlement_pending"
assert unchanged is not None
assert unchanged.balance == 100_000
async with AsyncSession(integration_engine, expire_on_commit=False) as retry:
stored = await retry.get(LightningInvoice, invoice.id)
assert stored is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
await check_invoice_payment(stored, retry)
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
paid = await verify.get(LightningInvoice, invoice.id)
credited = await verify.get(ApiKey, key_hash)
assert paid is not None
assert paid.status == "paid"
assert credited is not None
assert credited.balance == 200_000
wallet.mint.assert_awaited_once_with(100, quote_id=invoice.payment_hash)
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_expiry_cas_cannot_overwrite_concurrent_paid_invoice(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _lightning_invoice(expires_at=0)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(invoice)
await seed.commit()
async with AsyncSession(integration_engine, expire_on_commit=False) as caller:
stale = await caller.get(LightningInvoice, invoice.id)
assert stale is not None
await caller.commit()
async with AsyncSession(integration_engine, expire_on_commit=False) as paid:
result = await paid.exec( # type: ignore[call-overload]
update(LightningInvoice)
.where(col(LightningInvoice.id) == invoice.id)
.values(status="paid", paid_at=123)
)
assert result.rowcount == 1
await paid.commit()
expired = await _expire_invoice_if_authoritatively_unpaid(
stale, caller, True
)
assert expired is False
assert stale.status == "paid"
assert stale.paid_at == 123
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "paid"
assert stored.paid_at == 123
@pytest.mark.asyncio
async def test_paid_quote_worker_does_not_mint_after_expiry_claim_wins(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
invoice = _lightning_invoice(expires_at=0)
async with AsyncSession(integration_engine, expire_on_commit=False) as seed:
seed.add(invoice)
await seed.commit()
quote_started = asyncio.Event()
release_quote = asyncio.Event()
async def paid_quote_after_expiry(*_args: object, **_kwargs: object) -> Mock:
quote_started.set()
await release_quote.wait()
return Mock(paid=True)
wallet = Mock(
get_mint_quote=AsyncMock(side_effect=paid_quote_after_expiry),
mint=AsyncMock(),
)
async with AsyncSession(integration_engine, expire_on_commit=False) as worker:
observed_pending = await worker.get(LightningInvoice, invoice.id)
assert observed_pending is not None
with patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)):
settlement_task = asyncio.create_task(
check_invoice_payment(observed_pending, worker)
)
await quote_started.wait()
async with AsyncSession(
integration_engine, expire_on_commit=False
) as expirer:
expiry_view = await expirer.get(LightningInvoice, invoice.id)
assert expiry_view is not None
await expirer.commit()
assert await _expire_invoice_if_authoritatively_unpaid(
expiry_view, expirer, True
)
release_quote.set()
assert await settlement_task is False
wallet.mint.assert_not_awaited()
async with AsyncSession(integration_engine, expire_on_commit=False) as verify:
stored = await verify.get(LightningInvoice, invoice.id)
assert stored is not None
assert stored.status == "expired"
@@ -0,0 +1,178 @@
"""Money-safety regression coverage for automatic wallet payouts."""
from __future__ import annotations
import asyncio
from collections.abc import Callable, Coroutine
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core import db
from routstr.core.db import ApiKey
from routstr.core.settings import settings
from routstr.wallet import credit_balance, periodic_payout
PRIMARY_MINT = "http://primary:3338"
REFUND_MINT = "http://refund:3338"
PAYOUT_INTERVAL = 987
class _LoopBreak(Exception):
"""Stop the otherwise-infinite payout loop after one cycle."""
def _one_payout_cycle() -> Callable[[float], Coroutine[Any, Any, None]]:
intervals_seen = 0
async def sleep(seconds: float) -> None:
nonlocal intervals_seen
if seconds == PAYOUT_INTERVAL:
intervals_seen += 1
if intervals_seen == 2:
raise _LoopBreak()
return sleep
@pytest.mark.asyncio
async def test_cross_mint_liability_is_not_paid_as_owner_profit(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
"""Refund preferences must not make primary-mint customer funds payable."""
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(
ApiKey(
hashed_key="cross-mint-key",
balance=50_000,
refund_mint_url=REFUND_MINT,
refund_currency="sat",
)
)
await setup.commit()
primary_proof = MagicMock(amount=50)
raw_send = AsyncMock(return_value=50)
def proofs_for_mint(
_wallet: object, mint_url: str, unit: str, **_kwargs: object
) -> list[MagicMock]:
if mint_url == PRIMARY_MINT and unit == "sat":
return [primary_proof]
return []
with (
patch.object(settings, "cashu_mints", [REFUND_MINT]),
patch.object(settings, "primary_mint", PRIMARY_MINT),
patch.object(settings, "receive_ln_address", "owner@ln.test"),
patch.object(settings, "payout_interval_seconds", PAYOUT_INTERVAL),
patch.object(settings, "min_payout_sat", 10),
patch("routstr.wallet.asyncio.sleep", _one_payout_cycle()),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(side_effect=proofs_for_mint),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, _wallet: proofs),
),
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
):
with pytest.raises(_LoopBreak):
await periodic_payout()
raw_send.assert_not_awaited()
@pytest.mark.asyncio
async def test_payout_does_not_send_proofs_whose_liability_commit_is_in_flight(
integration_engine: AsyncEngine,
patched_db_engine: None,
) -> None:
"""Proof visibility before liability commit must not expose customer funds."""
key = ApiKey(
hashed_key="in-flight-topup-key",
balance=0,
refund_mint_url=PRIMARY_MINT,
refund_currency="sat",
)
async with AsyncSession(integration_engine, expire_on_commit=False) as setup:
setup.add(key)
await setup.commit()
proofs: list[MagicMock] = []
proof_visible = asyncio.Event()
finish_redemption = asyncio.Event()
liability_read = asyncio.Event()
async def redeem_token(
token: str,
destination_mint: str | None = None,
destination_unit: str | None = None,
) -> tuple[int, str, str]:
proofs.append(MagicMock(amount=200))
proof_visible.set()
await finish_redemption.wait()
return 200, "sat", PRIMARY_MINT
real_total_liability = db.total_user_liability
async def read_liability(_session: AsyncSession) -> int:
async with db.create_session() as snapshot_session:
value = await real_total_liability(snapshot_session)
liability_read.set()
return value
raw_send = AsyncMock(return_value=200)
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", PRIMARY_MINT),
patch.object(settings, "receive_ln_address", "owner@ln.test"),
patch.object(settings, "payout_interval_seconds", PAYOUT_INTERVAL),
patch.object(settings, "min_payout_sat", 10),
patch("routstr.wallet.asyncio.sleep", _one_payout_cycle()),
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=redeem_token)),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(side_effect=lambda *_args, **_kwargs: list(proofs)),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda visible, _wallet: visible),
),
patch(
"routstr.wallet.db.total_user_liability",
AsyncMock(side_effect=read_liability),
),
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
):
async with AsyncSession(integration_engine, expire_on_commit=False) as credit_session:
stored_key = await credit_session.get(ApiKey, key.hashed_key)
assert stored_key is not None
credit_task = asyncio.create_task(
credit_balance("cashu-token", stored_key, credit_session)
)
await asyncio.wait_for(proof_visible.wait(), timeout=2)
payout_task = asyncio.create_task(periodic_payout())
try:
await asyncio.wait_for(liability_read.wait(), timeout=0.1)
liability_was_read_while_crediting = True
except TimeoutError:
liability_was_read_while_crediting = False
finish_redemption.set()
await asyncio.wait_for(credit_task, timeout=2)
with pytest.raises(_LoopBreak):
await asyncio.wait_for(payout_task, timeout=2)
assert liability_was_read_while_crediting is False
raw_send.assert_not_awaited()
@@ -0,0 +1,602 @@
"""Real-database tests for the PPQ auto top-up claim lifecycle.
These exercise the claim against actual SQL rather than mocked sessions,
because the guarantees under test are all about what the database will and
will not let two concurrent writers do.
"""
import time
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlmodel import select
from routstr.core.db import CashuTransaction, create_session
from routstr.upstream.auto_topup import (
PPQ_PHASE_CLAIMED,
PPQ_PHASE_IN_FLIGHT,
PPQ_PHASE_RECONCILE,
_claim_ppq_topup,
_ppq_payment_id,
_ppq_payment_usd,
_ppq_request_id,
_ppq_spent_last_24h_usd,
_ppq_state_id_for_provider,
_record_ppq_invoice,
_set_ppq_state_terminal,
get_ppq_auto_topup_state,
release_ppq_auto_topup_state,
)
pytestmark = pytest.mark.asyncio
def _row(provider_id: int = 1) -> MagicMock:
row = MagicMock()
row.id = provider_id
return row
async def _seed_provider(provider_id: int = 1, slug: str = "ppq") -> None:
"""Claim creation is fenced on the provider row existing; seed it."""
from routstr.core.db import UpstreamProviderRow
async with create_session() as session:
session.add(
UpstreamProviderRow(
id=provider_id,
slug=slug,
provider_type="ppqai",
base_url="https://api.ppq.ai",
api_key="secret",
enabled=True,
)
)
await session.commit()
async def _state_row(provider_id: int = 1) -> CashuTransaction | None:
async with create_session() as session:
return await session.get(
CashuTransaction, _ppq_state_id_for_provider(provider_id)
)
async def _seed_claim(
provider_id: int,
phase: str,
invoice_id: str,
lease_expires_at: int,
quote_id: str = "quote-1",
) -> str:
"""Seed a claim row and return its state token (the full request_id)."""
token = _ppq_request_id(
"operation-1", lease_expires_at, phase, invoice_id, quote_id
)
async with create_session() as session:
session.add(
CashuTransaction(
id=_ppq_state_id_for_provider(provider_id),
token="lnbc-invoice",
amount=102,
unit="sat",
type="out",
request_id=token,
mint_url="https://mint.test",
collected=False,
source="ppq_auto_topup",
)
)
await session.commit()
return token
async def test_second_claim_is_refused_while_the_first_is_active(
patched_db_engine: Any,
) -> None:
await _seed_provider()
assert await _claim_ppq_topup(_row()) is not None
# The whole point of the claim: a concurrent cycle must not get one.
assert await _claim_ppq_topup(_row()) is None
async with create_session() as session:
rows = (await session.exec(select(CashuTransaction))).all()
assert len(rows) == 1
async def test_claim_is_reusable_once_the_previous_attempt_finished(
patched_db_engine: Any,
) -> None:
await _seed_provider()
first = await _claim_ppq_topup(_row())
assert first is not None
assert await _set_ppq_state_terminal(_row(), first, collected=True, swept=False)
second = await _claim_ppq_topup(_row())
assert second is not None and second != first
async def test_recording_the_invoice_moves_the_claim_in_flight(
patched_db_engine: Any,
) -> None:
await _seed_provider()
operation_id = await _claim_ppq_topup(_row())
assert operation_id is not None
state = await get_ppq_auto_topup_state(1)
assert state["phase"] == PPQ_PHASE_CLAIMED
assert state["releasable"] is True
assert state["invoice_id"] is None
lease = await _record_ppq_invoice(
_row(),
operation_id,
invoice="lnbc-invoice",
invoice_id="invoice-1",
quote_id="quote-1",
amount=102,
amount_usd=10,
unit="sat",
mint_url="https://mint.test",
)
assert lease > int(time.time())
state = await get_ppq_auto_topup_state(1)
assert state["phase"] == PPQ_PHASE_IN_FLIGHT
assert state["invoice_id"] == "invoice-1"
# A payment is committed to a mint, so an admin must not sweep it.
assert state["releasable"] is False
# The raw BOLT11 invoice must never reach the admin API.
assert "token" not in state
async def test_release_refuses_an_in_flight_claim(patched_db_engine: Any) -> None:
token = await _seed_claim(
1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) + 900
)
outcome = await release_ppq_auto_topup_state(1, state_token=token)
assert outcome.released is False
assert outcome.reason == "payment_in_flight"
row = await _state_row()
assert row is not None and row.swept is False
async def test_release_refuses_a_stale_state_token(patched_db_engine: Any) -> None:
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900)
outcome = await release_ppq_auto_topup_state(1, state_token="ppq:stale:token")
assert outcome.released is False
assert outcome.reason == "stale_state"
row = await _state_row()
assert row is not None and row.swept is False
async def test_release_accepts_a_reconcile_claim(patched_db_engine: Any) -> None:
token = await _seed_claim(
1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900
)
outcome = await release_ppq_auto_topup_state(1, state_token=token)
assert outcome.released is True
row = await _state_row()
assert row is not None and row.swept is True
async def test_expired_in_flight_claim_becomes_releasable(
patched_db_engine: Any,
) -> None:
# A worker that died mid-payment must not lock the provider forever.
token = await _seed_claim(1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) - 1)
assert (await get_ppq_auto_topup_state(1))["releasable"] is True
outcome = await release_ppq_auto_topup_state(1, state_token=token)
assert outcome.released is True
async def test_release_reports_no_active_claim_once_swept(
patched_db_engine: Any,
) -> None:
token = await _seed_claim(
1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900
)
assert (await release_ppq_auto_topup_state(1, state_token=token)).released
outcome = await release_ppq_auto_topup_state(1, state_token=token)
assert outcome.released is False
assert outcome.reason == "no_active_claim"
async def test_terminal_write_fails_after_the_claim_was_released(
patched_db_engine: Any,
) -> None:
"""The symptom an admin release leaves behind for the owning worker."""
token = await _seed_claim(
1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900
)
assert (await release_ppq_auto_topup_state(1, state_token=token)).released
assert (
await _set_ppq_state_terminal(
_row(), "operation-1", collected=True, swept=False
)
is False
)
async def test_ppq_claim_rows_are_excluded_from_the_admin_transaction_list(
patched_db_engine: Any,
) -> None:
from routstr.core.admin import get_transactions_api
await _seed_provider()
await _claim_ppq_topup(_row())
async with create_session() as session:
session.add(
CashuTransaction(
id="real-transaction",
token="cashuAreal",
amount=50,
unit="sat",
type="out",
source="x-cashu",
)
)
await session.commit()
result = await get_transactions_api()
ids = {t["id"] for t in result["transactions"]} # type: ignore[index,union-attr]
assert "real-transaction" in ids
assert _ppq_state_id_for_provider(1) not in ids
async def test_ppq_payment_audit_row_is_visible_and_survives_next_claim(
patched_db_engine: Any,
) -> None:
from routstr.core.admin import get_transactions_api
await _seed_provider()
operation_id = await _claim_ppq_topup(_row())
assert operation_id is not None
await _record_ppq_invoice(
_row(),
operation_id,
invoice="lnbc-secret-invoice",
invoice_id="invoice-1",
quote_id="quote-1",
amount=102,
amount_usd=10,
unit="sat",
mint_url="https://mint.test",
)
assert await _set_ppq_state_terminal(
_row(), operation_id, collected=True, swept=False
)
result = await get_transactions_api(source="ppq_auto_topup")
transactions = result["transactions"]
assert len(transactions) == 1
audit = transactions[0]
assert audit["id"] == _ppq_payment_id(operation_id)
assert audit["token"] == "ppq-invoice:invoice-1:usd:10"
assert audit["collected"] is True
assert "lnbc-secret-invoice" not in audit["token"]
# Reusing the deterministic claim lock must not overwrite history.
assert await _claim_ppq_topup(_row()) is not None
async with create_session() as session:
assert await session.get(CashuTransaction, audit["id"]) is not None
async def test_reconcile_settles_a_recorded_invoice(patched_db_engine: Any) -> None:
from routstr.upstream.auto_topup import _reconcile_ppq_state
await _seed_claim(1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) + 900)
provider = MagicMock()
provider.check_topup_status = AsyncMock(return_value=True)
# Still suppresses this cycle, but the claim is now finished.
assert await _reconcile_ppq_state(_row(), provider) is True
row = await _state_row()
assert row is not None and row.collected is True
async def test_stale_token_from_before_a_phase_change_cannot_release(
patched_db_engine: Any,
) -> None:
"""The blocker scenario: admin reviews `claimed`, payment turns ambiguous.
The operation id is identical in both states, so an id-based fence would
let the stale confirmation land. The full state token must not.
"""
await _seed_provider()
operation_id = await _claim_ppq_topup(_row())
assert operation_id is not None
reviewed = await get_ppq_auto_topup_state(1)
assert reviewed["phase"] == PPQ_PHASE_CLAIMED
# Worker records the invoice: same operation, new phase, proofs committed.
await _record_ppq_invoice(
_row(),
operation_id,
invoice="lnbc-invoice",
invoice_id="invoice-1",
quote_id="quote-1",
amount=102,
amount_usd=10,
unit="sat",
mint_url="https://mint.test",
)
outcome = await release_ppq_auto_topup_state(
1, state_token=str(reviewed["state_token"])
)
assert outcome.released is False
assert outcome.reason == "stale_state"
row = await _state_row()
assert row is not None and row.swept is False
async def test_concurrent_claims_only_one_wins(patched_db_engine: Any) -> None:
import asyncio
await _seed_provider()
results = await asyncio.gather(
*(_claim_ppq_topup(_row()) for _ in range(5)), return_exceptions=True
)
winners = [r for r in results if isinstance(r, str)]
assert len(winners) == 1
async with create_session() as session:
rows = (await session.exec(select(CashuTransaction))).all()
assert len(rows) == 1
async def test_reconcile_releases_claim_when_mint_reports_unpaid(
patched_db_engine: Any,
) -> None:
from routstr.upstream.auto_topup import _reconcile_ppq_state
# Lease expired, PPQ never credited: only the mint's own "unpaid" answer
# may hand the claim back.
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) - 1)
provider = MagicMock()
provider.check_topup_status = AsyncMock(return_value=False)
with patch(
"routstr.upstream.auto_topup.check_bolt11_payment_status",
AsyncMock(return_value="unpaid"),
) as status:
suppressed = await _reconcile_ppq_state(_row(), provider)
status.assert_awaited_once_with("https://mint.test", "sat", "quote-1")
assert suppressed is False
row = await _state_row()
assert row is not None and row.swept is True
async def test_reconcile_keeps_claim_when_mint_answer_is_not_final(
patched_db_engine: Any,
) -> None:
from routstr.upstream.auto_topup import _reconcile_ppq_state
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) - 1)
provider = MagicMock()
provider.check_topup_status = AsyncMock(return_value=False)
for answer in ("paid", "pending", "unknown"):
with patch(
"routstr.upstream.auto_topup.check_bolt11_payment_status",
AsyncMock(return_value=answer),
):
assert await _reconcile_ppq_state(_row(), provider) is True
row = await _state_row()
assert row is not None and row.swept is False, answer
async def test_release_endpoint_maps_refusals_to_409(patched_db_engine: Any) -> None:
from fastapi import HTTPException
from routstr.core.admin import (
ReleasePPQAutoTopupRequest,
release_ppq_auto_topup_api,
)
provider_row = MagicMock()
provider_row.provider_type = "ppqai"
token = await _seed_claim(
1, PPQ_PHASE_IN_FLIGHT, "invoice-1", int(time.time()) + 900
)
with patch(
"routstr.core.admin._require_ppq_provider",
AsyncMock(return_value=provider_row),
):
with pytest.raises(HTTPException) as excinfo:
await release_ppq_auto_topup_api(
1,
ReleasePPQAutoTopupRequest(
confirmed_safe_to_retry=True, state_token=token
),
)
assert excinfo.value.status_code == 409
assert "in flight" in excinfo.value.detail
with pytest.raises(HTTPException) as excinfo:
await release_ppq_auto_topup_api(
1,
ReleasePPQAutoTopupRequest(
confirmed_safe_to_retry=True, state_token="ppq:wrong"
),
)
assert excinfo.value.status_code == 409
assert "changed since" in excinfo.value.detail
async def test_provider_delete_is_blocked_by_an_active_claim(
patched_db_engine: Any,
) -> None:
from fastapi import HTTPException
from routstr.core.admin import delete_upstream_provider
from routstr.core.db import UpstreamProviderRow
async with create_session() as session:
session.add(
UpstreamProviderRow(
id=1,
slug="ppq",
provider_type="ppqai",
base_url="https://api.ppq.ai",
api_key="secret",
enabled=True,
)
)
await session.commit()
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900)
with pytest.raises(HTTPException) as excinfo:
await delete_upstream_provider("1")
assert excinfo.value.status_code == 409
# Provider must still exist.
async with create_session() as session:
assert await session.get(UpstreamProviderRow, 1) is not None
async def test_claim_is_refused_when_the_provider_row_is_gone(
patched_db_engine: Any,
) -> None:
"""The worker's half of the delete race: no provider row, no claim."""
assert await _claim_ppq_topup(_row()) is None
async with create_session() as session:
rows = (await session.exec(select(CashuTransaction))).all()
assert rows == []
async def test_claim_is_refused_after_a_provider_type_change(
patched_db_engine: Any,
) -> None:
from routstr.core.db import UpstreamProviderRow
await _seed_provider()
async with create_session() as session:
provider = await session.get(UpstreamProviderRow, 1)
assert provider is not None
provider.provider_type = "openai"
session.add(provider)
await session.commit()
assert await _claim_ppq_topup(_row()) is None
async def test_disabled_provider_with_claim_still_reconciles(
patched_db_engine: Any,
) -> None:
"""A claim tracks committed money; eligibility must not stop reconciling."""
from routstr.core.db import UpstreamProviderRow
from routstr.upstream.auto_topup import _reconcile_all_ppq_claims
await _seed_provider()
async with create_session() as session:
provider = await session.get(UpstreamProviderRow, 1)
assert provider is not None
provider.enabled = False
session.add(provider)
await session.commit()
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) + 900)
ppq = MagicMock()
ppq.check_topup_status = AsyncMock(return_value=True)
with patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=ppq,
):
await _reconcile_all_ppq_claims()
row = await _state_row()
assert row is not None and row.collected is True
async def test_claim_without_api_key_still_reconciles_via_the_mint(
patched_db_engine: Any,
) -> None:
from routstr.core.db import UpstreamProviderRow
from routstr.upstream.auto_topup import _reconcile_all_ppq_claims
await _seed_provider()
async with create_session() as session:
provider = await session.get(UpstreamProviderRow, 1)
assert provider is not None
provider.api_key = ""
session.add(provider)
await session.commit()
# Lease expired, so the mint may be consulted.
await _seed_claim(1, PPQ_PHASE_RECONCILE, "invoice-1", int(time.time()) - 1)
with patch(
"routstr.upstream.auto_topup.check_bolt11_payment_status",
AsyncMock(return_value="unpaid"),
) as status:
await _reconcile_all_ppq_claims()
# No API key: PPQ was never polled, but the mint was, and its definitive
# "unpaid" released the claim.
status.assert_awaited_once()
row = await _state_row()
assert row is not None and row.swept is True
def test_ppq_payment_usd_prefers_stamped_amount() -> None:
# Stamped rows must not move with the BTC price.
assert _ppq_payment_usd(102, "sat", "ppq-invoice:a:usd:10", 0.5) == 10.0
def test_ppq_payment_usd_falls_back_to_current_price() -> None:
# Rows recorded before the stamp existed convert sats at today's price.
assert _ppq_payment_usd(2000, "sat", "ppq-invoice:legacy", 0.001) == 2.0
assert _ppq_payment_usd(2_000_000, "msat", "ppq-invoice:legacy", 0.001) == 2.0
def test_ppq_payment_usd_survives_malformed_stamp() -> None:
assert _ppq_payment_usd(3000, "sat", "ppq-invoice:x:usd:oops", 0.001) == 3.0
async def test_daily_spend_ignores_provably_unattempted_payments(
patched_db_engine: Any,
) -> None:
def _payment(
id_: str, token: str, collected: bool, swept: bool
) -> CashuTransaction:
return CashuTransaction(
id=id_,
token=token,
amount=1,
unit="sat",
type="out",
source="ppq_auto_topup",
collected=collected,
swept=swept,
)
async with create_session() as session:
# Settled, in-flight, and provably-unattempted payments plus a
# pre-stamp row: only the unattempted one must be excluded.
session.add(_payment("pay-usd-1", "ppq-invoice:a:usd:100", True, False))
session.add(_payment("pay-usd-2", "ppq-invoice:b:usd:50", False, False))
session.add(_payment("pay-usd-3", "ppq-invoice:c:usd:25", False, True))
legacy = _payment("pay-usd-4", "ppq-invoice:legacy", True, False)
legacy.amount = 2000
session.add(legacy)
await session.commit()
assert await _ppq_spent_last_24h_usd(0.001) == 152.0
@@ -0,0 +1,65 @@
"""Integration coverage for proxy database-session lifetime."""
from __future__ import annotations
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import Response
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr import proxy as proxy_module
from routstr.core.db import ApiKey
@pytest.mark.asyncio
async def test_authenticated_proxy_releases_db_connection_before_upstream_headers(
integration_engine: AsyncEngine,
integration_session: AsyncSession,
patched_db_engine: None,
) -> None:
"""Slow upstream header waits must not retain a checked-out DB connection."""
key = ApiKey(
hashed_key="proxy-pool-key",
balance=1_000_000,
refund_mint_url="http://primary:3338",
refund_currency="sat",
)
integration_session.add(key)
await integration_session.commit()
request = MagicMock()
request.method = "POST"
request.headers = {"authorization": "Bearer test-key"}
request.body = AsyncMock(return_value=json.dumps({"model": "test-model"}).encode())
request.url.path = "/v1/chat/completions"
request.state.request_id = "pool-hold-regression"
model = MagicMock()
upstream = MagicMock()
upstream.provider_type = "test"
upstream.prepare_headers.return_value = {}
async def wait_for_headers(*args: object, **kwargs: object) -> Response:
assert integration_engine.pool.checkedout() == 0 # type: ignore[attr-defined]
return Response(status_code=200)
upstream.forward_request = AsyncMock(side_effect=wait_for_headers)
with (
patch("routstr.proxy.get_candidates", return_value=[(model, upstream)]),
patch("routstr.proxy.get_max_cost_for_model", AsyncMock(return_value=100)),
patch(
"routstr.proxy.calculate_discounted_max_cost",
AsyncMock(return_value=100),
),
patch("routstr.proxy.check_token_balance"),
patch("routstr.proxy.get_bearer_token_key", AsyncMock(return_value=key)),
):
response = await proxy_module._proxy(
request, "v1/chat/completions", integration_session
)
assert response.status_code == 200
@@ -126,8 +126,11 @@ async def test_parent_and_child_keys_are_not_pruned(
@pytest.mark.asyncio
async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
"""A key referenced by a pending topup invoice is never pruned mid-topup."""
@pytest.mark.parametrize("status", ["pending", "settlement_pending"])
async def test_retryable_invoice_protects_key(
patched_db_engine: None, status: str
) -> None:
"""A key referenced by a retryable topup invoice is never pruned mid-topup."""
key = _dead_key(LONG_AGO)
invoice = LightningInvoice(
id=f"inv_{uuid.uuid4().hex}",
@@ -135,7 +138,7 @@ async def test_pending_invoice_protects_key(patched_db_engine: None) -> None:
amount_sats=10,
description="topup",
payment_hash=uuid.uuid4().hex,
status="pending",
status=status,
api_key_hash=key.hashed_key,
purpose="topup",
expires_at=NOW + 10_000,
@@ -141,7 +141,7 @@ async def test_revert_with_zero_reserved_balance_is_noop(
Previously this would drive reserved_balance negative. With the floor guard,
it should return False and leave reserved_balance at 0.
"""
from routstr.auth import revert_pay_for_request
from routstr.auth import pay_for_request, revert_pay_for_request
unique_key = f"test_revert_key_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
@@ -151,8 +151,12 @@ async def test_revert_with_zero_reserved_balance_is_noop(
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 100, integration_session)
test_key.reserved_balance = 0
integration_session.add(test_key)
await integration_session.commit()
# Try to revert more than available — should be a no-op
# A stale cleanup already released the aggregate reservation.
result = await revert_pay_for_request(test_key, integration_session, 100)
await integration_session.refresh(test_key)
@@ -161,8 +165,8 @@ async def test_revert_with_zero_reserved_balance_is_noop(
assert test_key.reserved_balance == 0, (
f"Reserved balance should remain 0, got: {test_key.reserved_balance}"
)
assert test_key.total_requests == 0, (
f"Total requests should remain 0, got: {test_key.total_requests}"
assert test_key.total_requests == 1, (
f"Total requests should remain 1, got: {test_key.total_requests}"
)
@@ -171,17 +175,18 @@ async def test_revert_with_sufficient_reserved_balance_succeeds(
integration_session: AsyncSession,
) -> None:
"""Test that revert_pay_for_request works correctly when there is enough reserved balance."""
from routstr.auth import revert_pay_for_request
from routstr.auth import pay_for_request, revert_pay_for_request
unique_key = f"test_revert_ok_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=5000,
reserved_balance=500,
total_requests=3,
reserved_balance=0,
total_requests=2,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
result = await revert_pay_for_request(test_key, integration_session, 500)
@@ -202,17 +207,21 @@ async def test_revert_partial_reserved_balance_is_noop(
integration_session: AsyncSession,
) -> None:
"""Test that reverting more than the current reserved_balance is a no-op."""
from routstr.auth import revert_pay_for_request
from routstr.auth import pay_for_request, revert_pay_for_request
unique_key = f"test_revert_partial_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=5000,
reserved_balance=50,
total_requests=1,
reserved_balance=0,
total_requests=0,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
test_key.reserved_balance = 50
integration_session.add(test_key)
await integration_session.commit()
# Try to revert 500 when only 50 is reserved — should be no-op
result = await revert_pay_for_request(test_key, integration_session, 500)
@@ -237,20 +246,28 @@ async def test_double_revert_prevented(
This simulates the double-revert scenario where both upstream/base.py
and proxy.py attempt to revert the same reservation.
"""
from routstr.auth import revert_pay_for_request
from routstr.auth import (
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
)
unique_key = f"test_double_revert_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=10000,
reserved_balance=500,
total_requests=5,
reserved_balance=0,
total_requests=4,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
snapshot = await get_reservation_snapshot(test_key, integration_session)
# First revert — should succeed
result1 = await revert_pay_for_request(test_key, integration_session, 500)
result1 = await revert_pay_for_request(
test_key, integration_session, 500, snapshot
)
await integration_session.refresh(test_key)
assert result1 is True
@@ -258,7 +275,9 @@ async def test_double_revert_prevented(
assert test_key.total_requests == 4
# Second revert of the same amount — should be no-op
result2 = await revert_pay_for_request(test_key, integration_session, 500)
result2 = await revert_pay_for_request(
test_key, integration_session, 500, snapshot
)
await integration_session.refresh(test_key)
assert result2 is False, "Second revert should be a no-op"
@@ -279,22 +298,30 @@ async def test_sequential_reverts_never_go_negative(
Simulates the double-revert scenario where multiple code paths
attempt to revert the same reservation.
"""
from routstr.auth import revert_pay_for_request
from routstr.auth import (
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
)
unique_key = f"test_multi_revert_{uuid.uuid4().hex[:8]}"
test_key = ApiKey(
hashed_key=unique_key,
balance=10000,
reserved_balance=500,
total_requests=5,
reserved_balance=0,
total_requests=4,
)
integration_session.add(test_key)
await integration_session.commit()
await pay_for_request(test_key, 500, integration_session)
snapshot = await get_reservation_snapshot(test_key, integration_session)
# Run 5 sequential reverts for the same 500 reservation
results = []
for _ in range(5):
r = await revert_pay_for_request(test_key, integration_session, 500)
r = await revert_pay_for_request(
test_key, integration_session, 500, snapshot
)
results.append(r)
await integration_session.refresh(test_key)
@@ -317,7 +344,11 @@ async def test_child_key_revert_floor_guard(
integration_session: AsyncSession,
) -> None:
"""Test that child key reserved_balance also has floor guard on revert."""
from routstr.auth import revert_pay_for_request
from routstr.auth import (
get_reservation_snapshot,
pay_for_request,
revert_pay_for_request,
)
parent_key_hash = f"test_parent_{uuid.uuid4().hex[:8]}"
child_key_hash = f"test_child_{uuid.uuid4().hex[:8]}"
@@ -325,22 +356,26 @@ async def test_child_key_revert_floor_guard(
parent_key = ApiKey(
hashed_key=parent_key_hash,
balance=10000,
reserved_balance=500,
total_requests=3,
reserved_balance=0,
total_requests=2,
)
child_key = ApiKey(
hashed_key=child_key_hash,
balance=0,
reserved_balance=500,
total_requests=3,
reserved_balance=0,
total_requests=2,
parent_key_hash=parent_key_hash,
)
integration_session.add(parent_key)
integration_session.add(child_key)
await integration_session.commit()
await pay_for_request(child_key, 500, integration_session)
snapshot = await get_reservation_snapshot(child_key, integration_session)
# First revert succeeds
result1 = await revert_pay_for_request(child_key, integration_session, 500)
result1 = await revert_pay_for_request(
child_key, integration_session, 500, snapshot
)
await integration_session.refresh(parent_key)
await integration_session.refresh(child_key)
@@ -349,7 +384,9 @@ async def test_child_key_revert_floor_guard(
assert child_key.reserved_balance == 0
# Second revert is a no-op for both parent and child
result2 = await revert_pay_for_request(child_key, integration_session, 500)
result2 = await revert_pay_for_request(
child_key, integration_session, 500, snapshot
)
await integration_session.refresh(parent_key)
await integration_session.refresh(child_key)
+14 -4
View File
@@ -20,6 +20,7 @@ from collections.abc import Callable
from unittest.mock import AsyncMock, Mock, patch
import pytest
from cashu.core.base import MeltQuoteState
from httpx import AsyncClient, Response
from routstr.core.settings import settings
@@ -28,7 +29,9 @@ from routstr.core.settings import settings
# with the testmint stub that bypasses swapping (see conftest.py).
from routstr.wallet import recieve_token as _real_recieve_token
PRIMARY_MINT = "http://primary:3338"
# Match the authenticated fixture's persisted refund mint: existing-key topups
# are intentionally constrained to that mint for collateral provenance.
PRIMARY_MINT = "http://localhost:3338"
def _make_swap_mocks(
@@ -81,7 +84,9 @@ def _make_swap_mocks(
quote=f"melt_quote_{invoice}", amount=invoice, fee_reserve=_next_fee()
)
)
mock_token_wallet.melt = AsyncMock(return_value=Mock())
mock_token_wallet.melt = AsyncMock(
return_value=Mock(state=MeltQuoteState.paid)
)
return mock_token, mock_token_wallet, mock_primary_wallet
@@ -89,7 +94,12 @@ def _make_swap_mocks(
def _wallet_router(primary_wallet: Mock, token_wallet: Mock) -> Callable[..., Mock]:
"""Route get_wallet calls to the primary or foreign wallet mock by URL."""
def fake_get_wallet(mint_url: str, unit: str = "sat", load: bool = True) -> Mock:
def fake_get_wallet(
mint_url: str,
unit: str = "sat",
load: bool = True,
**kwargs: object,
) -> Mock:
return primary_wallet if mint_url == PRIMARY_MINT else token_wallet
return fake_get_wallet
@@ -139,7 +149,7 @@ async def test_topup_retries_when_melt_demands_more_than_quoted(
"Mint Error: not enough inputs provided for melt. "
"Provided: 179, needed: 180 (Code: 11000)"
),
Mock(),
Mock(state=MeltQuoteState.paid),
]
response = await _post_topup(
@@ -0,0 +1,115 @@
"""Restart reconciliation for ambiguous melts, against a real cashu wallet DB.
The ambiguous-melt path in ``execute_bolt11_payment`` re-reserves proofs with
``set_reserved_for_melt(..., quote_id=...)`` after cashu's ``melt()`` clears
both the reservation and the ``melt_id`` on a transport error. These tests
prove, on cashu's actual sqlite store rather than mocks, that the recovery
survives a process restart: a fresh wallet instance on the same database can
still find the proofs by ``melt_id`` the lookup ``get_melt_quote()`` uses to
invalidate them on "paid" or release them on "unpaid".
"""
from pathlib import Path
import pytest
from cashu.core.base import Proof
from cashu.wallet import crud
from cashu.wallet.wallet import Wallet
pytestmark = pytest.mark.asyncio
QUOTE_ID = "quote-restart-1"
def _proof(secret: str, amount: int = 64) -> Proof:
return Proof(
id="009a1f293253e41e",
amount=amount,
secret=secret,
C="02bc9097997d81afb2cc7346b5e4345a9346bd2a506eb7958598a72f0cf85163ea",
)
async def _wallet(db_dir: Path) -> Wallet:
# with_db builds the instance and runs migrations locally; nothing here
# talks to a mint.
return await Wallet.with_db("https://mint.test", str(db_dir))
async def _seed_ambiguous_melt(wallet: Wallet) -> list[Proof]:
"""Reproduce the exact sequence of an ambiguous melt failure.
1. Proofs exist and are selected for a melt.
2. cashu's melt() reserves them with the quote id, then hits a transport
error and rolls that back reservation gone, melt_id gone.
3. Our recovery in execute_bolt11_payment re-reserves with the quote id.
"""
proofs = [_proof("secret-a"), _proof("secret-b", amount=32)]
for proof in proofs:
await crud.store_proof(proof, db=wallet.db)
await wallet.set_reserved_for_melt(proofs, reserved=True, quote_id=QUOTE_ID)
# cashu's `except` block in melt():
await wallet.set_reserved_for_melt(proofs, reserved=False, quote_id=None)
# our recovery:
await wallet.set_reserved_for_melt(proofs, reserved=True, quote_id=QUOTE_ID)
return proofs
async def test_melt_recovery_is_findable_by_quote_after_restart(
tmp_path: Path,
) -> None:
wallet = await _wallet(tmp_path)
await _seed_ambiguous_melt(wallet)
# "Restart": a brand-new wallet on the same database file, as after a
# process crash between the melt and any reconciliation.
restarted = await _wallet(tmp_path)
found = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
# This is get_melt_quote()'s own lookup. If it comes back empty, a "paid"
# answer can never invalidate these proofs and an "unpaid" answer can
# never release them — the strand the send-style re-reserve caused.
assert sorted(p.secret for p in found) == ["secret-a", "secret-b"]
assert all(p.reserved for p in found)
assert all(p.melt_id == QUOTE_ID for p in found)
async def test_send_style_reservation_would_not_be_reconcilable(
tmp_path: Path,
) -> None:
"""The defect the fix removed, demonstrated on the real store."""
wallet = await _wallet(tmp_path)
proofs = [_proof("secret-send")]
for proof in proofs:
await crud.store_proof(proof, db=wallet.db)
await wallet.set_reserved_for_melt(proofs, reserved=True, quote_id=QUOTE_ID)
await wallet.set_reserved_for_melt(proofs, reserved=False, quote_id=None)
# The old recovery: reserve as a send, no quote association.
await wallet.set_reserved_for_send(proofs, reserved=True)
restarted = await _wallet(tmp_path)
found = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
assert found == [] # reconciliation would never see these proofs
async def test_unpaid_reconciliation_releases_recovered_proofs_after_restart(
tmp_path: Path,
) -> None:
"""The full recovery arc: crash, restart, mint says unpaid, funds usable."""
wallet = await _wallet(tmp_path)
await _seed_ambiguous_melt(wallet)
restarted = await _wallet(tmp_path)
found = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
assert len(found) == 2
# What get_melt_quote() does on an "unpaid" answer.
await restarted.set_reserved_for_melt(found, reserved=False, quote_id=None)
released = await crud.get_proofs(db=restarted.db, melt_id=QUOTE_ID)
assert released == []
all_proofs = await crud.get_proofs(db=restarted.db)
assert len(all_proofs) == 2
assert all(not p.reserved for p in all_proofs) # spendable again
+35
View File
@@ -0,0 +1,35 @@
from contextlib import asynccontextmanager
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from routstr.core.admin import get_transactions_api
from routstr.core.db import CashuTransaction
@pytest.mark.asyncio
async def test_transactions_api_excludes_internal_sweep_claim_timestamp() -> None:
transaction = CashuTransaction(
token="cashu-token",
amount=10,
unit="sat",
type="out",
sweep_started_at=123,
)
count_result = MagicMock()
count_result.one.return_value = 1
transactions_result = MagicMock()
transactions_result.all.return_value = [transaction]
session = MagicMock()
session.exec = AsyncMock(side_effect=[count_result, transactions_result])
@asynccontextmanager
async def create_session(): # type: ignore[no-untyped-def]
yield session
with patch("routstr.core.admin.create_session", create_session):
response = await get_transactions_api()
assert response["total"] == 1
assert response["transactions"][0]["token"] == "cashu-token"
assert "sweep_started_at" not in response["transactions"][0]
+92 -22
View File
@@ -1,8 +1,12 @@
import base64
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from fastapi import HTTPException
import routstr.wallet as wallet_module
from routstr.core import admin
@@ -13,20 +17,12 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
) -> None:
primary_mint = "https://primary.example"
effective_mint = requested_mint or primary_mint
wallet = object()
proofs = [SimpleNamespace(amount=40), SimpleNamespace(amount=60)]
token = "cashuBoutgoing"
get_wallet = AsyncMock(return_value=wallet)
get_proofs = Mock(return_value=proofs)
filter_proofs = AsyncMock(return_value=proofs)
send_token = AsyncMock(return_value=token)
store_transaction = AsyncMock(return_value=True)
monkeypatch.setattr(admin, "get_wallet", get_wallet)
monkeypatch.setattr(admin, "get_proofs_per_mint_and_unit", get_proofs)
monkeypatch.setattr(admin, "slow_filter_spend_proofs", filter_proofs)
monkeypatch.setattr(admin, "send_token", send_token)
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=effective_mint))
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
monkeypatch.setattr(admin.settings, "primary_mint", primary_mint)
@@ -35,10 +31,7 @@ async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
admin.WithdrawRequest(amount=75, mint_url=requested_mint, unit="sat"),
)
assert result == {"token": token}
get_wallet.assert_awaited_once_with(effective_mint, "sat")
get_proofs.assert_called_once_with(wallet, effective_mint, "sat", not_reserved=True)
filter_proofs.assert_awaited_once_with(proofs, wallet)
assert result == {"token": token, "mint_url": effective_mint}
send_token.assert_awaited_once_with(75, "sat", effective_mint)
store_transaction.assert_awaited_once_with(
token=token,
@@ -56,17 +49,10 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mint = "https://primary.example"
proofs = [SimpleNamespace(amount=100)]
token = "cashuBrecoverable"
monkeypatch.setattr(admin, "get_wallet", AsyncMock(return_value=object()))
monkeypatch.setattr(
admin, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
)
monkeypatch.setattr(
admin, "slow_filter_spend_proofs", AsyncMock(return_value=proofs)
)
monkeypatch.setattr(admin, "send_token", AsyncMock(return_value=token))
monkeypatch.setattr(admin, "token_mint_url", Mock(return_value=mint))
monkeypatch.setattr(
admin,
"store_cashu_transaction",
@@ -78,5 +64,89 @@ async def test_withdraw_returns_issued_token_when_audit_storage_fails(
result = await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
assert result == {"token": token}
assert result == {"token": token, "mint_url": mint}
critical.assert_called_once()
@pytest.mark.asyncio
async def test_withdraw_falls_back_from_insufficient_preferred_mint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
requested_mint = "https://primary.example"
actual_mint = "https://secondary.example"
proofs = [SimpleNamespace(amount=100, reserved=False, id="00")]
token_payload = {
"token": [
{
"mint": actual_mint,
"proofs": [
{
"id": "00",
"amount": 75,
"secret": "secret",
"C": "02" + "00" * 32,
}
],
}
],
"unit": "sat",
}
token = "cashuA" + base64.urlsafe_b64encode(
json.dumps(token_payload).encode()
).decode()
wallet = SimpleNamespace(
keysets={},
proofs=proofs,
select_to_send=AsyncMock(return_value=(proofs, 0)),
serialize_proofs=AsyncMock(return_value=token),
set_reserved_for_send=AsyncMock(),
)
find_funded = AsyncMock(return_value=actual_mint)
store_transaction = AsyncMock(return_value=True)
monkeypatch.setattr(wallet_module, "find_trusted_mint_with_funds", find_funded)
monkeypatch.setattr(wallet_module, "get_wallet", AsyncMock(return_value=wallet))
monkeypatch.setattr(
wallet_module, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
)
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
result = await admin.withdraw(
Mock(), admin.WithdrawRequest(amount=75, mint_url=requested_mint)
)
assert result == {"token": token, "mint_url": actual_mint}
find_funded.assert_awaited_once_with(
75, "sat", requested_mint, force_reload=True
)
wallet.select_to_send.assert_awaited_once()
store_transaction.assert_awaited_once_with(
token=token,
amount=75,
unit="sat",
mint_url=actual_mint,
typ="out",
collected=False,
source="admin",
)
@pytest.mark.asyncio
async def test_withdraw_maps_true_aggregate_insufficient_funds_to_400(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
admin,
"send_token",
AsyncMock(
side_effect=ValueError(
"No trusted mint has 75 sat available; balances={'mint': 0}"
)
),
)
with pytest.raises(HTTPException) as exc_info:
await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
assert exc_info.value.status_code == 400
assert exc_info.value.detail == "Insufficient wallet balance"
+48
View File
@@ -270,6 +270,54 @@ async def test_internal_error_with_invalid_keyword_does_not_masquerade(
assert await session.get(ApiKey, hashed_key) is None
@pytest.mark.asyncio
async def test_primary_msat_token_sets_provenance_without_cashu_mint_duplicate(
session: AsyncSession,
) -> None:
token = "cashuAprimary_msat_token"
token_obj = SimpleNamespace(mint="http://primary:3338", unit="msat")
credit = AsyncMock(return_value=1_000)
from routstr.core.settings import settings
with (
patch.object(settings, "primary_mint", token_obj.mint),
patch.object(settings, "primary_mint_unit", "msat"),
patch.object(settings, "cashu_mints", []),
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
patch("routstr.auth.credit_balance", new=credit),
):
key = await validate_bearer_key(token, session)
assert key.refund_mint_url == token_obj.mint
assert key.refund_currency == "msat"
credit.assert_awaited_once()
@pytest.mark.asyncio
async def test_primary_token_unit_mismatch_is_rejected_before_redemption(
session: AsyncSession,
) -> None:
token = "cashuAprimary_wrong_unit"
token_obj = SimpleNamespace(mint="http://primary:3338", unit="sat")
credit = AsyncMock(return_value=1_000)
from routstr.core.settings import settings
with (
patch.object(settings, "primary_mint", token_obj.mint),
patch.object(settings, "primary_mint_unit", "msat"),
patch.object(settings, "cashu_mints", []),
patch("routstr.auth.deserialize_token_from_string", return_value=token_obj),
patch("routstr.auth.credit_balance", new=credit),
):
with pytest.raises(HTTPException) as exc_info:
await validate_bearer_key(token, session)
assert exc_info.value.status_code == 400
credit.assert_not_awaited()
@pytest.mark.asyncio
async def test_malformed_cashu_token_returns_400_invalid_token(
session: AsyncSession,
+595 -2
View File
@@ -4,7 +4,29 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from routstr.core.db import CashuTransaction
from routstr.upstream.auto_topup import _check_and_topup
from routstr.upstream.auto_topup import (
_check_and_topup,
_parse_ppq_request_id,
_run_auto_topup_cycle,
validate_ppq_auto_topup_settings,
)
from routstr.upstream.ppqai import PPQAIUpstreamProvider
from routstr.wallet import Bolt11PaymentAmbiguous, Bolt11PaymentNotAttempted
def test_ppq_claim_parser_rejects_invalid_expiry() -> None:
assert (
_parse_ppq_request_id("ppq:operation:not-a-timestamp:claimed:invoice:none")
is None
)
@pytest.mark.asyncio
async def test_ppq_balance_rejects_boolean_api_value() -> None:
provider = PPQAIUpstreamProvider("secret")
provider.check_balance = AsyncMock(return_value={"balance": False}) # type: ignore[method-assign]
assert await provider.get_balance() is None
def _row() -> MagicMock:
@@ -12,6 +34,7 @@ def _row() -> MagicMock:
row.id = "provider-1"
row.base_url = "https://provider.test"
row.api_key = "secret"
row.provider_type = "routstr"
row.provider_settings = json.dumps(
{
"auto_topup": True,
@@ -66,6 +89,10 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
"routstr.upstream.auto_topup.store_cashu_transaction",
AsyncMock(return_value=True),
) as store,
patch(
"routstr.upstream.auto_topup.token_mint_url",
return_value="https://fallback-mint.test",
),
patch("routstr.upstream.auto_topup.create_session", return_value=session),
):
await _check_and_topup(_row())
@@ -74,7 +101,7 @@ async def test_auto_topup_persists_before_sending_and_marks_success_collected()
token="cashu-token",
amount=50,
unit="sat",
mint_url="https://mint.test",
mint_url="https://fallback-mint.test",
typ="out",
collected=False,
source="auto_topup",
@@ -138,6 +165,572 @@ async def test_auto_topup_does_not_send_untracked_token() -> None:
"routstr.upstream.auto_topup.store_cashu_transaction",
AsyncMock(side_effect=RuntimeError("database unavailable")),
),
patch(
"routstr.upstream.auto_topup.release_token_reservation",
AsyncMock(),
) as reclaim,
):
await _check_and_topup(_row())
reclaim.assert_awaited_once_with("cashu-token")
provider.topup.assert_not_awaited()
def _ppq_row() -> MagicMock:
row = MagicMock()
row.id = "ppq-provider-1"
row.base_url = "https://api.ppq.ai"
row.api_key = "secret"
row.provider_type = "ppqai"
row.provider_settings = json.dumps(
{
"auto_topup": True,
"topup_threshold": 5.0,
"topup_amount_limit": 10,
}
)
return row
@pytest.mark.asyncio
async def test_ppq_auto_topup_pays_invoice_and_confirms_settlement() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock(
return_value=MagicMock(
invoice_id="invoice-1",
payment_request="lnbc-invoice",
amount=10,
currency="USD",
expires_at=None,
)
)
provider.check_topup_status = AsyncMock(return_value=True)
plan = MagicMock()
plan.invoice_amount_sats = 100
plan.maximum_spend_sats = 102
plan.quote.amount = 100
plan.quote.fee_reserve = 2
plan.mint_url = "https://mint-rich.test"
plan.unit = "sat"
row = _ppq_row()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup._claim_ppq_topup",
AsyncMock(return_value="operation-1"),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=10_000),
),
patch(
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
AsyncMock(return_value=0.0),
),
patch(
"routstr.upstream.auto_topup.prepare_bolt11_payment",
AsyncMock(return_value=plan),
) as prepare,
patch(
"routstr.upstream.auto_topup.execute_bolt11_payment",
AsyncMock(return_value=(101, "https://mint-rich.test", "sat")),
) as execute,
patch("routstr.upstream.auto_topup._record_ppq_invoice", AsyncMock()) as record,
patch(
"routstr.upstream.auto_topup._record_ppq_payment_spent", AsyncMock()
) as record_spent,
patch(
"routstr.upstream.auto_topup._set_ppq_state_terminal", AsyncMock()
) as terminal,
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
):
await _check_and_topup(row)
provider.initiate_topup.assert_awaited_once_with(10)
prepare.assert_awaited_once_with("lnbc-invoice")
execute.assert_awaited_once_with(plan)
record.assert_awaited_once()
record_spent.assert_awaited_once_with("operation-1", 101)
provider.check_topup_status.assert_awaited_once_with("invoice-1")
terminal.assert_awaited_once_with(row, "operation-1", collected=True, swept=False)
@pytest.mark.asyncio
async def test_ppq_ambiguous_melt_keeps_claim_and_emits_critical_alert() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock(
return_value=MagicMock(
invoice_id="invoice-1",
payment_request="lnbc-invoice",
amount=10,
currency="USD",
expires_at=None,
)
)
plan = MagicMock(maximum_spend_sats=102, mint_url="https://mint.test", unit="sat")
plan.quote.amount = 100
plan.quote.fee_reserve = 2
row = _ppq_row()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup._claim_ppq_topup",
AsyncMock(return_value="operation-1"),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=10_000),
),
patch(
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
AsyncMock(return_value=0.0),
),
patch(
"routstr.upstream.auto_topup.prepare_bolt11_payment",
AsyncMock(return_value=plan),
),
patch(
"routstr.upstream.auto_topup.execute_bolt11_payment",
AsyncMock(side_effect=Bolt11PaymentAmbiguous("ambiguous melt")),
),
patch(
"routstr.upstream.auto_topup._record_ppq_invoice",
AsyncMock(return_value=2_000_000_000),
),
patch(
"routstr.upstream.auto_topup._mark_ppq_reconcile", AsyncMock()
) as reconcile_mark,
patch(
"routstr.upstream.auto_topup._set_ppq_state_terminal", AsyncMock()
) as terminal,
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
patch("routstr.upstream.auto_topup.logger.critical") as critical,
):
with pytest.raises(Bolt11PaymentAmbiguous, match="ambiguous melt"):
await _check_and_topup(row)
# The claim is never released — it moves to reconcile for the admin.
terminal.assert_not_awaited()
reconcile_mark.assert_awaited_once()
critical.assert_called_once()
assert "admin reconciliation" in critical.call_args.args[0]
@pytest.mark.asyncio
async def test_ppq_payment_not_attempted_releases_claim_for_retry() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock(
return_value=MagicMock(
invoice_id="invoice-1",
payment_request="lnbc-invoice",
amount=10,
currency="USD",
expires_at=None,
)
)
plan = MagicMock(maximum_spend_sats=102, mint_url="https://mint.test", unit="sat")
plan.quote.amount = 100
plan.quote.fee_reserve = 2
plan.quote.quote = "quote-1"
terminal = AsyncMock(return_value=True)
row = _ppq_row()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=10_000),
),
patch(
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
AsyncMock(return_value=0.0),
),
patch(
"routstr.upstream.auto_topup._claim_ppq_topup",
AsyncMock(return_value="operation-1"),
),
patch(
"routstr.upstream.auto_topup.prepare_bolt11_payment",
AsyncMock(return_value=plan),
),
patch(
"routstr.upstream.auto_topup._record_ppq_invoice",
AsyncMock(return_value=2_000_000_000),
),
patch(
"routstr.upstream.auto_topup.execute_bolt11_payment",
AsyncMock(side_effect=Bolt11PaymentNotAttempted("unpaid")),
),
patch("routstr.upstream.auto_topup._set_ppq_state_terminal", terminal),
patch(
"routstr.upstream.auto_topup._mark_ppq_reconcile", AsyncMock()
) as reconcile,
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
pytest.raises(Bolt11PaymentNotAttempted, match="unpaid"),
):
await _check_and_topup(row)
terminal.assert_awaited_once_with(row, "operation-1", collected=False, swept=True)
reconcile.assert_not_awaited()
@pytest.mark.asyncio
async def test_ppq_status_error_after_payment_marks_reconcile_and_alerts() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock(
return_value=MagicMock(
invoice_id="invoice-1",
payment_request="lnbc-invoice",
amount=10,
currency="USD",
expires_at=None,
)
)
provider.check_topup_status = AsyncMock(side_effect=RuntimeError("PPQ 502"))
plan = MagicMock(maximum_spend_sats=102, mint_url="https://mint.test", unit="sat")
plan.quote.amount = 100
plan.quote.fee_reserve = 2
plan.quote.quote = "quote-1"
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=10_000),
),
patch(
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
AsyncMock(return_value=0.0),
),
patch(
"routstr.upstream.auto_topup._claim_ppq_topup",
AsyncMock(return_value="operation-1"),
),
patch(
"routstr.upstream.auto_topup.prepare_bolt11_payment",
AsyncMock(return_value=plan),
),
patch(
"routstr.upstream.auto_topup._record_ppq_invoice",
AsyncMock(return_value=2_000_000_000),
),
patch(
"routstr.upstream.auto_topup.execute_bolt11_payment",
AsyncMock(return_value=(101, "https://mint.test", "sat")),
),
patch(
"routstr.upstream.auto_topup._record_ppq_payment_spent", AsyncMock()
) as spent,
patch(
"routstr.upstream.auto_topup._mark_ppq_reconcile", AsyncMock()
) as reconcile,
patch(
"routstr.upstream.auto_topup._set_ppq_state_terminal", AsyncMock()
) as terminal,
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
patch("routstr.upstream.auto_topup.logger.critical") as critical,
):
await _check_and_topup(_ppq_row())
spent.assert_awaited_once_with("operation-1", 101)
reconcile.assert_awaited_once()
terminal.assert_not_awaited()
assert "settlement polling failed" in critical.call_args.args[0]
@pytest.mark.asyncio
async def test_ppq_preflight_funding_check_happens_before_invoice_creation() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=1),
),
patch("routstr.upstream.auto_topup._claim_ppq_topup", AsyncMock()) as claim,
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
):
await _check_and_topup(_ppq_row())
provider.initiate_topup.assert_not_awaited()
claim.assert_not_awaited()
@pytest.mark.asyncio
async def test_active_claim_at_cycle_start_suppresses_topup_for_whole_cycle() -> None:
row = _ppq_row()
row.id = 1
session = AsyncMock()
result = MagicMock()
result.all.return_value = [row]
session.exec.return_value = result
context = MagicMock()
context.__aenter__ = AsyncMock(return_value=session)
context.__aexit__ = AsyncMock(return_value=None)
with (
patch(
"routstr.upstream.auto_topup._reconcile_all_ppq_claims",
AsyncMock(return_value={1}),
),
patch("routstr.upstream.auto_topup.create_session", return_value=context),
patch("routstr.upstream.auto_topup._check_and_topup", AsyncMock()) as check,
):
await _run_auto_topup_cycle()
check.assert_not_awaited()
@pytest.mark.asyncio
async def test_ppq_auto_topup_skips_when_balance_meets_threshold() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=5.0)
provider.initiate_topup = AsyncMock()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
):
await _check_and_topup(_ppq_row())
provider.initiate_topup.assert_not_awaited()
@pytest.mark.asyncio
async def test_ppq_auto_topup_skips_when_daily_spend_cap_reached() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=10_000_000),
),
# 1000 USD already spent, exactly the daily cap: the next 10 USD
# top-up must be refused.
patch(
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
AsyncMock(return_value=1000.0),
),
patch(
"routstr.upstream.auto_topup._claim_ppq_topup",
AsyncMock(),
) as claim,
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
):
await _check_and_topup(_ppq_row())
claim.assert_not_awaited()
provider.initiate_topup.assert_not_awaited()
@pytest.mark.asyncio
async def test_ppq_pending_attempt_suppresses_duplicate_topup() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock()
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=True),
),
):
await _check_and_topup(_ppq_row())
provider.get_balance.assert_not_awaited()
@pytest.mark.asyncio
async def test_ppq_auto_topup_rejects_non_finite_balance() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=float("nan"))
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch("routstr.upstream.auto_topup._claim_ppq_topup", AsyncMock()) as claim,
):
await _check_and_topup(_ppq_row())
claim.assert_not_awaited()
@pytest.mark.asyncio
async def test_settled_topup_alerts_when_its_claim_was_already_released() -> None:
provider = MagicMock()
provider.get_balance = AsyncMock(return_value=2.5)
provider.initiate_topup = AsyncMock(
return_value=MagicMock(
invoice_id="invoice-1",
payment_request="lnbc-invoice",
amount=10,
currency="USD",
expires_at=None,
)
)
provider.check_topup_status = AsyncMock(return_value=True)
plan = MagicMock()
plan.maximum_spend_sats = 102
plan.quote.amount = 100
plan.quote.fee_reserve = 2
plan.mint_url = "https://mint-rich.test"
plan.unit = "sat"
with (
patch(
"routstr.upstream.auto_topup.PPQAIUpstreamProvider.from_db_row",
return_value=provider,
),
patch(
"routstr.upstream.auto_topup._reconcile_ppq_state",
AsyncMock(return_value=False),
),
patch(
"routstr.upstream.auto_topup._claim_ppq_topup",
AsyncMock(return_value="operation-1"),
),
patch(
"routstr.upstream.auto_topup.prepare_bolt11_payment",
AsyncMock(return_value=plan),
),
patch(
"routstr.upstream.auto_topup.maximum_owner_cashu_balance_sats",
AsyncMock(return_value=10_000),
),
patch(
"routstr.upstream.auto_topup._ppq_spent_last_24h_usd",
AsyncMock(return_value=0.0),
),
patch(
"routstr.upstream.auto_topup.execute_bolt11_payment",
AsyncMock(return_value=(101, "https://mint-rich.test", "sat")),
),
patch("routstr.upstream.auto_topup._record_ppq_invoice", AsyncMock()),
patch("routstr.upstream.auto_topup._record_ppq_payment_spent", AsyncMock()),
patch(
"routstr.upstream.auto_topup._set_ppq_state_terminal",
AsyncMock(return_value=False),
),
patch("routstr.upstream.auto_topup.sats_usd_price", return_value=0.001),
patch("routstr.upstream.auto_topup.logger") as log,
):
await _check_and_topup(_ppq_row())
assert any(
"claim was already released" in call.args[0]
for call in log.critical.call_args_list
)
@pytest.mark.parametrize(
("settings", "expected"),
[
({"auto_topup": False, "topup_threshold": -1}, None),
(
{"auto_topup": True, "topup_threshold": 5, "topup_amount_limit": 10},
None,
),
(
{"auto_topup": True, "topup_threshold": None, "topup_amount_limit": 10},
"threshold",
),
(
{"auto_topup": True, "topup_threshold": 5, "topup_amount_limit": 0.5},
"whole number",
),
(
{"auto_topup": True, "topup_threshold": 5, "topup_amount_limit": 5000},
"between",
),
(
{"auto_topup": True, "topup_threshold": True, "topup_amount_limit": 10},
"threshold",
),
],
)
def test_ppq_auto_topup_settings_validation(
settings: dict, expected: str | None
) -> None:
problem = validate_ppq_auto_topup_settings(settings)
if expected is None:
assert problem is None
else:
assert problem is not None and expected in problem
def test_ppq_auto_topup_settings_validation_survives_huge_json_integers() -> None:
# json.loads happily produces integers past float range; float() raises
# OverflowError there instead of returning inf.
problem = validate_ppq_auto_topup_settings(
{"auto_topup": True, "topup_threshold": 10**400, "topup_amount_limit": 10}
)
assert problem is not None and "threshold" in problem
+160
View File
@@ -221,6 +221,78 @@ def _make_api_key(
return key
@pytest.mark.asyncio
async def test_apikey_refund_returns_persisted_token_after_cache_loss() -> None:
key = _make_api_key(balance=0, refund_currency="sat")
refund_token = "cashuApersisted_refund_token"
refund_tx = _make_cashu_tx(
token=refund_token,
amount=5,
unit="sat",
type="out",
request_id=None,
)
refund_tx.source = "apikey"
refund_tx.api_key_hashed_key = key.hashed_key
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.exec = AsyncMock(return_value=_exec_result(refund_tx))
session.add = MagicMock()
session.commit = AsyncMock()
with (
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
patch("routstr.balance.send_token", AsyncMock()) as mock_send_token,
):
result = await refund_wallet_endpoint(
authorization="Bearer sk-testhash",
x_cashu=None,
session=session,
)
assert result == {"token": refund_token, "sats": "5"}
assert refund_tx.collected is True
session.add.assert_called_once_with(refund_tx)
session.commit.assert_awaited_once()
mock_send_token.assert_not_awaited()
@pytest.mark.asyncio
async def test_apikey_refund_rejects_persisted_token_after_sweep() -> None:
from fastapi import HTTPException
key = _make_api_key(balance=0, refund_currency="sat")
refund_tx = _make_cashu_tx(
token="cashuAswept_apikey_refund",
amount=5,
unit="sat",
request_id=None,
swept=True,
)
refund_tx.source = "apikey"
refund_tx.api_key_hashed_key = key.hashed_key
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.exec = AsyncMock(return_value=_exec_result(refund_tx))
session.add = MagicMock()
session.commit = AsyncMock()
with patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)):
with pytest.raises(HTTPException) as exc_info:
await refund_wallet_endpoint(
authorization="Bearer sk-testhash",
x_cashu=None,
session=session,
)
assert exc_info.value.status_code == 410
assert exc_info.value.detail == "Refund has been swept"
session.add.assert_not_called()
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_apikey_refund_stores_cashu_transaction_with_apikey_source() -> None:
key = _make_api_key(balance=5000, refund_currency="sat")
@@ -534,6 +606,29 @@ async def test_topup_mint_unreachable_returns_503(error: Exception) -> None:
assert exc_info.value.detail == "Cashu mint is unreachable"
@pytest.mark.asyncio
async def test_topup_unreachable_source_mint_explains_why_fallback_is_impossible() -> None:
from fastapi import HTTPException
from routstr.wallet import SourceMintConnectionError
key = _make_api_key(balance=1000)
session = MagicMock()
error = SourceMintConnectionError("Issuing Cashu mint is unreachable")
with (
patch("routstr.balance.get_billing_key", AsyncMock(return_value=key)),
patch("routstr.balance.credit_balance", AsyncMock(side_effect=error)),
):
with pytest.raises(HTTPException) as exc_info:
await topup_wallet_endpoint(
cashu_token="cashuAtoken", key=key, session=session
)
assert exc_info.value.status_code == 503
assert "cannot be redeemed at another mint" in exc_info.value.detail
@pytest.mark.asyncio
async def test_topup_already_spent_still_returns_400() -> None:
"""Regression: the mint-unreachable short-circuit must not swallow the
@@ -686,3 +781,68 @@ async def test_topup_unexpected_non_valueerror_returns_500() -> None:
assert exc_info.value.status_code == 500
assert exc_info.value.detail == "Internal server error"
@pytest.mark.asyncio
async def test_apikey_refund_ambiguous_melt_does_not_restore_balance() -> None:
"""An ambiguous LNURL melt may still settle: the debit must be kept."""
from fastapi import HTTPException
from routstr.payment.lnurl import MeltOutcomeAmbiguousError
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
session.commit = AsyncMock()
with (
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
patch("routstr.balance._refund_cache_set", AsyncMock()),
patch(
"routstr.balance.send_to_lnurl",
AsyncMock(side_effect=MeltOutcomeAmbiguousError("outcome is ambiguous")),
),
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
):
with pytest.raises(HTTPException) as exc_info:
await refund_wallet_endpoint(
authorization="Bearer sk-testhash",
x_cashu=None,
session=session,
)
assert exc_info.value.status_code == 502
mock_restore.assert_not_awaited()
@pytest.mark.asyncio
async def test_apikey_refund_clean_failure_still_restores_balance() -> None:
"""A definitively failed melt must keep restoring the debited balance."""
from fastapi import HTTPException
key = _make_api_key(balance=5000, refund_address="user@ln.example.com")
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.exec = AsyncMock(return_value=MagicMock(rowcount=1))
session.commit = AsyncMock()
with (
patch("routstr.balance._refund_cache_get", AsyncMock(return_value=None)),
patch("routstr.balance._refund_cache_set", AsyncMock()),
patch(
"routstr.balance.send_to_lnurl",
AsyncMock(side_effect=RuntimeError("mint rejected melt")),
),
patch("routstr.balance._restore_balance", AsyncMock()) as mock_restore,
):
with pytest.raises(HTTPException):
await refund_wallet_endpoint(
authorization="Bearer sk-testhash",
x_cashu=None,
session=session,
)
mock_restore.assert_awaited_once()
+40
View File
@@ -0,0 +1,40 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from fastapi import FastAPI
from httpx import ASGITransport, AsyncClient
from routstr import balance as balance_module
from routstr.core.db import get_session
@pytest.mark.asyncio
async def test_create_balance_accepts_large_cashu_token_in_post_body(
monkeypatch: pytest.MonkeyPatch,
) -> None:
token = "cashuA" + "x" * 20_000
key = SimpleNamespace(hashed_key="hashed", balance=123_000)
validate_bearer_key = AsyncMock(return_value=key)
session = AsyncMock()
monkeypatch.setattr(balance_module, "validate_bearer_key", validate_bearer_key)
async def override_get_session(): # type: ignore[no-untyped-def]
yield session
app = FastAPI()
app.include_router(balance_module.balance_router)
app.dependency_overrides[get_session] = override_get_session
async with AsyncClient(
transport=ASGITransport(app=app), # type: ignore[arg-type]
base_url="http://test",
) as client:
response = await client.post(
"/v1/balance/create",
json={"initial_balance_token": token},
)
assert response.status_code == 200
assert response.json() == {"api_key": "sk-hashed", "balance": 123_000}
validate_bearer_key.assert_awaited_once_with(token, session)
@@ -0,0 +1,115 @@
"""Real-DB coverage for db.balances_by_mint_and_unit.
Verifies the grouped liability query used by fetch_all_balances: it sums
balances per (mint_url, unit), filters to the requested mints/units, excludes
NULL mint/currency rows, and returns nothing for empty inputs.
"""
from typing import AsyncGenerator
import pytest
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlalchemy.pool import StaticPool
from sqlmodel import SQLModel
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import (
ApiKey,
balance_for_mint_and_unit,
balances_by_mint_and_unit,
)
def _make_engine() -> AsyncEngine:
return create_async_engine(
"sqlite+aiosqlite://",
poolclass=StaticPool,
connect_args={"check_same_thread": False},
)
@pytest.fixture
async def session() -> "AsyncGenerator[AsyncSession, None]":
engine = _make_engine()
async with engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
db_session = AsyncSession(engine, expire_on_commit=False)
try:
yield db_session
finally:
await db_session.close()
await engine.dispose()
async def _add_key(
session: AsyncSession,
hashed_key: str,
balance: int,
mint_url: str | None,
currency: str | None,
) -> None:
session.add(
ApiKey(
hashed_key=hashed_key,
balance=balance,
refund_mint_url=mint_url,
refund_currency=currency,
)
)
await session.commit()
@pytest.mark.asyncio
async def test_sums_and_groups_by_mint_and_unit(session: AsyncSession) -> None:
await _add_key(session, "a", 1000, "http://m1", "sat")
await _add_key(session, "b", 500, "http://m1", "sat")
await _add_key(session, "c", 7000, "http://m1", "msat")
await _add_key(session, "d", 200, "http://m2", "sat")
result = await balances_by_mint_and_unit(
session, ["http://m1", "http://m2"], ["sat", "msat"]
)
assert result[("http://m1", "sat")] == 1500
assert result[("http://m1", "msat")] == 7000
assert result[("http://m2", "sat")] == 200
@pytest.mark.asyncio
async def test_filters_out_unrequested_mints_and_units(session: AsyncSession) -> None:
await _add_key(session, "a", 1000, "http://wanted", "sat")
await _add_key(session, "b", 999, "http://other", "sat")
await _add_key(session, "c", 888, "http://wanted", "usd")
result = await balances_by_mint_and_unit(session, ["http://wanted"], ["sat"])
assert result == {("http://wanted", "sat"): 1000}
@pytest.mark.asyncio
async def test_excludes_rows_with_null_mint_or_currency(session: AsyncSession) -> None:
await _add_key(session, "a", 1000, "http://m1", "sat")
await _add_key(session, "b", 4242, None, None)
result = await balances_by_mint_and_unit(session, ["http://m1"], ["sat"])
assert result == {("http://m1", "sat"): 1000}
@pytest.mark.asyncio
async def test_scalar_balance_for_one_mint_and_unit(session: AsyncSession) -> None:
await _add_key(session, "a", 1000, "http://m1", "sat")
await _add_key(session, "b", 500, "http://m1", "sat")
await _add_key(session, "c", 9000, "http://m1", "msat")
await _add_key(session, "d", 700, "http://m2", "sat")
assert await balance_for_mint_and_unit(session, "http://m1", "sat") == 1500
assert await balance_for_mint_and_unit(session, "http://missing", "sat") == 0
@pytest.mark.asyncio
async def test_empty_inputs_return_empty_mapping(session: AsyncSession) -> None:
await _add_key(session, "a", 1000, "http://m1", "sat")
assert await balances_by_mint_and_unit(session, [], ["sat"]) == {}
assert await balances_by_mint_and_unit(session, ["http://m1"], []) == {}
+132 -4
View File
@@ -15,6 +15,7 @@ os.environ.setdefault("LIGHTNING_ADDRESS", "test@stm.to")
from routstr.core.settings import settings
from routstr.payment.cost_calculation import CostData, MaxCostData, calculate_cost
from routstr.payment.models import Architecture, Model, Pricing
@pytest.fixture(autouse=True)
@@ -527,11 +528,136 @@ async def test_openrouter_upstream_inference_cost_components_are_used() -> None:
result = await calculate_cost(response, max_cost=100000)
assert isinstance(result, CostData)
assert result.input_msats == 994
assert result.output_msats == 3477
assert result.input_msats == 995
assert result.output_msats == 3476
assert result.cache_read_msats == 758
assert result.cache_creation_msats == 0
assert result.input_msats + result.output_msats == result.total_msats == 4471
@pytest.mark.asyncio
async def test_usd_cache_breakdown_matches_token_priced_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Authoritative USD totals must retain model-specific cache-rate ratios."""
monkeypatch.setattr(settings, "fixed_pricing", False)
model = Model(
id="cache-priced-model",
name="cache-priced-model",
created=0,
description="",
context_length=8192,
architecture=Architecture(
modality="text",
input_modalities=["text"],
output_modalities=["text"],
tokenizer="test",
instruct_type=None,
),
pricing=Pricing(prompt=0.01, completion=0.02),
sats_pricing=Pricing(
prompt=0.01,
completion=0.02,
input_cache_read=0.001,
input_cache_write=0.01,
),
per_request_limits=None,
top_provider=None,
)
usage = {
"prompt_tokens": 1000,
"completion_tokens": 100,
"prompt_tokens_details": {"cached_tokens": 900},
}
token_result = await calculate_cost(
{"model": model.id, "usage": usage},
max_cost=100_000,
model_obj=model,
)
usd_result = await calculate_cost(
{
"model": model.id,
"usage": {
**usage,
"cost": 0.000195,
"cost_details": {
"input_cost": 0.000095,
"output_cost": 0.0001,
},
},
},
max_cost=100_000,
model_obj=model,
provider_fee=1.0,
)
assert isinstance(token_result, CostData)
assert isinstance(usd_result, CostData)
assert usd_result.total_msats == token_result.total_msats == 3900
assert usd_result.input_msats + usd_result.output_msats == usd_result.total_msats
assert usd_result.cache_read_msats == token_result.cache_read_msats == 900
@pytest.mark.asyncio
async def test_usd_cache_breakdown_does_not_absorb_total_rounding_remainder(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Sub-msat cache components truncate like the token-priced path."""
monkeypatch.setattr(settings, "fixed_pricing", False)
model = Model(
id="sub-msat-cache-model",
name="sub-msat-cache-model",
created=0,
description="",
context_length=8192,
architecture=Architecture(
modality="text",
input_modalities=["text"],
output_modalities=["text"],
tokenizer="test",
instruct_type=None,
),
pricing=Pricing(prompt=0.001, completion=0.001),
sats_pricing=Pricing(
prompt=0.001,
completion=0.001,
input_cache_write=0.0006,
),
per_request_limits=None,
top_provider=None,
)
usage = {
"input_tokens": 0,
"output_tokens": 0,
"cache_creation_input_tokens": 1,
}
token_result = await calculate_cost(
{"model": model.id, "usage": usage},
max_cost=100_000,
model_obj=model,
)
usd_result = await calculate_cost(
{
"model": model.id,
"usage": {
**usage,
"cost": 0.00000003,
"cost_details": {"input_cost": 0.00000003},
},
},
max_cost=100_000,
model_obj=model,
provider_fee=1.0,
)
assert isinstance(token_result, CostData)
assert isinstance(usd_result, CostData)
assert usd_result.total_msats == token_result.total_msats == 1
assert usd_result.cache_creation_msats == token_result.cache_creation_msats == 0
# ============================================================================
# PPQ.AI BYOK: upstream_inference_cost + BYOK fee billing
#
@@ -568,12 +694,14 @@ async def test_ppq_byok_bills_upstream_inference_cost_plus_fee() -> None:
# msats), not the fee alone (~0.0023 USD → ~45k msats). ~20× correction.
assert result.total_msats == 940274
assert result.input_msats + result.output_msats == result.total_msats
assert result.input_msats == 926546
assert result.output_msats == 13728
assert result.input_msats == 926547
assert result.output_msats == 13727
assert result.total_usd == pytest.approx(0.047013667305)
# Token normalisation (OpenAI dialect: cached included in prompt_tokens)
assert result.input_tokens == 5070 # 164371 - 159301
assert result.cache_read_input_tokens == 159301
assert result.cache_read_msats == 897966
assert result.cache_creation_msats == 0
assert result.output_tokens == 99
+113
View File
@@ -0,0 +1,113 @@
"""Response-contract tests for Routstr cost metadata across paid paths."""
import json
import os
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
os.environ.setdefault("UPSTREAM_API_KEY", "test")
from routstr.core.db import ApiKey # noqa: E402
from routstr.upstream.base import BaseUpstreamProvider # noqa: E402
COST_DATA = {
"base_msats": 0,
"input_msats": 1_200,
"output_msats": 300,
"total_msats": 1_500,
"total_usd": 0.0001,
"input_tokens": 10,
"output_tokens": 3,
"cache_read_input_tokens": 8,
"cache_creation_input_tokens": 2,
"cache_read_msats": 80,
"cache_creation_msats": 40,
}
def _provider() -> BaseUpstreamProvider:
return BaseUpstreamProvider(base_url="http://test", api_key="upstream-key")
def _key() -> ApiKey:
return ApiKey(hashed_key="abcdef0123" * 4, balance=1_000_000)
def _session() -> Any:
session = MagicMock()
session.refresh = AsyncMock()
return session
def _upstream_response(payload: dict) -> httpx.Response:
return httpx.Response(
200,
json=payload,
request=httpx.Request("POST", "http://test"),
)
def _assert_cost_contract(response: Any) -> None:
body = json.loads(response.body)
assert body["usage"]["cost"] == {
"base_msats": 0,
"input_msats": 1_200,
"output_msats": 300,
"total_msats": 1_500,
"total_usd": 0.0001,
"cache_read_input_tokens": 8,
"cache_creation_input_tokens": 2,
"cache_read_msats": 80,
"cache_creation_msats": 40,
}
assert response.headers["X-Routstr-Cost-Msats"] == "1500"
assert response.headers["X-Routstr-Input-Cost-Msats"] == "1200"
assert response.headers["X-Routstr-Output-Cost-Msats"] == "300"
@pytest.mark.asyncio
async def test_balance_chat_completion_uses_shared_cost_contract() -> None:
provider = _provider()
with patch(
"routstr.upstream.base.adjust_payment_for_tokens",
new=AsyncMock(return_value=dict(COST_DATA)),
):
response = await provider.handle_non_streaming_chat_completion(
_upstream_response(
{
"model": "test-model",
"usage": {"prompt_tokens": 10, "completion_tokens": 3},
}
),
_key(),
_session(),
deducted_max_cost=10_000,
)
_assert_cost_contract(response)
@pytest.mark.asyncio
async def test_balance_responses_completion_uses_shared_cost_contract() -> None:
provider = _provider()
with patch(
"routstr.upstream.base.adjust_payment_for_tokens",
new=AsyncMock(return_value=dict(COST_DATA)),
):
response = await provider.handle_non_streaming_responses_completion(
_upstream_response(
{
"model": "test-model",
"usage": {"input_tokens": 10, "output_tokens": 3},
}
),
_key(),
_session(),
deducted_max_cost=10_000,
)
_assert_cost_contract(response)
+11 -14
View File
@@ -4,7 +4,7 @@ Tests admin endpoints that are testable without full app setup:
withdraw validation, authentication guards, and slug validation.
"""
from unittest.mock import Mock, patch
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import HTTPException, Request
@@ -46,22 +46,19 @@ async def test_withdraw_rejects_insufficient_balance() -> None:
request = Request(scope={"type": "http", "method": "POST"})
with patch("routstr.core.admin.get_wallet") as mock_wallet, \
patch("routstr.core.admin.get_proofs_per_mint_and_unit") as mock_proofs, \
patch("routstr.core.admin.slow_filter_spend_proofs") as mock_filter:
mock_w = Mock()
mock_w.keysets = {}
mock_w.proofs = []
mock_wallet.return_value = mock_w
mock_proofs.return_value = []
mock_filter.return_value = []
with patch(
"routstr.core.admin.send_token",
new=AsyncMock(
side_effect=ValueError(
"No trusted mint has 1000000 sat available; balances={}"
)
),
):
with pytest.raises(HTTPException) as exc_info:
await withdraw(request, WithdrawRequest(amount=1000000, unit="sat"))
assert exc_info.value.status_code == 400
assert "Insufficient" in str(exc_info.value.detail)
assert exc_info.value.status_code == 400
assert "Insufficient" in str(exc_info.value.detail)
# ===========================================================================
+85
View File
@@ -0,0 +1,85 @@
from unittest.mock import MagicMock, patch
import pytest
from sqlalchemy.pool import StaticPool
from routstr.core import db
from routstr.core.db import create_db_engine
from routstr.core.settings import settings
@pytest.mark.asyncio
async def test_engine_uses_validated_bounded_pool_settings(
monkeypatch: pytest.MonkeyPatch, tmp_path: object
) -> None:
monkeypatch.setattr(settings, "database_pool_size", 12)
monkeypatch.setattr(settings, "database_max_overflow", 3)
monkeypatch.setattr(settings, "database_pool_timeout", 2.5)
monkeypatch.setattr(settings, "database_pool_recycle", 900)
monkeypatch.setattr(settings, "database_pool_pre_ping", False)
engine = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/pool.db")
try:
assert engine.pool.size() == 12 # type: ignore[attr-defined]
assert engine.pool._max_overflow == 3 # type: ignore[attr-defined]
assert engine.pool._timeout == 2.5 # type: ignore[attr-defined]
assert engine.pool._recycle == 900
assert engine.pool._pre_ping is False
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_memory_sqlite_keeps_static_pool(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings, "database_pool_pre_ping", True)
engine = create_db_engine("sqlite+aiosqlite://")
try:
assert isinstance(engine.pool, StaticPool)
assert engine.pool._pre_ping is True
finally:
await engine.dispose()
def test_non_sqlite_backend_enables_pre_ping_automatically(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings, "database_pool_pre_ping", False)
fake_engine = MagicMock()
with (
patch.object(db, "create_async_engine", return_value=fake_engine) as factory,
patch.object(db.event, "listen") as listen,
):
created = create_db_engine("postgresql+asyncpg://user:pass@db/node")
assert created is fake_engine
assert factory.call_args.kwargs["pool_pre_ping"] is True
assert listen.call_count == 2
@pytest.mark.asyncio
async def test_every_created_engine_warns_for_long_checkouts(
monkeypatch: pytest.MonkeyPatch, tmp_path: object
) -> None:
monkeypatch.setattr(settings, "database_pool_hold_warn_seconds", 0.0)
monkeypatch.setattr(settings, "database_pool_pre_ping", False)
first = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/first.db")
second = create_db_engine(f"sqlite+aiosqlite:///{tmp_path}/second.db")
try:
with patch.object(db.logger, "warning") as warning:
async with first.connect() as connection:
await connection.exec_driver_sql("SELECT 1")
async with second.connect() as connection:
await connection.exec_driver_sql("SELECT 1")
assert warning.call_count == 2
assert all(
call.kwargs["extra"]["threshold_seconds"] == 0.0
for call in warning.call_args_list
)
finally:
await first.dispose()
await second.dispose()
+63 -40
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from typing import AsyncGenerator
from typing import Any, AsyncGenerator
from unittest.mock import AsyncMock, MagicMock
import pytest
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
@@ -8,7 +9,8 @@ from sqlalchemy.pool import StaticPool
from sqlmodel import SQLModel, select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.core.db import ApiKey
from routstr.auth import get_reservation_snapshot, pay_for_request
from routstr.core.db import ApiKey, ReservationRelease
from routstr.upstream.ehbp import (
finalize_ehbp_actual_cost_payment,
finalize_ehbp_max_cost_payment,
@@ -43,18 +45,38 @@ async def _api_key(session: AsyncSession, hashed_key: str) -> ApiKey | None:
).one_or_none()
def _fail_nth_api_key_update(
session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
target_update: int,
) -> None:
"""Return rowcount=0 for one API-key UPDATE without mutating the database."""
original_exec = session.exec
api_key_updates = 0
async def exec_with_failure(
statement: Any, *args: Any, **kwargs: Any
) -> Any:
nonlocal api_key_updates
table = getattr(statement, "table", None)
if getattr(table, "name", None) == "api_keys":
api_key_updates += 1
if api_key_updates == target_update:
return MagicMock(rowcount=0)
return await original_exec(statement, *args, **kwargs)
monkeypatch.setattr(session, "exec", exec_with_failure)
@pytest.mark.asyncio
async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve(
session: AsyncSession,
) -> None:
key = ApiKey(
hashed_key="ehbp-actual",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
key = ApiKey(hashed_key="ehbp-actual", balance=10_000)
session.add(key)
await session.commit()
await pay_for_request(key, 3_000, session)
reservation = await get_reservation_snapshot(key, session)
await finalize_ehbp_actual_cost_payment(
key,
@@ -68,6 +90,7 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
"input_msats": 500,
"output_msats": 700,
},
reservation_snapshot=reservation,
)
updated = await _api_key(session, "ehbp-actual")
@@ -82,28 +105,22 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
session: AsyncSession,
) -> None:
parent = ApiKey(
hashed_key="ehbp-parent",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
parent = ApiKey(hashed_key="ehbp-parent", balance=10_000)
child = ApiKey(
hashed_key="ehbp-child",
balance=0,
reserved_balance=3_000,
reserved_at=123,
parent_key_hash="ehbp-parent",
hashed_key="ehbp-child", balance=0, parent_key_hash="ehbp-parent"
)
session.add(parent)
session.add(child)
await session.commit()
await pay_for_request(child, 3_000, session)
reservation = await get_reservation_snapshot(child, session)
await finalize_ehbp_max_cost_payment(
child,
session,
max_cost_for_model=3_000,
model_id="tinfoil/model",
reservation_snapshot=reservation,
)
updated_parent = await _api_key(session, "ehbp-parent")
@@ -123,17 +140,16 @@ async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
@pytest.mark.asyncio
async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matches_no_rows(
session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
key = ApiKey(
hashed_key="ehbp-missing-parent",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
key = ApiKey(hashed_key="ehbp-missing-parent", balance=10_000)
session.add(key)
await session.commit()
await session.delete(key)
await session.commit()
await pay_for_request(key, 3_000, session)
reservation = await get_reservation_snapshot(key, session)
_fail_nth_api_key_update(session, monkeypatch, target_update=1)
rollback_spy = AsyncMock(wraps=session.rollback)
monkeypatch.setattr(session, "rollback", rollback_spy)
await finalize_ehbp_actual_cost_payment(
key,
@@ -141,45 +157,52 @@ async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matche
reserved_cost_for_model=3_000,
model_id="tinfoil/model",
cost_info={"total_msats": 1_200},
reservation_snapshot=reservation,
)
assert await _api_key(session, "ehbp-missing-parent") is None
rollback_spy.assert_awaited_once()
updated = await _api_key(session, "ehbp-missing-parent")
assert updated is not None
assert updated.balance == 10_000
assert updated.reserved_balance == 3_000
assert updated.total_spent == 0
release = await session.get(ReservationRelease, reservation.release_id)
assert release is not None
assert release.status == "active"
@pytest.mark.asyncio
async def test_finalize_max_cost_payment_rolls_back_parent_when_child_update_matches_no_rows(
session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
parent = ApiKey(
hashed_key="ehbp-rollback-parent",
balance=10_000,
reserved_balance=3_000,
reserved_at=123,
)
parent = ApiKey(hashed_key="ehbp-rollback-parent", balance=10_000)
child = ApiKey(
hashed_key="ehbp-missing-child",
balance=0,
reserved_balance=3_000,
reserved_at=123,
parent_key_hash="ehbp-rollback-parent",
)
session.add(parent)
session.add(child)
await session.commit()
await session.delete(child)
await session.commit()
await pay_for_request(child, 3_000, session)
reservation = await get_reservation_snapshot(child, session)
_fail_nth_api_key_update(session, monkeypatch, target_update=2)
await finalize_ehbp_max_cost_payment(
child,
session,
max_cost_for_model=3_000,
model_id="tinfoil/model",
reservation_snapshot=reservation,
)
updated_parent = await _api_key(session, "ehbp-rollback-parent")
assert updated_parent is not None
assert updated_parent.balance == 10_000
assert updated_parent.reserved_balance == 3_000
assert updated_parent.reserved_at == 123
assert updated_parent.total_spent == 0
assert await _api_key(session, "ehbp-missing-child") is None
updated_child = await _api_key(session, "ehbp-missing-child")
assert updated_child is not None
assert updated_child.reserved_balance == 3_000
assert updated_child.total_spent == 0
+238 -10
View File
@@ -1,5 +1,5 @@
import asyncio
from collections.abc import AsyncIterator
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
@@ -13,9 +13,19 @@ from routstr import wallet
from routstr.core import db
@asynccontextmanager
async def _session_context(session: Mock) -> AsyncIterator[Mock]:
yield session
class _SessionContext:
def __init__(self, session: Mock) -> None:
self.session = session
async def __aenter__(self) -> Mock:
return self.session
async def __aexit__(self, *args: object) -> None:
return None
def _session_context(session: Mock) -> _SessionContext:
return _SessionContext(session)
@pytest.mark.asyncio
@@ -47,7 +57,7 @@ async def test_fee_payout_checkpoint_is_atomic_and_durable() -> None:
@pytest.mark.asyncio
async def test_fee_payout_checkpoints_before_sending() -> None:
async def test_fee_payout_prepares_wallet_then_checkpoints_before_sending() -> None:
session = Mock()
fee = SimpleNamespace(
accumulated_msats=5_000,
@@ -57,6 +67,10 @@ async def test_fee_payout_checkpoints_before_sending() -> None:
payout_wallet = Mock()
events: list[str] = []
async def prepare(*_args: object, **_kwargs: object) -> Mock:
events.append("prepare")
return payout_wallet
async def checkpoint(*_args: object) -> bool:
events.append("checkpoint")
return True
@@ -77,18 +91,92 @@ async def test_fee_payout_checkpoints_before_sending() -> None:
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
patch(
"routstr.wallet.db.create_session", return_value=_session_context(session)
),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch("routstr.wallet.db.reset_routstr_fee", side_effect=checkpoint),
patch("routstr.wallet.db.complete_routstr_fee_payout", side_effect=complete),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=payout_wallet)),
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=prepare)),
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
patch("routstr.wallet.raw_send_to_lnurl", side_effect=send),
):
with pytest.raises(asyncio.CancelledError):
await wallet.periodic_routstr_fee_payout()
assert events == ["checkpoint", "send", "complete"]
assert events == ["prepare", "checkpoint", "send", "complete"]
@pytest.mark.asyncio
async def test_fee_payout_preparation_failure_does_not_checkpoint() -> None:
session = Mock()
fee = SimpleNamespace(
accumulated_msats=5_000,
payout_in_progress_msats=0,
payout_started_at=None,
)
checkpoint = AsyncMock()
with (
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
patch(
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch(
"routstr.wallet.db.create_session", return_value=_session_context(session)
),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch("routstr.wallet.db.reset_routstr_fee", checkpoint),
patch(
"routstr.wallet.get_wallet",
AsyncMock(side_effect=RuntimeError("wallet unavailable")),
),
):
with pytest.raises(asyncio.CancelledError):
await wallet.periodic_routstr_fee_payout()
checkpoint.assert_not_awaited()
@pytest.mark.asyncio
async def test_fee_payout_lost_checkpoint_race_does_not_send() -> None:
session = Mock()
fee = SimpleNamespace(
accumulated_msats=5_000,
payout_in_progress_msats=0,
payout_started_at=None,
)
send = AsyncMock()
with (
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
patch(
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch(
"routstr.wallet.db.create_session", return_value=_session_context(session)
),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch(
"routstr.wallet.db.reset_routstr_fee",
AsyncMock(return_value=False),
),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
patch("routstr.wallet.raw_send_to_lnurl", send),
patch("routstr.wallet.logger.warning") as warning,
):
with pytest.raises(asyncio.CancelledError):
await wallet.periodic_routstr_fee_payout()
send.assert_not_awaited()
warning.assert_called_once_with("Routstr fee payout was already claimed")
@pytest.mark.asyncio
@@ -107,7 +195,9 @@ async def test_fee_payout_does_not_retry_an_unresolved_checkpoint() -> None:
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
patch(
"routstr.wallet.db.create_session", return_value=_session_context(session)
),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock()) as checkpoint,
patch("routstr.wallet.get_wallet", AsyncMock()) as get_wallet,
@@ -141,7 +231,9 @@ async def test_fee_payout_keeps_checkpoint_when_send_outcome_is_unknown() -> Non
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
patch(
"routstr.wallet.db.create_session", return_value=_session_context(session)
),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
patch("routstr.wallet.db.complete_routstr_fee_payout", complete),
@@ -158,3 +250,139 @@ async def test_fee_payout_keeps_checkpoint_when_send_outcome_is_unknown() -> Non
complete.assert_not_awaited()
critical.assert_called_once()
@pytest.mark.asyncio
async def test_fee_payout_cancellation_during_send_alerts_and_propagates() -> None:
session = Mock()
fee = SimpleNamespace(
accumulated_msats=5_000,
payout_in_progress_msats=0,
payout_started_at=None,
)
complete = AsyncMock()
with (
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
patch("routstr.wallet.asyncio.sleep", AsyncMock(return_value=None)),
patch(
"routstr.wallet.db.create_session", return_value=_session_context(session)
),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
patch("routstr.wallet.db.complete_routstr_fee_payout", complete),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
patch(
"routstr.wallet.raw_send_to_lnurl",
AsyncMock(side_effect=asyncio.CancelledError()),
),
patch("routstr.wallet.logger.critical") as critical,
):
with pytest.raises(asyncio.CancelledError):
await wallet.periodic_routstr_fee_payout()
complete.assert_not_awaited()
critical.assert_called_once()
assert critical.call_args.args[0] == (
"Routstr fee payout outcome is unknown; manual reconciliation required"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure_site", ["session", "completion"])
async def test_fee_payout_completion_failures_use_sent_checkpoint_alert(
failure_site: str,
) -> None:
session = Mock()
fee = SimpleNamespace(
accumulated_msats=5_000,
payout_in_progress_msats=0,
payout_started_at=None,
)
completion = AsyncMock()
if failure_site == "session":
create_session = Mock(
side_effect=[
_session_context(session),
_session_context(session),
RuntimeError("pool unavailable"),
]
)
else:
create_session = Mock(return_value=_session_context(session))
completion.side_effect = RuntimeError("checkpoint unavailable")
with (
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
patch(
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch("routstr.wallet.db.create_session", create_session),
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
patch("routstr.wallet.db.complete_routstr_fee_payout", completion),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
patch("routstr.wallet.raw_send_to_lnurl", AsyncMock(return_value=5)),
patch("routstr.wallet.logger.critical") as critical,
):
with pytest.raises(asyncio.CancelledError):
await wallet.periodic_routstr_fee_payout()
critical.assert_called_once()
assert critical.call_args.args[0] == (
"Routstr fee payout sent but checkpoint was not completed"
)
@pytest.mark.asyncio
async def test_fee_payout_releases_db_connection_during_send(tmp_path: object) -> None:
"""With pool_size=1, the payout must not hold a connection while the
external LNURL send is in flight, or the completion step would starve."""
engine = create_async_engine(
f"sqlite+aiosqlite:///{tmp_path}/payout.db", pool_size=1, max_overflow=0
)
async with engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
async with AsyncSession(engine) as session:
session.add(db.RoutstrFee(id=1, accumulated_msats=5_000_000))
await session.commit()
@asynccontextmanager
async def create_session() -> AsyncGenerator[AsyncSession, None]:
async with AsyncSession(engine, expire_on_commit=False) as session:
yield session
async def send(*_args: object, **_kwargs: object) -> int:
assert engine.pool.checkedout() == 0 # type: ignore[attr-defined]
return 5
try:
with (
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
patch(
"routstr.wallet.asyncio.sleep",
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
),
patch("routstr.wallet.db.create_session", create_session),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
patch("routstr.wallet.raw_send_to_lnurl", side_effect=send),
):
with pytest.raises(asyncio.CancelledError):
await wallet.periodic_routstr_fee_payout()
async with AsyncSession(engine) as session:
fee = await db.get_routstr_fee(session)
assert fee.payout_in_progress_msats == 0
assert fee.total_paid_msats == 5_000_000
finally:
await engine.dispose()
+70
View File
@@ -4,6 +4,9 @@ import subprocess
import sys
from pathlib import Path
from alembic.config import Config
from alembic.script import ScriptDirectory
def _run_alembic(root: Path, database_url: str, revision: str) -> None:
env = os.environ.copy()
@@ -18,6 +21,40 @@ def _run_alembic(root: Path, database_url: str, revision: str) -> None:
)
def test_fresh_node_migrates_fee_payout_schema_to_head(tmp_path: Path) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "fresh-node.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
_run_alembic(root, database_url, "head")
with sqlite3.connect(database_path) as connection:
version = connection.execute(
"SELECT version_num FROM alembic_version"
).fetchone()
columns = {
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
}
fee = connection.execute(
"SELECT id, accumulated_msats, total_paid_msats, last_paid_at, "
"payout_in_progress_msats, payout_started_at FROM routstr_fees"
).fetchone()
migration_config = Config(str(root / "alembic.ini"))
assert version == (
ScriptDirectory.from_config(migration_config).get_current_head(),
)
assert {
"id",
"accumulated_msats",
"total_paid_msats",
"last_paid_at",
"payout_in_progress_msats",
"payout_started_at",
} <= columns
assert fee == (1, 0, 0, None, 0, None)
def test_fee_payout_checkpoint_migration_preserves_existing_row(
tmp_path: Path,
) -> None:
@@ -44,3 +81,36 @@ def test_fee_payout_checkpoint_migration_preserves_existing_row(
).fetchone()
assert row == (5000, 1000, 123, 0, None)
def test_fee_payout_checkpoint_repair_restores_columns_missing_at_old_head(
tmp_path: Path,
) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "migration.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
old_head = "7f2843d3f4e4"
_run_alembic(root, database_url, old_head)
# Reproduce a database that was stamped to head after a duplicate-column or
# unknown-revision recovery skipped part of the migration chain.
with sqlite3.connect(database_path) as connection:
connection.execute("ALTER TABLE routstr_fees DROP COLUMN payout_started_at")
connection.execute(
"ALTER TABLE routstr_fees DROP COLUMN payout_in_progress_msats"
)
connection.commit()
_run_alembic(root, database_url, "head")
with sqlite3.connect(database_path) as connection:
columns = {
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
}
row = connection.execute(
"SELECT payout_in_progress_msats, payout_started_at "
"FROM routstr_fees WHERE id = 1"
).fetchone()
assert {"payout_in_progress_msats", "payout_started_at"} <= columns
assert row == (0, None)
+392 -10
View File
@@ -1,11 +1,33 @@
import asyncio
from collections.abc import AsyncGenerator, Generator
from contextlib import asynccontextmanager
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from sqlalchemy.ext.asyncio import create_async_engine
from sqlmodel import SQLModel
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.wallet import fetch_all_balances
@pytest.fixture(autouse=True)
def clear_balance_fetch_state() -> Generator[None, None, None]:
from routstr import wallet
wallet._balance_fetch_failures.clear()
wallet._balance_fetch_locks.clear()
wallet._mint_supported_units.clear()
wallet._MintRateGuard._guards.clear()
yield
wallet._balance_fetch_failures.clear()
wallet._balance_fetch_locks.clear()
wallet._mint_supported_units.clear()
wallet._MintRateGuard._guards.clear()
@asynccontextmanager
async def _fake_session(): # type: ignore[no-untyped-def]
yield MagicMock()
@@ -23,11 +45,13 @@ def _patches( # type: ignore[no-untyped-def]
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
AsyncMock(side_effect=lambda proofs, wallet, **kwargs: proofs),
),
patch(
"routstr.wallet.db.balances_for_mint_and_unit",
AsyncMock(return_value=user_balance_msats),
"routstr.wallet.db.balances_by_mint_and_unit",
AsyncMock(
return_value={("http://primary:3338", "sat"): user_balance_msats}
),
),
patch("routstr.wallet.db.create_session", _fake_session),
]
@@ -38,8 +62,9 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
"""With empty cashu_mints, balances are still fetched for primary_mint."""
from routstr.core.settings import settings
with patch.object(settings, "cashu_mints", []), patch.object(
settings, "primary_mint", "http://primary:3338"
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
):
for p in _patches(proof_amount=1000):
p.start()
@@ -54,13 +79,314 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
assert total_wallet == 1000
@pytest.mark.asyncio
async def test_fetch_all_balances_uses_units_advertised_by_mint() -> None:
from routstr.core.settings import settings
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch(
"routstr.wallet._get_supported_mint_units",
AsyncMock(return_value=["sat"]),
) as supported_units,
):
for p in _patches(proof_amount=1000):
p.start()
try:
details, *_ = await fetch_all_balances()
finally:
patch.stopall()
supported_units.assert_awaited_once_with("http://mint:3338")
assert [detail["unit"] for detail in details] == ["sat"]
@pytest.mark.asyncio
async def test_unit_discovery_failure_returns_structured_balance_error() -> None:
from routstr.core.settings import settings
get_wallet = AsyncMock()
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch(
"routstr.wallet._get_supported_mint_units",
AsyncMock(side_effect=httpx.ConnectError("mint unavailable")),
),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
):
details, *_ = await fetch_all_balances()
assert details[0]["unit"] == settings.primary_mint_unit
assert details[0]["error_code"] == "unreachable"
assert details[0]["retry_after_seconds"] > 0
get_wallet.assert_not_awaited()
@pytest.mark.asyncio
async def test_supported_mint_units_come_from_active_keysets() -> None:
from routstr.core.settings import settings
from routstr.wallet import _get_supported_mint_units
# Cashu versions/mints may deserialize keyset units as either strings or
# Unit enum-like objects. Both representations must be accepted.
sat = MagicMock(active=True, unit="sat")
msat = MagicMock(active=False, unit="msat")
usd = MagicMock(active=True)
usd.unit.name = "usd"
wallet = MagicMock()
wallet._get_keysets = AsyncMock(return_value=[usd, msat, sat])
with (
patch.object(settings, "primary_mint_unit", "sat"),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
):
units = await _get_supported_mint_units("http://mint:3338")
cached_units = await _get_supported_mint_units("http://mint:3338")
assert units == ["sat", "usd"]
assert cached_units == units
wallet._get_keysets.assert_awaited_once()
@pytest.mark.asyncio
async def test_fetch_all_balances_backs_off_after_connection_failure() -> None:
from routstr.core.settings import settings
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
first = await fetch_all_balances(units=["sat"])
second = await fetch_all_balances(units=["sat"])
assert first[0][0]["error"] == "mint unavailable"
assert first[0][0]["error_code"] == "unreachable"
assert first[0][0]["retry_after_seconds"] == 60
assert second[0][0]["error"] == "mint unavailable"
assert second[0][0]["error_code"] == "unreachable"
assert get_wallet.await_count == 1
warning.assert_called_once()
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.mint.time.monotonic", return_value=71),
patch("routstr.wallet.logger.warning"),
):
await fetch_all_balances(units=["sat"])
assert get_wallet.await_count == 2
@pytest.mark.asyncio
async def test_fetch_all_balances_reports_rate_limit_status() -> None:
from routstr.core.settings import settings
request = httpx.Request("GET", "http://mint:3338/v1/keysets")
response = httpx.Response(429, request=request, headers={"Retry-After": "45"})
error = httpx.HTTPStatusError("rate limited", request=request, response=response)
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch("routstr.wallet.get_wallet", AsyncMock(side_effect=error)),
patch("routstr.wallet.db.create_session", _fake_session),
):
details, *_ = await fetch_all_balances(units=["sat"])
assert details[0]["error_code"] == "rate_limited"
assert details[0]["retry_after_seconds"] == 60
@pytest.mark.asyncio
async def test_balance_failure_applies_mint_cooldown_to_other_units() -> None:
from routstr.core.settings import settings
from routstr.wallet import _mint_cooldown_remaining
mint = "http://mint:3338"
get_wallet = AsyncMock(side_effect=httpx.ConnectError("mint unavailable"))
with (
patch.object(settings, "cashu_mints", [mint]),
patch.object(settings, "primary_mint", mint),
patch("routstr.wallet.get_wallet", get_wallet),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.mint.time.monotonic", return_value=10),
patch("routstr.wallet.logger.warning") as warning,
):
details, *_ = await fetch_all_balances(units=["sat", "msat"])
cooldown = _mint_cooldown_remaining(mint)
assert get_wallet.await_count == 1
assert warning.call_count == 1
assert cooldown == 60
assert details[0]["error"] == "mint unavailable"
assert details[0]["error_code"] == "unreachable"
assert details[1]["error"] == "Mint is unreachable"
assert details[1]["error_code"] == "unreachable"
@pytest.mark.asyncio
async def test_fetch_all_balances_closes_db_session_before_concurrent_mint_io() -> None:
"""Slow mint checks must never run while the balance DB session is open."""
from routstr.core.settings import settings
session_open = False
mint_calls = 0
@asynccontextmanager
async def tracked_session(): # type: ignore[no-untyped-def]
nonlocal session_open
session_open = True
try:
yield MagicMock()
finally:
session_open = False
async def slow_filter(proofs, wallet): # type: ignore[no-untyped-def]
nonlocal mint_calls
assert session_open is False
mint_calls += 1
await asyncio.sleep(0)
return proofs
with (
patch.object(settings, "cashu_mints", ["http://one:3338", "http://two:3338"]),
patch.object(settings, "primary_mint", "http://one:3338"),
patch("routstr.wallet.db.create_session", tracked_session),
patch(
"routstr.wallet.db.balances_by_mint_and_unit",
AsyncMock(return_value={}),
create=True,
),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=1)]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=slow_filter),
),
):
details, *_ = await fetch_all_balances(units=["sat", "msat"])
assert mint_calls == 4
assert all("error" not in detail for detail in details)
@pytest.mark.asyncio
async def test_fetch_all_balances_bounds_parallel_mint_checks() -> None:
"""A slow mint fleet cannot create an unbounded external-I/O fan-out."""
from routstr.core.settings import settings
active = 0
peak = 0
async def slow_filter(proofs, wallet): # type: ignore[no-untyped-def]
nonlocal active, peak
active += 1
peak = max(peak, active)
await asyncio.sleep(0.01)
active -= 1
return proofs
with (
patch.object(
settings,
"cashu_mints",
[f"http://mint-{index}:3338" for index in range(8)],
),
patch.object(settings, "primary_mint", ""),
patch.object(settings, "mint_operation_concurrency", 2),
patch("routstr.wallet.db.create_session", _fake_session),
patch(
"routstr.wallet.db.balances_by_mint_and_unit",
AsyncMock(return_value={}),
),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=slow_filter),
),
):
details, *_ = await fetch_all_balances(units=["sat"])
assert len(details) == 8
assert peak == 2
@pytest.mark.asyncio
async def test_slow_mints_do_not_exhaust_a_single_connection_pool(
tmp_path: Path,
) -> None:
"""Concurrent slow balance refreshes release the sole DB connection promptly."""
from routstr.core.settings import settings
engine = create_async_engine(
f"sqlite+aiosqlite:///{tmp_path / 'pool-pressure.db'}",
pool_size=1,
max_overflow=0,
pool_timeout=0.2,
)
async with engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
@asynccontextmanager
async def single_pool_session() -> AsyncGenerator[AsyncSession, None]:
async with AsyncSession(engine, expire_on_commit=False) as session:
yield session
async def slow_filter(proofs, wallet): # type: ignore[no-untyped-def]
await asyncio.sleep(0.3)
return proofs
try:
with (
patch.object(settings, "cashu_mints", ["http://slow:3338"]),
patch.object(settings, "primary_mint", "http://slow:3338"),
patch.object(settings, "mint_operation_concurrency", 1),
patch("routstr.wallet.db.create_session", single_pool_session),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=slow_filter),
),
):
results = await asyncio.gather(
*(fetch_all_balances(units=["sat"]) for _ in range(6))
)
assert all("error" not in result[0][0] for result in results)
assert engine.pool.checkedout() == 0 # type: ignore[attr-defined]
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_fetch_all_balances_reports_liability_when_wallet_is_empty() -> None:
"""An empty wallet must not hide outstanding user liabilities."""
from routstr.core.settings import settings
with patch.object(settings, "cashu_mints", []), patch.object(
settings, "primary_mint", "http://primary:3338"
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
):
for p in _patches(proof_amount=0, user_balance_msats=5000):
p.start()
@@ -84,9 +410,10 @@ async def test_fetch_all_balances_no_duplicate_primary_mint() -> None:
"""primary_mint already in cashu_mints is not inspected twice."""
from routstr.core.settings import settings
with patch.object(
settings, "cashu_mints", ["http://primary:3338"]
), patch.object(settings, "primary_mint", "http://primary:3338"):
with (
patch.object(settings, "cashu_mints", ["http://primary:3338"]),
patch.object(settings, "primary_mint", "http://primary:3338"),
):
for p in _patches(proof_amount=1000):
p.start()
try:
@@ -98,3 +425,58 @@ async def test_fetch_all_balances_no_duplicate_primary_mint() -> None:
assert [d["mint_url"] for d in details] == ["http://primary:3338"]
assert total_wallet == 1000
@pytest.mark.asyncio
async def test_fetch_all_balances_degrades_when_liability_read_fails() -> None:
from routstr.core.settings import settings
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
patch("routstr.wallet.db.create_session", _fake_session),
patch(
"routstr.wallet.db.balances_by_mint_and_unit",
AsyncMock(side_effect=RuntimeError("db pool exhausted")),
),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=1000)]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
),
):
details, total_wallet, total_user, owner = await fetch_all_balances(
units=["sat"]
)
assert details[0]["error"] == "db pool exhausted"
assert details[0]["wallet_balance"] == 1000
assert details[0]["user_balance"] == 0
assert details[0]["owner_balance"] == 0
assert (total_wallet, total_user, owner) == (1000, 0, 0)
@pytest.mark.asyncio
async def test_liability_error_keeps_more_specific_mint_error() -> None:
from routstr.core.settings import settings
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
patch("routstr.wallet.db.create_session", _fake_session),
patch(
"routstr.wallet.db.balances_by_mint_and_unit",
AsyncMock(side_effect=RuntimeError("db pool exhausted")),
),
patch(
"routstr.wallet.get_wallet",
AsyncMock(side_effect=RuntimeError("mint down")),
),
):
details, *_ = await fetch_all_balances(units=["sat"])
assert details[0]["error"] == "mint down"
+466
View File
@@ -0,0 +1,466 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import MintQuoteState, Proof
from routstr.lightning import (
InvoiceRecoverRequest,
_invoice_settlement_locks,
_is_outputs_already_signed,
_mint_invoice_quote,
check_invoice_payment,
get_invoice_status,
recover_invoice,
)
from routstr.wallet import Wallet
def _invoice(**overrides: object) -> SimpleNamespace:
values = {
"id": "invoice-1",
"payment_hash": "quote-1",
"amount_sats": 100,
"purpose": "create",
"status": "pending",
"paid_at": None,
"api_key_hash": None,
"mint_url": "http://mint:3338",
"balance_limit": None,
"balance_limit_reset": None,
"validity_date": None,
"created_at": 1,
"expires_at": 2,
}
values.update(overrides)
return SimpleNamespace(**values)
def _proof(amount: int, mint_id: str, *, reserved: bool = False) -> Proof:
return Proof(amount=amount, mint_id=mint_id, reserved=reserved)
def _recovery_wallet(
error: Exception,
*,
proofs_before: list[Proof] | None = None,
proofs_after: list[Proof] | None = None,
) -> Mock:
async def load_proofs(*, reload: bool) -> None:
if wallet.load_proofs.await_count >= 2 and proofs_after is not None:
wallet.proofs = list(proofs_after)
wallet = Mock(
mint=AsyncMock(side_effect=error),
keysets={"keyset-1": Mock()},
restore_tokens_for_keyset=AsyncMock(),
load_proofs=AsyncMock(side_effect=load_proofs),
proofs=list(proofs_before or []),
)
return wallet
@pytest.mark.asyncio
async def test_invoice_mint_recovers_quote_linked_outputs_already_signed() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs have already been signed before (Code: 11003)"),
proofs_after=[_proof(100, "quote-1")],
)
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
wallet.restore_tokens_for_keyset.assert_awaited_once_with(
"keyset-1", to=1, batch=25
)
assert wallet.load_proofs.await_count == 2
@pytest.mark.asyncio
async def test_invoice_mint_accepts_preloaded_quote_linked_proofs() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("must not mint"),
proofs_before=[_proof(64, "quote-1"), _proof(36, "quote-1")],
)
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
wallet.mint.assert_not_awaited()
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_invoice_mint_does_not_accept_unrelated_11003_text() -> None:
invoice = _invoice()
error = Exception("backend request 11003 failed")
wallet = _recovery_wallet(error)
with pytest.raises(Exception) as caught:
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
assert caught.value is error
wallet.restore_tokens_for_keyset.assert_not_awaited()
@pytest.mark.asyncio
async def test_installed_cashu_error_shape_recognizes_realistic_11003_phrase() -> None:
request = httpx.Request("POST", "http://mint:3338/v1/mint/bolt11")
response = httpx.Response(
400,
request=request,
json={"detail": "outputs have already been signed before", "code": 11003},
)
with pytest.raises(Exception) as caught:
Wallet.raise_on_error_request(response)
assert _is_outputs_already_signed(caught.value)
@pytest.mark.asyncio
@pytest.mark.parametrize("recovered", [0, 99])
async def test_invoice_mint_rejects_empty_or_short_quote_recovery(
recovered: int,
) -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs already signed (Code: 11003)"),
proofs_after=[_proof(recovered, "quote-1")] if recovered else [],
)
with pytest.raises(RuntimeError, match="expected at least 100"):
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_invoice_mint_rejects_unrelated_concurrent_balance_growth() -> None:
invoice = _invoice()
wallet = _recovery_wallet(
Exception("Mint Error: outputs already signed (Code: 11003)"),
proofs_after=[_proof(10_000, "different-quote")],
)
with pytest.raises(RuntimeError, match="quote-linked recovery returned 0"):
await _mint_invoice_quote(wallet, invoice) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_quote_not_found_is_definitively_unpaid() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
wallet = Mock(
get_mint_quote=AsyncMock(
side_effect=Exception("Mint Error: quote not found (Code: 0)")
)
)
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert result is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
"message",
[
"Mint Error: quote not found (Code: 10000)",
"Mint Error: quote not found (Code: 01)",
"Mint Error: quote not found (Code: 0x10)",
],
)
async def test_quote_not_found_without_exact_code_0_is_not_definitively_unpaid(
message: str,
) -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(side_effect=Exception(message)))
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert result is False
@pytest.mark.asyncio
async def test_quote_not_found_case_insensitive() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
wallet = Mock(
get_mint_quote=AsyncMock(
side_effect=Exception("MINT ERROR: Quote Not Found (code 0)")
)
)
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert result is True
@pytest.mark.asyncio
async def test_non_pending_invoice_is_not_minted() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(status="expired")
session = AsyncMock()
with patch("routstr.lightning.get_wallet", AsyncMock()) as get_wallet:
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
get_wallet.assert_not_awaited()
session.commit.assert_awaited_once()
assert _invoice_settlement_locks == {}
@pytest.mark.asyncio
async def test_ambiguous_invoice_mint_timeout_remains_recoverable() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
state_session = AsyncMock()
state_session.exec.return_value.rowcount = 1
@asynccontextmanager
async def owned_session() -> AsyncIterator[AsyncMock]:
yield state_session
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", owned_session),
patch(
"routstr.lightning._mint_invoice_quote",
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert invoice.status == "settlement_pending"
state_session.commit.assert_awaited_once()
session.rollback.assert_not_awaited()
# One commit closes the initial read transaction before external I/O.
session.commit.assert_awaited_once()
@pytest.mark.asyncio
async def test_quote_not_found_after_payment_confirmation_is_not_unpaid() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
state_session = AsyncMock()
state_session.exec.return_value.rowcount = 1
@asynccontextmanager
async def owned_session() -> AsyncIterator[AsyncMock]:
yield state_session
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", owned_session),
patch(
"routstr.lightning._mint_invoice_quote",
AsyncMock(
side_effect=Exception("Mint Error: quote not found (Code: 0)")
),
),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert result is False
assert invoice.status == "settlement_pending"
@pytest.mark.asyncio
async def test_quote_lookup_timeout_is_not_definitively_unpaid() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(expires_at=0)
session = AsyncMock()
wallet = Mock(
get_mint_quote=AsyncMock(side_effect=httpx.TimeoutException("quote timeout"))
)
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
result = await check_invoice_payment(invoice, session) # type: ignore[arg-type]
assert result is False
@pytest.mark.asyncio
async def test_overdue_invoice_does_not_expire_after_ambiguous_quote_lookup() -> None:
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
session.get.return_value = invoice
check = AsyncMock(return_value=False)
with patch("routstr.lightning.check_invoice_payment", check):
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
assert response.status == "pending"
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_overdue_invoice_expires_only_after_definitive_unpaid_quote() -> None:
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
session.get.return_value = invoice
check = AsyncMock(return_value=True)
async def expire(
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
) -> bool:
assert definitive is True
candidate.status = "expired"
return True
with (
patch("routstr.lightning.check_invoice_payment", check),
patch(
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
side_effect=expire,
) as expire_invoice,
):
response = await get_invoice_status(invoice.id, session) # type: ignore[arg-type]
assert response.status == "expired"
expire_invoice.assert_awaited_once_with(invoice, session, True)
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_recover_applies_authoritative_expiry_helper() -> None:
invoice = _invoice(status="pending", expires_at=0)
session = AsyncMock()
result = Mock()
result.first.return_value = invoice
session.exec.return_value = result
check = AsyncMock(return_value=True)
async def expire(
candidate: SimpleNamespace, _session: AsyncMock, definitive: bool
) -> bool:
assert definitive is True
candidate.status = "expired"
return True
with (
patch("routstr.lightning.check_invoice_payment", check),
patch(
"routstr.lightning._expire_invoice_if_authoritatively_unpaid",
side_effect=expire,
) as expire_invoice,
):
response = await recover_invoice(
InvoiceRecoverRequest(bolt11="lnbc-test"), session # type: ignore[arg-type]
)
assert response.status == "expired"
expire_invoice.assert_awaited_once_with(invoice, session, True)
@pytest.mark.asyncio
async def test_paid_state_write_failure_still_reports_non_expirable_outcome() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice(expires_at=0)
session = AsyncMock()
wallet = Mock(
get_mint_quote=AsyncMock(
return_value=Mock(paid=True, state=MintQuoteState.paid)
)
)
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch(
"routstr.lightning._mint_invoice_quote",
AsyncMock(side_effect=httpx.TimeoutException("response lost")),
),
patch(
"routstr.lightning.create_session",
side_effect=RuntimeError("database unavailable"),
),
patch("routstr.lightning._reload_invoice_view", AsyncMock()),
):
definitively_unpaid = await check_invoice_payment(
invoice, session # type: ignore[arg-type]
)
assert definitively_unpaid is False
assert invoice.status == "pending"
@pytest.mark.asyncio
async def test_settlement_pending_invoice_does_not_expire() -> None:
invoice = _invoice(status="settlement_pending", expires_at=0)
session = AsyncMock()
session.get.return_value = invoice
check = AsyncMock()
with patch("routstr.lightning.check_invoice_payment", check):
response = await get_invoice_status(
invoice.id, session # type: ignore[arg-type]
)
check.assert_awaited_once_with(invoice, session)
assert response.status == "settlement_pending"
session.commit.assert_not_awaited()
@pytest.mark.asyncio
async def test_concurrent_invoice_checks_finalize_once_in_process() -> None:
_invoice_settlement_locks.clear()
invoice = _invoice()
session = AsyncMock()
wallet = Mock(get_mint_quote=AsyncMock(return_value=Mock(paid=True)))
async def refresh(obj: SimpleNamespace) -> None:
return None
session.refresh = AsyncMock(side_effect=refresh)
@asynccontextmanager
async def owned_session() -> AsyncIterator[AsyncMock]:
owned = AsyncMock()
owned.exec.return_value.rowcount = 1
yield owned
with (
patch("routstr.lightning.get_wallet", AsyncMock(return_value=wallet)),
patch("routstr.lightning.create_session", owned_session),
patch("routstr.lightning._mint_invoice_quote", AsyncMock()),
patch(
"routstr.lightning._finalize_invoice_settlement",
AsyncMock(return_value=(True, "b" * 64)),
) as finalize,
):
await asyncio.gather(
check_invoice_payment(invoice, session), # type: ignore[arg-type]
check_invoice_payment(invoice, session), # type: ignore[arg-type]
)
assert invoice.status == "paid"
finalize.assert_awaited_once()
assert _invoice_settlement_locks == {}
+179 -46
View File
@@ -1,70 +1,202 @@
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
timeout as LNURLError instead of hanging.
"""
"""LNURL melt attempts must not misclassify ambiguous payment outcomes."""
import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from cashu.core.base import MeltQuoteState
from routstr.payment import lnurl
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
from routstr.core.settings import settings
from routstr.mint import MintCooldownError, MintRateGuard
from routstr.payment.lnurl import (
MeltOutcomeAmbiguousError,
raw_send_to_lnurl,
)
LNURL_DATA = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
def _wallet() -> tuple[MagicMock, list[MagicMock]]:
proofs = [MagicMock(amount=1000)]
wallet = MagicMock(url="https://mint.test")
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
return wallet, proofs
def _lnurl_patches() -> tuple[Any, Any]:
return (
patch(
"routstr.payment.lnurl.get_lnurl_data",
AsyncMock(return_value=LNURL_DATA),
),
patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
),
)
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
proofs = [MagicMock(amount=1000)]
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
async def test_raw_send_to_lnurl_timeout_keeps_unpaid_outcome_ambiguous() -> None:
wallet, proofs = _wallet()
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5) # far longer than the patched timeout
await asyncio.sleep(5)
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.unpaid)
)
data_patch, invoice_patch = _lnurl_patches()
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
with (
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
):
with pytest.raises(LNURLError, match="Melt timed out"):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
wallet.get_melt_quote.assert_awaited_once_with("q")
wallet.set_reserved_for_melt.assert_not_called()
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
"""A prompt melt still returns the net amount, unaffected by the guard."""
proofs = [MagicMock(amount=1000)]
async def test_raw_send_to_lnurl_timeout_reconciled_paid_is_success() -> None:
wallet, proofs = _wallet()
wallet = MagicMock()
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
wallet.melt = AsyncMock(return_value=MagicMock())
async def _hang(**kwargs: object) -> None:
await asyncio.sleep(5)
lnurl_data = {
"callback_url": "https://ln.tld/cb",
"min_sendable": 1_000,
"max_sendable": 100_000_000,
}
wallet.melt = AsyncMock(side_effect=_hang)
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.paid)
)
data_patch, invoice_patch = _lnurl_patches()
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
), patch(
"routstr.payment.lnurl.get_lnurl_invoice",
AsyncMock(return_value=("lnbc1...", {})),
with (
patch.object(settings, "mint_operation_timeout_seconds", 0.05),
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
assert paid > 0
wallet.get_melt_quote.assert_awaited_once_with("q")
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_pending_response_stays_ambiguous() -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.pending))
wallet.get_melt_quote = AsyncMock(
return_value=MagicMock(state=MeltQuoteState.pending)
)
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_operation_timeout_seconds", 5),
data_patch,
invoice_patch,
pytest.raises(MeltOutcomeAmbiguousError, match="outcome is ambiguous"),
):
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
wallet.get_melt_quote.assert_awaited_once_with("q")
@pytest.mark.asyncio
@pytest.mark.parametrize("rate_error", ["cooldown", "http_429"])
async def test_raw_send_to_lnurl_rate_rejection_unreserves_proofs(
rate_error: str,
) -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock()
wallet.set_reserved_for_send = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
async def run_operation(factory: Any, *, op_name: str, **_: object) -> Any:
if op_name == "lnurl_melt":
if rate_error == "cooldown":
raise MintCooldownError(str(wallet.url), 60)
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
response = httpx.Response(429, request=request)
raise httpx.HTTPStatusError(
"rate limited", request=request, response=response
)
return await factory()
with (
data_patch,
invoice_patch,
patch(
"routstr.payment.lnurl.run_mint_operation",
side_effect=run_operation,
),
pytest.raises((MintCooldownError, httpx.HTTPStatusError)),
):
await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
wallet.melt.assert_not_awaited()
wallet.set_reserved_for_send.assert_awaited_once_with(
proofs, reserved=False
)
@pytest.mark.asyncio
async def test_real_mint_wrapper_http_429_unreserves_proofs() -> None:
wallet, proofs = _wallet()
request = httpx.Request("POST", f"{wallet.url}/v1/melt/bolt11")
response = httpx.Response(429, request=request)
wallet.melt = AsyncMock(
side_effect=httpx.HTTPStatusError(
"rate limited", request=request, response=response
)
)
wallet.set_reserved_for_send = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_retry_max_attempts", 0),
data_patch,
invoice_patch,
pytest.raises(httpx.HTTPStatusError),
):
await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
)
wallet.melt.assert_awaited_once()
wallet.set_reserved_for_send.assert_awaited_once_with(
proofs, reserved=False
)
MintRateGuard._guards.pop(str(wallet.url), None)
@pytest.mark.asyncio
async def test_raw_send_to_lnurl_succeeds_on_explicit_paid_response() -> None:
wallet, proofs = _wallet()
wallet.melt = AsyncMock(return_value=MagicMock(state=MeltQuoteState.paid))
wallet.get_melt_quote = AsyncMock()
data_patch, invoice_patch = _lnurl_patches()
with (
patch.object(settings, "mint_operation_timeout_seconds", 5),
data_patch,
invoice_patch,
):
paid = await raw_send_to_lnurl(
wallet, proofs, "owner@ln.tld", "sat", amount=1000
@@ -72,3 +204,4 @@ async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
assert paid > 0
wallet.melt.assert_awaited_once()
wallet.get_melt_quote.assert_not_awaited()
+91
View File
@@ -0,0 +1,91 @@
from unittest.mock import AsyncMock, Mock
import pytest
from cashu.core.base import MeltQuoteState, ProofSpentState
from routstr.wallet import (
TokenConsumedError,
_confirm_melt_paid,
_reconcile_ambiguous_melt,
)
@pytest.mark.asyncio
async def test_paid_quote_is_authoritative_when_proof_lookup_would_fail() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.paid)),
check_proof_state=AsyncMock(side_effect=RuntimeError("proof API unavailable")),
)
assert await _reconcile_ambiguous_melt(wallet, "quote-1", [Mock()]) is True
wallet.check_proof_state.assert_not_awaited()
@pytest.mark.asyncio
async def test_timeout_snapshot_unpaid_unspent_remains_non_retryable() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.unpaid)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=ProofSpentState.unspent)])
),
)
with pytest.raises(TokenConsumedError, match="ambiguous"):
await _reconcile_ambiguous_melt(wallet, "quote-2", [Mock()])
@pytest.mark.asyncio
async def test_successful_pending_melt_response_requires_reconciliation() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=MeltQuoteState.pending)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=ProofSpentState.pending)])
),
)
with pytest.raises(TokenConsumedError, match="ambiguous"):
await _confirm_melt_paid(
wallet,
"quote-pending",
[Mock()],
Mock(state=MeltQuoteState.pending),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("quote_state", "proof_state"),
[
(MeltQuoteState.pending, ProofSpentState.pending),
(MeltQuoteState.unpaid, ProofSpentState.spent),
(MeltQuoteState.unpaid, ProofSpentState.pending),
],
)
async def test_ambiguous_or_consumed_melt_is_never_reported_unspent(
quote_state: MeltQuoteState, proof_state: ProofSpentState
) -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(return_value=Mock(state=quote_state)),
check_proof_state=AsyncMock(
return_value=Mock(states=[Mock(state=proof_state)])
),
)
with pytest.raises(TokenConsumedError, match="reconciliation required"):
await _reconcile_ambiguous_melt(wallet, "quote-3", [Mock()])
@pytest.mark.asyncio
async def test_failed_melt_reconciliation_is_non_retryable() -> None:
wallet = Mock(
url="http://source-mint:3338",
get_melt_quote=AsyncMock(side_effect=RuntimeError("mint unavailable")),
check_proof_state=AsyncMock(),
)
with pytest.raises(TokenConsumedError, match="outcome is unknown"):
await _reconcile_ambiguous_melt(wallet, "quote-4", [Mock()])
+29 -1
View File
@@ -18,6 +18,7 @@ from fastapi.responses import Response, StreamingResponse
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
os.environ.setdefault("UPSTREAM_API_KEY", "test")
from routstr.auth import ReservationSnapshot # noqa: E402
from routstr.core.db import ApiKey # noqa: E402
from routstr.payment.cost_calculation import CostData # noqa: E402
from routstr.payment.models import Architecture, Model, Pricing # noqa: E402
@@ -450,7 +451,8 @@ async def test_non_streaming_dispatches_via_litellm_and_returns_anthropic_respon
assert payload["model"] == "openai/gpt-4o-mini" # mapped back to requested
assert payload["usage"]["input_tokens"] == 5
assert payload["usage"]["output_tokens"] == 3
assert payload["usage"]["cost"] == 0.0001
assert payload["usage"]["cost"]["total_msats"] == 1234
assert payload["usage"]["cost"]["total_usd"] == 0.0001
assert payload["usage"]["cost_sats"] == 1
@@ -498,6 +500,12 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
yield {"type": "message_stop"}
fake_cost = {"total_msats": 4321, "total_usd": 0.00015}
reservation = ReservationSnapshot(
release_id="messages-stream",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=10_000,
)
captured_cost_call: dict[str, Any] = {}
@@ -508,9 +516,11 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
max_cost: int,
model_obj: Any = None,
provider_fee: Any = None,
reservation_snapshot: Any = None,
) -> dict:
captured_cost_call["combined_data"] = combined_data
captured_cost_call["max_cost"] = max_cost
captured_cost_call["reservation_snapshot"] = reservation_snapshot
return fake_cost
fake_session = MagicMock()
@@ -544,6 +554,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
session=session,
max_cost_for_model=10_000,
model_obj=model,
reservation_snapshot=reservation,
)
assert isinstance(result, StreamingResponse)
@@ -566,6 +577,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
assert combined["usage"]["input_tokens"] == 5
assert combined["usage"]["output_tokens"] == 7
assert combined["model"] == "openai/gpt-4o-mini"
assert captured_cost_call["reservation_snapshot"] is reservation
@pytest.mark.asyncio
@@ -593,6 +605,12 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
fake_cost = {"total_msats": 999, "total_usd": 0.0001}
reservation = ReservationSnapshot(
release_id="messages-byte-stream",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=10_000,
)
captured: dict[str, Any] = {}
async def fake_adjust(
@@ -602,8 +620,10 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
max_cost: int,
model_obj: Any = None,
provider_fee: Any = None,
reservation_snapshot: Any = None,
) -> dict:
captured["combined_data"] = combined_data
captured["reservation_snapshot"] = reservation_snapshot
return fake_cost
fake_session = MagicMock()
@@ -636,6 +656,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
session=session,
max_cost_for_model=10_000,
model_obj=model,
reservation_snapshot=reservation,
)
assert isinstance(result, StreamingResponse)
@@ -659,6 +680,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
assert combined["usage"]["input_tokens"] == 3
assert combined["usage"]["output_tokens"] == 4
assert combined["model"] == "openai/gpt-4o-mini"
assert captured["reservation_snapshot"] is reservation
# ---------------------------------------------------------------------------
@@ -833,6 +855,9 @@ async def test_x_cashu_streaming_replays_events_and_sets_refund_header() -> None
assert isinstance(result, StreamingResponse)
assert result.headers.get("X-Cashu") == "cashuSTREAM"
assert result.headers.get("X-Routstr-Cost-Msats") == "1500000"
assert result.headers.get("X-Routstr-Input-Cost-Msats") == "1000000"
assert result.headers.get("X-Routstr-Output-Cost-Msats") == "500000"
# 1_500_000 msats → 1500 sats. Refund = 5000 - 1500 = 3500.
mock_refund.assert_awaited_once()
refund_call = mock_refund.await_args
@@ -851,6 +876,9 @@ async def test_x_cashu_streaming_replays_events_and_sets_refund_header() -> None
assert "event: message_start" in joined
assert "event: message_delta" in joined
assert "event: message_stop" in joined
assert '"total_msats": 1500000' in joined
assert '"input_msats": 1000000' in joined
assert '"output_msats": 500000' in joined
# ---------------------------------------------------------------------------
+121
View File
@@ -0,0 +1,121 @@
import asyncio
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from cashu.core.base import Unit
from routstr.mint import (
MintCooldownError,
MintRateGuard,
MintRateLimitedError,
fail_fast_mint_operations,
)
from routstr.wallet import Wallet
@pytest.mark.asyncio
async def test_cooldown_fails_fast_while_wallet_mutation_scope_is_held() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(3600, reason="rate_limited")
operation = AsyncMock(return_value="should not run")
with (
patch("routstr.mint.asyncio.sleep", AsyncMock()) as sleep,
pytest.raises(MintCooldownError) as caught,
):
async with fail_fast_mint_operations():
await guard.run(operation)
assert caught.value.retry_after_seconds > 0
operation.assert_not_awaited()
sleep.assert_not_awaited()
@pytest.mark.asyncio
async def test_expired_cooldown_allows_probe_in_wallet_mutation_scope() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(0, reason="rate_limited")
operation = AsyncMock(return_value="recovered")
async with fail_fast_mint_operations():
result = await guard.run(operation)
assert result == "recovered"
operation.assert_awaited_once()
assert guard._needs_probe is False
@pytest.mark.asyncio
async def test_fail_fast_does_not_wait_behind_existing_probe() -> None:
guard = MintRateGuard("http://mint:3338", max_concurrency=1)
guard.apply_cooldown(0, reason="rate_limited")
probe_started = asyncio.Event()
release_probe = asyncio.Event()
async def probe() -> str:
probe_started.set()
await release_probe.wait()
return "recovered"
first = asyncio.create_task(guard.run(probe))
await probe_started.wait()
try:
async with fail_fast_mint_operations():
with pytest.raises(MintCooldownError):
await asyncio.wait_for(guard.run(AsyncMock()), timeout=0.05)
finally:
release_probe.set()
assert await first == "recovered"
@pytest.mark.asyncio
async def test_cashu_429_dispatches_through_wallet_override() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
429,
request=request,
json={"detail": "too many requests", "code": 42900},
)
wallet = object.__new__(Wallet)
wallet.url = "http://mint:3338"
wallet.db = Mock()
wallet.keysets = {"loaded": Mock()}
wallet.mint_info = Mock()
wallet.mint_info.requires_blind_auth_path.return_value = False
wallet.mint_info.requires_clear_auth_path.return_value = False
wallet.auth_db = None
wallet.auth_keyset_id = None
real_client = httpx.AsyncClient
def client_factory(*args: object, **kwargs: object) -> httpx.AsyncClient:
return real_client(
transport=httpx.MockTransport(handler),
base_url=str(kwargs["base_url"]),
)
with (
patch("cashu.wallet.v1_api.httpx.AsyncClient", side_effect=client_factory),
pytest.raises(MintRateLimitedError),
):
await wallet.mint_quote(1, Unit.sat)
async def test_guard_concurrency_change_preserves_cooldown_state() -> None:
from routstr.core.settings import settings
mint_url = "https://mint.test-concurrency-carryover"
with patch.object(settings, "mint_max_concurrency", 2):
guard = MintRateGuard.get(mint_url)
guard.apply_cooldown(120.0, reason="rate_limited")
guard._consecutive_rate_limits = 3
with patch.object(settings, "mint_max_concurrency", 5):
rebuilt = MintRateGuard.get(mint_url)
assert rebuilt is not guard
assert rebuilt.cooldown_remaining() > 0
assert rebuilt._cooldown_reason == "rate_limited"
assert rebuilt._consecutive_rate_limits == 3
+50
View File
@@ -0,0 +1,50 @@
"""Persisted mint preferences must not bypass the configured trusted set."""
from unittest.mock import AsyncMock, patch
import pytest
from routstr.core.settings import settings
from routstr.lightning import _request_mint_with_fallback
TRUSTED = "https://good-mint.example.com"
UNTRUSTED = "https://removed-mint.example.com"
async def test_untrusted_allowed_mints_fall_back_to_trusted_set() -> None:
attempted: list[str] = []
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
attempted.append(mint_url)
raise ConnectionError("unreachable in test")
with (
patch.object(settings, "primary_mint", TRUSTED),
patch.object(settings, "cashu_mints", [TRUSTED]),
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
):
with pytest.raises(Exception):
await _request_mint_with_fallback(10, allowed_mints=[UNTRUSTED])
assert UNTRUSTED not in attempted
assert attempted == [TRUSTED]
async def test_trusted_allowed_mints_are_used_verbatim() -> None:
attempted: list[str] = []
async def fake_get_wallet(mint_url: str, unit: str, **kwargs: object) -> None:
attempted.append(mint_url)
raise ConnectionError("unreachable in test")
with (
patch.object(settings, "primary_mint", TRUSTED),
patch.object(settings, "cashu_mints", [TRUSTED, "https://other.example.com"]),
patch("routstr.lightning.get_wallet", AsyncMock(side_effect=fake_get_wallet)),
patch("routstr.lightning.mint_cooldown_remaining", return_value=0.0),
):
with pytest.raises(Exception):
await _request_mint_with_fallback(10, allowed_mints=[TRUSTED])
assert attempted == [TRUSTED]
+47
View File
@@ -0,0 +1,47 @@
import os
import sqlite3
import subprocess
import sys
from pathlib import Path
def _run_alembic(root: Path, database_url: str, command: str, revision: str) -> None:
env = os.environ.copy()
env["DATABASE_URL"] = database_url
subprocess.run(
[sys.executable, "-m", "alembic", command, revision],
cwd=root,
env=env,
check=True,
capture_output=True,
text=True,
)
def _lightning_invoice_columns(database_path: Path) -> set[str]:
with sqlite3.connect(database_path) as connection:
return {
row[1]
for row in connection.execute("PRAGMA table_info(lightning_invoices)")
}
def test_mint_url_migration_upgrades_and_downgrades_from_main_head(
tmp_path: Path,
) -> None:
root = Path(__file__).resolve().parents[2]
database_path = tmp_path / "mint-url-migration.db"
database_url = f"sqlite+aiosqlite:///{database_path}"
previous_head = "64ed5594df1f"
_run_alembic(root, database_url, "upgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "ecfa0d6e2a36")
assert "mint_url" in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "downgrade", previous_head)
assert "mint_url" not in _lightning_invoice_columns(database_path)
_run_alembic(root, database_url, "upgrade", "head")
assert "mint_url" in _lightning_invoice_columns(database_path)
File diff suppressed because it is too large Load Diff
+30
View File
@@ -125,3 +125,33 @@ async def test_get_max_cost_for_model_tolerance() -> None:
"gpt-4", session=mock_session, model_obj=mock_model
)
assert cost == 450000 # 500 sats * 1000 * 0.9 = 450000
async def test_discounted_max_cost_floors_at_min_request_msat() -> None:
from routstr.payment.helpers import calculate_discounted_max_cost
pricing = Mock()
pricing.prompt = 0.001
pricing.completion = 0.001
pricing.max_prompt_cost = 100.0
pricing.max_completion_cost = 100.0
model_obj = Mock()
model_obj.sats_pricing = pricing
model_obj.top_provider = None
model_obj.context_length = None
body = {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 1,
}
with (
patch.object(settings, "fixed_pricing", False),
patch.object(settings, "tolerance_percentage", 0),
patch.object(settings, "min_request_msat", 1000),
):
cost = await calculate_discounted_max_cost(150_000, body, model_obj)
assert cost == 1000
+126 -48
View File
@@ -59,23 +59,29 @@ async def test_periodic_payout_includes_primary_mint_not_in_cashu_mints() -> Non
get_wallet = AsyncMock(return_value=MagicMock())
raw_send = AsyncMock(return_value=1000)
with patch.object(settings, "cashu_mints", []), patch.object(
settings, "primary_mint", "http://primary:3338"
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
settings, "payout_interval_seconds", _INTERVAL
), patch.object(settings, "min_payout_sat", 10), patch(
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
), patch("routstr.wallet.db.create_session", _fake_session), patch(
"routstr.wallet.get_wallet", get_wallet
), patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
), patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
), patch(
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
), patch("routstr.wallet.raw_send_to_lnurl", raw_send):
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
patch.object(settings, "payout_interval_seconds", _INTERVAL),
patch.object(settings, "min_payout_sat", 10),
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.get_wallet", get_wallet),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
),
patch(
"routstr.wallet.db.total_user_liability",
AsyncMock(return_value=0),
),
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
):
with pytest.raises(_LoopBreak):
await periodic_payout()
@@ -84,12 +90,66 @@ async def test_periodic_payout_includes_primary_mint_not_in_cashu_mints() -> Non
assert raw_send.await_count >= 1
@pytest.mark.asyncio
async def test_periodic_payout_releases_session_before_slow_mint_send() -> None:
"""The DB connection is returned before the external LNURL call starts."""
from routstr.core.settings import settings
session_open = False
sends_completed = 0
@asynccontextmanager
async def tracked_session(): # type: ignore[no-untyped-def]
nonlocal session_open
session_open = True
try:
yield MagicMock()
finally:
session_open = False
async def raw_send(*args: object, **kwargs: object) -> int:
nonlocal sends_completed
assert session_open is False
sends_completed += 1
return 1000
with (
patch.object(settings, "cashu_mints", []),
patch.object(settings, "primary_mint", "http://primary:3338"),
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
patch.object(settings, "payout_interval_seconds", _INTERVAL),
patch.object(settings, "min_payout_sat", 10),
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
patch("routstr.wallet.db.create_session", tracked_session),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
),
patch(
"routstr.wallet.db.total_user_liability",
AsyncMock(return_value=0),
),
patch("routstr.wallet.raw_send_to_lnurl", AsyncMock(side_effect=raw_send)),
):
with pytest.raises(_LoopBreak):
await periodic_payout()
assert sends_completed == 2
@pytest.mark.asyncio
async def test_periodic_payout_isolates_failing_mint() -> None:
"""A failing mint does not prevent payout for the other mints."""
from routstr.core.settings import settings
async def _get_wallet(mint_url: str, unit: str) -> MagicMock:
async def _get_wallet(
mint_url: str, unit: str, force_reload: bool = False
) -> MagicMock:
if mint_url == "http://bad:3338":
raise RuntimeError("mint unreachable")
return MagicMock()
@@ -97,23 +157,29 @@ async def test_periodic_payout_isolates_failing_mint() -> None:
get_wallet = AsyncMock(side_effect=_get_wallet)
raw_send = AsyncMock(return_value=1000)
with patch.object(
settings, "cashu_mints", ["http://bad:3338", "http://good:3338"]
), patch.object(settings, "primary_mint", "http://good:3338"), patch.object(
settings, "receive_ln_address", "owner@ln.tld"
), patch.object(settings, "payout_interval_seconds", _INTERVAL), patch.object(
settings, "min_payout_sat", 10
), patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()), patch(
"routstr.wallet.db.create_session", _fake_session
), patch("routstr.wallet.get_wallet", get_wallet), patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
), patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
), patch(
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
), patch("routstr.wallet.raw_send_to_lnurl", raw_send):
with (
patch.object(settings, "cashu_mints", ["http://bad:3338", "http://good:3338"]),
patch.object(settings, "primary_mint", "http://good:3338"),
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
patch.object(settings, "payout_interval_seconds", _INTERVAL),
patch.object(settings, "min_payout_sat", 10),
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
patch("routstr.wallet.db.create_session", _fake_session),
patch("routstr.wallet.get_wallet", get_wallet),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
),
patch(
"routstr.wallet.db.total_user_liability",
AsyncMock(return_value=0),
),
patch("routstr.wallet.raw_send_to_lnurl", raw_send),
):
with pytest.raises(_LoopBreak):
await periodic_payout()
@@ -128,27 +194,39 @@ async def test_periodic_payout_isolates_failing_mint() -> None:
@pytest.mark.asyncio
async def test_periodic_payout_handles_session_creation_failure() -> None:
"""A db.create_session failure is logged and the payout loop continues."""
"""A db.create_session failure is logged per mint/unit and the loop continues."""
from routstr.core.settings import settings
create_session = MagicMock(side_effect=RuntimeError("db unavailable"))
logger = MagicMock()
with patch.object(settings, "cashu_mints", ["http://mint:3338"]), patch.object(
settings, "primary_mint", "http://mint:3338"
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
settings, "payout_interval_seconds", _INTERVAL
), patch(
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
), patch(
"routstr.wallet.db.create_session", create_session
), patch("routstr.wallet.logger", logger):
with (
patch.object(settings, "cashu_mints", ["http://mint:3338"]),
patch.object(settings, "primary_mint", "http://mint:3338"),
patch.object(settings, "receive_ln_address", "owner@ln.tld"),
patch.object(settings, "payout_interval_seconds", _INTERVAL),
patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()),
patch("routstr.wallet.db.create_session", create_session),
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
patch(
"routstr.wallet.get_proofs_per_mint_and_unit",
MagicMock(return_value=[MagicMock(amount=100_000)]),
),
patch(
"routstr.wallet.slow_filter_spend_proofs",
AsyncMock(side_effect=lambda proofs, wallet: proofs),
),
patch("routstr.wallet.logger", logger),
):
with pytest.raises(_LoopBreak):
await periodic_payout()
create_session.assert_called_once()
logger.error.assert_called_once()
# The liability session is opened per mint/unit (sat + msat), and each
# DB failure retains the cycle-specific alert wording while remaining
# isolated to its own iteration.
assert create_session.call_count == 2
assert logger.error.call_count == 2
message = logger.error.call_args.args[0]
extra = logger.error.call_args.kwargs["extra"]
assert message == "Error in periodic payout cycle: RuntimeError"
assert extra == {"error": "db unavailable"}
assert extra["error"] == "db unavailable"
@@ -0,0 +1,43 @@
from collections.abc import AsyncIterator
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.responses import StreamingResponse
from routstr import proxy as proxy_module
@pytest.mark.asyncio
async def test_proxy_closes_request_session_before_returning_response() -> None:
"""Route completion must release DB resources before response delivery."""
request = MagicMock()
request.method = "GET"
request.headers = {"accept": "application/json"}
request.url.path = "/not-an-api-route"
request.state.request_id = "test-request"
session = AsyncMock()
response = await proxy_module.proxy(request, "not-an-api-route", session=session)
assert response.status_code == 404
session.close.assert_awaited_once()
@pytest.mark.asyncio
async def test_proxy_session_is_closed_before_first_stream_chunk() -> None:
request = MagicMock()
session = AsyncMock()
async def stream() -> AsyncIterator[bytes]:
session.close.assert_awaited_once()
yield b"chunk"
upstream_response = StreamingResponse(stream())
with patch("routstr.proxy._proxy", AsyncMock(return_value=upstream_response)):
response = await proxy_module.proxy(
request, "v1/chat/completions", session=session
)
assert isinstance(response, StreamingResponse)
chunks = [chunk async for chunk in response.body_iterator]
assert chunks == [b"chunk"]
+281 -5
View File
@@ -1,4 +1,6 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from unittest.mock import AsyncMock, patch
@@ -7,6 +9,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlmodel import SQLModel, select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr import wallet
from routstr.core.db import CashuTransaction
from routstr.wallet import refund_sweep_once
@@ -39,6 +42,43 @@ async def _load(
return {row.token: row for row in result.all()}
@pytest.mark.asyncio
async def test_refund_sweep_releases_db_session_during_token_redemption(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
await _insert(
session_factory,
CashuTransaction(
token="eligible", amount=1, unit="sat", type="out", created_at=800
),
)
session_open = False
@asynccontextmanager
async def tracked_session() -> AsyncIterator[AsyncSession]:
nonlocal session_open
async with session_factory() as session:
session_open = True
try:
yield session
finally:
session_open = False
async def receive_token(token: str) -> None:
assert token == "eligible"
assert session_open is False
with (
patch("routstr.wallet.db.create_session", tracked_session),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.time.time", return_value=1000),
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=receive_token)),
):
await refund_sweep_once()
assert (await _load(session_factory))["eligible"].swept is True
@pytest.mark.asyncio
async def test_refund_sweep_only_processes_expired_eligible_outgoing_tokens(
session_factory: async_sessionmaker[AsyncSession],
@@ -90,14 +130,17 @@ async def test_refund_sweep_only_processes_expired_eligible_outgoing_tokens(
@pytest.mark.asyncio
@pytest.mark.parametrize(
("error", "collected"),
("error", "collected", "claim_started_at"),
[
(RuntimeError("token already spent"), True),
(RuntimeError("mint unavailable"), False),
(RuntimeError("token already spent"), True, None),
(RuntimeError("mint unavailable"), False, 1000),
],
)
async def test_refund_sweep_records_terminal_but_not_transient_failures(
session_factory: async_sessionmaker[AsyncSession], error: Exception, collected: bool
async def test_refund_sweep_records_spent_and_unknown_outcomes_safely(
session_factory: async_sessionmaker[AsyncSession],
error: Exception,
collected: bool,
claim_started_at: int | None,
) -> None:
await _insert(
session_factory,
@@ -116,3 +159,236 @@ async def test_refund_sweep_records_terminal_but_not_transient_failures(
refund = (await _load(session_factory))["refund"]
assert refund.collected is collected
assert refund.swept is False
assert refund.sweep_started_at == claim_started_at
@pytest.mark.asyncio
async def test_post_spend_failure_retains_claim_and_stale_retry_records_sweep(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
await _insert(
session_factory,
CashuTransaction(
token="post-spend-failure",
amount=1,
unit="sat",
type="out",
created_at=800,
),
)
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
patch("routstr.wallet.time.time", return_value=1000),
patch(
"routstr.wallet.recieve_token",
AsyncMock(
side_effect=wallet.TokenConsumedError(
"Mint on primary failed after successful melt"
)
),
),
):
await refund_sweep_once()
retained = (await _load(session_factory))["post-spend-failure"]
assert retained.swept is False
assert retained.collected is False
assert retained.sweep_started_at == 1000
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
patch("routstr.wallet.time.time", return_value=1300),
patch(
"routstr.wallet.recieve_token",
AsyncMock(side_effect=RuntimeError("token already spent")),
),
):
await refund_sweep_once()
recovered = (await _load(session_factory))["post-spend-failure"]
assert recovered.swept is True
assert recovered.collected is False
assert recovered.sweep_started_at is None
@pytest.mark.asyncio
async def test_refund_sweep_retains_claim_on_cancellation_during_redemption(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
await _insert(
session_factory,
CashuTransaction(
token="cancelled", amount=1, unit="sat", type="out", created_at=800
),
)
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.time.time", return_value=1000),
patch(
"routstr.wallet.recieve_token",
AsyncMock(side_effect=asyncio.CancelledError()),
),
):
with pytest.raises(asyncio.CancelledError):
await refund_sweep_once()
refund = (await _load(session_factory))["cancelled"]
assert refund.swept is False
assert refund.sweep_started_at == 1000
@pytest.mark.asyncio
async def test_checkpoint_failure_retains_claim_and_stale_retry_records_sweep(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
await _insert(
session_factory,
CashuTransaction(
token="checkpoint-failure",
amount=1,
unit="sat",
type="out",
created_at=800,
),
)
real_set_state = wallet._set_refund_sweep_state
async def fail_swept_checkpoint(
refund_id: str,
*,
predicates: tuple[object, ...] = (),
**values: object,
) -> int:
if values.get("swept") is True:
raise RuntimeError("checkpoint unavailable")
return await real_set_state(refund_id, predicates=predicates, **values)
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
patch("routstr.wallet.time.time", return_value=1000),
patch(
"routstr.wallet.recieve_token", AsyncMock(return_value=(1, "sat", "mint"))
),
patch(
"routstr.wallet._set_refund_sweep_state",
side_effect=fail_swept_checkpoint,
),
patch("routstr.wallet.logger.critical") as critical,
):
await refund_sweep_once()
retained = (await _load(session_factory))["checkpoint-failure"]
assert retained.swept is False
assert retained.collected is False
assert retained.sweep_started_at == 1000
critical.assert_called_once()
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
patch("routstr.wallet.time.time", return_value=1300),
patch(
"routstr.wallet.recieve_token",
AsyncMock(side_effect=RuntimeError("token already spent")),
),
):
await refund_sweep_once()
recovered = (await _load(session_factory))["checkpoint-failure"]
assert recovered.swept is True
assert recovered.collected is False
assert recovered.sweep_started_at is None
@pytest.mark.asyncio
@pytest.mark.parametrize("redemption_succeeds", [True, False])
async def test_expired_worker_cannot_overwrite_or_release_newer_claim(
session_factory: async_sessionmaker[AsyncSession],
redemption_succeeds: bool,
) -> None:
await _insert(
session_factory,
CashuTransaction(
token="reclaimed",
amount=1,
unit="sat",
type="out",
created_at=800,
),
)
async def replace_claim(_token: str) -> tuple[int, str, str]:
async with session_factory() as session:
result = await session.exec(
select(CashuTransaction).where(CashuTransaction.token == "reclaimed")
)
transaction = result.one()
transaction.sweep_started_at = 1100
session.add(transaction)
await session.commit()
if not redemption_succeeds:
raise RuntimeError("mint unavailable")
return (1, "sat", "mint")
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.time.time", return_value=1000),
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=replace_claim)),
):
await refund_sweep_once()
reclaimed = (await _load(session_factory))["reclaimed"]
assert reclaimed.swept is False
assert reclaimed.collected is False
assert reclaimed.sweep_started_at == 1100
@pytest.mark.asyncio
async def test_refund_sweep_recovers_stale_claim_without_misreporting_collection(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
await _insert(
session_factory,
CashuTransaction(
token="stale",
amount=1,
unit="sat",
type="out",
created_at=800,
sweep_started_at=100,
),
CashuTransaction(
token="active",
amount=1,
unit="sat",
type="out",
created_at=800,
sweep_started_at=950,
),
)
receive = AsyncMock(side_effect=RuntimeError("token already spent"))
with (
patch("routstr.wallet.db.create_session", side_effect=session_factory),
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
patch("routstr.wallet.settings.refund_sweep_claim_timeout_seconds", 200),
patch("routstr.wallet.time.time", return_value=1000),
patch("routstr.wallet.recieve_token", receive),
):
await refund_sweep_once()
receive.assert_awaited_once_with("stale")
loaded = await _load(session_factory)
assert loaded["stale"].swept is True
assert loaded["stale"].collected is False
assert loaded["stale"].sweep_started_at is None
assert loaded["active"].swept is False
assert loaded["active"].sweep_started_at == 950
+95
View File
@@ -62,6 +62,101 @@ def test_payout_settings_have_sensible_defaults() -> None:
assert s.payout_interval_seconds == 900
def test_database_pool_defaults_provide_concurrency_headroom() -> None:
s = Settings()
assert s.database_pool_size == 10
assert s.database_max_overflow == 20
assert s.database_pool_timeout == 15.0
assert s.database_pool_recycle == 1800
assert s.database_pool_pre_ping is False
assert s.database_pool_hold_warn_seconds == 10.0
@pytest.mark.parametrize(
("field", "bad_value"),
[
("database_pool_size", 0),
("database_max_overflow", -1),
("database_pool_timeout", 0),
("database_pool_recycle", -1),
("database_pool_hold_warn_seconds", 0),
],
)
def test_database_pool_settings_reject_invalid_values(
field: str, bad_value: int
) -> None:
with pytest.raises(ValidationError):
Settings.parse_obj({field: bad_value})
@pytest.mark.asyncio
async def test_database_pool_fields_are_env_only_not_persisted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""DB pool sizing is infrastructure the node needs *before* it can read the
DB, so it can never be configured from the DB it must never be written to
the settings blob, and a stale/injected DB value must never shadow env.
"""
monkeypatch.setenv("DATABASE_POOL_SIZE", "7")
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with AsyncSession(engine, expire_on_commit=False) as session:
s = await SettingsService.initialize(session)
# The env value is live for runtime consumers...
assert s.database_pool_size == 7
# ...but pool sizing is never written to the settings blob.
blob = await _read_settings_blob(session)
for field in (
"database_pool_size",
"database_max_overflow",
"database_pool_timeout",
"database_pool_recycle",
"database_pool_pre_ping",
"database_pool_hold_warn_seconds",
):
assert field not in blob
# Even a stale blob that somehow carries a pool value must not win: env
# stays authoritative on the next initialize.
await session.exec( # type: ignore
text("UPDATE settings SET data = :d WHERE id = 1").bindparams(
d=json.dumps({"database_pool_size": 99})
)
)
await session.commit()
again = await SettingsService.initialize(session)
assert again.database_pool_size == 7
@pytest.mark.asyncio
async def test_update_does_not_apply_env_only_fields_to_live_settings(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""DB pool sizing is env-only: a settings update must neither persist it nor
mutate the live value. The engine pool is already built at boot from env, so
a UI/API update carrying a pool value must not make the live setting diverge
from the running pool.
"""
monkeypatch.delenv("DATABASE_POOL_SIZE", raising=False)
monkeypatch.setattr(settings, "database_pool_size", 10)
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with AsyncSession(engine, expire_on_commit=False) as session:
await SettingsService.initialize(session)
await SettingsService.update(
{"database_pool_size": 99, "name": "PoolTweaker"}, session
)
# A non-env-only field still updates normally...
assert settings.name == "PoolTweaker"
# ...but the env-only pool size stays at the boot value.
assert settings.database_pool_size == 10
# ...and it is never written to the settings blob.
blob = await _read_settings_blob(session)
assert "database_pool_size" not in blob
@pytest.mark.parametrize(
"field,bad_value",
[
+52 -8
View File
@@ -16,13 +16,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlalchemy.pool import StaticPool
from sqlmodel import SQLModel
from sqlmodel import SQLModel, select
from sqlmodel.ext.asyncio.session import AsyncSession
from routstr.auth import pay_for_request
from routstr.balance import refund_wallet_endpoint
from routstr.core.db import (
ApiKey,
ReservationRelease,
release_stale_reservations,
reset_all_reserved_balances,
)
@@ -70,7 +71,9 @@ async def test_pay_for_request_sets_reserved_at(session: AsyncSession) -> None:
@pytest.mark.asyncio
async def test_pay_for_request_sets_reserved_at_on_child_key(session: AsyncSession) -> None:
async def test_pay_for_request_sets_reserved_at_on_child_key(
session: AsyncSession,
) -> None:
parent = ApiKey(hashed_key="parentkey", balance=10_000)
child = ApiKey(hashed_key="childkey", balance=0, parent_key_hash="parentkey")
session.add(parent)
@@ -150,6 +153,39 @@ async def test_release_stale_reservations_releases_old(session: AsyncSession) ->
assert key.reserved_at is None
@pytest.mark.asyncio
async def test_targeted_parent_cleanup_releases_child_owned_reservation(
session: AsyncSession,
) -> None:
parent = ApiKey(hashed_key="stale-parent", balance=5_000)
child = ApiKey(
hashed_key="stale-child", parent_key_hash=parent.hashed_key, balance=0
)
session.add_all([parent, child])
await session.commit()
await pay_for_request(child, 1_000, session)
reservation = (
await session.exec(
select(ReservationRelease).where(
ReservationRelease.key_hash == child.hashed_key
)
)
).one()
reservation.created_at = int(time.time()) - 1_000
session.add(reservation)
await session.commit()
released = await release_stale_reservations(
session, max_age_seconds=300, key_hash=parent.hashed_key
)
assert released == 1
await session.refresh(parent)
await session.refresh(child)
assert parent.reserved_balance == 0
assert child.reserved_balance == 0
@pytest.mark.asyncio
async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) -> None:
key = ApiKey(
@@ -170,7 +206,9 @@ async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) ->
@pytest.mark.asyncio
async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncSession) -> None:
async def test_release_stale_reservations_skips_null_reserved_at(
session: AsyncSession,
) -> None:
# Reservations without a timestamp may belong to instances running older
# code (rolling deploy) — the background sweeper must not touch them.
key = ApiKey(
@@ -190,7 +228,9 @@ async def test_release_stale_reservations_skips_null_reserved_at(session: AsyncS
@pytest.mark.asyncio
async def test_reset_all_reserved_balances_clears_reserved_at(session: AsyncSession) -> None:
async def test_reset_all_reserved_balances_clears_reserved_at(
session: AsyncSession,
) -> None:
key = ApiKey(
hashed_key="resetkey",
balance=5_000,
@@ -352,6 +392,7 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
upstream.forward_request = AsyncMock(side_effect=asyncio.CancelledError())
session = MagicMock()
reservation_snapshot = MagicMock()
revert_mock = AsyncMock(return_value=True)
with (
@@ -369,13 +410,16 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
"get_reservation_snapshot",
AsyncMock(return_value=reservation_snapshot),
),
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
):
with pytest.raises(asyncio.CancelledError):
await proxy_module.proxy(request, "v1/chat/completions", session=session)
revert_mock.assert_awaited_once_with(key, session, 1_000)
revert_mock.assert_awaited_once_with(key, session, 1000, reservation_snapshot)
+7
View File
@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.upstream.base import BaseUpstreamProvider
@@ -67,6 +68,12 @@ async def test_stream_with_id_injection() -> None:
max_cost_for_model=100,
background_tasks=background_tasks,
requested_model="test-model",
reservation_snapshot=ReservationSnapshot(
release_id="test-release",
key_hash="test_hash",
billing_key_hash="test_hash",
reserved_msats=100,
),
)
results = []
@@ -0,0 +1,448 @@
import asyncio
from collections.abc import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlmodel import SQLModel
from sqlmodel.ext.asyncio.session import AsyncSession
import routstr.auth as auth_module
from routstr.auth import (
ReservationSnapshot,
adjust_payment_for_tokens,
get_reservation_snapshot,
pay_for_request,
release_reservation,
)
from routstr.core.db import ApiKey, ReservationRelease
from routstr.payment.cost_calculation import MaxCostData
from routstr.upstream.base import BaseUpstreamProvider
async def _engine() -> AsyncEngine:
engine = create_async_engine("sqlite+aiosqlite://")
async with engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
return engine
@pytest.mark.asyncio
async def test_release_reservation_is_durable_and_idempotent() -> None:
engine = await _engine()
key = ApiKey(hashed_key="key", balance=1_000)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 500, session)
snapshot = await get_reservation_snapshot(key, session)
record = await session.get(ReservationRelease, snapshot.release_id)
assert record is not None and record.status == "active"
assert await release_reservation(snapshot, session, 500) is True
assert await release_reservation(snapshot, session, 500) is True
await session.refresh(key)
await session.refresh(record)
assert key.reserved_balance == 0
assert key.reserved_at is None
assert record.status == "released"
await engine.dispose()
@pytest.mark.asyncio
async def test_release_only_owns_its_concurrent_reservation() -> None:
engine = await _engine()
key = ApiKey(hashed_key="key", balance=1_000)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 400, session)
first = await get_reservation_snapshot(key, session)
await pay_for_request(key, 400, session)
second = await get_reservation_snapshot(key, session)
assert first.release_id != second.release_id
assert await release_reservation(first, session, 400) is True
assert await release_reservation(first, session, 400) is True
await session.refresh(key)
assert key.reserved_balance == 400
assert await release_reservation(second, session, 400) is True
await session.refresh(key)
assert key.reserved_balance == 0
await engine.dispose()
@pytest.mark.asyncio
async def test_release_updates_parent_and_child_atomically() -> None:
engine = await _engine()
parent = ApiKey(hashed_key="parent", balance=1_000)
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add_all([parent, child])
await session.commit()
await pay_for_request(child, 500, session)
snapshot = await get_reservation_snapshot(child, session)
assert await release_reservation(snapshot, session, 500) is True
await session.refresh(parent)
await session.refresh(child)
assert (parent.reserved_balance, child.reserved_balance) == (0, 0)
assert (parent.reserved_at, child.reserved_at) == (None, None)
await engine.dispose()
@pytest.mark.asyncio
async def test_release_rolls_back_partial_parent_child_update() -> None:
engine = await _engine()
parent = ApiKey(hashed_key="parent", balance=1_000)
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add_all([parent, child])
await session.commit()
await pay_for_request(child, 500, session)
snapshot = await get_reservation_snapshot(child, session)
child.reserved_balance = 100
session.add(child)
await session.commit()
assert await release_reservation(snapshot, session, 500) is False
await session.refresh(parent)
await session.refresh(child)
record = await session.get(ReservationRelease, snapshot.release_id)
assert (parent.reserved_balance, child.reserved_balance) == (500, 100)
assert record is not None and record.status == "active"
await engine.dispose()
@pytest.mark.asyncio
async def test_post_commit_failure_cannot_release_charged_reservation() -> None:
engine = await _engine()
key = ApiKey(hashed_key="key", balance=1_000)
cost = MaxCostData(
base_msats=500,
input_msats=0,
output_msats=0,
total_msats=500,
)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 500, session)
snapshot = await get_reservation_snapshot(key, session)
with (
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
patch.object(
session,
"refresh",
AsyncMock(side_effect=SQLAlchemyError("post-commit refresh failed")),
),
):
with pytest.raises(SQLAlchemyError, match="post-commit refresh failed"):
await adjust_payment_for_tokens(key, {}, session, 500)
await session.rollback()
assert await release_reservation(snapshot, session, 500) is False
charged_key = await session.get(ApiKey, "key")
record = await session.get(ReservationRelease, snapshot.release_id)
assert charged_key is not None
assert (charged_key.balance, charged_key.reserved_balance) == (500, 0)
assert record is not None and record.status == "charged"
await engine.dispose()
@pytest.mark.asyncio
async def test_generic_background_settlement_uses_explicit_reservation() -> None:
engine = await _engine()
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key", provider_fee=1.0
)
key = ApiKey(hashed_key="generic-key", balance=1_000)
cost = MaxCostData(
base_msats=500,
input_msats=0,
output_msats=0,
total_msats=500,
)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(key)
await session.commit()
await pay_for_request(key, 500, session)
snapshot = await get_reservation_snapshot(key, session)
context_token = auth_module._current_reservation.set(None)
try:
with (
patch(
"routstr.upstream.base.create_session",
side_effect=lambda: AsyncSession(engine, expire_on_commit=False),
),
patch(
"routstr.upstream.base.adjust_payment_for_tokens",
auth_module.adjust_payment_for_tokens,
),
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
):
await provider._finalize_generic_streaming_payment(
key.hashed_key,
500,
"audio/speech",
model_obj=None,
provider_fee=provider.provider_fee,
reservation_snapshot=snapshot,
)
finally:
auth_module._current_reservation.reset(context_token)
async with AsyncSession(engine, expire_on_commit=False) as session:
settled_key = await session.get(ApiKey, key.hashed_key)
record = await session.get(ReservationRelease, snapshot.release_id)
assert settled_key is not None
assert (settled_key.balance, settled_key.reserved_balance) == (500, 0)
assert record is not None and record.status == "charged"
await engine.dispose()
@pytest.mark.asyncio
async def test_streaming_release_is_terminal_and_suppresses_background_charge() -> None:
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key"
)
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
yield b"data: [DONE]\n\n"
upstream_response = MagicMock()
upstream_response.status_code = 200
upstream_response.headers = {"content-type": "text/event-stream"}
upstream_response.aiter_bytes = aiter_bytes
key = MagicMock(spec=ApiKey)
key.hashed_key = "test-key-hash"
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.rollback = AsyncMock()
session_context = MagicMock()
session_context.__aenter__ = AsyncMock(return_value=session)
session_context.__aexit__ = AsyncMock(return_value=None)
release = AsyncMock(return_value=True)
reservation_snapshot = MagicMock()
reservation_snapshot.reserved_msats = 500
background_tasks = MagicMock()
with (
patch(
"routstr.upstream.base.adjust_payment_for_tokens",
AsyncMock(side_effect=SQLAlchemyError("database unavailable")),
),
patch(
"routstr.upstream.base.get_reservation_snapshot",
AsyncMock(return_value=reservation_snapshot),
),
patch("routstr.upstream.base.release_reservation", release),
patch("routstr.upstream.base.create_session", return_value=session_context),
):
response = await provider.handle_streaming_chat_completion(
response=upstream_response,
key=key,
max_cost_for_model=500,
background_tasks=background_tasks,
)
with pytest.raises(SQLAlchemyError, match="database unavailable"):
async for _ in response.body_iterator:
pass
session.rollback.assert_awaited_once()
release.assert_awaited_once_with(reservation_snapshot, session, 500)
background_tasks.add_task.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"release_outcome",
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
)
async def test_responses_streaming_releases_and_raises_on_billing_failure(
release_outcome: bool | BaseException,
) -> None:
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key"
)
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
yield (
b'data: {"type":"response.completed","response":{"model":"test",'
b'"usage":{"input_tokens":1,"output_tokens":1}}}\n\n'
)
yield b"data: [DONE]\n\n"
upstream_response = MagicMock(
status_code=200,
headers={"content-type": "text/event-stream"},
)
upstream_response.aiter_bytes = aiter_bytes
key = MagicMock(spec=ApiKey)
key.hashed_key = "responses-key"
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.rollback = AsyncMock()
session_context = MagicMock()
session_context.__aenter__ = AsyncMock(return_value=session)
session_context.__aexit__ = AsyncMock(return_value=None)
snapshot = ReservationSnapshot(
release_id="responses-release",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=500,
)
release = (
AsyncMock(side_effect=release_outcome)
if isinstance(release_outcome, BaseException)
else AsyncMock(return_value=release_outcome)
)
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
with (
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
patch("routstr.upstream.base.release_reservation", release),
patch("routstr.upstream.base.create_session", return_value=session_context),
):
response = await provider.handle_streaming_responses_completion(
response=upstream_response,
key=key,
max_cost_for_model=500,
reservation_snapshot=snapshot,
)
emitted = bytearray()
with pytest.raises(SQLAlchemyError, match="database unavailable"):
async for chunk in response.body_iterator:
if isinstance(chunk, str):
emitted.extend(chunk.encode())
else:
emitted.extend(bytes(chunk))
assert b'"total_msats": 0' not in emitted
adjust.assert_awaited_once()
session.rollback.assert_awaited_once()
release.assert_awaited_once_with(snapshot, session, 500)
@pytest.mark.asyncio
@pytest.mark.parametrize("via_litellm", [False, True])
@pytest.mark.parametrize(
"release_outcome",
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
)
async def test_messages_streaming_releases_and_raises_on_billing_failure(
via_litellm: bool,
release_outcome: bool | BaseException,
) -> None:
provider = BaseUpstreamProvider(
base_url="https://api.example.com", api_key="test-key"
)
key = MagicMock(spec=ApiKey)
key.hashed_key = "messages-key"
session = MagicMock()
session.get = AsyncMock(return_value=key)
session.rollback = AsyncMock()
session_context = MagicMock()
session_context.__aenter__ = AsyncMock(return_value=session)
session_context.__aexit__ = AsyncMock(return_value=None)
snapshot = ReservationSnapshot(
release_id=f"messages-{'litellm' if via_litellm else 'native'}",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=500,
)
release = (
AsyncMock(side_effect=release_outcome)
if isinstance(release_outcome, BaseException)
else AsyncMock(return_value=release_outcome)
)
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
async def native_chunks() -> AsyncGenerator[bytes, None]:
yield (
b'event: message_start\ndata: {"type":"message_start","message":'
b'{"model":"test","usage":{"input_tokens":1,"output_tokens":0}}}\n\n'
)
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
async def litellm_chunks() -> AsyncGenerator[dict, None]:
yield {
"type": "message_start",
"message": {
"model": "test",
"usage": {"input_tokens": 1, "output_tokens": 0},
},
}
yield {"type": "message_stop"}
with (
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
patch("routstr.upstream.base.release_reservation", release),
patch("routstr.upstream.base.create_session", return_value=session_context),
):
if via_litellm:
response = provider._stream_litellm_messages(
iterator=litellm_chunks(),
key=key,
max_cost_for_model=500,
requested_model=None,
reservation_snapshot=snapshot,
)
else:
upstream_response = MagicMock(
status_code=200,
headers={"content-type": "text/event-stream"},
)
upstream_response.aiter_bytes = native_chunks
response = await provider.handle_streaming_messages_completion(
response=upstream_response,
key=key,
max_cost_for_model=500,
reservation_snapshot=snapshot,
)
with pytest.raises(SQLAlchemyError, match="database unavailable"):
async for _ in response.body_iterator:
pass
adjust.assert_awaited_once()
session.rollback.assert_awaited_once()
release.assert_awaited_once_with(snapshot, session, 500)
@pytest.mark.asyncio
async def test_cross_key_reservation_snapshot_is_rejected_without_mutation() -> None:
engine = await _engine()
first = ApiKey(hashed_key="first", balance=1_000)
second = ApiKey(hashed_key="second", balance=1_000)
async with AsyncSession(engine, expire_on_commit=False) as session:
session.add(first)
session.add(second)
await session.commit()
await pay_for_request(first, 500, session)
snapshot = await get_reservation_snapshot(first, session)
with pytest.raises(RuntimeError, match="does not belong"):
await adjust_payment_for_tokens(
second,
{"model": "test", "usage": None},
session,
500,
reservation_snapshot=snapshot,
)
await session.refresh(first)
await session.refresh(second)
assert first.reserved_balance == 500
assert second.reserved_balance == 0
await engine.dispose()
@@ -24,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.upstream import base
from routstr.upstream.base import BaseUpstreamProvider
@@ -67,6 +68,12 @@ async def _drive(chunks: list[bytes], requested_model: str | None = None) -> lis
max_cost_for_model=100,
background_tasks=MagicMock(),
requested_model=requested_model,
reservation_snapshot=ReservationSnapshot(
release_id="test-release",
key_hash="test_hash",
billing_key_hash="test_hash",
reserved_msats=100,
),
)
out: list[bytes] = []
+14 -4
View File
@@ -331,6 +331,7 @@ async def test_5xx_wrapped_rate_limit_is_classified(
@pytest.mark.asyncio
async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
from routstr import proxy as proxy_module
from routstr.auth import ReservationSnapshot
from routstr.core.db import ApiKey
from routstr.core.exceptions import UpstreamError
@@ -359,6 +360,12 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
)
session = MagicMock()
reservation = ReservationSnapshot(
release_id="rate-limit-release",
key_hash=key.hashed_key,
billing_key_hash=key.hashed_key,
reserved_msats=1_000,
)
revert_mock = AsyncMock(return_value=True)
with (
@@ -376,10 +383,13 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
AsyncMock(return_value=1_000),
),
patch.object(proxy_module, "check_token_balance", MagicMock()),
patch.object(
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
),
patch.object(proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)),
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
patch.object(
proxy_module,
"get_reservation_snapshot",
AsyncMock(return_value=reservation),
),
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
):
response = await proxy_module.proxy(
@@ -396,4 +406,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
assert RAW_ORG_ID not in serialized
assert "org-[REDACTED]" in serialized
# Single upstream failed -> reservation reverted exactly once (no double-charge).
revert_mock.assert_awaited_once_with(key, session, 1_000)
revert_mock.assert_awaited_once_with(key, session, 1000, reservation)
+1586 -48
View File
File diff suppressed because it is too large Load Diff
+9 -3
View File
@@ -67,8 +67,13 @@ async def test_non_streaming_includes_cost_sats() -> None:
)
body = json.loads(response.body)
assert "cost_sats" in body["usage"]
assert body["usage"]["cost_sats"] == 5 # 5000 msats // 1000
assert body["usage"]["cost"]["total_msats"] == 5000
assert body["usage"]["cost"]["input_msats"] == 3000
assert body["usage"]["cost"]["output_msats"] == 2000
assert response.headers["x-routstr-cost-msats"] == "5000"
assert response.headers["x-routstr-input-cost-msats"] == "3000"
assert response.headers["x-routstr-output-cost-msats"] == "2000"
@pytest.mark.asyncio
@@ -96,7 +101,7 @@ async def test_non_streaming_cost_sats_value_rounds_down() -> None:
@pytest.mark.asyncio
async def test_non_streaming_preserves_existing_usage_fields() -> None:
async def test_non_streaming_preserves_tokens_and_replaces_upstream_cost() -> None:
provider = _make_provider()
cost_data = _make_cost_data(total_msats=3000)
@@ -127,7 +132,8 @@ async def test_non_streaming_preserves_existing_usage_fields() -> None:
assert usage["prompt_tokens"] == 100
assert usage["completion_tokens"] == 50
assert usage["total_tokens"] == 150
assert usage["cost"] == 0.00015
assert usage["cost"]["total_msats"] == 3000
assert usage["cost"]["total_usd"] == 0.00025
assert usage["cost_sats"] == 3
+27 -4
View File
@@ -105,6 +105,23 @@ export function DetailedWalletBalance({
const formatMintLabel = (detail: BalanceDetail) =>
`${detail.mint_url.replace('https://', '').replace('http://', '')}${detail.unit.toUpperCase()}`;
const formatBalanceError = (detail: BalanceDetail) => {
const labels: Record<string, string> = {
rate_limited: 'rate limited',
unreachable: 'unreachable',
cooldown: 'cooling down',
mint_error: 'mint error',
};
const label =
(detail.error_code ? labels[detail.error_code] : undefined) ??
detail.error ??
'error';
const retryAfter = detail.retry_after_seconds;
return retryAfter && retryAfter > 0
? `${label} (retry in ${Math.ceil(retryAfter)}s)`
: label;
};
return (
<>
<Card>
@@ -262,9 +279,12 @@ export function DetailedWalletBalance({
<TableCell className='max-w-md font-mono text-xs break-all whitespace-normal'>
{formatMintLabel(detail)}
</TableCell>
<TableCell className='text-right font-mono'>
<TableCell
className='text-right font-mono'
title={detail.error}
>
{detail.error
? 'error'
? formatBalanceError(detail)
: formatAmount(walletMsat)}
</TableCell>
<TableCell className='text-right font-mono'>
@@ -306,9 +326,12 @@ export function DetailedWalletBalance({
<p className='text-muted-foreground text-xs'>
Wallet
</p>
<p className='font-mono text-sm'>
<p
className='font-mono text-sm'
title={detail.error}
>
{detail.error
? 'error'
? formatBalanceError(detail)
: formatAmount(walletMsat)}
</p>
</div>
@@ -91,25 +91,27 @@ export function CashuPaymentWorkflow({
setIsCreatingKey(true);
try {
const params = new URLSearchParams({
const requestPayload: {
initial_balance_token: string;
balance_limit?: number;
balance_limit_reset?: string;
validity_date?: number;
} = {
initial_balance_token: initialToken.trim(),
});
if (balanceLimit) params.append('balance_limit', balanceLimit);
};
if (balanceLimit) requestPayload.balance_limit = Number(balanceLimit);
if (balanceLimitReset)
params.append('balance_limit_reset', balanceLimitReset);
requestPayload.balance_limit_reset = balanceLimitReset;
if (validityDate) {
const timestamp = Math.floor(
requestPayload.validity_date = Math.floor(
new Date(validityDate + 'T23:59:59').getTime() / 1000
);
params.append('validity_date', timestamp.toString());
}
const response = await fetch(
`${baseUrl}/v1/balance/create?${params.toString()}`,
{
method: 'GET',
headers: { 'Content-Type': 'application/json' },
}
);
const response = await fetch(`${baseUrl}/v1/balance/create`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(requestPayload),
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(errorText || 'Failed to create API key');
+158 -1
View File
@@ -1,3 +1,5 @@
import { AdminService } from '@/lib/api/services/admin';
import type { PPQAutoTopupState } from '@/lib/api/services/admin';
import type {
AdminModel,
ProviderModels,
@@ -20,12 +22,16 @@ import {
Trash2,
Key,
RotateCcw,
AlertTriangle,
Unlock,
Loader2,
} from 'lucide-react';
import { ProviderBalance } from '@/components/provider-balance';
import { ProviderModelsPanel } from '@/components/provider-models-panel';
import { RoutstrCreateKeySection } from '@/components/providers/RoutstrCreateKeySection';
import { RoutstrProviderService } from '@/lib/api/services/routstr-provider';
import { useMutation, useQueryClient } from '@tanstack/react-query';
import { getErrorStatus } from '@/lib/api/client';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { useState } from 'react';
import { toast } from 'sonner';
import { cn } from '@/lib/utils';
@@ -36,6 +42,16 @@ import {
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog';
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from '@/components/ui/alert-dialog';
interface ProviderCardProps {
provider: UpstreamProvider;
@@ -77,8 +93,71 @@ export function ProviderCard({
}: ProviderCardProps) {
const queryClient = useQueryClient();
const [isKeyModalOpen, setIsKeyModalOpen] = useState(false);
const [isReleaseDialogOpen, setIsReleaseDialogOpen] = useState(false);
// The claim as the query cache held it when the admin opened the dialog.
// The mutation sends this token rather than re-reading the query at submit
// time: a background refetch after the dialog opened must not swap in a
// state the admin never saw. The server rejects a stale token with a 409,
// which is the authoritative guard.
const [reviewedState, setReviewedState] = useState<PPQAutoTopupState | null>(
null
);
const hasDetails = Boolean(provider.api_version) || isExpanded;
const isRoutstr = provider.provider_type === 'routstr';
const isPPQ = provider.provider_type === 'ppqai';
const { data: ppqAutoTopupState, isError: ppqStateFetchFailed } = useQuery({
queryKey: ['ppq-auto-topup-state', provider.id],
queryFn: () => AdminService.getPPQAutoTopupState(provider.id),
enabled: isPPQ,
refetchInterval: 30000,
});
// A claim the server will not let us release: a worker is between reserving
// proofs and hearing back from the mint, and sweeping it would let the next
// cycle pay a second invoice.
const isPPQPaymentInFlight =
Boolean(ppqAutoTopupState?.active) &&
ppqAutoTopupState?.releasable === false;
const openReleaseDialog = () => {
setReviewedState(ppqAutoTopupState ?? null);
setIsReleaseDialogOpen(true);
};
const releasePPQMutation = useMutation({
mutationFn: () =>
AdminService.releasePPQAutoTopup(
provider.id,
reviewedState?.state_token ?? null
),
onSuccess: () => {
queryClient.invalidateQueries({
queryKey: ['ppq-auto-topup-state', provider.id],
});
setIsReleaseDialogOpen(false);
setReviewedState(null);
toast.success('PPQ auto top-up claim released');
},
onError: (error: Error) => {
queryClient.invalidateQueries({
queryKey: ['ppq-auto-topup-state', provider.id],
});
if (getErrorStatus(error) === 409) {
// The claim changed since it was reviewed; the stale snapshot is
// useless, so force a fresh review.
setIsReleaseDialogOpen(false);
setReviewedState(null);
toast.error(
'PPQ claim changed since it was reviewed; reopen to see the new state'
);
return;
}
// Transient failure: keep the dialog and the reviewed snapshot so the
// admin can retry without re-navigating.
toast.error(`Failed to release PPQ claim: ${error.message}`);
},
});
const refundMutation = useMutation({
mutationFn: () => RoutstrProviderService.refundBalance(provider.id),
@@ -113,6 +192,35 @@ export function ProviderCard({
>
{provider.enabled ? 'Enabled' : 'Disabled'}
</Badge>
{ppqAutoTopupState?.active && (
<Badge
variant='outline'
className={cn(
'w-fit gap-1',
isPPQPaymentInFlight
? 'border-blue-500 text-blue-700 dark:text-blue-400'
: 'border-amber-500 text-amber-700 dark:text-amber-400'
)}
>
{isPPQPaymentInFlight ? (
<Loader2 className='h-3 w-3 animate-spin' />
) : (
<AlertTriangle className='h-3 w-3' />
)}
{isPPQPaymentInFlight
? 'Paying invoice'
: 'Auto top-up needs review'}
</Badge>
)}
{isPPQ && ppqStateFetchFailed && (
<Badge
variant='outline'
className='border-destructive text-destructive w-fit gap-1'
>
<AlertTriangle className='h-3 w-3' />
Top-up status unavailable
</Badge>
)}
</div>
<CardDescription className='break-all'>
{provider.base_url}
@@ -153,6 +261,19 @@ export function ProviderCard({
</Button>
)}
{isPPQ && ppqAutoTopupState?.active && !isPPQPaymentInFlight && (
<Button
variant='outline'
size='sm'
onClick={openReleaseDialog}
className='justify-center gap-1.5 border-amber-500 text-amber-700 dark:text-amber-400'
title='Release only after manually verifying the Lightning payment outcome'
>
<Unlock className='h-4 w-4' />
<span>Release top-up</span>
</Button>
)}
{isRoutstr && provider.api_key && (
<Button
variant='outline'
@@ -212,6 +333,42 @@ export function ProviderCard({
</div>
</CardHeader>
<AlertDialog
open={isReleaseDialogOpen}
onOpenChange={setIsReleaseDialogOpen}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Release PPQ auto top-up?</AlertDialogTitle>
<AlertDialogDescription>
Only continue after checking PPQ and the Cashu mint and confirming
the previous Lightning payment cannot later settle. Releasing an
ambiguous payment can allow a duplicate top-up.
{reviewedState?.invoice_id
? ` Invoice: ${reviewedState.invoice_id}`
: ''}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={(e) => {
// Radix closes the dialog on action click by default; the
// mutation handlers decide whether it closes (kept open on
// transient errors so the admin can retry).
e.preventDefault();
releasePPQMutation.mutate();
}}
disabled={releasePPQMutation.isPending}
>
{releasePPQMutation.isPending
? 'Releasing...'
: 'I verified it is safe to retry'}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
<Dialog open={isKeyModalOpen} onOpenChange={setIsKeyModalOpen}>
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[500px]'>
<DialogHeader>
@@ -12,6 +12,7 @@ import {
DialogTitle,
} from '@/components/ui/dialog';
import { ProviderFormFields } from '@/components/provider-form-fields';
import { ppqAutoTopupSettingsInvalid } from '@/components/providers/PPQAutoTopupSettings';
interface ProviderFormDialogContentProps {
mode: 'create' | 'edit';
@@ -52,6 +53,11 @@ export function ProviderFormDialogContent({
isSubmitting,
availableMints,
}: ProviderFormDialogContentProps) {
// The server re-validates these bounds; this only stops submitting a form
// whose inline errors are already visible.
const hasInvalidSettings =
formData.provider_type === 'ppqai' &&
ppqAutoTopupSettingsInvalid(formData.provider_settings || {});
return (
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[500px]'>
<DialogHeader>
@@ -80,7 +86,7 @@ export function ProviderFormDialogContent({
</Button>
<Button
onClick={onSubmit}
disabled={isSubmitting}
disabled={isSubmitting || hasInvalidSettings}
className='w-full sm:w-auto'
>
{isSubmitting ? submittingLabel : submitLabel}
+16
View File
@@ -17,6 +17,7 @@ import {
SelectValue,
} from '@/components/ui/select';
import { Switch } from '@/components/ui/switch';
import { PPQAutoTopupSettings } from '@/components/providers/PPQAutoTopupSettings';
import { RoutstrNodeSettings } from '@/components/providers/RoutstrNodeSettings';
import { RoutstrCreateKeySection } from '@/components/providers/RoutstrCreateKeySection';
@@ -78,6 +79,8 @@ export function ProviderFormFields({
provider_type: value,
base_url: getDefaultBaseUrl(value),
provider_fee: value === 'openrouter' ? 1.06 : 1.01,
provider_settings:
value === prev.provider_type ? prev.provider_settings : {},
}));
}}
>
@@ -118,6 +121,19 @@ export function ProviderFormFields({
/>
)}
{formData.provider_type === 'ppqai' && (
<PPQAutoTopupSettings
settings={formData.provider_settings || {}}
onSettingsChange={(settings) =>
setFormData((prev) => ({
...prev,
provider_settings: settings,
}))
}
idPrefix={mode === 'edit' ? 'edit' : ''}
/>
)}
<div className='grid gap-2'>
<Label htmlFor={`${idPrefix}slug`}>
Slug {mode === 'create' ? '(optional, auto-generated)' : ''}
@@ -0,0 +1,191 @@
'use client';
import { Input } from '@/components/ui/input';
import { Label } from '@/components/ui/label';
import { Switch } from '@/components/ui/switch';
interface ProviderSettings {
auto_topup?: boolean;
topup_threshold?: number;
topup_amount_limit?: number;
[key: string]: unknown;
}
interface PPQAutoTopupSettingsProps {
settings: ProviderSettings;
onSettingsChange: (settings: ProviderSettings) => void;
idPrefix?: string;
}
/**
* Field-level validation shared with the dialog's submit gating. The server
* enforces the same bounds authoritatively; this only keeps a knowingly
* invalid form from being submitted.
*
* Validation only applies while auto top-up is enabled: a disabled toggle
* hides the fields, and stale out-of-range values behind it must not block
* submission invisibly. When enabled, both fields are required a blank
* field would otherwise submit and fail server-side.
*/
export function ppqAutoTopupSettingsErrors(settings: ProviderSettings): {
thresholdError?: string;
amountError?: string;
} {
if (!settings.auto_topup) {
return {};
}
const threshold = settings.topup_threshold;
const amount = settings.topup_amount_limit;
return {
thresholdError:
threshold === undefined
? 'Required when auto top-up is enabled'
: threshold <= 0
? 'Must be greater than 0'
: undefined,
amountError:
amount === undefined
? 'Required when auto top-up is enabled'
: amount < 1 || amount > 500
? 'Must be between 1 and 500 USD'
: undefined,
};
}
export function ppqAutoTopupSettingsInvalid(
settings: ProviderSettings
): boolean {
const { thresholdError, amountError } = ppqAutoTopupSettingsErrors(settings);
return Boolean(thresholdError || amountError);
}
export function PPQAutoTopupSettings({
settings,
onSettingsChange,
idPrefix = '',
}: PPQAutoTopupSettingsProps) {
const prefix = idPrefix ? `${idPrefix}_` : '';
const update = (patch: Partial<ProviderSettings>) =>
onSettingsChange({ ...settings, ...patch });
/**
* Clearing the field yields '' and parse* yields NaN, which JSON.stringify
* turns into null. Drop the key instead so the server rejects a missing
* value rather than storing a broken one.
*/
const updateNumber = (
key: 'topup_threshold' | 'topup_amount_limit',
raw: string,
parse: (value: string) => number
) => {
const next = { ...settings };
const parsed = parse(raw);
if (raw === '' || Number.isNaN(parsed)) {
delete next[key];
} else {
next[key] = parsed;
}
onSettingsChange(next);
};
const { thresholdError, amountError } = ppqAutoTopupSettingsErrors(settings);
return (
<div className='bg-muted/30 grid gap-4 rounded-lg border p-4'>
<Label className='text-sm font-semibold'>PPQ Auto Top-up</Label>
<div className='flex items-center justify-between'>
<Label htmlFor={`${prefix}ppq_auto_topup`} className='text-sm'>
Enable Auto Top-up
</Label>
<Switch
id={`${prefix}ppq_auto_topup`}
checked={!!settings.auto_topup}
onCheckedChange={(checked) => update({ auto_topup: checked })}
/>
</div>
{settings.auto_topup && (
<div className='border-primary/20 grid gap-4 border-l-2 pt-2 pl-4'>
<div className='grid gap-2'>
<Label
htmlFor={`${prefix}ppq_topup_threshold`}
className='text-xs font-medium'
>
When credits are below (USD)
</Label>
<Input
id={`${prefix}ppq_topup_threshold`}
type='number'
min='0.01'
step='0.01'
className='h-9'
placeholder='e.g. 5'
value={settings.topup_threshold ?? ''}
aria-invalid={Boolean(thresholdError)}
aria-describedby={
thresholdError
? `${prefix}ppq_topup_threshold_error`
: undefined
}
onChange={(e) =>
updateNumber('topup_threshold', e.target.value, parseFloat)
}
/>
{thresholdError && (
<p
id={`${prefix}ppq_topup_threshold_error`}
className='text-destructive text-[10px]'
>
{thresholdError}
</p>
)}
</div>
<div className='grid gap-2'>
<Label
htmlFor={`${prefix}ppq_topup_amount_limit`}
className='text-xs font-medium'
>
Purchase this amount (USD)
</Label>
<Input
id={`${prefix}ppq_topup_amount_limit`}
type='number'
min='1'
max='500'
step='1'
className='h-9'
placeholder='e.g. 10'
value={settings.topup_amount_limit ?? ''}
aria-invalid={Boolean(amountError)}
aria-describedby={
amountError
? `${prefix}ppq_topup_amount_limit_error`
: undefined
}
onChange={(e) =>
updateNumber('topup_amount_limit', e.target.value, (v) =>
parseInt(v, 10)
)
}
/>
{amountError && (
<p
id={`${prefix}ppq_topup_amount_limit_error`}
className='text-destructive text-[10px]'
>
{amountError}
</p>
)}
</div>
<p className='text-muted-foreground text-[10px]'>
Pays PPQ&apos;s Lightning invoice from the sufficiently funded Cashu
mint with the highest available balance.
</p>
</div>
)}
</div>
);
}
@@ -161,13 +161,11 @@ export function RoutstrCreateKeySection({
setIsCreatingCashu(true);
try {
const params = new URLSearchParams({
initial_balance_token: cashuToken.trim(),
const resp = await fetch(`${cleanUrl}/v1/balance/create`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ initial_balance_token: cashuToken.trim() }),
});
const resp = await fetch(
`${cleanUrl}/v1/balance/create?${params.toString()}`,
{ method: 'GET', headers: { 'Content-Type': 'application/json' } }
);
if (!resp.ok) {
const errorText = await resp.text();
+15
View File
@@ -144,3 +144,18 @@ export class ApiError extends Error {
this.data = data;
}
}
/**
* HTTP status of a caught request error, whatever shape it arrived in.
* apiClient methods rethrow raw Axios errors, so callers must not rely on
* `instanceof ApiError` alone to read a status code.
*/
export function getErrorStatus(error: unknown): number | undefined {
if (error instanceof ApiError) {
return error.status;
}
if (axios.isAxiosError(error)) {
return error.response?.status;
}
return undefined;
}
+50
View File
@@ -1013,6 +1013,32 @@ export class AdminService {
}>(`/admin/api/upstream-providers/${providerId}/balance`);
}
static async getPPQAutoTopupState(
providerId: number
): Promise<PPQAutoTopupState> {
return await apiClient.get<PPQAutoTopupState>(
`/admin/api/upstream-providers/${providerId}/ppq-auto-topup`
);
}
/**
* `stateToken` must be the `state_token` snapshotted when the admin opened
* the confirmation not re-read at submit time. The server rejects a
* release whose claim changed in any way since that snapshot.
*/
static async releasePPQAutoTopup(
providerId: number,
stateToken: string | null
): Promise<{
ok: boolean;
released: boolean;
}> {
return await apiClient.post<{ ok: boolean; released: boolean }>(
`/admin/api/upstream-providers/${providerId}/ppq-auto-topup/release`,
{ confirmed_safe_to_retry: true, state_token: stateToken }
);
}
// ── CLI Tokens ──
static async listCliTokens(): Promise<CliTokenListItem[]> {
@@ -1244,6 +1270,30 @@ export interface TransactionsResponse {
total: number;
}
export interface PPQAutoTopupState {
ok: boolean;
active: boolean;
/**
* Opaque version of the claim as reviewed. Echo it back verbatim to
* release; the server rejects a release whose claim changed in any way
* (new attempt, phase change, renewed lease) since this was read.
*/
state_token?: string | null;
/** Identifies the attempt currently holding the claim. Informational. */
operation_id?: string | null;
/** 'claimed' | 'in_flight' | 'reconcile'. Null when the claim is malformed. */
phase?: string | null;
/** False while a payment is in flight — the server rejects a release then. */
releasable?: boolean;
expires_at?: number | null;
invoice_id?: string | null;
created_at?: number;
amount?: number;
unit?: string;
mint_url?: string | null;
malformed?: boolean;
}
export interface LightningInvoice {
id: string;
bolt11: string;
+3
View File
@@ -36,10 +36,13 @@ export interface BalanceDetail {
user_balance: number;
owner_balance: number;
error?: string;
error_code?: 'rate_limited' | 'unreachable' | 'cooldown' | 'mint_error';
retry_after_seconds?: number;
}
export interface WithdrawResponse {
token: string;
mint_url: string;
}
export interface CreateChildKeyResponse {