From 2b4f70442af28093526a8e07ae50d8be6ce39eaf Mon Sep 17 00:00:00 2001 From: thefux Date: Wed, 22 Jul 2026 00:06:33 +0000 Subject: [PATCH 01/26] fix: recover stale Cashu reservations safely --- routstr/wallet.py | 17 +- scripts/reconcile_reserved_proofs.py | 233 +++++++++++++++++++++++ scripts/retry_minibits_reconciliation.sh | 62 ++++++ tests/unit/test_fetch_all_balances.py | 31 ++- 4 files changed, 334 insertions(+), 9 deletions(-) create mode 100755 scripts/reconcile_reserved_proofs.py create mode 100755 scripts/retry_minibits_reconciliation.sh diff --git a/routstr/wallet.py b/routstr/wallet.py index 03d45fb8..5ea4025b 100644 --- a/routstr/wallet.py +++ b/routstr/wallet.py @@ -219,8 +219,10 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int """Internal send function - returns amount and serialized token""" effective_mint_url = mint_url or settings.primary_mint wallet: Wallet = await get_wallet(effective_mint_url, unit) - proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit) + all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit) + proofs = [proof for proof in all_proofs if not proof.reserved] proofs_for_mint = sum(p.amount for p in proofs) + reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved) # Fallback: proofs from untrusted source mints are swapped to primary_mint # during receive, so the user's preferred refund_mint_url may have no proofs @@ -232,8 +234,10 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int ) effective_mint_url = settings.primary_mint wallet = await get_wallet(effective_mint_url, unit) - proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit) + all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit) + proofs = [proof for proof in all_proofs if not proof.reserved] proofs_for_mint = sum(p.amount for p in proofs) + reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved) all_mint_urls = list({k.mint_url for k in wallet.keysets.values()}) proof_summary = { @@ -247,7 +251,8 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int raw_proofs_by_keyset[p.id] = raw_proofs_by_keyset.get(p.id, 0) + p.amount logger.info( f"send: proof inventory | mint={effective_mint_url} unit={unit} amount={amount} " - f"primary_mint={settings.primary_mint} proofs_for_mint={proofs_for_mint} " + f"primary_mint={settings.primary_mint} liquid_proofs_for_mint={proofs_for_mint} " + f"reserved_proofs_for_mint={reserved_for_mint} " f"all_mints={all_mint_urls} by_keyset={proof_summary} " f"raw_proofs_by_keyset_id={raw_proofs_by_keyset} " f"total_wallet_proofs={sum(p.amount for p in wallet.proofs)}" @@ -848,7 +853,7 @@ async def fetch_all_balances( "unit": unit, "wallet_balance": proofs_balance, "user_balance": user_balance, - "owner_balance": proofs_balance - user_balance if proofs_balance != 0 else 0, + "owner_balance": proofs_balance - user_balance, } return result except Exception as e: @@ -904,9 +909,7 @@ async def fetch_all_balances( total_wallet_balance_sats += proofs_balance_sats total_user_balance_sats += user_balance_sats - owner_balance = 0 - if total_wallet_balance_sats != 0: - owner_balance = total_wallet_balance_sats - total_user_balance_sats + owner_balance = total_wallet_balance_sats - total_user_balance_sats return ( balance_details, diff --git a/scripts/reconcile_reserved_proofs.py b/scripts/reconcile_reserved_proofs.py new file mode 100755 index 00000000..4ae3fa48 --- /dev/null +++ b/scripts/reconcile_reserved_proofs.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Reconcile Routstr's reserved Cashu proofs against their mints. + +Safe default is dry-run. --apply mutates wallet.sqlite3 and keys.db. +Run only while no process is using these databases and after verified backups. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sqlite3 +import time +from collections import defaultdict +from pathlib import Path + +import httpx +from cashu.core.base import Proof +from cashu.wallet.helpers import deserialize_token_from_string + + +def proof_from_row(row: sqlite3.Row) -> Proof: + return Proof(amount=row["amount"], C=row["C"], secret=row["secret"], id=row["id"]) + + +async def fetch_states( + client: httpx.AsyncClient, + mint_url: str, + proofs: list[Proof], + batch_size: int, +) -> dict[str, str]: + states: dict[str, str] = {} + endpoint = mint_url.rstrip("/") + "/v1/checkstate" + for offset in range(0, len(proofs), batch_size): + batch = proofs[offset : offset + batch_size] + last_error: Exception | None = None + for attempt in range(4): + try: + response = await client.post(endpoint, json={"Ys": [proof.Y for proof in batch]}) + response.raise_for_status() + for item in response.json().get("states", []): + states[item["Y"]] = item["state"] + last_error = None + break + except (httpx.TimeoutException, httpx.TransportError) as exc: + last_error = exc + await asyncio.sleep(2**attempt) + if last_error is not None: + raise last_error + return states + + +def load_pending_refund_secrets(keys: sqlite3.Connection) -> tuple[set[str], dict[str, set[str]]]: + pending: set[str] = set() + transaction_secrets: dict[str, set[str]] = {} + rows = keys.execute( + """ + SELECT id, token + FROM cashu_transactions + WHERE type = 'out' AND collected = 0 AND swept = 0 + """ + ).fetchall() + for row in rows: + try: + token = deserialize_token_from_string(row["token"]) + except Exception: + continue + secrets = {proof.secret for proof in token.proofs} + transaction_secrets[row["id"]] = secrets + pending.update(secrets) + return pending, transaction_secrets + + +async def run(args: argparse.Namespace) -> int: + root = Path(args.root).resolve() + keys_path = root / "keys.db" + wallet_path = root / ".wallet" / "wallet.sqlite3" + if not keys_path.exists() or not wallet_path.exists(): + raise SystemExit("keys.db or .wallet/wallet.sqlite3 not found") + + keys = sqlite3.connect(keys_path) + wallet = sqlite3.connect(wallet_path) + keys.row_factory = sqlite3.Row + wallet.row_factory = sqlite3.Row + keys.execute("PRAGMA foreign_keys=ON") + wallet.execute("PRAGMA foreign_keys=ON") + + if keys.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise SystemExit("keys.db integrity check failed") + if wallet.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise SystemExit("wallet.sqlite3 integrity check failed") + + pending_secrets, transaction_secrets = load_pending_refund_secrets(keys) + rows = wallet.execute( + """ + SELECT p.rowid AS proof_rowid, p.*, k.mint_url, k.unit + FROM proofs p + JOIN keysets k ON k.id = p.id + WHERE COALESCE(p.reserved, 0) != 0 + ORDER BY k.mint_url, k.unit, p.time_reserved + """ + ).fetchall() + grouped: dict[tuple[str, str], list[sqlite3.Row]] = defaultdict(list) + for row in rows: + grouped[(row["mint_url"], row["unit"])].append(row) + + report: dict[str, object] = { + "mode": "apply" if args.apply else "dry-run", + "root": str(root), + "started_at": int(time.time()), + "pending_refund_secrets": len(pending_secrets), + "mints": {}, + "errors": {}, + } + state_by_secret: dict[str, str] = {} + + async with httpx.AsyncClient(timeout=args.timeout) as client: + for (mint_url, unit), mint_rows in grouped.items(): + proofs = [proof_from_row(row) for row in mint_rows] + try: + states = await fetch_states(client, mint_url, proofs, args.batch_size) + except Exception as exc: + report["errors"][f"{mint_url}|{unit}"] = f"{type(exc).__name__}: {exc}" + continue + + summary: dict[str, dict[str, int]] = defaultdict(lambda: {"proofs": 0, "amount": 0}) + actions = {"delete_spent": 0, "release_untracked_unspent": 0, "preserve_pending": 0, "preserve_unknown": 0} + for row, proof in zip(mint_rows, proofs): + state = states.get(proof.Y, "MISSING") + state_by_secret[row["secret"]] = state + summary[state]["proofs"] += 1 + summary[state]["amount"] += row["amount"] + + if state == "SPENT": + actions["delete_spent"] += 1 + if args.apply: + wallet.execute( + """ + INSERT OR IGNORE INTO proofs_used + (amount, C, secret, time_used, id, derivation_path, mint_id, melt_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + row["amount"], row["C"], row["secret"], + row["time_reserved"] or int(time.time()), row["id"], + row["derivation_path"], row["mint_id"], row["melt_id"], + ), + ) + wallet.execute("DELETE FROM proofs WHERE rowid = ?", (row["proof_rowid"],)) + elif state == "UNSPENT" and row["secret"] not in pending_secrets: + actions["release_untracked_unspent"] += 1 + if args.apply: + wallet.execute( + "UPDATE proofs SET reserved = 0, send_id = NULL, time_reserved = NULL WHERE rowid = ?", + (row["proof_rowid"],), + ) + elif state == "UNSPENT": + actions["preserve_pending"] += 1 + else: + actions["preserve_unknown"] += 1 + + report["mints"][f"{mint_url}|{unit}"] = { + "states": dict(summary), + "actions": actions, + } + + # Mark pending outgoing tokens collected only when every proof the mint reported is SPENT. + collected_transactions: list[str] = [] + for transaction_id, secrets in transaction_secrets.items(): + known = [state_by_secret.get(secret) for secret in secrets] + if known and all(state == "SPENT" for state in known): + collected_transactions.append(transaction_id) + if args.apply: + keys.execute( + "UPDATE cashu_transactions SET collected = 1 WHERE id = ? AND collected = 0 AND swept = 0", + (transaction_id,), + ) + report["mark_collected_transactions"] = len(collected_transactions) + + malformed = keys.execute( + "SELECT COUNT(*), COALESCE(SUM(balance), 0) FROM api_keys WHERE refund_mint_url = ?", + ("https://mint.minibits.cash/Bi",), + ).fetchone() + report["canonicalize_minibits_url"] = {"keys": malformed[0], "balance_msat": malformed[1]} + if args.apply: + keys.execute( + "UPDATE api_keys SET refund_mint_url = ? WHERE refund_mint_url = ?", + ("https://mint.minibits.cash/Bitcoin", "https://mint.minibits.cash/Bi"), + ) + + negative_rows = keys.execute( + "SELECT hashed_key, balance FROM api_keys WHERE balance < 0 ORDER BY balance" + ).fetchall() + report["negative_balances"] = { + "keys": len(negative_rows), + "amount_msat": sum(row["balance"] for row in negative_rows), + "key_prefixes": [row["hashed_key"][:12] for row in negative_rows], + } + if args.apply: + keys.execute("UPDATE api_keys SET balance = 0 WHERE balance < 0") + + if args.apply: + wallet.commit() + keys.commit() + else: + wallet.rollback() + keys.rollback() + + report["finished_at"] = int(time.time()) + output = root / f"reconciliation-{'applied' if args.apply else 'dry-run'}-{report['finished_at']}.json" + output.write_text(json.dumps(report, indent=2, sort_keys=True)) + os.chmod(output, 0o600) + print(json.dumps(report, indent=2, sort_keys=True)) + print(f"report={output}") + + wallet.close() + keys.close() + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", default=".") + parser.add_argument("--apply", action="store_true") + parser.add_argument("--batch-size", type=int, default=300) + parser.add_argument("--timeout", type=float, default=45.0) + return asyncio.run(run(parser.parse_args())) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/retry_minibits_reconciliation.sh b/scripts/retry_minibits_reconciliation.sh new file mode 100755 index 00000000..8851ea14 --- /dev/null +++ b/scripts/retry_minibits_reconciliation.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${HOME}/proxy" +MARKER="${ROOT}/.minibits-reconciliation-complete" +LOCK="${ROOT}/.minibits-reconciliation.lock" +LOG="${ROOT}/logs/minibits-reconciliation.log" +TAG="ROUTSTR-MINIBITS-RECONCILE" + +cd "$ROOT" +[[ -e "$MARKER" ]] && exit 0 +exec 9>"$LOCK" +flock -n 9 || exit 0 + +printf '%s starting reconciliation retry\n' "$(date -Is)" >>"$LOG" +if ! .venv/bin/python scripts/reconcile_reserved_proofs.py --root . --timeout 90 --apply >>"$LOG" 2>&1; then + printf '%s reconciliation command failed\n' "$(date -Is)" >>"$LOG" + exit 0 +fi + +latest=$(ls -1t reconciliation-applied-*.json 2>/dev/null | head -1 || true) +[[ -n "$latest" ]] || exit 0 +if .venv/bin/python - "$latest" <<'PY' +import json, sys +report = json.load(open(sys.argv[1])) +error_keys = report.get("errors", {}) +raise SystemExit(any(key.startswith("https://mint.minibits.cash/Bitcoin|") for key in error_keys)) +PY +then + if .venv/bin/python - <<'PY' >>"$LOG" 2>&1 +import sqlite3 +keys = sqlite3.connect("keys.db") +wallet = sqlite3.connect(".wallet/wallet.sqlite3") +positive_balances = keys.execute( + "SELECT COALESCE(SUM(CASE WHEN balance > 0 THEN balance ELSE 0 END), 0) FROM api_keys" +).fetchone()[0] +pending_refunds = keys.execute( + "SELECT COALESCE(SUM(CASE WHEN unit = 'sat' THEN amount * 1000 ELSE amount END), 0) " + "FROM cashu_transactions WHERE type = 'out' AND collected = 0 AND swept = 0" +).fetchone()[0] +fees = keys.execute("SELECT COALESCE(SUM(accumulated_msats), 0) FROM routstr_fees").fetchone()[0] +liquid_msats = wallet.execute( + "SELECT COALESCE(SUM(CASE WHEN k.unit = 'msat' THEN p.amount ELSE p.amount * 1000 END), 0) " + "FROM proofs p JOIN keysets k ON k.id = p.id WHERE COALESCE(p.reserved, 0) = 0" +).fetchone()[0] +obligations = positive_balances + pending_refunds + fees +print(f"liquid_msats={liquid_msats} obligations_msats={obligations} surplus_msats={liquid_msats-obligations}") +keys.close(); wallet.close() +raise SystemExit(0 if liquid_msats >= obligations else 1) +PY + then + touch "$MARKER" + chmod 600 "$MARKER" + printf '%s Minibits reconciliation completed and solvency verified; starting Routstr\n' "$(date -Is)" >>"$LOG" + docker compose up -d routstr >>"$LOG" 2>&1 + (crontab -l 2>/dev/null | grep -v "$TAG" || true) | crontab - + else + printf '%s reconciliation completed but liquid assets remain below obligations; node stays stopped\n' "$(date -Is)" >>"$LOG" + fi +else + printf '%s Minibits remains unavailable; retry retained\n' "$(date -Is)" >>"$LOG" +fi diff --git a/tests/unit/test_fetch_all_balances.py b/tests/unit/test_fetch_all_balances.py index dcd99107..433e84d4 100644 --- a/tests/unit/test_fetch_all_balances.py +++ b/tests/unit/test_fetch_all_balances.py @@ -11,7 +11,9 @@ async def _fake_session(): # type: ignore[no-untyped-def] yield MagicMock() -def _patches(proof_amount: int = 1000): # type: ignore[no-untyped-def] +def _patches( # type: ignore[no-untyped-def] + proof_amount: int = 1000, user_balance_msats: int = 0 +): proof = MagicMock(amount=proof_amount) return [ patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())), @@ -25,7 +27,7 @@ def _patches(proof_amount: int = 1000): # type: ignore[no-untyped-def] ), patch( "routstr.wallet.db.balances_for_mint_and_unit", - AsyncMock(return_value=0), + AsyncMock(return_value=user_balance_msats), ), patch("routstr.wallet.db.create_session", _fake_session), ] @@ -52,6 +54,31 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None: assert total_wallet == 1000 +@pytest.mark.asyncio +async def test_fetch_all_balances_reports_liability_when_wallet_is_empty() -> None: + """An empty wallet must not hide outstanding user liabilities.""" + from routstr.core.settings import settings + + with patch.object(settings, "cashu_mints", []), patch.object( + settings, "primary_mint", "http://primary:3338" + ): + for p in _patches(proof_amount=0, user_balance_msats=5000): + p.start() + try: + details, total_wallet, total_user, owner = await fetch_all_balances( + units=["sat"] + ) + finally: + patch.stopall() + + assert details[0]["wallet_balance"] == 0 + assert details[0]["user_balance"] == 5 + assert details[0]["owner_balance"] == -5 + assert total_wallet == 0 + assert total_user == 5 + assert owner == -5 + + @pytest.mark.asyncio async def test_fetch_all_balances_no_duplicate_primary_mint() -> None: """primary_mint already in cashu_mints is not inspected twice.""" From 6c762f0c3d527e8d4247f8de33e280e9a3dfe652 Mon Sep 17 00:00:00 2001 From: thefux Date: Wed, 22 Jul 2026 00:09:36 +0000 Subject: [PATCH 02/26] fix: type reconciliation report mappings --- scripts/reconcile_reserved_proofs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/reconcile_reserved_proofs.py b/scripts/reconcile_reserved_proofs.py index 4ae3fa48..8148b145 100755 --- a/scripts/reconcile_reserved_proofs.py +++ b/scripts/reconcile_reserved_proofs.py @@ -106,13 +106,15 @@ async def run(args: argparse.Namespace) -> int: for row in rows: grouped[(row["mint_url"], row["unit"])].append(row) + mint_reports: dict[str, object] = {} + errors: dict[str, str] = {} report: dict[str, object] = { "mode": "apply" if args.apply else "dry-run", "root": str(root), "started_at": int(time.time()), "pending_refund_secrets": len(pending_secrets), - "mints": {}, - "errors": {}, + "mints": mint_reports, + "errors": errors, } state_by_secret: dict[str, str] = {} @@ -122,7 +124,7 @@ async def run(args: argparse.Namespace) -> int: try: states = await fetch_states(client, mint_url, proofs, args.batch_size) except Exception as exc: - report["errors"][f"{mint_url}|{unit}"] = f"{type(exc).__name__}: {exc}" + errors[f"{mint_url}|{unit}"] = f"{type(exc).__name__}: {exc}" continue summary: dict[str, dict[str, int]] = defaultdict(lambda: {"proofs": 0, "amount": 0}) @@ -161,7 +163,7 @@ async def run(args: argparse.Namespace) -> int: else: actions["preserve_unknown"] += 1 - report["mints"][f"{mint_url}|{unit}"] = { + mint_reports[f"{mint_url}|{unit}"] = { "states": dict(summary), "actions": actions, } From 86adee9b1bf381e3f886a4b41acb87fe153d2d74 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Fri, 26 Jun 2026 14:32:18 +0200 Subject: [PATCH 03/26] feat(vault): add Fernet/scrypt secret primitives Encrypt/decrypt secrets at rest with Fernet keyed by ROUTSTR_SECRET_KEY, and hash/verify admin passwords with scrypt. Self-contained helpers with no consumers yet. Co-Authored-By: Claude Opus 4.8 --- routstr/core/vault.py | 131 ++++++++++++++++++++++++++++++++++++ tests/unit/test_vault.py | 141 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 routstr/core/vault.py create mode 100644 tests/unit/test_vault.py diff --git a/routstr/core/vault.py b/routstr/core/vault.py new file mode 100644 index 00000000..1f39da91 --- /dev/null +++ b/routstr/core/vault.py @@ -0,0 +1,131 @@ +"""Encrypt/hash/fingerprint helpers for secrets at rest (issue #553). + +Thin wrapper over ``cryptography`` so nothing else in the codebase touches +Fernet/scrypt/HMAC directly: + +- :func:`encrypt`/:func:`decrypt` — Fernet symmetric encryption, keyed by the + mandatory ``ROUTSTR_SECRET_KEY``. Ciphertext is self-describing (``fernet:v1:`` + prefix) so a value can be told apart from legacy plaintext and so reading it + under the wrong key surfaces as a hard error rather than silent corruption. +- :func:`hash_password`/:func:`verify_password` — salted scrypt hashing. This is + *key-independent*: it never reads ``ROUTSTR_SECRET_KEY``, so password login and + the recovery script keep working even when the key is missing. + +A missing or malformed ``ROUTSTR_SECRET_KEY`` fails fast with the generation +command in the message. +""" + +import base64 +import hashlib +import hmac +import os +import secrets + +from cryptography.fernet import Fernet + +_PREFIX = "fernet:v1:" +_GEN_COMMAND = ( + 'python -c "from cryptography.fernet import Fernet; ' + 'print(Fernet.generate_key().decode())"' +) + +# Minimum admin-password length, enforced wherever a password is set/changed +# (admin endpoints + the recovery script) so the policy lives in one place. +MIN_PASSWORD_LENGTH = 8 + +# scrypt parameters; packed into each hash so verification is parameter-free. +_SCRYPT_N = 2**14 +_SCRYPT_R = 8 +_SCRYPT_P = 1 +_SCRYPT_DKLEN = 32 +_SCRYPT_SALT_BYTES = 16 + + +def _require_secret_key() -> str: + key = os.environ.get("ROUTSTR_SECRET_KEY") + if not key: + raise RuntimeError( + "ROUTSTR_SECRET_KEY is not set. It is required to encrypt secrets at " + "rest. Generate one with:\n " + _GEN_COMMAND + ) + return key + + +def get_fernet() -> Fernet: + """Build a :class:`Fernet` from ``ROUTSTR_SECRET_KEY`` (fails fast).""" + key = _require_secret_key() + try: + return Fernet(key.encode()) + except (ValueError, TypeError) as exc: + raise RuntimeError( + "ROUTSTR_SECRET_KEY is malformed; it must be a url-safe base64 " + "32-byte Fernet key. Generate one with:\n " + _GEN_COMMAND + ) from exc + + +def encrypt(plaintext: str) -> str: + """Encrypt ``plaintext`` into a self-describing ``fernet:v1:`` token.""" + token = get_fernet().encrypt(plaintext.encode()).decode() + return _PREFIX + token + + +def is_encrypted(value: str) -> bool: + """True if ``value`` carries the ``fernet:v1:`` prefix this module emits.""" + return value.startswith(_PREFIX) + + +def decrypt(ciphertext: str) -> str: + """Decrypt a ``fernet:v1:`` token. + + Raises ``ValueError`` for an unprefixed value (so legacy plaintext is never + mistaken for ciphertext) and ``InvalidToken`` when the value was written + under a different ``ROUTSTR_SECRET_KEY``. + """ + if not is_encrypted(ciphertext): + raise ValueError("value is not fernet:v1: ciphertext") + token = ciphertext[len(_PREFIX) :] + return get_fernet().decrypt(token.encode()).decode() + + +def hash_password(password: str) -> str: + """Salted scrypt hash, self-describing as ``scrypt:n:r:p:salt:hash``.""" + salt = secrets.token_bytes(_SCRYPT_SALT_BYTES) + derived = hashlib.scrypt( + password.encode(), + salt=salt, + n=_SCRYPT_N, + r=_SCRYPT_R, + p=_SCRYPT_P, + dklen=_SCRYPT_DKLEN, + ) + return ":".join( + [ + "scrypt", + str(_SCRYPT_N), + str(_SCRYPT_R), + str(_SCRYPT_P), + base64.b64encode(salt).decode(), + base64.b64encode(derived).decode(), + ] + ) + + +def verify_password(password: str, stored: str) -> bool: + """Constant-time check of ``password`` against a :func:`hash_password` value.""" + try: + scheme, n, r, p, salt_b64, hash_b64 = stored.split(":") + if scheme != "scrypt": + return False + salt = base64.b64decode(salt_b64) + expected = base64.b64decode(hash_b64) + derived = hashlib.scrypt( + password.encode(), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + ) + except (ValueError, TypeError): + return False + return hmac.compare_digest(derived, expected) diff --git a/tests/unit/test_vault.py b/tests/unit/test_vault.py new file mode 100644 index 00000000..21621d54 --- /dev/null +++ b/tests/unit/test_vault.py @@ -0,0 +1,141 @@ +"""Tests for ``routstr.core.vault`` — the secret encrypt/hash/fingerprint helpers. + +Specifies the primitives that the rest of the secret-storage work (issue #553) +builds on, independent of any database or app wiring: + +- ``encrypt``/``decrypt`` — Fernet symmetric encryption emitting self-describing + ``fernet:v1:`` ciphertext, so a value can be told apart from legacy plaintext + and from ciphertext written under a different ``ROUTSTR_SECRET_KEY`` (which + surfaces as a hard ``InvalidToken`` rather than silent corruption). +- ``hash_password``/``verify_password`` — salted scrypt hashing that is + *key-independent* (does not depend on ``ROUTSTR_SECRET_KEY``), so password + login and the recovery script keep working even if the key is lost. +- a missing/malformed ``ROUTSTR_SECRET_KEY`` fails fast with the generation + command in the message. +""" + +import pytest +from cryptography.fernet import InvalidToken + +from routstr.core import vault + +# Two distinct, valid Fernet keys held fixed so ciphertext/fingerprints are +# reproducible across runs and we can exercise the wrong-key path. +KEY_A = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU=" +KEY_B = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ=" + + +def _use_key(monkeypatch: pytest.MonkeyPatch, key: str) -> None: + monkeypatch.setenv("ROUTSTR_SECRET_KEY", key) + + +# --- encrypt / decrypt ----------------------------------------------------- + + +def test_encrypt_decrypt_round_trips(monkeypatch: pytest.MonkeyPatch) -> None: + _use_key(monkeypatch, KEY_A) + assert vault.decrypt(vault.encrypt("nsec1secret")) == "nsec1secret" + + +def test_encrypt_emits_self_describing_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _use_key(monkeypatch, KEY_A) + assert vault.encrypt("x").startswith("fernet:v1:") + + +def test_encrypt_is_non_deterministic(monkeypatch: pytest.MonkeyPatch) -> None: + # Fernet embeds a random IV/timestamp: equal plaintext -> different + # ciphertext. This is exactly why upstream-key equality needs a blind index. + _use_key(monkeypatch, KEY_A) + assert vault.encrypt("same") != vault.encrypt("same") + + +def test_is_encrypted_distinguishes_ciphertext_from_plaintext( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _use_key(monkeypatch, KEY_A) + assert vault.is_encrypted(vault.encrypt("x")) is True + assert vault.is_encrypted("sk-plaintext-api-key") is False + assert vault.is_encrypted("") is False + + +def test_decrypt_rejects_unprefixed_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Guards the migration paths: a legacy plaintext value must never be + # mistaken for ciphertext and "decrypted". + _use_key(monkeypatch, KEY_A) + with pytest.raises(ValueError): + vault.decrypt("not-encrypted") + + +def test_decrypt_with_wrong_key_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The fail-fast signal: ciphertext written under KEY_A cannot be read under + # KEY_B -> InvalidToken (bootstrap turns this into a clear startup error). + _use_key(monkeypatch, KEY_A) + token = vault.encrypt("secret") + _use_key(monkeypatch, KEY_B) + with pytest.raises(InvalidToken): + vault.decrypt(token) + + +# --- password hashing (key-independent) ------------------------------------ + + +def test_hash_and_verify_password(monkeypatch: pytest.MonkeyPatch) -> None: + _use_key(monkeypatch, KEY_A) + stored = vault.hash_password("correct horse") + assert vault.verify_password("correct horse", stored) is True + assert vault.verify_password("wrong", stored) is False + + +def test_password_hash_is_salted(monkeypatch: pytest.MonkeyPatch) -> None: + _use_key(monkeypatch, KEY_A) + a = vault.hash_password("pw") + b = vault.hash_password("pw") + assert a != b + assert vault.verify_password("pw", a) is True + assert vault.verify_password("pw", b) is True + + +def test_verify_password_rejects_malformed_stored_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A garbage or non-scrypt stored value must verify to False, never raise. + _use_key(monkeypatch, KEY_A) + assert vault.verify_password("pw", "") is False + assert vault.verify_password("pw", "not-a-hash") is False + assert vault.verify_password("pw", "bcrypt:1:2:3:x:y") is False + + +def test_password_hashing_is_key_independent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # scrypt does not use ROUTSTR_SECRET_KEY, so login and the recovery script + # work even when the key is missing. + monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + stored = vault.hash_password("pw") + assert vault.verify_password("pw", stored) is True + + +# --- fail-fast on missing/malformed key ------------------------------------ + + +def test_missing_key_fails_fast_with_generation_command( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + with pytest.raises(RuntimeError) as exc: + vault.encrypt("x") + msg = str(exc.value) + assert "ROUTSTR_SECRET_KEY" in msg + assert "Fernet.generate_key" in msg + + +def test_malformed_key_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key") + with pytest.raises(RuntimeError): + vault.encrypt("x") From 6c5c3f0b5ff0337d762af819e1e980ad01a688b4 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Fri, 26 Jun 2026 14:32:38 +0200 Subject: [PATCH 04/26] feat(db): add encrypted Secret store and migration Introduce a singleton Secret model holding the admin password hash and the Fernet-encrypted nsec, with a hand-written migration for the secrets table. Add suite-wide pytest config pinning a valid ROUTSTR_SECRET_KEY. Co-Authored-By: Claude Opus 4.8 --- .../c6f8d2e4a1b3_add_secrets_table.py | 42 +++++++++ routstr/core/db.py | 61 ++++++++++++ tests/conftest.py | 15 +++ tests/integration/test_secret_model.py | 93 +++++++++++++++++++ 4 files changed, 211 insertions(+) create mode 100644 migrations/versions/c6f8d2e4a1b3_add_secrets_table.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/test_secret_model.py diff --git a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py new file mode 100644 index 00000000..f867dd5f --- /dev/null +++ b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py @@ -0,0 +1,42 @@ +"""add secrets table + +Revision ID: c6f8d2e4a1b3 +Revises: b5e7c9d1f3a2 +Create Date: 2026-06-24 00:00:00.000000 + +Creates the node-level singleton secret store (issue #553). Schema only; moving +any legacy plaintext into the encrypted/hashed columns happens at bootstrap, +where the live ROUTSTR_SECRET_KEY is available. +""" + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +revision = "c6f8d2e4a1b3" +down_revision = "b5e7c9d1f3a2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "secrets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "admin_password_hash", + sqlmodel.sql.sqltypes.AutoString(), + nullable=True, + ), + sa.Column( + "encrypted_nsec", + sqlmodel.sql.sqltypes.AutoString(), + nullable=True, + ), + sa.Column("updated_at", sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("secrets") diff --git a/routstr/core/db.py b/routstr/core/db.py index 24236a74..827cc24f 100644 --- a/routstr/core/db.py +++ b/routstr/core/db.py @@ -443,6 +443,21 @@ class RoutstrFee(SQLModel, table=True): # type: ignore payout_started_at: int | None = Field(default=None) +class Secret(SQLModel, table=True): # type: ignore + """Node-level secrets, stored encrypted/hashed at rest (singleton, id=1). + + The asymmetric column names document the encoding: ``_hash`` is one-way + (scrypt, verify only) while ``encrypted_`` is reversible (Fernet). Per-provider + upstream keys live on ``upstream_providers``, not here. See ``routstr.core.vault``. + """ + + __tablename__ = "secrets" + id: int = Field(default=1, primary_key=True) + admin_password_hash: str | None = Field(default=None) + encrypted_nsec: str | None = Field(default=None) + updated_at: int | None = Field(default=None) + + class CliToken(SQLModel, table=True): # type: ignore """Long-lived authorization token for CLI/agent use against admin endpoints.""" @@ -481,6 +496,52 @@ async def get_routstr_fee(session: AsyncSession) -> RoutstrFee: return fee +async def get_secret(session: AsyncSession) -> Secret: + secret = await session.get(Secret, 1) + if secret is None: + secret = Secret(id=1) + session.add(secret) + try: + await session.commit() + except IntegrityError: + # Another worker created the singleton row between our read and + # insert (multiple workers booting against one shared DB). Roll back + # and read the row they committed instead of failing startup. + await session.rollback() + secret = await session.get(Secret, 1) + if secret is None: + raise + return secret + await session.refresh(secret) + return secret + + +async def set_admin_password(session: AsyncSession, password: str) -> None: + """Store the admin password as a one-way hash on the Secret singleton.""" + from .vault import hash_password + + secret = await get_secret(session) + secret.admin_password_hash = hash_password(password) + secret.updated_at = int(time.time()) + session.add(secret) + await session.commit() + + +async def set_nsec(session: AsyncSession, nsec: str) -> None: + """Store the node's nsec, Fernet-encrypted, on the Secret singleton. + + An empty string clears it (the node then holds no Nostr identity and signs + no events). + """ + from .vault import encrypt + + secret = await get_secret(session) + secret.encrypted_nsec = encrypt(nsec) if nsec else None + secret.updated_at = int(time.time()) + session.add(secret) + await session.commit() + + async def reset_routstr_fee(session: AsyncSession, paid_msats: int) -> bool: """Checkpoint a fee payout before making the external payment.""" stmt = ( diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..8112487d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +"""Shared pytest configuration for the whole suite. + +A fixed, valid ``ROUTSTR_SECRET_KEY`` is set before any app import so that +secret encryption is deterministic across the suite and the mandatory-key +fail-fast does not break app-boot tests. Tests that need a different key (or an +absent one) override this per-test via ``monkeypatch``. +""" + +import os + +# Valid Fernet keys; KEY_A is the suite default, KEY_B is for wrong-key tests. +TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU=" +TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ=" + +os.environ.setdefault("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY) diff --git a/tests/integration/test_secret_model.py b/tests/integration/test_secret_model.py new file mode 100644 index 00000000..77133d52 --- /dev/null +++ b/tests/integration/test_secret_model.py @@ -0,0 +1,93 @@ +"""Tests for the ``Secret`` singleton model (issue #553). + +Specifies the node-level secret store: a single row (``id=1``, like +``RoutstrFee``) holding the one-way admin-password hash and the encrypted nsec. +``get_secret`` is get-or-create, so callers always get the singleton without +worrying whether it has been initialised yet. Encoding of the values themselves +lives in ``routstr.core.vault``; here we only assert the row persists and stays +a singleton. +""" + +import time +from typing import Any + +import pytest +from sqlmodel import select +from sqlmodel.ext.asyncio.session import AsyncSession + +from routstr.core.db import Secret, get_secret + + +@pytest.mark.asyncio +async def test_get_secret_creates_singleton( + integration_session: AsyncSession, +) -> None: + secret = await get_secret(integration_session) + assert secret.id == 1 + # Fresh row carries no secret material yet. + assert secret.admin_password_hash is None + assert secret.encrypted_nsec is None + assert secret.updated_at is None + + +@pytest.mark.asyncio +async def test_get_secret_is_idempotent( + integration_session: AsyncSession, +) -> None: + first = await get_secret(integration_session) + second = await get_secret(integration_session) + assert first.id == second.id == 1 + rows = (await integration_session.exec(select(Secret))).all() + assert len(rows) == 1 + + +@pytest.mark.asyncio +async def test_secret_fields_round_trip( + integration_session: AsyncSession, +) -> None: + secret = await get_secret(integration_session) + secret.admin_password_hash = "scrypt:16384:8:1:c2FsdA==:aGFzaA==" + secret.encrypted_nsec = "fernet:v1:gAAAAA" + secret.updated_at = int(time.time()) + integration_session.add(secret) + await integration_session.commit() + + integration_session.expunge_all() + reloaded = await get_secret(integration_session) + assert reloaded.admin_password_hash == "scrypt:16384:8:1:c2FsdA==:aGFzaA==" + assert reloaded.encrypted_nsec == "fernet:v1:gAAAAA" + assert reloaded.updated_at is not None + + +@pytest.mark.asyncio +async def test_get_secret_tolerates_concurrent_first_insert( + integration_engine: Any, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A second worker wins the race and commits the singleton row first. + async with AsyncSession(integration_engine, expire_on_commit=False) as other: + other.add(Secret(id=1, admin_password_hash="scrypt:from-other-worker")) + await other.commit() + + # Reproduce the race window: our session's first read still sees no row, so + # it attempts to INSERT a duplicate id=1. The real IntegrityError that follows + # must be recovered (roll back, re-read) rather than crashing startup. + real_get = integration_session.get + calls = {"n": 0} + + async def stale_first_read(model: Any, pk: Any) -> Any: + calls["n"] += 1 + if calls["n"] == 1: + return None + return await real_get(model, pk) + + monkeypatch.setattr(integration_session, "get", stale_first_read) + + secret = await get_secret(integration_session) + + # Recovered the other worker's row; no crash, still a single row. + assert secret.id == 1 + assert secret.admin_password_hash == "scrypt:from-other-worker" + rows = (await integration_session.exec(select(Secret))).all() + assert len(rows) == 1 From c40723c1e2eff064fb1b502e6be1ecaee354df25 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Fri, 26 Jun 2026 14:32:46 +0200 Subject: [PATCH 05/26] feat(settings): bootstrap secrets at startup and require ROUTSTR_SECRET_KEY for stored nsec Persist and load admin password and nsec from the encrypted Secret store on boot: generate a temporary admin password on first run (logged once), encrypt a provided nsec, and fail fast if a stored nsec cannot be decrypted with the current key. Stop clobbering live secret settings with empty env values. Co-Authored-By: Claude Opus 4.8 --- routstr/core/main.py | 15 +- routstr/core/settings.py | 206 ++++++++++++++-- tests/integration/test_secret_bootstrap.py | 268 +++++++++++++++++++++ tests/unit/test_settings.py | 108 ++++++++- 4 files changed, 564 insertions(+), 33 deletions(-) create mode 100644 tests/integration/test_secret_bootstrap.py diff --git a/routstr/core/main.py b/routstr/core/main.py index c9598a1c..ca1a5a93 100644 --- a/routstr/core/main.py +++ b/routstr/core/main.py @@ -37,7 +37,7 @@ from .exceptions import general_exception_handler, http_exception_handler from .logging import get_logger, setup_logging from .middleware import LoggingMiddleware from .not_found import _NOT_FOUND_HTML, not_found_catch_all # noqa: F401 -from .settings import SettingsService +from .settings import SettingsService, bootstrap_secrets from .settings import settings as global_settings from .version import __version__ @@ -85,17 +85,20 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: # Initialize application settings (env -> computed -> DB precedence) async with create_session() as session: + # Move secrets into the encrypted/hashed store and decrypt the nsec + # into the in-memory settings BEFORE initializing settings: the + # initialize step strips secrets from the persisted blob, so legacy + # plaintext (env or old blob) must be migrated into the Secret store + # first or the only copy of a blob-only secret would be lost. + # Generates and logs an admin password on a fresh node; fails fast if + # a stored secret can't be decrypted. + await bootstrap_secrets(session) s = await SettingsService.initialize(session) if s.reset_reserved_balance_on_startup: from .db import reset_all_reserved_balances await reset_all_reserved_balances(session) - if not s.admin_password: - logger.warning( - f"Admin password is not set. Visit {s.http_url or 'http://localhost:8000'}/admin to set the password." - ) - # Apply app metadata from settings try: app.title = s.name diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 3a144e10..8b779e42 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio import json import os +import secrets +import time from datetime import datetime, timezone from typing import Any @@ -26,7 +28,6 @@ class Settings(BaseSettings): # Core upstream_base_url: str = Field(default="", env="UPSTREAM_BASE_URL") upstream_api_key: str = Field(default="", env="UPSTREAM_API_KEY") - admin_password: str = Field(default="", env="ADMIN_PASSWORD") # Node info name: str = Field(default="ARoutstrNode", env="NAME") @@ -132,10 +133,67 @@ def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]: return normalized +# Secrets are credentials, not config: they live in the encrypted/hashed Secret +# store (and decrypted in-memory for runtime use), never in the persisted +# settings blob. ``admin_password`` is gone from the model entirely; +# ``nsec``/``upstream_api_key`` remain live fields but are stripped from every +# blob write so they are never written back to plaintext. See +# ``bootstrap_secrets`` and ``routstr.core.vault``. +SECRET_FIELDS = frozenset({"admin_password", "nsec", "upstream_api_key"}) + + +def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]: + """Return a copy of ``data`` without any secret fields (for persistence).""" + return {k: v for k, v in data.items() if k not in SECRET_FIELDS} + + +def _apply_to_live_settings(data: dict[str, Any]) -> None: + """Apply ``data`` onto the live ``settings`` for all in-process importers. + + Secrets are owned by ``bootstrap_secrets`` (which decrypts the nsec into + memory before this runs) — they are never persisted to the blob, so ``data`` + re-derived from the secret-free blob carries empty secret values. Skip those + empty overwrites so a live secret is never clobbered; a non-empty value + (legacy env, or a not-yet-stripped blob mid-migration) is still applied. + """ + live = settings.dict() + for k, v in data.items(): + if k in SECRET_FIELDS and not v and live.get(k): + continue + setattr(settings, k, v) + + def _compute_primary_mint(cashu_mints: list[str]) -> str: return cashu_mints[0] if cashu_mints else "https://mint.minibits.cash/Bitcoin" +def derive_npub_from_nsec(nsec: str) -> str | None: + """Derive the npub (bech32) from an nsec or 64-char hex private key, or None. + + Parsing is delegated to :func:`routstr.nostr.listing.nsec_to_keypair`, the + single place that knows the nsec/hex formats (and already returns ``None`` on + any unusable input); this only bech32-encodes the resulting public key. The + contract stays "return None on unusable input", so a bad key never crashes + boot. + """ + try: + from nostr.key import PublicKey # type: ignore + + from ..nostr.listing import nsec_to_keypair + except ImportError: + return None + + keypair = nsec_to_keypair(nsec) + if keypair is None: + return None + _privkey_hex, pubkey_hex = keypair + + try: + return PublicKey(bytes.fromhex(pubkey_hex)).bech32() + except (ValueError, AttributeError): + return None + + def resolve_bootstrap() -> Settings: base = Settings() # Reads env with custom parse_env_var # Back-compat env mapping @@ -190,23 +248,9 @@ def resolve_bootstrap() -> Settings: pass # Derive NPUB from NSEC if not provided if not base.npub and base.nsec: - try: - from nostr.key import PrivateKey # type: ignore - - if base.nsec.startswith("nsec"): - pk = PrivateKey.from_nsec(base.nsec) - elif len(base.nsec) == 64: - pk = PrivateKey(bytes.fromhex(base.nsec)) - else: - pk = None - if pk is not None: - try: - base.npub = pk.public_key.bech32() - except Exception: - # Fallback to hex if bech32 not available - base.npub = pk.public_key.hex() - except Exception: - pass + npub = derive_npub_from_nsec(base.nsec) + if npub: + base.npub = npub if not base.cors_origins: base.cors_origins = ["*"] if not base.primary_mint: @@ -256,15 +300,14 @@ class SettingsService: text( "INSERT INTO settings (id, data, updated_at) VALUES (1, :data, :updated_at)" ).bindparams( - data=json.dumps(env_resolved.dict()), + data=json.dumps(_strip_secret_fields(env_resolved.dict())), updated_at=datetime.now(timezone.utc), ) ) await db_session.commit() cls._current = settings # Update the existing instance in-place for all live importers - for k, v in env_resolved.dict().items(): - setattr(settings, k, v) + _apply_to_live_settings(env_resolved.dict()) return cls._current db_id, db_data, _updated_at = row @@ -291,20 +334,24 @@ class SettingsService: merged_dict.get("cashu_mints", []) ) - if db_json_raw != merged_dict: + # Persist without secrets; compare against the stripped target so a + # legacy blob that still carries plaintext secrets gets rewritten + # (and thereby sunset) even when its non-secret values are unchanged. + persisted = _strip_secret_fields(merged_dict) + if db_json_raw != persisted: await db_session.exec( # type: ignore text( "UPDATE settings SET data = :data, updated_at = :updated_at WHERE id = 1" ).bindparams( - data=json.dumps(merged_dict), + data=json.dumps(persisted), updated_at=datetime.now(timezone.utc), ) ) await db_session.commit() # Update the existing instance in-place for all live importers - for k, v in merged_dict.items(): - setattr(settings, k, v) + # (keeps the decrypted nsec/upstream_api_key live in memory). + _apply_to_live_settings(merged_dict) cls._current = settings return cls._current @@ -326,7 +373,7 @@ class SettingsService: text( "UPDATE settings SET data = :data, updated_at = :updated_at WHERE id = 1" ).bindparams( - data=json.dumps(candidate.dict()), + data=json.dumps(_strip_secret_fields(candidate.dict())), updated_at=datetime.now(timezone.utc), ) ) @@ -355,3 +402,110 @@ class SettingsService: setattr(settings, k, v) cls._current = settings return settings + + +async def _read_raw_settings_blob(db_session: AsyncSession) -> dict[str, Any]: + """Best-effort read of the raw persisted settings JSON (may not exist yet).""" + from sqlmodel import text + + try: + result = await db_session.exec( # type: ignore + text("SELECT data FROM settings WHERE id = 1") + ) + row = result.first() + except Exception: + return {} + if row is None: + return {} + (data_str,) = row + try: + data = json.loads(data_str) if isinstance(data_str, str) else dict(data_str) + except Exception: + return {} + return data if isinstance(data, dict) else {} + + +def _legacy_plaintext( + raw_blob: dict[str, Any], env_name: str, blob_key: str +) -> str | None: + """Legacy plaintext for a secret: env first, then the old settings blob.""" + env_value = os.environ.get(env_name) + if env_value: + return env_value + blob_value = raw_blob.get(blob_key) + if isinstance(blob_value, str) and blob_value: + return blob_value + return None + + +async def bootstrap_secrets(db_session: AsyncSession) -> None: + """Move node secrets into the encrypted/hashed Secret store at startup. + + Per secret: + * column already set -> use it (the nsec is decrypted into the in-memory + ``settings``; a wrong ROUTSTR_SECRET_KEY surfaces as a clear fail-fast). + * column empty but legacy plaintext exists (env, or the old settings + blob) -> transform it (hash the password / encrypt the nsec) into the + column. + * nothing (admin password only) -> generate a strong random password, + hash it, and log it once with the /admin URL. + """ + from cryptography.fernet import InvalidToken + + from . import vault + from .db import get_secret + from .logging import get_logger + + logger = get_logger(__name__) + + raw_blob = await _read_raw_settings_blob(db_session) + secret = await get_secret(db_session) + changed = False + + # Admin password — one-way scrypt hash. + if secret.admin_password_hash is None: + legacy_password = _legacy_plaintext( + raw_blob, "ADMIN_PASSWORD", "admin_password" + ) + if legacy_password: + secret.admin_password_hash = vault.hash_password(legacy_password) + else: + generated = secrets.token_urlsafe(24) + secret.admin_password_hash = vault.hash_password(generated) + admin_url = (settings.http_url or "http://localhost:8000").rstrip("/") + logger.warning( + "No admin password set; generated a temporary one (shown only " + "now): %s\nLog in at %s/admin and change it from the dashboard " + "settings.", + generated, + admin_url, + ) + changed = True + + # Nostr nsec — reversible Fernet encryption. + if secret.encrypted_nsec is not None: + try: + settings.nsec = vault.decrypt(secret.encrypted_nsec) + except InvalidToken as exc: + raise RuntimeError( + "Stored nsec cannot be decrypted with the current " + "ROUTSTR_SECRET_KEY. The key changed, or this database came from " + "another node. Restore the original ROUTSTR_SECRET_KEY to recover." + ) from exc + else: + legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec") + if legacy_nsec: + secret.encrypted_nsec = vault.encrypt(legacy_nsec) + settings.nsec = legacy_nsec + changed = True + + # Derive npub from whatever nsec we now hold, if not already known. + if settings.nsec and not settings.npub: + npub = derive_npub_from_nsec(settings.nsec) + if npub: + settings.npub = npub + + if changed: + secret.updated_at = int(time.time()) + db_session.add(secret) + await db_session.commit() diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py new file mode 100644 index 00000000..3b45fd61 --- /dev/null +++ b/tests/integration/test_secret_bootstrap.py @@ -0,0 +1,268 @@ +"""Tests for ``bootstrap_secrets`` — moving node secrets into the Secret store. + +Specifies the per-secret bootstrap that runs at startup (issue #553). For both +the admin password and the nsec it follows the same three branches: use the +column if already set, otherwise migrate any legacy plaintext (env first, then +the old settings blob), otherwise — admin password only — generate and log one. +A column written under a different ROUTSTR_SECRET_KEY fails fast rather than +silently corrupting state. +""" + +import json +from contextlib import asynccontextmanager +from typing import Any, AsyncGenerator + +import pytest +from sqlmodel import text +from sqlmodel.ext.asyncio.session import AsyncSession + +from routstr.core import vault +from routstr.core.db import get_secret +from routstr.core.settings import ( + SettingsService, + bootstrap_secrets, + derive_npub_from_nsec, + settings, +) + +# Valid Fernet keys; must match the suite default in tests/conftest.py. +TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU=" +TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ=" + +NSEC_HEX = "1" * 64 + + +@pytest.fixture +def clean_secret_env(monkeypatch: pytest.MonkeyPatch) -> None: + """No ambient legacy secrets, and a known in-memory settings baseline.""" + monkeypatch.delenv("ADMIN_PASSWORD", raising=False) + monkeypatch.delenv("NSEC", raising=False) + monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY) + monkeypatch.setattr(settings, "nsec", "") + monkeypatch.setattr(settings, "npub", "") + monkeypatch.setattr(settings, "http_url", "") + + +async def _create_settings_blob(session: AsyncSession, data: dict) -> None: + await session.exec( # type: ignore + text( + "CREATE TABLE IF NOT EXISTS settings " + "(id INTEGER PRIMARY KEY, data TEXT NOT NULL, " + "updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)" + ) + ) + await session.exec( # type: ignore + text("INSERT INTO settings (id, data) VALUES (1, :data)").bindparams( + data=json.dumps(data) + ) + ) + await session.commit() + + +# --- admin password -------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_generates_admin_password_when_none( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + await bootstrap_secrets(integration_session) + secret = await get_secret(integration_session) + assert secret.admin_password_hash is not None + assert secret.admin_password_hash.startswith("scrypt:") + + +@pytest.mark.asyncio +async def test_admin_password_generation_is_idempotent( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + await bootstrap_secrets(integration_session) + first = (await get_secret(integration_session)).admin_password_hash + await bootstrap_secrets(integration_session) + second = (await get_secret(integration_session)).admin_password_hash + assert first is not None and first == second + + +@pytest.mark.asyncio +async def test_hashes_legacy_admin_password_from_env( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("ADMIN_PASSWORD", "hunter2") + await bootstrap_secrets(integration_session) + secret = await get_secret(integration_session) + assert secret.admin_password_hash is not None + assert vault.verify_password("hunter2", secret.admin_password_hash) is True + + +@pytest.mark.asyncio +async def test_hashes_legacy_admin_password_from_blob( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + # No ADMIN_PASSWORD in env, but the old settings blob carries one. + await _create_settings_blob(integration_session, {"admin_password": "blobpw"}) + await bootstrap_secrets(integration_session) + secret = await get_secret(integration_session) + assert vault.verify_password("blobpw", secret.admin_password_hash or "") is True + + +# --- nsec ------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_encrypts_legacy_nsec_from_env_and_derives_npub( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NSEC", NSEC_HEX) + await bootstrap_secrets(integration_session) + secret = await get_secret(integration_session) + assert secret.encrypted_nsec is not None + assert vault.is_encrypted(secret.encrypted_nsec) is True + assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX + # In-memory runtime value is the decrypted nsec, and npub is derived from it. + assert settings.nsec == NSEC_HEX + assert settings.npub == derive_npub_from_nsec(NSEC_HEX) + + +@pytest.mark.asyncio +async def test_decrypts_existing_nsec_column( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + secret = await get_secret(integration_session) + secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + integration_session.add(secret) + await integration_session.commit() + stored = secret.encrypted_nsec + + await bootstrap_secrets(integration_session) + reloaded = await get_secret(integration_session) + assert settings.nsec == NSEC_HEX + # The column is reused, not re-encrypted. + assert reloaded.encrypted_nsec == stored + + +@pytest.mark.asyncio +async def test_fail_fast_when_nsec_encrypted_with_different_key( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Encrypt the column under the alternate key, then bootstrap under the + # suite key -> the value cannot be decrypted -> clear startup failure. + monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY_ALT) + secret = await get_secret(integration_session) + secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + integration_session.add(secret) + await integration_session.commit() + + monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY) + with pytest.raises(RuntimeError, match="ROUTSTR_SECRET_KEY"): + await bootstrap_secrets(integration_session) + + +# --- boot ordering: rescue legacy blob secrets before they are stripped ---- + + +@pytest.mark.asyncio +async def test_blob_only_nsec_is_migrated_before_blob_is_stripped( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + # Legacy node whose nsec lives ONLY in the settings blob (never in env). + # bootstrap_secrets must run *before* SettingsService.initialize strips the + # blob, or the only copy of the secret would be lost. + await _create_settings_blob( + integration_session, {"nsec": NSEC_HEX, "name": "LegacyNode"} + ) + + await bootstrap_secrets(integration_session) + await SettingsService.initialize(integration_session) + + secret = await get_secret(integration_session) + # The plaintext nsec has been moved into the encrypted Secret store... + assert secret.encrypted_nsec is not None + assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX + assert settings.nsec == NSEC_HEX + # ...and stripped from the persisted settings blob. + row = await integration_session.exec( # type: ignore + text("SELECT data FROM settings WHERE id = 1") + ) + blob = json.loads(row.first()[0]) + assert "nsec" not in blob + assert blob["name"] == "LegacyNode" + + +@pytest.mark.asyncio +async def test_initialize_does_not_clobber_store_only_nsec( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + # Steady state after migration: the nsec lives ONLY in the encrypted Secret + # store (NSEC removed from env, blob already stripped on a previous boot). + # bootstrap decrypts it into memory; initialize then re-derives settings from + # the secret-free blob and must NOT wipe the live nsec back to empty (or the + # node would silently stop signing Nostr announcements). + await _create_settings_blob(integration_session, {"name": "LegacyNode"}) + secret = await get_secret(integration_session) + secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + integration_session.add(secret) + await integration_session.commit() + + await bootstrap_secrets(integration_session) + assert settings.nsec == NSEC_HEX # bootstrap decrypted it into memory + + await SettingsService.initialize(integration_session) + # The live secret survives initialize even though no env/blob carries it... + assert settings.nsec == NSEC_HEX + # ...and is still never written back to the persisted blob. + row = await integration_session.exec( # type: ignore + text("SELECT data FROM settings WHERE id = 1") + ) + assert "nsec" not in json.loads(row.first()[0]) + + +@pytest.mark.asyncio +async def test_startup_runs_bootstrap_before_settings_initialize( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The two tests above prove the migration outcome *given* the call order; + # they hardcode that order themselves. This one guards the order at its real + # call site — the application lifespan — so a reorder in main.py (which would + # strip a blob-only secret before bootstrap could rescue it) is caught. + import routstr.core.main as main + + order: list[str] = [] + + class _Abort(Exception): + pass + + @asynccontextmanager + async def fake_create_session() -> AsyncGenerator[None, None]: + yield None + + async def fake_bootstrap(session: Any) -> None: + order.append("bootstrap") + + async def fake_initialize(session: Any) -> None: + order.append("initialize") + # Stop startup here, before the background-task fan-out (prices, nostr, + # upstreams) that we don't want to run in a unit test. + raise _Abort() + + async def noop_init_db() -> None: + return None + + monkeypatch.setattr(main, "configure_litellm", lambda: None) + monkeypatch.setattr(main, "register_deepseek_v4_pricing", lambda: None) + monkeypatch.setattr(main, "run_migrations", lambda: None) + monkeypatch.setattr(main, "init_db", noop_init_db) + monkeypatch.setattr(main, "create_session", fake_create_session) + monkeypatch.setattr(main, "bootstrap_secrets", fake_bootstrap) + monkeypatch.setattr(main.SettingsService, "initialize", fake_initialize) + + with pytest.raises(_Abort): + async with main.lifespan(main.app): + pass + + assert order == ["bootstrap", "initialize"] diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index ccdb71bc..ffe6352f 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -1,3 +1,4 @@ +import json import os import pytest @@ -6,7 +7,15 @@ from sqlalchemy.ext.asyncio import create_async_engine from sqlmodel import text from sqlmodel.ext.asyncio.session import AsyncSession -from routstr.core.settings import Settings, SettingsService +from routstr.core.settings import Settings, SettingsService, settings + +NSEC_HEX = "1" * 64 + + +async def _read_settings_blob(session: AsyncSession) -> dict: + """Return the raw persisted settings JSON (id=1) as a dict.""" + row = await session.exec(text("SELECT data FROM settings WHERE id = 1")) # type: ignore + return json.loads(row.first()[0]) @pytest.mark.asyncio @@ -120,3 +129,100 @@ async def test_settings_initialize_discards_unknown_keys() -> None: assert '"enable_analytics_sharing": true' in stored_data assert "nostr_analytics_enabled" not in stored_data assert "unknown_key" not in stored_data + + +# ── Secret fields are never written to the settings blob (issue #553) ──────── + + +def test_settings_model_drops_admin_password_field() -> None: + # admin_password now lives only as a one-way hash in the Secret store; it is + # no longer a settings field at all. + assert "admin_password" not in Settings.__fields__ + # nsec and upstream_api_key remain runtime values held in memory. + assert "nsec" in Settings.__fields__ + assert "upstream_api_key" in Settings.__fields__ + + +@pytest.mark.asyncio +async def test_secret_fields_kept_in_memory_but_not_persisted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NSEC", NSEC_HEX) + monkeypatch.setenv("UPSTREAM_API_KEY", "sk-upstream") + # Reset the live globals so monkeypatch reverts them after the test. + monkeypatch.setattr(settings, "nsec", "") + monkeypatch.setattr(settings, "upstream_api_key", "") + + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with AsyncSession(engine, expire_on_commit=False) as session: + s = await SettingsService.initialize(session) + + # Runtime consumers still see the live secret values. + assert s.nsec == NSEC_HEX + assert s.upstream_api_key == "sk-upstream" + + # ...but they are never written to the settings blob. + blob = await _read_settings_blob(session) + assert "nsec" not in blob + assert "upstream_api_key" not in blob + assert "admin_password" not in blob + # Non-secret derived/public values are still persisted. + assert blob["npub"] == s.npub + + +@pytest.mark.asyncio +async def test_existing_blob_secrets_are_stripped_on_initialize( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("NSEC", raising=False) + monkeypatch.delenv("UPSTREAM_API_KEY", raising=False) + monkeypatch.delenv("ADMIN_PASSWORD", raising=False) + + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with AsyncSession(engine, expire_on_commit=False) as session: + await SettingsService.initialize(session) + + # Simulate a legacy row that still carries plaintext secrets in the blob. + await session.exec( # type: ignore + text("UPDATE settings SET data = :d WHERE id = 1").bindparams( + d=json.dumps( + { + "name": "LegacyNode", + "admin_password": "pw", + "nsec": NSEC_HEX, + "upstream_api_key": "sk-legacy", + } + ) + ) + ) + await session.commit() + + await SettingsService.initialize(session) + + blob = await _read_settings_blob(session) + assert "admin_password" not in blob + assert "nsec" not in blob + assert "upstream_api_key" not in blob + # Non-secret values survive the migration. + assert blob["name"] == "LegacyNode" + + +@pytest.mark.asyncio +async def test_update_does_not_persist_secret_fields( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NSEC", NSEC_HEX) + monkeypatch.setenv("UPSTREAM_API_KEY", "sk-upstream") + monkeypatch.setattr(settings, "nsec", "") + monkeypatch.setattr(settings, "upstream_api_key", "") + + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with AsyncSession(engine, expire_on_commit=False) as session: + await SettingsService.initialize(session) + await SettingsService.update({"name": "Updated"}, session) + + blob = await _read_settings_blob(session) + assert blob["name"] == "Updated" + assert "nsec" not in blob + assert "upstream_api_key" not in blob + assert "admin_password" not in blob From a024d5be5eab0e10cf630fb2cb4166774435ff93 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Fri, 26 Jun 2026 14:32:53 +0200 Subject: [PATCH 06/26] feat(admin): manage nsec via API and redact secrets in responses Add an admin endpoint to set, rotate and clear the nsec, authenticate against the stored password hash, and redact secret values (nsec shown as [REDACTED]) in settings responses. Wire the admin UI to the new endpoint. Co-Authored-By: Claude Opus 4.8 --- routstr/core/admin.py | 114 +++++++++----- tests/integration/test_admin_auth.py | 141 ++++++++++++++++++ tests/integration/test_admin_nsec_endpoint.py | 113 ++++++++++++++ .../test_admin_settings_endpoint.py | 82 ++++++++++ ui/components/settings/admin-settings.tsx | 27 +++- ui/lib/api/services/admin.ts | 9 ++ 6 files changed, 445 insertions(+), 41 deletions(-) create mode 100644 tests/integration/test_admin_auth.py create mode 100644 tests/integration/test_admin_nsec_endpoint.py create mode 100644 tests/integration/test_admin_settings_endpoint.py diff --git a/routstr/core/admin.py b/routstr/core/admin.py index 71a6e348..7823fd99 100644 --- a/routstr/core/admin.py +++ b/routstr/core/admin.py @@ -20,6 +20,7 @@ from ..wallet import ( send_token, slow_filter_spend_proofs, ) +from . import vault from .db import ( ApiKey, CashuTransaction, @@ -28,6 +29,9 @@ from .db import ( ModelRow, UpstreamProviderRow, create_session, + get_secret, + set_admin_password, + set_nsec, ) from .db import ( store_cashu_transaction_with_retry as store_cashu_transaction, @@ -35,7 +39,7 @@ from .db import ( from .log_manager import log_manager from .logging import get_logger from .provider_slugs import allocate_unique_provider_slug -from .settings import SettingsService, settings +from .settings import SettingsService, derive_npub_from_nsec, settings logger = get_logger(__name__) @@ -206,8 +210,6 @@ async def get_settings(request: Request) -> dict: data = settings.dict() if "upstream_api_key" in data: data["upstream_api_key"] = "[REDACTED]" if data["upstream_api_key"] else "" - if "admin_password" in data: - data["admin_password"] = "[REDACTED]" if data["admin_password"] else "" if "nsec" in data: data["nsec"] = "[REDACTED]" if data["nsec"] else "" return data @@ -224,9 +226,10 @@ class PasswordUpdate(BaseModel): @admin_router.patch("/api/settings", dependencies=[Depends(require_admin_api)]) async def update_settings(request: Request, update: SettingsUpdate) -> dict: - # Remove sensitive fields from general settings update + # Secrets are not editable through the general settings endpoint; they have + # dedicated rotation paths and never reach the settings blob. settings_data = update.root.copy() - sensitive_fields = ["admin_password", "upstream_api_key", "nsec"] + sensitive_fields = ["upstream_api_key", "nsec"] for field in sensitive_fields: if field in settings_data: del settings_data[field] @@ -241,8 +244,6 @@ async def update_settings(request: Request, update: SettingsUpdate) -> dict: data = new_settings.dict() if "upstream_api_key" in data: data["upstream_api_key"] = "[REDACTED]" if data["upstream_api_key"] else "" - if "admin_password" in data: - data["admin_password"] = "[REDACTED]" if data["admin_password"] else "" if "nsec" in data: data["nsec"] = "[REDACTED]" if data["nsec"] else "" return data @@ -250,43 +251,85 @@ async def update_settings(request: Request, update: SettingsUpdate) -> dict: @admin_router.patch("/api/password", dependencies=[Depends(require_admin_api)]) async def update_password(request: Request, password_update: PasswordUpdate) -> dict: - current_password = settings.admin_password - - if not current_password: - raise HTTPException(status_code=500, detail="Admin password not configured") - - if password_update.current_password != current_password: - raise HTTPException(status_code=401, detail="Current password is incorrect") - - # Validate new password - new_password = password_update.new_password.strip() - if len(new_password) < 6: - raise HTTPException( - status_code=400, detail="New password must be at least 6 characters" - ) - - # Update password async with create_session() as session: - await SettingsService.update({"admin_password": new_password}, session) + secret = await get_secret(session) + + if not secret.admin_password_hash: + raise HTTPException( + status_code=500, detail="Admin password not configured" + ) + + if not vault.verify_password( + password_update.current_password, secret.admin_password_hash + ): + raise HTTPException( + status_code=401, detail="Current password is incorrect" + ) + + # Validate new password + new_password = password_update.new_password.strip() + if len(new_password) < vault.MIN_PASSWORD_LENGTH: + raise HTTPException( + status_code=400, + detail=( + "New password must be at least " + f"{vault.MIN_PASSWORD_LENGTH} characters" + ), + ) + + await set_admin_password(session, new_password) return {"ok": True, "message": "Password updated successfully"} +class NsecUpdate(BaseModel): + nsec: str + + +@admin_router.patch("/api/nsec", dependencies=[Depends(require_admin_api)]) +async def update_nsec(request: Request, payload: NsecUpdate) -> dict[str, object]: + # The node's Nostr identity is a secret: it is stored encrypted in the + # Secret store, never in the settings blob, so it gets its own endpoint + # rather than riding the general settings PATCH (which strips it). An empty + # nsec clears the identity. + nsec = payload.nsec.strip() + npub = "" + if nsec: + derived = derive_npub_from_nsec(nsec) + if not derived: + raise HTTPException(status_code=400, detail="Invalid nsec") + npub = derived + + async with create_session() as session: + await set_nsec(session, nsec) + + # Reflect the change in the live runtime so Nostr signing/announcements pick + # it up without a restart (mirrors what bootstrap_secrets sets at boot). + settings.nsec = nsec + settings.npub = npub + return {"ok": True, "npub": npub} + + class SetupRequest(BaseModel): password: str @admin_router.post("/api/setup") async def initial_setup(request: Request, payload: SetupRequest) -> dict[str, object]: - if settings.admin_password: - raise HTTPException(status_code=409, detail="Admin password already set") - pw = (payload.password or "").strip() - if len(pw) < 8: - raise HTTPException( - status_code=400, detail="Password must be at least 8 characters" - ) async with create_session() as session: - await SettingsService.update({"admin_password": pw}, session) + secret = await get_secret(session) + if secret.admin_password_hash: + raise HTTPException(status_code=409, detail="Admin password already set") + pw = (payload.password or "").strip() + if len(pw) < vault.MIN_PASSWORD_LENGTH: + raise HTTPException( + status_code=400, + detail=( + "Password must be at least " + f"{vault.MIN_PASSWORD_LENGTH} characters" + ), + ) + await set_admin_password(session, pw) return {"ok": True} @@ -298,12 +341,13 @@ class AdminLoginRequest(BaseModel): async def admin_login( request: Request, payload: AdminLoginRequest ) -> dict[str, object]: - admin_pw = settings.admin_password + async with create_session() as session: + secret = await get_secret(session) - if not admin_pw: + if not secret.admin_password_hash: raise HTTPException(status_code=500, detail="Admin password not configured") - if payload.password != admin_pw: + if not vault.verify_password(payload.password, secret.admin_password_hash): raise HTTPException(status_code=401, detail="Invalid password") token = secrets.token_urlsafe(32) diff --git a/tests/integration/test_admin_auth.py b/tests/integration/test_admin_auth.py new file mode 100644 index 00000000..e7743d45 --- /dev/null +++ b/tests/integration/test_admin_auth.py @@ -0,0 +1,141 @@ +"""Tests for admin password auth backed by the hashed Secret store (issue #553). + +Login, password change and first-run setup verify against the one-way +``Secret.admin_password_hash`` (scrypt) instead of a plaintext settings field, +which also closes the old ``!=`` timing-attack comparison. The flow is exercised +end-to-end through the public admin endpoints: setup writes the first hash, +login checks against it, and a password change re-hashes so the old password +stops working and the new one starts. +""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient, Response + + +async def _setup_password(client: AsyncClient, password: str) -> None: + resp = await client.post("/admin/api/setup", json={"password": password}) + assert resp.status_code == 200, resp.text + + +async def _login(client: AsyncClient, password: str) -> Response: + return await client.post("/admin/api/login", json={"password": password}) + + +# --- login ----------------------------------------------------------------- + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_login_500_when_no_password_configured( + integration_client: AsyncClient, +) -> None: + resp = await _login(integration_client, "anything") + assert resp.status_code == 500 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_login_succeeds_with_correct_password( + integration_client: AsyncClient, +) -> None: + await _setup_password(integration_client, "correct horse") + resp = await _login(integration_client, "correct horse") + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert isinstance(body["token"], str) and body["token"] + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_login_rejects_wrong_password( + integration_client: AsyncClient, +) -> None: + await _setup_password(integration_client, "correct horse") + resp = await _login(integration_client, "wrong horse") + assert resp.status_code == 401 + + +# --- first-run setup ------------------------------------------------------- + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_setup_rejects_short_password( + integration_client: AsyncClient, +) -> None: + resp = await integration_client.post("/admin/api/setup", json={"password": "short"}) + assert resp.status_code == 400 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_setup_409_when_password_already_set( + integration_client: AsyncClient, +) -> None: + await _setup_password(integration_client, "first password") + resp = await integration_client.post( + "/admin/api/setup", json={"password": "second password"} + ) + assert resp.status_code == 409 + + +# --- password change ------------------------------------------------------- + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_update_password_rehashes_so_only_new_works( + integration_client: AsyncClient, +) -> None: + await _setup_password(integration_client, "old password") + login = await _login(integration_client, "old password") + token = login.json()["token"] + integration_client.headers["Authorization"] = f"Bearer {token}" + + resp = await integration_client.patch( + "/admin/api/password", + json={"current_password": "old password", "new_password": "new password"}, + ) + assert resp.status_code == 200, resp.text + + # Drop admin auth so the login calls aren't treated as authenticated noise. + integration_client.headers.pop("Authorization", None) + assert (await _login(integration_client, "old password")).status_code == 401 + assert (await _login(integration_client, "new password")).status_code == 200 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_update_password_rejects_wrong_current( + integration_client: AsyncClient, +) -> None: + await _setup_password(integration_client, "old password") + login = await _login(integration_client, "old password") + token = login.json()["token"] + integration_client.headers["Authorization"] = f"Bearer {token}" + + resp = await integration_client.patch( + "/admin/api/password", + json={"current_password": "not the password", "new_password": "new password"}, + ) + assert resp.status_code == 401 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_update_password_rejects_short_new( + integration_client: AsyncClient, +) -> None: + await _setup_password(integration_client, "old password") + login = await _login(integration_client, "old password") + token = login.json()["token"] + integration_client.headers["Authorization"] = f"Bearer {token}" + + resp = await integration_client.patch( + "/admin/api/password", + json={"current_password": "old password", "new_password": "x"}, + ) + assert resp.status_code == 400 diff --git a/tests/integration/test_admin_nsec_endpoint.py b/tests/integration/test_admin_nsec_endpoint.py new file mode 100644 index 00000000..23d11c36 --- /dev/null +++ b/tests/integration/test_admin_nsec_endpoint.py @@ -0,0 +1,113 @@ +"""Tests for the admin nsec rotation endpoint (issue #553). + +The Nostr identity is a secret: it lives encrypted in the Secret store, never in +the settings blob, so it cannot be set through the general settings PATCH. This +dedicated endpoint is the supported way to set/rotate/clear it — it encrypts the +key at rest, updates the live runtime identity (so signing picks it up without a +restart), and derives the npub. Invalid keys are rejected. +""" + +from __future__ import annotations + +import secrets +import time +from collections.abc import AsyncGenerator + +import pytest +import pytest_asyncio +from httpx import AsyncClient + +from routstr.core import vault +from routstr.core.admin import admin_sessions +from routstr.core.db import AsyncSession, get_secret +from routstr.core.settings import derive_npub_from_nsec, settings + +# A valid 64-char hex private key (accepted by nsec_to_keypair, as in bootstrap). +NSEC_HEX = "1" * 64 + + +@pytest_asyncio.fixture +async def admin_client( + integration_client: AsyncClient, +) -> AsyncGenerator[AsyncClient, None]: + """An integration_client pre-authenticated with an admin session token.""" + token = secrets.token_urlsafe(24) + admin_sessions[token] = int(time.time()) + 3600 + integration_client.headers["Authorization"] = f"Bearer {token}" + yield integration_client + admin_sessions.pop(token, None) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_update_nsec_stores_encrypted_and_derives_npub( + admin_client: AsyncClient, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "nsec", "") + monkeypatch.setattr(settings, "npub", "") + + resp = await admin_client.patch("/admin/api/nsec", json={"nsec": NSEC_HEX}) + assert resp.status_code == 200 + + expected_npub = derive_npub_from_nsec(NSEC_HEX) + assert resp.json() == {"ok": True, "npub": expected_npub} + + # Stored encrypted at rest, decryptable back to the original key. + integration_session.expunge_all() + secret = await get_secret(integration_session) + assert secret.encrypted_nsec is not None + assert vault.is_encrypted(secret.encrypted_nsec) + assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX + + # Live runtime identity updated so Nostr signing reflects it without restart. + assert settings.nsec == NSEC_HEX + assert settings.npub == expected_npub + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_update_nsec_rejects_invalid_key( + admin_client: AsyncClient, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "nsec", "") + monkeypatch.setattr(settings, "npub", "") + + resp = await admin_client.patch( + "/admin/api/nsec", json={"nsec": "not-a-real-nsec"} + ) + assert resp.status_code == 400 + + # Nothing stored, live identity untouched. + integration_session.expunge_all() + secret = await get_secret(integration_session) + assert secret.encrypted_nsec is None + assert settings.nsec == "" + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_update_nsec_clears_identity_with_empty_value( + admin_client: AsyncClient, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Start from a node that has an identity... + monkeypatch.setattr(settings, "nsec", "") + monkeypatch.setattr(settings, "npub", "") + set_resp = await admin_client.patch("/admin/api/nsec", json={"nsec": NSEC_HEX}) + assert set_resp.status_code == 200 + + # ...then clear it. + clear_resp = await admin_client.patch("/admin/api/nsec", json={"nsec": ""}) + assert clear_resp.status_code == 200 + assert clear_resp.json() == {"ok": True, "npub": ""} + + integration_session.expunge_all() + secret = await get_secret(integration_session) + assert secret.encrypted_nsec is None + assert settings.nsec == "" + assert settings.npub == "" diff --git a/tests/integration/test_admin_settings_endpoint.py b/tests/integration/test_admin_settings_endpoint.py new file mode 100644 index 00000000..c97a9a97 --- /dev/null +++ b/tests/integration/test_admin_settings_endpoint.py @@ -0,0 +1,82 @@ +"""Tests for the admin settings endpoint's handling of secrets (issue #553). + +``admin_password`` is no longer a settings field (it lives only as a one-way +hash in the Secret store), so it must never appear in the GET/PATCH payloads. +``nsec`` and ``upstream_api_key`` remain live in-memory runtime values but are +redacted on read and ignored on write — they cannot be set through the general +settings endpoint, only through their dedicated rotation paths. +""" + +from __future__ import annotations + +import secrets +import time +from collections.abc import AsyncGenerator + +import pytest +import pytest_asyncio +from httpx import AsyncClient + +from routstr.core.admin import admin_sessions +from routstr.core.db import AsyncSession +from routstr.core.settings import SettingsService, settings + + +@pytest_asyncio.fixture +async def admin_client( + integration_client: AsyncClient, +) -> AsyncGenerator[AsyncClient, None]: + """An integration_client pre-authenticated with an admin session token.""" + token = secrets.token_urlsafe(24) + admin_sessions[token] = int(time.time()) + 3600 + integration_client.headers["Authorization"] = f"Bearer {token}" + yield integration_client + admin_sessions.pop(token, None) + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_get_settings_omits_admin_password_and_redacts_secrets( + admin_client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "nsec", "nsec-secret") + monkeypatch.setattr(settings, "upstream_api_key", "sk-secret") + + resp = await admin_client.get("/admin/api/settings") + assert resp.status_code == 200 + + data = resp.json() + assert "admin_password" not in data + assert data["nsec"] == "[REDACTED]" + assert data["upstream_api_key"] == "[REDACTED]" + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_patch_settings_ignores_secret_fields( + admin_client: AsyncClient, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The PATCH path persists through SettingsService, which needs an + # initialized current snapshot and a settings row in the shared test DB. + await SettingsService.initialize(integration_session) + monkeypatch.setattr(settings, "nsec", "original-nsec") + + resp = await admin_client.patch( + "/admin/api/settings", + json={ + "name": "Renamed", + "nsec": "attacker-nsec", + "upstream_api_key": "attacker-key", + "admin_password": "attacker-pw", + }, + ) + assert resp.status_code == 200 + + data = resp.json() + assert data["name"] == "Renamed" + assert "admin_password" not in data + assert data["nsec"] == "[REDACTED]" + # The live secret was not overwritten through the general settings endpoint. + assert settings.nsec == "original-nsec" diff --git a/ui/components/settings/admin-settings.tsx b/ui/components/settings/admin-settings.tsx index f5ee51d9..68045b3f 100644 --- a/ui/components/settings/admin-settings.tsx +++ b/ui/components/settings/admin-settings.tsx @@ -116,9 +116,24 @@ export function AdminSettings() { setSaving(true); setError(''); - const updatedData = await AdminService.updateSettings(settings); - setSettings(updatedData as SettingsData); - setInitialSettings(updatedData as SettingsData); + // The nsec is a secret with its own endpoint (the general settings PATCH + // strips it); only send it when the operator actually changed it, so an + // untouched redacted value is never written back. The npub is derived + // server-side from the new nsec — fold it into the settings payload so the + // persisted blob stays consistent with the stored key. + let settingsPayload = settings; + if (hasFieldChanged('nsec')) { + const result = await AdminService.updateNsec( + (settings.nsec as string) || '' + ); + settingsPayload = { ...settings, npub: result.npub }; + } + + const updatedData = (await AdminService.updateSettings( + settingsPayload + )) as SettingsData; + setSettings(updatedData); + setInitialSettings(updatedData); toast.success('Settings saved successfully'); } catch (err) { const message = @@ -140,8 +155,8 @@ export function AdminSettings() { return; } - if (passwordData.new_password.length < 6) { - setPasswordError('New password must be at least 6 characters'); + if (passwordData.new_password.length < 8) { + setPasswordError('New password must be at least 8 characters'); return; } @@ -944,7 +959,7 @@ export function AdminSettings() { new_password: e.target.value, })) } - placeholder='Enter new password (min 6 characters)' + placeholder='Enter new password (min 8 characters)' /> diff --git a/ui/lib/api/services/admin.ts b/ui/lib/api/services/admin.ts index 5fa168cf..13c4366d 100644 --- a/ui/lib/api/services/admin.ts +++ b/ui/lib/api/services/admin.ts @@ -801,6 +801,15 @@ export class AdminService { ); } + static async updateNsec( + nsec: string + ): Promise<{ ok: boolean; npub: string }> { + return await apiClient.patch<{ ok: boolean; npub: string }>( + '/admin/api/nsec', + { nsec } + ); + } + static async login(password: string): Promise<{ ok: boolean; token: string; From 8aa2fa5c4aaa515707ada05a40ee754c03f3b6ff Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Fri, 26 Jun 2026 14:33:01 +0200 Subject: [PATCH 07/26] feat(scripts): add admin-password reset CLI Provide an offline recovery command that sets a new admin password directly in the Secret store, for operators locked out of the admin UI. Co-Authored-By: Claude Opus 4.8 --- scripts/__init__.py | 1 + scripts/reset_admin_password.py | 104 ++++++++++++++++++ .../integration/test_reset_admin_password.py | 76 +++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 scripts/__init__.py create mode 100644 scripts/reset_admin_password.py create mode 100644 tests/integration/test_reset_admin_password.py diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..5d5fdfa0 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Operational and recovery scripts for routstr-core (not a shipped package).""" diff --git a/scripts/reset_admin_password.py b/scripts/reset_admin_password.py new file mode 100644 index 00000000..e2c8f46f --- /dev/null +++ b/scripts/reset_admin_password.py @@ -0,0 +1,104 @@ +"""Recover admin access by resetting the stored admin password (issue #553). + +The lockout escape hatch for an operator who has lost the admin password. It +talks to the ``secrets`` table directly and deliberately does *not* require +``ROUTSTR_SECRET_KEY``: the admin password is scrypt-hashed (key-independent), +so recovery works even when the encryption key is missing or has changed. + +Two explicit, mutually exclusive actions — running with no arguments only prints +help, so the password can't be reset by accident: + + python scripts/reset_admin_password.py --password + Hash and store now. + + python scripts/reset_admin_password.py --regenerate + Clear the stored hash; the next node startup generates a fresh random + password and logs it once (with the /admin URL). +""" + +import argparse +import asyncio +import sys +import time + +from sqlmodel.ext.asyncio.session import AsyncSession + +from routstr.core.db import create_session, get_secret, set_admin_password +from routstr.core.vault import MIN_PASSWORD_LENGTH + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="reset_admin_password", + description="Reset the node's admin password (recovery from lockout).", + ) + action = parser.add_mutually_exclusive_group() + action.add_argument( + "--password", + metavar="NEW_PASSWORD", + help=f"set this as the new admin password (min {MIN_PASSWORD_LENGTH} chars)", + ) + action.add_argument( + "--regenerate", + action="store_true", + help="clear the password so the next startup generates and logs a new one", + ) + return parser + + +async def apply_reset( + session: AsyncSession, + *, + password: str | None = None, + regenerate: bool = False, +) -> str: + """Perform the requested reset against ``session``; return a status message.""" + if password is not None: + if len(password) < MIN_PASSWORD_LENGTH: + raise ValueError( + f"New password must be at least {MIN_PASSWORD_LENGTH} characters" + ) + await set_admin_password(session, password) + return "Admin password updated." + + if regenerate: + secret = await get_secret(session) + secret.admin_password_hash = None + secret.updated_at = int(time.time()) + session.add(secret) + await session.commit() + return ( + "Admin password cleared. The next node startup will generate a new " + "one and log it once with the /admin URL." + ) + + return "" + + +async def _run(password: str | None, regenerate: bool) -> str: + async with create_session() as session: + return await apply_reset( + session, password=password, regenerate=regenerate + ) + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.password is None and not args.regenerate: + parser.print_help() + return 0 + + try: + message = asyncio.run(_run(args.password, args.regenerate)) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + print(message) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/test_reset_admin_password.py b/tests/integration/test_reset_admin_password.py new file mode 100644 index 00000000..d0e479be --- /dev/null +++ b/tests/integration/test_reset_admin_password.py @@ -0,0 +1,76 @@ +"""Tests for the ``reset_admin_password`` recovery script (issue #553). + +The script is the lockout escape hatch: it works without ``ROUTSTR_SECRET_KEY`` +(scrypt hashing is key-independent). Two explicit, mutually exclusive actions — +``--password`` sets a new hash now, ``--regenerate`` clears the hash so the next +boot generates and logs a fresh one. A bare invocation is informational only and +must never touch the database (so nobody resets their password by accident). +""" + +import pytest +from sqlmodel.ext.asyncio.session import AsyncSession + +from routstr.core import vault +from routstr.core.db import get_secret, set_admin_password +from scripts.reset_admin_password import apply_reset, build_parser, main + + +@pytest.mark.asyncio +async def test_password_sets_a_verifiable_hash( + integration_session: AsyncSession, +) -> None: + await apply_reset(integration_session, password="recover-me-123") + + secret = await get_secret(integration_session) + assert secret.admin_password_hash is not None + assert vault.verify_password("recover-me-123", secret.admin_password_hash) is True + assert secret.updated_at is not None + + +@pytest.mark.asyncio +async def test_regenerate_clears_the_hash( + integration_session: AsyncSession, +) -> None: + # Start from a node that already has an admin password set. + await set_admin_password(integration_session, "old-password-9") + assert (await get_secret(integration_session)).admin_password_hash is not None + + await apply_reset(integration_session, regenerate=True) + + secret = await get_secret(integration_session) + # Cleared -> the next boot's bootstrap_secrets generates and logs a new one. + assert secret.admin_password_hash is None + assert secret.updated_at is not None + + +@pytest.mark.asyncio +async def test_password_below_min_length_is_rejected( + integration_session: AsyncSession, +) -> None: + await set_admin_password(integration_session, "old-password-9") + + with pytest.raises(ValueError, match="8 characters"): + await apply_reset(integration_session, password="short") + + # The existing password is untouched by the rejected reset. + secret = await get_secret(integration_session) + assert vault.verify_password("old-password-9", secret.admin_password_hash or "") + + +def test_password_and_regenerate_are_mutually_exclusive() -> None: + parser = build_parser() + with pytest.raises(SystemExit): + parser.parse_args(["--password", "abcd1234", "--regenerate"]) + + +def test_no_args_prints_help_and_never_opens_a_session( + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _fail() -> None: + raise AssertionError("a bare invocation must not touch the database") + + monkeypatch.setattr("scripts.reset_admin_password.create_session", _fail) + + assert main([]) == 0 + assert "usage" in capsys.readouterr().out.lower() From 14748c28fbd01673e86023d47a3dd139f1be850d Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Fri, 26 Jun 2026 14:33:09 +0200 Subject: [PATCH 08/26] docs: document ROUTSTR_SECRET_KEY and first-run admin password Explain that ROUTSTR_SECRET_KEY is now mandatory (with the generation command) and describe the first-run flow where a temporary admin password is logged once. Co-Authored-By: Claude Opus 4.8 --- .env.example | 13 +++++++++++-- README.md | 19 +++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 81138cc6..e0dbae73 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,17 @@ UPSTREAM_API_KEY=your-upstream-api-key # Tinfoil (confidential inference enclaves, EHBP) # TINFOIL_API_KEY=your-tinfoil-api-key -# ADMIN_PASSWORD=secure-admin-password +# Secret key used to encrypt secrets at rest (REQUIRED). The node refuses to +# start without it. Generate one with: +# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +ROUTSTR_SECRET_KEY= + +# The admin password and the Nostr identity (nsec) are NOT set here. The admin +# password is generated and logged once on first start (read it from the logs to +# sign in); both are managed afterwards from the admin UI and stored encrypted in +# the database. ADMIN_PASSWORD / NSEC are still read once as a legacy seed for +# existing deployments, but new nodes should set them in the UI — a value left in +# .env is ignored once the node has been configured. # Database # DATABASE_URL=sqlite+aiosqlite:///keys.db @@ -13,7 +23,6 @@ UPSTREAM_API_KEY=your-upstream-api-key # Node Information # NAME=My Routstr Node # DESCRIPTION=Fast AI API access with Bitcoin payments -# NSEC=nsec1... # HTTP_URL=https://api.mynode.com # ONION_URL=http://mynode.onion (auto fetched from compose) # RELAYS="wss://relay.damus.io,wss://relay.nostr.band,wss://eden.nostr.land,wss://relay.routstr.com" diff --git a/README.md b/README.md index da66b107..767bacae 100644 --- a/README.md +++ b/README.md @@ -55,19 +55,34 @@ If you are a node runner, start a Routstr Core instance using Docker Compose: 1. **Prepare your `.env`**: ```bash - ADMIN_PASSWORD=mysecretpassword + # Required: encrypts secrets at rest. The node won't start without it. + ROUTSTR_SECRET_KEY= NAME="My AI Node" DESCRIPTION="Fast access to models" NSEC=yournsec RECEIVE_LN_ADDRESS=yourname@wallet.com ``` + Generate `ROUTSTR_SECRET_KEY` once and keep it stable — changing it makes + previously encrypted secrets unreadable: + ```bash + python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" + ``` + 2. **Start the services**: ```bash docker compose up -d ``` -3. **Configure**: +3. **Get your admin password**: + On first start the node generates an admin password and logs it once with the + `/admin` URL. Read it from the logs: + ```bash + docker compose logs routstr | grep -i admin + ``` + (Lost it? Reset with `python scripts/reset_admin_password.py --regenerate`.) + +4. **Configure**: Open [http://localhost:8000/admin/](http://localhost:8000/admin/) to connect your AI providers and set pricing. For full instructions, see the **[Provider Quick Start Guide](https://docs.routstr.com/provider/quickstart/)**. From afcb3f7cdacaa6e5d074b67dfe956efb700fc09e Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Sat, 27 Jun 2026 20:08:28 +0200 Subject: [PATCH 09/26] fix(settings): keep upstream_api_key in the settings blob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit upstream_api_key was added to SECRET_FIELDS, so it was stripped from every blob write — but unlike nsec, nothing migrates it into encrypted storage. A node carrying it only in the DB blob would load it into memory once, rewrite the blob without it, and lose it on the next restart, breaking upstream auth. It is node-scoped config that really belongs on a provider, not a vault secret, and it has no encrypted home yet. Remove it from SECRET_FIELDS so it stays in the blob exactly as before; redaction on read and ignore-on-write in the admin settings endpoint are unchanged. Encrypting it is follow-up work. Co-Authored-By: Claude Opus 4.8 --- routstr/core/settings.py | 15 +++--- .../test_admin_settings_endpoint.py | 6 +-- tests/unit/test_settings.py | 48 ++++++++++++++----- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 8b779e42..b8eacdd6 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -135,11 +135,14 @@ def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]: # Secrets are credentials, not config: they live in the encrypted/hashed Secret # store (and decrypted in-memory for runtime use), never in the persisted -# settings blob. ``admin_password`` is gone from the model entirely; -# ``nsec``/``upstream_api_key`` remain live fields but are stripped from every -# blob write so they are never written back to plaintext. See -# ``bootstrap_secrets`` and ``routstr.core.vault``. -SECRET_FIELDS = frozenset({"admin_password", "nsec", "upstream_api_key"}) +# settings blob. ``admin_password`` is gone from the model entirely; ``nsec`` +# remains a live field but is stripped from every blob write so it is never +# written back to plaintext. ``upstream_api_key`` is intentionally *not* here: +# it has no encrypted home yet (it is node-scoped today but really belongs on a +# provider), so stripping it would lose it on the next restart. It stays in the +# blob as before; encrypting it is follow-up work. See ``bootstrap_secrets`` and +# ``routstr.core.vault``. +SECRET_FIELDS = frozenset({"admin_password", "nsec"}) def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]: @@ -350,7 +353,7 @@ class SettingsService: await db_session.commit() # Update the existing instance in-place for all live importers - # (keeps the decrypted nsec/upstream_api_key live in memory). + # (keeps the decrypted nsec live in memory). _apply_to_live_settings(merged_dict) cls._current = settings return cls._current diff --git a/tests/integration/test_admin_settings_endpoint.py b/tests/integration/test_admin_settings_endpoint.py index c97a9a97..15d05159 100644 --- a/tests/integration/test_admin_settings_endpoint.py +++ b/tests/integration/test_admin_settings_endpoint.py @@ -2,9 +2,9 @@ ``admin_password`` is no longer a settings field (it lives only as a one-way hash in the Secret store), so it must never appear in the GET/PATCH payloads. -``nsec`` and ``upstream_api_key`` remain live in-memory runtime values but are -redacted on read and ignored on write — they cannot be set through the general -settings endpoint, only through their dedicated rotation paths. +``nsec`` (in-memory at runtime) and ``upstream_api_key`` (still in the settings +blob) are both redacted on read and ignored on write — they cannot be set +through the general settings endpoint, only through their dedicated paths. """ from __future__ import annotations diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index ffe6352f..6d458470 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -138,7 +138,8 @@ def test_settings_model_drops_admin_password_field() -> None: # admin_password now lives only as a one-way hash in the Secret store; it is # no longer a settings field at all. assert "admin_password" not in Settings.__fields__ - # nsec and upstream_api_key remain runtime values held in memory. + # nsec remains a runtime value held in memory; upstream_api_key is ordinary + # config that still lives in the persisted blob. assert "nsec" in Settings.__fields__ assert "upstream_api_key" in Settings.__fields__ @@ -148,28 +149,53 @@ async def test_secret_fields_kept_in_memory_but_not_persisted( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("NSEC", NSEC_HEX) - monkeypatch.setenv("UPSTREAM_API_KEY", "sk-upstream") # Reset the live globals so monkeypatch reverts them after the test. monkeypatch.setattr(settings, "nsec", "") - monkeypatch.setattr(settings, "upstream_api_key", "") engine = create_async_engine("sqlite+aiosqlite:///:memory:") async with AsyncSession(engine, expire_on_commit=False) as session: s = await SettingsService.initialize(session) - # Runtime consumers still see the live secret values. + # Runtime consumers still see the live secret value. assert s.nsec == NSEC_HEX - assert s.upstream_api_key == "sk-upstream" - # ...but they are never written to the settings blob. + # ...but it is never written to the settings blob. blob = await _read_settings_blob(session) assert "nsec" not in blob - assert "upstream_api_key" not in blob assert "admin_password" not in blob # Non-secret derived/public values are still persisted. assert blob["npub"] == s.npub +@pytest.mark.asyncio +async def test_upstream_api_key_survives_persistence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # upstream_api_key is provider-scoped config, not a vault secret: it has no + # encrypted home yet, so it must stay in the settings blob. Stripping it + # would load it once, rewrite the blob without it, and lose it on the next + # restart. Guard the on-disk survival path: blob-only value, no env. + monkeypatch.delenv("UPSTREAM_API_KEY", raising=False) + monkeypatch.setattr(settings, "upstream_api_key", "") + + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with AsyncSession(engine, expire_on_commit=False) as session: + await SettingsService.initialize(session) + await session.exec( # type: ignore + text("UPDATE settings SET data = :d WHERE id = 1").bindparams( + d=json.dumps({"name": "LegacyNode", "upstream_api_key": "sk-only-in-db"}) + ) + ) + await session.commit() + + # A reload must not drop the key from the blob... + await SettingsService.initialize(session) + blob = await _read_settings_blob(session) + assert blob["upstream_api_key"] == "sk-only-in-db" + # ...and it stays live for the proxy hot path. + assert settings.upstream_api_key == "sk-only-in-db" + + @pytest.mark.asyncio async def test_existing_blob_secrets_are_stripped_on_initialize( monkeypatch: pytest.MonkeyPatch, @@ -202,9 +228,10 @@ async def test_existing_blob_secrets_are_stripped_on_initialize( blob = await _read_settings_blob(session) assert "admin_password" not in blob assert "nsec" not in blob - assert "upstream_api_key" not in blob - # Non-secret values survive the migration. + # Non-secret values survive the migration, including upstream_api_key, + # which is not vaulted yet and so must stay in the blob. assert blob["name"] == "LegacyNode" + assert blob["upstream_api_key"] == "sk-legacy" @pytest.mark.asyncio @@ -212,9 +239,7 @@ async def test_update_does_not_persist_secret_fields( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("NSEC", NSEC_HEX) - monkeypatch.setenv("UPSTREAM_API_KEY", "sk-upstream") monkeypatch.setattr(settings, "nsec", "") - monkeypatch.setattr(settings, "upstream_api_key", "") engine = create_async_engine("sqlite+aiosqlite:///:memory:") async with AsyncSession(engine, expire_on_commit=False) as session: @@ -224,5 +249,4 @@ async def test_update_does_not_persist_secret_fields( blob = await _read_settings_blob(session) assert blob["name"] == "Updated" assert "nsec" not in blob - assert "upstream_api_key" not in blob assert "admin_password" not in blob From 56a67c0a86e56d2cccd020bbadd2b3e7ae6e51ab Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Sat, 27 Jun 2026 20:33:51 +0200 Subject: [PATCH 10/26] fix(settings): fail fast when an nsec is set but ROUTSTR_SECRET_KEY is missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encryption of the Nostr identity at rest is mandatory. When a legacy nsec is present (env or blob) but no ROUTSTR_SECRET_KEY is set, bootstrap previously fell into vault.encrypt and surfaced its generic "key not set" error. Raise an explicit, nsec-contextual error first so the boot failure is intentional and actionable — it names the missing key and prints the generation command — rather than relying on vault throwing incidentally. No secret is dropped: the node refuses to start until the operator sets the key. Co-Authored-By: Claude Opus 4.8 --- routstr/core/settings.py | 11 +++++++++++ tests/integration/test_secret_bootstrap.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/routstr/core/settings.py b/routstr/core/settings.py index b8eacdd6..728a42d3 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -498,6 +498,17 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: else: legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec") if legacy_nsec: + # The node has a Nostr identity to protect. Encryption at rest is + # mandatory: without a key we fail fast (clear, actionable boot + # error) rather than silently persisting the nsec in plaintext. + if not os.environ.get("ROUTSTR_SECRET_KEY"): + raise RuntimeError( + "An nsec is configured but ROUTSTR_SECRET_KEY is not set. " + "The key is required to encrypt the Nostr identity at rest. " + "Generate one and set it in the environment:\n" + ' python -c "from cryptography.fernet import Fernet; ' + 'print(Fernet.generate_key().decode())"' + ) secret.encrypted_nsec = vault.encrypt(legacy_nsec) settings.nsec = legacy_nsec changed = True diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 3b45fd61..9aa957ee 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -163,6 +163,27 @@ async def test_fail_fast_when_nsec_encrypted_with_different_key( await bootstrap_secrets(integration_session) +# --- encryption is mandatory: a node with an nsec needs ROUTSTR_SECRET_KEY ----- + + +@pytest.mark.asyncio +async def test_legacy_nsec_without_secret_key_fails_fast( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A node with a Nostr identity must not boot without a key to encrypt it: + # encryption at rest is mandatory, not opt-in. The failure names the missing + # key and hands over the generation command rather than crashing opaquely or + # silently persisting the nsec in plaintext. No env/blob copy is dropped — the + # node refuses until the operator sets the key. + monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + monkeypatch.setenv("NSEC", NSEC_HEX) + + with pytest.raises(RuntimeError, match="ROUTSTR_SECRET_KEY"): + await bootstrap_secrets(integration_session) + + # --- boot ordering: rescue legacy blob secrets before they are stripped ---- From 4872c318d58834182fab58e77ce89f0ecc2c4890 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Sat, 27 Jun 2026 20:36:07 +0200 Subject: [PATCH 11/26] fix(settings): keep npub consistent with a store-only nsec on initialize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the nsec lives only in the encrypted Secret store (env carries no NSEC) and the settings blob holds no npub, bootstrap_secrets decrypted the nsec and derived the npub into memory, but SettingsService.initialize then re-derived settings from the npub-less blob and overwrote the live npub back to empty — leaving a private key with no matching public key, so the node silently stopped announcing a usable Nostr identity. Derive npub from the live nsec during initialize when the merged settings carry none, so the public key stays consistent with the identity and is persisted to the blob. Co-Authored-By: Claude Opus 4.8 --- routstr/core/settings.py | 10 +++++++ tests/integration/test_secret_bootstrap.py | 33 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 728a42d3..623cc11a 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -337,6 +337,16 @@ class SettingsService: merged_dict.get("cashu_mints", []) ) + # Keep npub consistent with the live nsec. bootstrap_secrets may hold + # the decrypted nsec (from the encrypted store) even when neither env + # nor the blob carries an nsec/npub; derive from that live value so + # initialize never wipes a known public key back to empty, leaving a + # private key with no matching npub. + if not merged_dict.get("npub") and settings.nsec: + derived_npub = derive_npub_from_nsec(settings.nsec) + if derived_npub: + merged_dict["npub"] = derived_npub + # Persist without secrets; compare against the stripped target so a # legacy blob that still carries plaintext secrets gets rewritten # (and thereby sunset) even when its non-secret values are unchanged. diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 9aa957ee..4a8a51db 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -243,6 +243,39 @@ async def test_initialize_does_not_clobber_store_only_nsec( assert "nsec" not in json.loads(row.first()[0]) +@pytest.mark.asyncio +async def test_initialize_keeps_npub_matching_store_only_nsec( + clean_secret_env: None, integration_session: AsyncSession +) -> None: + # Steady state with mandatory encryption: the nsec lives ONLY in the + # encrypted Secret store (env carries no NSEC) and the blob has no npub. + # bootstrap decrypts the nsec and derives the npub into memory; initialize + # then re-derives settings from the npub-less blob and must NOT wipe the npub + # back to empty, or the node holds a private key with no matching public key + # and silently stops announcing a usable Nostr identity. + expected_npub = derive_npub_from_nsec(NSEC_HEX) + assert expected_npub # guard: the test key must yield a real npub + + await _create_settings_blob(integration_session, {"name": "LegacyNode"}) + secret = await get_secret(integration_session) + secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + integration_session.add(secret) + await integration_session.commit() + + await bootstrap_secrets(integration_session) + assert settings.npub == expected_npub # bootstrap derived it + + await SettingsService.initialize(integration_session) + # The npub still matches the live nsec... + assert settings.nsec == NSEC_HEX + assert settings.npub == expected_npub + # ...and is persisted to the blob (it is public, not a stripped secret). + row = await integration_session.exec( # type: ignore + text("SELECT data FROM settings WHERE id = 1") + ) + assert json.loads(row.first()[0])["npub"] == expected_npub + + @pytest.mark.asyncio async def test_startup_runs_bootstrap_before_settings_initialize( monkeypatch: pytest.MonkeyPatch, From b8700dde401116dda98e05c7d39bf363c4e4046d Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Sat, 27 Jun 2026 21:00:08 +0200 Subject: [PATCH 12/26] harden(admin): cap scrypt work factor, keep generated password off disk Address review hardening items on the secret-storage path: - vault.verify_password caps N/r/p at the parameters this module emits, so a tampered or corrupt stored hash can't force an unbounded scrypt work factor (memory grows with N*r) and turn a login into an OOM/DoS. - bootstrap prints the generated first-run admin password to stdout instead of the logger, so it reaches the operator once without being persisted into the on-disk log files. - admin_login reads the password hash while the DB session is open rather than off a detached ORM instance after the context exits. - drop a stray debug print of the request payload in upsert_provider_model. Co-Authored-By: Claude Opus 4.8 --- routstr/core/admin.py | 8 +++++--- routstr/core/settings.py | 15 +++++++-------- routstr/core/vault.py | 12 +++++++++--- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/routstr/core/admin.py b/routstr/core/admin.py index 7823fd99..02b81611 100644 --- a/routstr/core/admin.py +++ b/routstr/core/admin.py @@ -343,11 +343,14 @@ async def admin_login( ) -> dict[str, object]: async with create_session() as session: secret = await get_secret(session) + # Read the hash while the session is open; the ORM object is detached + # once the context exits and its attributes can no longer be loaded. + password_hash = secret.admin_password_hash - if not secret.admin_password_hash: + if not password_hash: raise HTTPException(status_code=500, detail="Admin password not configured") - if not vault.verify_password(payload.password, secret.admin_password_hash): + if not vault.verify_password(payload.password, password_hash): raise HTTPException(status_code=401, detail="Invalid password") token = secrets.token_urlsafe(32) @@ -526,7 +529,6 @@ class ModelCreate(BaseModel): async def upsert_provider_model( provider_id: str, payload: ModelCreate ) -> dict[str, object]: - print(payload) logger.info( f"UPSERT_PROVIDER_MODEL called: provider_id={provider_id}, model_id={payload.id}" ) diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 623cc11a..5b9793ee 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -467,9 +467,6 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: from . import vault from .db import get_secret - from .logging import get_logger - - logger = get_logger(__name__) raw_blob = await _read_raw_settings_blob(db_session) secret = await get_secret(db_session) @@ -486,12 +483,14 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: generated = secrets.token_urlsafe(24) secret.admin_password_hash = vault.hash_password(generated) admin_url = (settings.http_url or "http://localhost:8000").rstrip("/") - logger.warning( + # Print to stdout rather than the logger: the operator must see this + # once (e.g. `docker compose logs`), but it must not be persisted + # into the on-disk log files the logger also writes to. + print( "No admin password set; generated a temporary one (shown only " - "now): %s\nLog in at %s/admin and change it from the dashboard " - "settings.", - generated, - admin_url, + f"now): {generated}\nLog in at {admin_url}/admin and change it " + "from the dashboard settings.", + flush=True, ) changed = True diff --git a/routstr/core/vault.py b/routstr/core/vault.py index 1f39da91..8bc04ff3 100644 --- a/routstr/core/vault.py +++ b/routstr/core/vault.py @@ -116,14 +116,20 @@ def verify_password(password: str, stored: str) -> bool: scheme, n, r, p, salt_b64, hash_b64 = stored.split(":") if scheme != "scrypt": return False + n_int, r_int, p_int = int(n), int(r), int(p) + # Cap the work factor at the parameters this module emits. scrypt's + # memory cost grows with N*r, so an oversized N/r in a tampered or + # corrupt stored hash could turn a single login into an OOM/DoS. + if n_int > _SCRYPT_N or r_int > _SCRYPT_R or p_int > _SCRYPT_P: + return False salt = base64.b64decode(salt_b64) expected = base64.b64decode(hash_b64) derived = hashlib.scrypt( password.encode(), salt=salt, - n=int(n), - r=int(r), - p=int(p), + n=n_int, + r=r_int, + p=p_int, dklen=len(expected), ) except (ValueError, TypeError): From 5d5c849180a8c3851c2fe6b4ed1d5952086320df Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Tue, 7 Jul 2026 16:17:03 +0200 Subject: [PATCH 13/26] fix(migrations): repoint secrets migration onto current Alembic head The add-secrets migration branched off b5e7c9d1f3a2, but the add-slug migration c6d7e8f9a0b1 has since landed on that same parent, leaving two Alembic heads. `alembic upgrade head` then refuses to run and the node fails to boot. Repoint down_revision onto the current head so the chain is linear again. Co-Authored-By: Claude Opus 4.8 --- migrations/versions/c6f8d2e4a1b3_add_secrets_table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py index f867dd5f..cd41d89c 100644 --- a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py +++ b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py @@ -1,8 +1,8 @@ """add secrets table Revision ID: c6f8d2e4a1b3 -Revises: b5e7c9d1f3a2 -Create Date: 2026-06-24 00:00:00.000000 +Revises: c6d7e8f9a0b1 +Create Date: 2026-07-07 00:00:00.000000 Creates the node-level singleton secret store (issue #553). Schema only; moving any legacy plaintext into the encrypted/hashed columns happens at bootstrap, @@ -14,7 +14,7 @@ import sqlmodel from alembic import op revision = "c6f8d2e4a1b3" -down_revision = "b5e7c9d1f3a2" +down_revision = "c6d7e8f9a0b1" branch_labels = None depends_on = None From 0415806c5ac8a074885382d74f732e3a598f2c00 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Tue, 7 Jul 2026 16:17:30 +0200 Subject: [PATCH 14/26] fix(config): keep the vault the sole owner of node secrets Two ways a stale legacy NSEC could override or resurrect an nsec the vault already owns (issue #553): - initialize() re-applied env/blob values onto live settings after bootstrap had decrypted the authoritative nsec, so a stale NSEC left in .env would clobber it on restart (e.g. after rotating the key in the admin UI). _apply_to_live_settings now never re-applies secret fields; bootstrap_secrets is their only writer. - An empty encrypted_nsec could not distinguish "never migrated" from "intentionally cleared", so clearing the identity via the admin API and restarting re-imported the old NSEC from env/blob. Record vault ownership in a new secrets.nsec_managed column (set on legacy import and on every set_nsec write); bootstrap skips the legacy import once the vault owns the nsec, so a cleared identity stays cleared. Co-Authored-By: Claude Opus 4.8 --- .../c6f8d2e4a1b3_add_secrets_table.py | 10 +++- routstr/core/db.py | 10 +++- routstr/core/settings.py | 22 +++++--- tests/integration/test_secret_bootstrap.py | 53 ++++++++++++++++++- tests/unit/test_settings.py | 7 +-- 5 files changed, 88 insertions(+), 14 deletions(-) diff --git a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py index cd41d89c..77a5a8d9 100644 --- a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py +++ b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py @@ -6,7 +6,9 @@ Create Date: 2026-07-07 00:00:00.000000 Creates the node-level singleton secret store (issue #553). Schema only; moving any legacy plaintext into the encrypted/hashed columns happens at bootstrap, -where the live ROUTSTR_SECRET_KEY is available. +where the live ROUTSTR_SECRET_KEY is available. ``nsec_managed`` records that the +vault has taken ownership of the nsec, so a cleared identity is never resurrected +from a stale legacy ``NSEC`` env var / settings blob on the next boot. """ import sqlalchemy as sa @@ -33,6 +35,12 @@ def upgrade() -> None: sqlmodel.sql.sqltypes.AutoString(), nullable=True, ), + sa.Column( + "nsec_managed", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), sa.Column("updated_at", sa.Integer(), nullable=True), sa.PrimaryKeyConstraint("id"), ) diff --git a/routstr/core/db.py b/routstr/core/db.py index 827cc24f..5fe29858 100644 --- a/routstr/core/db.py +++ b/routstr/core/db.py @@ -455,6 +455,11 @@ class Secret(SQLModel, table=True): # type: ignore id: int = Field(default=1, primary_key=True) admin_password_hash: str | None = Field(default=None) encrypted_nsec: str | None = Field(default=None) + # True once the vault owns the nsec (imported from legacy plaintext, or set + # via the admin API). A cleared nsec then stays cleared: bootstrap must not + # resurrect it from a stale legacy ``NSEC`` env var / settings blob, which an + # empty ``encrypted_nsec`` alone cannot distinguish from "never migrated". + nsec_managed: bool = Field(default=False) updated_at: int | None = Field(default=None) @@ -531,12 +536,15 @@ async def set_nsec(session: AsyncSession, nsec: str) -> None: """Store the node's nsec, Fernet-encrypted, on the Secret singleton. An empty string clears it (the node then holds no Nostr identity and signs - no events). + no events). Either way the vault now owns the nsec, so ``nsec_managed`` is + set: a cleared identity must not be resurrected from a stale legacy ``NSEC`` + on the next boot. """ from .vault import encrypt secret = await get_secret(session) secret.encrypted_nsec = encrypt(nsec) if nsec else None + secret.nsec_managed = True secret.updated_at = int(time.time()) session.add(secret) await session.commit() diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 5b9793ee..b3165cc2 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -153,15 +153,15 @@ def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]: def _apply_to_live_settings(data: dict[str, Any]) -> None: """Apply ``data`` onto the live ``settings`` for all in-process importers. - Secrets are owned by ``bootstrap_secrets`` (which decrypts the nsec into - memory before this runs) — they are never persisted to the blob, so ``data`` - re-derived from the secret-free blob carries empty secret values. Skip those - empty overwrites so a live secret is never clobbered; a non-empty value - (legacy env, or a not-yet-stripped blob mid-migration) is still applied. + Secrets are owned exclusively by ``bootstrap_secrets``, which runs first and + has already decrypted the authoritative nsec into memory (importing any + legacy plaintext on the way). Never re-apply secret fields from env/blob + here: a non-empty but stale ``NSEC`` env var would otherwise override an nsec + the vault has taken ownership of (e.g. after the operator rotates it in the + UI), and an empty one would wipe the live value. Skip them entirely. """ - live = settings.dict() for k, v in data.items(): - if k in SECRET_FIELDS and not v and live.get(k): + if k in SECRET_FIELDS: continue setattr(settings, k, v) @@ -504,7 +504,12 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: "ROUTSTR_SECRET_KEY. The key changed, or this database came from " "another node. Restore the original ROUTSTR_SECRET_KEY to recover." ) from exc - else: + elif not secret.nsec_managed: + # The vault has not taken ownership yet: import any legacy plaintext + # (env, or the old settings blob). Once managed, an empty encrypted_nsec + # means the identity was intentionally cleared via the admin API, so this + # branch is skipped and the nsec stays empty rather than being resurrected + # from a stale legacy copy. legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec") if legacy_nsec: # The node has a Nostr identity to protect. Encryption at rest is @@ -519,6 +524,7 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: 'print(Fernet.generate_key().decode())"' ) secret.encrypted_nsec = vault.encrypt(legacy_nsec) + secret.nsec_managed = True settings.nsec = legacy_nsec changed = True diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 4a8a51db..077a5f74 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -17,7 +17,7 @@ from sqlmodel import text from sqlmodel.ext.asyncio.session import AsyncSession from routstr.core import vault -from routstr.core.db import get_secret +from routstr.core.db import get_secret, set_nsec from routstr.core.settings import ( SettingsService, bootstrap_secrets, @@ -30,6 +30,9 @@ TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU=" TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ=" NSEC_HEX = "1" * 64 +# A different key, standing in for a stale value left behind in env/blob after +# the vault has taken ownership of the real one. +STALE_NSEC_HEX = "2" * 64 @pytest.fixture @@ -243,6 +246,54 @@ async def test_initialize_does_not_clobber_store_only_nsec( assert "nsec" not in json.loads(row.first()[0]) +@pytest.mark.asyncio +async def test_stale_env_nsec_does_not_override_vault_nsec( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The vault owns the nsec, but a stale NSEC (e.g. the operator rotated the + # key in the UI yet left the old value in .env) is still in the environment. + # bootstrap decrypts the store value; initialize must NOT let the stale env + # value clobber it, or a restart silently reverts to the old identity. + await set_nsec(integration_session, NSEC_HEX) + + monkeypatch.setenv("NSEC", STALE_NSEC_HEX) + await _create_settings_blob(integration_session, {"name": "LegacyNode"}) + + await bootstrap_secrets(integration_session) + await SettingsService.initialize(integration_session) + + # The vault value wins; the stale env value is ignored. + assert settings.nsec == NSEC_HEX + + +@pytest.mark.asyncio +async def test_cleared_nsec_stays_cleared_across_reboot( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # An identity was imported from env, then the operator cleared it via the + # admin API. The old NSEC is still in env. On the next boot the cleared + # identity must stay cleared, not get resurrected from the stale env value. + monkeypatch.setenv("NSEC", NSEC_HEX) + await bootstrap_secrets(integration_session) + assert settings.nsec == NSEC_HEX + + # Clear via the admin path (mirrors the endpoint: store empty, live empty). + await set_nsec(integration_session, "") + monkeypatch.setattr(settings, "nsec", "") + + # Reboot with the stale NSEC still present in env. + await bootstrap_secrets(integration_session) + + reloaded = await get_secret(integration_session) + assert reloaded.nsec_managed is True + assert reloaded.encrypted_nsec is None # not re-imported + assert settings.nsec == "" # stays cleared + + @pytest.mark.asyncio async def test_initialize_keeps_npub_matching_store_only_nsec( clean_secret_env: None, integration_session: AsyncSession diff --git a/tests/unit/test_settings.py b/tests/unit/test_settings.py index 6d458470..a553b6d4 100644 --- a/tests/unit/test_settings.py +++ b/tests/unit/test_settings.py @@ -148,9 +148,10 @@ def test_settings_model_drops_admin_password_field() -> None: async def test_secret_fields_kept_in_memory_but_not_persisted( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("NSEC", NSEC_HEX) - # Reset the live globals so monkeypatch reverts them after the test. - monkeypatch.setattr(settings, "nsec", "") + # bootstrap_secrets (which runs first at boot) owns the nsec and has already + # decrypted it into memory; simulate that live value. initialize must keep it + # in memory for runtime consumers yet never write it to the settings blob. + monkeypatch.setattr(settings, "nsec", NSEC_HEX) engine = create_async_engine("sqlite+aiosqlite:///:memory:") async with AsyncSession(engine, expire_on_commit=False) as session: From 1045f1061ba7024782de9e571cdc32659fb38e20 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Mon, 13 Jul 2026 17:28:31 +0200 Subject: [PATCH 15/26] fix(settings): derive npub from the vault nsec, not a stale env value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit initialize() only filled npub when it was empty, so an existing node with a stale NSEC still lingering in its env/blob kept that value's derived npub even after the vault took ownership of a different nsec. The node then held the vault's private key but announced the old env key's public key — a split identity that anything reading settings.npub would broadcast. npub is a pure derivation of nsec and is never configured on its own, so derive it from the live (vault) nsec and override rather than only fill. Co-Authored-By: Claude Opus 4.8 --- routstr/core/settings.py | 15 +++++++----- tests/integration/test_secret_bootstrap.py | 27 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/routstr/core/settings.py b/routstr/core/settings.py index b3165cc2..542aa80e 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -337,12 +337,15 @@ class SettingsService: merged_dict.get("cashu_mints", []) ) - # Keep npub consistent with the live nsec. bootstrap_secrets may hold - # the decrypted nsec (from the encrypted store) even when neither env - # nor the blob carries an nsec/npub; derive from that live value so - # initialize never wipes a known public key back to empty, leaving a - # private key with no matching npub. - if not merged_dict.get("npub") and settings.nsec: + # Keep npub consistent with the live nsec. bootstrap_secrets has + # already run and holds the single authoritative nsec (decrypted from + # the encrypted store, or freshly imported). merged_dict starts from + # the env/blob, which may carry a STALE nsec — and therefore a stale + # derived npub — after the vault took ownership. Derive from the live + # value and OVERRIDE, not just fill: otherwise the node keeps the + # vault's private key but announces the old env key's npub (npub is a + # pure derivation of nsec, never configured independently of it). + if settings.nsec: derived_npub = derive_npub_from_nsec(settings.nsec) if derived_npub: merged_dict["npub"] = derived_npub diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 077a5f74..18899dea 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -268,6 +268,33 @@ async def test_stale_env_nsec_does_not_override_vault_nsec( assert settings.nsec == NSEC_HEX +@pytest.mark.asyncio +async def test_stale_env_nsec_does_not_split_npub_from_vault_nsec( + clean_secret_env: None, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # As above, the vault owns the nsec while a stale NSEC lingers in env. The + # private key correctly comes from the vault, but the npub must too: if + # initialize derives the public key from the stale env nsec, the node ends up + # with a private key from the vault and a public key from the old env value, + # and anything reading settings.npub announces the wrong Nostr identity. + expected_npub = derive_npub_from_nsec(NSEC_HEX) + stale_npub = derive_npub_from_nsec(STALE_NSEC_HEX) + assert expected_npub and stale_npub and expected_npub != stale_npub # guard + + await set_nsec(integration_session, NSEC_HEX) + + monkeypatch.setenv("NSEC", STALE_NSEC_HEX) + await _create_settings_blob(integration_session, {"name": "LegacyNode"}) + + await bootstrap_secrets(integration_session) + await SettingsService.initialize(integration_session) + + assert settings.nsec == NSEC_HEX + assert settings.npub == expected_npub + + @pytest.mark.asyncio async def test_cleared_nsec_stays_cleared_across_reboot( clean_secret_env: None, From 7106dfe330ffa1a41de903479e89dfc60cb3dad0 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Mon, 13 Jul 2026 17:29:36 +0200 Subject: [PATCH 16/26] fix(vault): provision a master key on upgrade instead of refusing to boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node with a legacy plaintext nsec but no ROUTSTR_SECRET_KEY refused to boot: bootstrap_secrets raised and vault.encrypt required the env key. That turned encryption at rest into a hard breaking change on auto-upgrade. Encryption stays mandatory — the nsec is never persisted in plaintext — but key custody becomes flexible. When no ROUTSTR_SECRET_KEY is set, encrypt() generates a Fernet key, writes it owner-only (0600) to a key file, and prints a one-time back-it-up notice, so an upgrading node keeps running. The read path stays strict: decrypt()/get_fernet() never mint a key (a fresh key could not match existing ciphertext) and fail fast with the generation command when none is configured. A malformed env key still fails fast rather than silently self-provisioning a different key. The key file defaults beside the SQLite database (ROUTSTR_SECRET_KEY_FILE overrides), so it rides whatever volume already persists the data instead of a working-directory path a container recreate would drop. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + routstr/core/settings.py | 14 +- routstr/core/vault.py | 167 ++++++++++++++++++--- tests/integration/test_secret_bootstrap.py | 40 +++-- tests/unit/test_vault.py | 135 ++++++++++++++++- 5 files changed, 315 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 0d6c7236..903d526e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ __pycache__ .env keys.db +routstr_secret.key wallet.sqlite3 # Python build artifacts diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 542aa80e..133496f6 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -516,16 +516,10 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec") if legacy_nsec: # The node has a Nostr identity to protect. Encryption at rest is - # mandatory: without a key we fail fast (clear, actionable boot - # error) rather than silently persisting the nsec in plaintext. - if not os.environ.get("ROUTSTR_SECRET_KEY"): - raise RuntimeError( - "An nsec is configured but ROUTSTR_SECRET_KEY is not set. " - "The key is required to encrypt the Nostr identity at rest. " - "Generate one and set it in the environment:\n" - ' python -c "from cryptography.fernet import Fernet; ' - 'print(Fernet.generate_key().decode())"' - ) + # mandatory, but a missing key is provisioned, not fatal: + # vault.encrypt generates and persists a master key (with a loud + # one-time operator notice) when none was supplied, so an upgrading + # node keeps running. The nsec is never persisted in plaintext. secret.encrypted_nsec = vault.encrypt(legacy_nsec) secret.nsec_managed = True settings.nsec = legacy_nsec diff --git a/routstr/core/vault.py b/routstr/core/vault.py index 8bc04ff3..674d74c2 100644 --- a/routstr/core/vault.py +++ b/routstr/core/vault.py @@ -4,15 +4,22 @@ Thin wrapper over ``cryptography`` so nothing else in the codebase touches Fernet/scrypt/HMAC directly: - :func:`encrypt`/:func:`decrypt` — Fernet symmetric encryption, keyed by the - mandatory ``ROUTSTR_SECRET_KEY``. Ciphertext is self-describing (``fernet:v1:`` - prefix) so a value can be told apart from legacy plaintext and so reading it - under the wrong key surfaces as a hard error rather than silent corruption. + mandatory master key. Ciphertext is self-describing (``fernet:v1:`` prefix) so + a value can be told apart from legacy plaintext and so reading it under the + wrong key surfaces as a hard error rather than silent corruption. - :func:`hash_password`/:func:`verify_password` — salted scrypt hashing. This is - *key-independent*: it never reads ``ROUTSTR_SECRET_KEY``, so password login and - the recovery script keep working even when the key is missing. + *key-independent*: it never reads the master key, so password login and the + recovery script keep working even when the key is missing. -A missing or malformed ``ROUTSTR_SECRET_KEY`` fails fast with the generation -command in the message. +Key custody is flexible but encryption is not optional. The key comes from the +``ROUTSTR_SECRET_KEY`` env var, else a persisted key file +(``ROUTSTR_SECRET_KEY_FILE``, defaulting beside the SQLite database so it persists +on the same volume as the data); when neither is set, :func:`encrypt` generates +one to the key file and prints a one-time notice, so an existing node upgrades +without breaking instead of refusing to boot. Reading is strict — :func:`decrypt` +never generates a key (a new key could not match existing ciphertext) and fails +fast with the generation command when none is configured. A malformed +``ROUTSTR_SECRET_KEY`` is an operator error and always fails fast. """ import base64 @@ -20,8 +27,11 @@ import hashlib import hmac import os import secrets +from pathlib import Path from cryptography.fernet import Fernet +from sqlalchemy.engine import make_url +from sqlalchemy.exc import ArgumentError _PREFIX = "fernet:v1:" _GEN_COMMAND = ( @@ -29,6 +39,15 @@ _GEN_COMMAND = ( 'print(Fernet.generate_key().decode())"' ) +# Where an auto-generated master key is persisted when the operator supplies no +# ``ROUTSTR_SECRET_KEY``. Defaults beside the SQLite database so it rides whatever +# volume already persists the data (a container recreate would otherwise generate +# a fresh key and be unable to decrypt existing secrets); falls back to the +# working directory when the DB location is unknown. Override the exact path with +# ``ROUTSTR_SECRET_KEY_FILE``. +_KEY_FILE_ENV = "ROUTSTR_SECRET_KEY_FILE" +_DEFAULT_KEY_FILE = "routstr_secret.key" + # Minimum admin-password length, enforced wherever a password is set/changed # (admin endpoints + the recovery script) so the policy lives in one place. MIN_PASSWORD_LENGTH = 8 @@ -41,19 +60,106 @@ _SCRYPT_DKLEN = 32 _SCRYPT_SALT_BYTES = 16 -def _require_secret_key() -> str: - key = os.environ.get("ROUTSTR_SECRET_KEY") - if not key: - raise RuntimeError( - "ROUTSTR_SECRET_KEY is not set. It is required to encrypt secrets at " - "rest. Generate one with:\n " + _GEN_COMMAND - ) +def _database_dir() -> Path | None: + """Directory of the SQLite database file, or ``None`` when it has no on-disk + location (a non-SQLite URL or ``:memory:``). + + Read from ``DATABASE_URL`` at call time and parsed here rather than importing + ``routstr.core.db`` — that module builds the engine at import, which the + crypto layer must not drag in. Mirrors db.py's ``DATABASE_URL`` default. + """ + url_str = os.environ.get("DATABASE_URL", "sqlite+aiosqlite:///keys.db") + try: + url = make_url(url_str) + except ArgumentError: + return None + if url.get_backend_name() != "sqlite" or not url.database: + return None + if url.database == ":memory:": + return None + return Path(url.database).parent + + +def _key_file_path() -> Path: + """Where the auto-generated master key is read from / written to. + + ``ROUTSTR_SECRET_KEY_FILE`` wins; otherwise the key sits beside the SQLite + database so it persists on the same volume as the data, falling back to the + working directory when the DB location is unknown. + """ + override = os.environ.get(_KEY_FILE_ENV) + if override: + return Path(override) + directory = _database_dir() + return (directory or Path()) / _DEFAULT_KEY_FILE + + +def _read_key_file(path: Path) -> str | None: + try: + stored = path.read_text().strip() + except OSError: + return None + return stored or None + + +def _load_secret_key() -> str | None: + """The configured key without provisioning: env var, then the key file.""" + return os.environ.get("ROUTSTR_SECRET_KEY") or _read_key_file(_key_file_path()) + + +def _warn_generated_key(path: Path, key: str) -> None: + # stdout, not the logger: the operator must see this once (e.g. in + # ``docker compose logs``), but it must never be persisted into the on-disk + # log files the logger also writes. Mirrors the generated-admin-password + # notice so an upgrade cannot silently create an unbacked key. + print( + "No ROUTSTR_SECRET_KEY was set; generated one to encrypt node secrets at " + f"rest and saved it to {path}.\n" + "!! BACK UP THIS FILE. If it is lost, the encrypted secrets cannot be " + "recovered and will have to be re-entered.\n" + "To manage the key yourself (e.g. from a secrets manager) set it in the " + f"environment instead:\n ROUTSTR_SECRET_KEY={key}", + flush=True, + ) + + +def _generate_and_persist_key(path: Path) -> str: + """Generate a Fernet key, persist it owner-only, and warn once.""" + key = Fernet.generate_key().decode() + path.parent.mkdir(parents=True, exist_ok=True) + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + # Another worker won the race and wrote the key first; adopt theirs + # rather than clobber a key that secrets may already be encrypted under. + existing = _read_key_file(path) + if existing: + return existing + raise + with os.fdopen(fd, "w") as handle: + handle.write(key) + _warn_generated_key(path, key) return key -def get_fernet() -> Fernet: - """Build a :class:`Fernet` from ``ROUTSTR_SECRET_KEY`` (fails fast).""" - key = _require_secret_key() +def ensure_secret_key() -> str: + """Return the master key, provisioning one if the operator supplied none. + + Precedence: the ``ROUTSTR_SECRET_KEY`` env var, then the persisted key file, + otherwise a freshly generated key written to the key file (with a one-time + operator notice). This keeps encryption at rest mandatory while letting an + existing node upgrade without setting a key first. A malformed env key is + left to fail at :func:`get_fernet` — it is an operator error, not an unset + key, so it must not trigger silent self-provisioning. + """ + env_key = os.environ.get("ROUTSTR_SECRET_KEY") + if env_key: + return env_key + path = _key_file_path() + return _read_key_file(path) or _generate_and_persist_key(path) + + +def _fernet_from_key(key: str) -> Fernet: try: return Fernet(key.encode()) except (ValueError, TypeError) as exc: @@ -63,10 +169,31 @@ def get_fernet() -> Fernet: ) from exc +def get_fernet() -> Fernet: + """Build a :class:`Fernet` from the configured key (env var or key file). + + Strict: this never generates a key, so already-encrypted ciphertext is never + shadowed by a fresh key. A read with no key configured fails fast with the + generation command. + """ + key = _load_secret_key() + if not key: + raise RuntimeError( + "ROUTSTR_SECRET_KEY is not set. It is required to encrypt secrets at " + "rest. Generate one with:\n " + _GEN_COMMAND + ) + return _fernet_from_key(key) + + def encrypt(plaintext: str) -> str: - """Encrypt ``plaintext`` into a self-describing ``fernet:v1:`` token.""" - token = get_fernet().encrypt(plaintext.encode()).decode() - return _PREFIX + token + """Encrypt ``plaintext`` into a self-describing ``fernet:v1:`` token. + + Provisions a master key (env var, key file, or a freshly generated one) so an + upgrading node never has to set one before its first secret is stored; the + value is always encrypted, never persisted in plaintext. + """ + fernet = _fernet_from_key(ensure_secret_key()) + return _PREFIX + fernet.encrypt(plaintext.encode()).decode() def is_encrypted(value: str) -> bool: diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 18899dea..4a177ee2 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -10,6 +10,7 @@ silently corrupting state. import json from contextlib import asynccontextmanager +from pathlib import Path from typing import Any, AsyncGenerator import pytest @@ -166,25 +167,46 @@ async def test_fail_fast_when_nsec_encrypted_with_different_key( await bootstrap_secrets(integration_session) -# --- encryption is mandatory: a node with an nsec needs ROUTSTR_SECRET_KEY ----- +# --- encryption is mandatory, key custody is not: upgrade without a key -------- @pytest.mark.asyncio -async def test_legacy_nsec_without_secret_key_fails_fast( +async def test_legacy_nsec_without_secret_key_generates_and_encrypts( clean_secret_env: None, integration_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], ) -> None: - # A node with a Nostr identity must not boot without a key to encrypt it: - # encryption at rest is mandatory, not opt-in. The failure names the missing - # key and hands over the generation command rather than crashing opaquely or - # silently persisting the nsec in plaintext. No env/blob copy is dropped — the - # node refuses until the operator sets the key. + # A node upgrading with a legacy plaintext NSEC but no ROUTSTR_SECRET_KEY must + # NOT break. Encryption at rest stays mandatory (the nsec is never persisted + # in plaintext), but the key custody is flexible: bootstrap generates a master + # key, persists it to the key file, warns loudly, and encrypts the identity — + # so the node keeps running instead of refusing to boot. monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + key_file = tmp_path / "routstr_secret.key" + monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file)) monkeypatch.setenv("NSEC", NSEC_HEX) - with pytest.raises(RuntimeError, match="ROUTSTR_SECRET_KEY"): - await bootstrap_secrets(integration_session) + await bootstrap_secrets(integration_session) + + # A master key was generated and persisted... + assert key_file.exists() + # ...the nsec is encrypted at rest under it, never stored in plaintext... + secret = await get_secret(integration_session) + assert secret.encrypted_nsec is not None + assert vault.is_encrypted(secret.encrypted_nsec) is True + assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX + assert secret.nsec_managed is True + # ...the node holds the live identity (npub derived from it)... + assert settings.nsec == NSEC_HEX + assert settings.npub == derive_npub_from_nsec(NSEC_HEX) + # ...and the operator is loudly told a key was generated and must be backed up + # (path + value shown) so an upgrade cannot silently create an unbacked key. + out = capsys.readouterr().out + assert str(key_file) in out + assert key_file.read_text().strip() in out + assert "BACK UP" in out.upper() # --- boot ordering: rescue legacy blob secrets before they are stripped ---- diff --git a/tests/unit/test_vault.py b/tests/unit/test_vault.py index 21621d54..52f1d9fd 100644 --- a/tests/unit/test_vault.py +++ b/tests/unit/test_vault.py @@ -14,6 +14,8 @@ builds on, independent of any database or app wiring: command in the message. """ +from pathlib import Path + import pytest from cryptography.fernet import InvalidToken @@ -124,12 +126,17 @@ def test_password_hashing_is_key_independent( # --- fail-fast on missing/malformed key ------------------------------------ -def test_missing_key_fails_fast_with_generation_command( - monkeypatch: pytest.MonkeyPatch, +def test_decrypt_without_any_key_fails_fast_with_generation_command( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: + # Reading secrets is strict: with no key in env AND no key file, decrypt + # fails fast with the generation command rather than silently minting a new + # key (a fresh key could never match already-encrypted ciphertext). Only the + # encrypt path auto-provisions; the read path never does. monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(tmp_path / "absent.key")) with pytest.raises(RuntimeError) as exc: - vault.encrypt("x") + vault.decrypt("fernet:v1:not-real-ciphertext") msg = str(exc.value) assert "ROUTSTR_SECRET_KEY" in msg assert "Fernet.generate_key" in msg @@ -139,3 +146,125 @@ def test_malformed_key_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key") with pytest.raises(RuntimeError): vault.encrypt("x") + + +def test_malformed_env_key_does_not_self_provision( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # A malformed env key is an operator mistake, not an unset key: it must fail + # loudly, never silently generate a different key to a file (which would hide + # the mistake and could brick secrets the operator meant to key differently). + monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key") + key_file = tmp_path / "routstr_secret.key" + monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file)) + with pytest.raises(RuntimeError): + vault.encrypt("x") + assert not key_file.exists() + + +# --- auto-provisioned key file (non-breaking upgrade path) ----------------- + + +@pytest.fixture +def generated_key_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """No env key; the key file points at a fresh, empty tmp location. + + Exercises what an existing node hits when it upgrades without setting + ROUTSTR_SECRET_KEY: the master key is auto-generated and persisted here so + boot does not break, while secrets are still never written in plaintext. + """ + monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + key_file = tmp_path / "routstr_secret.key" + monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file)) + return key_file + + +def test_encrypt_without_key_generates_and_persists_key_file( + generated_key_file: Path, +) -> None: + # Encryption at rest stays mandatory, but a missing key is provisioned rather + # than fatal: encrypt generates a key, writes it to the key file (owner-only), + # and the value round-trips — decrypt, still with no env key, reads the same + # file key back. + key_file = generated_key_file + assert not key_file.exists() + + token = vault.encrypt("nsec1secret") + + assert token.startswith("fernet:v1:") + assert key_file.exists() + assert key_file.stat().st_mode & 0o077 == 0 # not group/other-accessible + assert vault.decrypt(token) == "nsec1secret" + + +def test_generated_key_warns_operator_with_path_and_value( + generated_key_file: Path, capsys: pytest.CaptureFixture[str] +) -> None: + # The whole point of auto-generating is that an upgrading operator MUST NOT + # miss it: the notice names the file to back up, prints the key so it can be + # promoted into a secrets manager, and shouts the back-up imperative. + key_file = generated_key_file + vault.encrypt("x") + + out = capsys.readouterr().out + assert "ROUTSTR_SECRET_KEY" in out + assert str(key_file) in out + assert key_file.read_text().strip() in out + assert "BACK UP" in out.upper() + + +def test_existing_key_file_is_reused_and_warns_only_once( + generated_key_file: Path, capsys: pytest.CaptureFixture[str] +) -> None: + # Once the key exists, later encrypts reuse it (never rotate a key that + # secrets were already encrypted under) and stay silent (no repeated notice). + key_file = generated_key_file + vault.encrypt("a") + first_key = key_file.read_text() + capsys.readouterr() # drain the one-time notice + + vault.encrypt("b") + + assert key_file.read_text() == first_key + assert capsys.readouterr().out == "" + + +def test_env_key_takes_precedence_over_key_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # An explicit env key wins over a persisted file key (an operator-supplied + # key from a secrets manager overrides the auto-generated one), and the file + # is left untouched. + key_file = tmp_path / "routstr_secret.key" + key_file.write_text(KEY_B) + monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file)) + monkeypatch.setenv("ROUTSTR_SECRET_KEY", KEY_A) + + token = vault.encrypt("x") + + assert vault.decrypt(token) == "x" # env key (A) decrypts it + monkeypatch.setenv("ROUTSTR_SECRET_KEY", KEY_B) + with pytest.raises(InvalidToken): + vault.decrypt(token) # the file key (B) does not + assert key_file.read_text() == KEY_B # file key never used or overwritten + + +def test_generated_key_defaults_beside_the_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # With no env key and no explicit key-file path, the key is provisioned next + # to the SQLite database, so it rides whatever volume already persists the + # data instead of landing in the working directory (where a container + # recreate would lose it and brick decryption). + monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + monkeypatch.delenv("ROUTSTR_SECRET_KEY_FILE", raising=False) + monkeypatch.chdir(tmp_path) # isolate the working-dir fallback from the repo + db_dir = tmp_path / "data" + db_dir.mkdir() + monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{db_dir}/routstr.db") + + token = vault.encrypt("beside-the-db") + + assert (db_dir / "routstr_secret.key").exists() + assert not (tmp_path / "routstr_secret.key").exists() # not the working dir + assert vault.decrypt(token) == "beside-the-db" From d4657dca5a64e8f87bdd4411f8be747d0eed4e9d Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Mon, 13 Jul 2026 17:30:05 +0200 Subject: [PATCH 17/26] docs: reflect the optional, auto-generated secret key The secret-key docs still said ROUTSTR_SECRET_KEY was required and that the node would not start without it. Update the README, .env.example, and the provider docs (configuration, quickstart, deployment) to the current behaviour: the key is optional; when unset the node generates one beside the database, so it persists on the same volume as the data, and prints a one-time back-it-up notice; set it explicitly to manage the key yourself. Switch the generation and reset snippets to `uv run python`, and add the containerised reset variant. Co-Authored-By: Claude Opus 4.8 --- .env.example | 9 ++++++--- README.md | 14 +++++++++----- docs/provider/configuration.md | 21 +++++++++++++++++++++ docs/provider/deployment.md | 11 +++++++++++ docs/provider/quickstart.md | 4 ++++ 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index e0dbae73..8ff04b35 100644 --- a/.env.example +++ b/.env.example @@ -5,9 +5,12 @@ UPSTREAM_API_KEY=your-upstream-api-key # Tinfoil (confidential inference enclaves, EHBP) # TINFOIL_API_KEY=your-tinfoil-api-key -# Secret key used to encrypt secrets at rest (REQUIRED). The node refuses to -# start without it. Generate one with: -# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +# Secret key used to encrypt node secrets at rest (optional). If unset, the node +# generates one on first start, writes it to routstr_secret.key (override the path +# with ROUTSTR_SECRET_KEY_FILE), and prints it once — back that file up, because +# losing the key makes previously encrypted secrets unreadable. Set it explicitly +# to manage the key yourself (recommended in production). Generate one with: +# uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" ROUTSTR_SECRET_KEY= # The admin password and the Nostr identity (nsec) are NOT set here. The admin diff --git a/README.md b/README.md index 767bacae..023ccd18 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,10 @@ If you are a node runner, start a Routstr Core instance using Docker Compose: 1. **Prepare your `.env`**: ```bash - # Required: encrypts secrets at rest. The node won't start without it. + # Optional: encrypts node secrets at rest. If unset, the node generates a key + # on first start, writes it to routstr_secret.key, and prints it once — back + # up that file. Set it explicitly to manage the key yourself (recommended in + # production). ROUTSTR_SECRET_KEY= NAME="My AI Node" DESCRIPTION="Fast access to models" @@ -63,10 +66,11 @@ If you are a node runner, start a Routstr Core instance using Docker Compose: RECEIVE_LN_ADDRESS=yourname@wallet.com ``` - Generate `ROUTSTR_SECRET_KEY` once and keep it stable — changing it makes - previously encrypted secrets unreadable: + If you don't set one, a key is generated and printed on first start — save it + somewhere safe (losing it makes previously encrypted secrets unreadable). To + supply your own, generate it once and keep it stable: ```bash - python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" + uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" ``` 2. **Start the services**: @@ -80,7 +84,7 @@ If you are a node runner, start a Routstr Core instance using Docker Compose: ```bash docker compose logs routstr | grep -i admin ``` - (Lost it? Reset with `python scripts/reset_admin_password.py --regenerate`.) + (Lost it? Reset with `docker compose exec routstr /.venv/bin/python scripts/reset_admin_password.py --regenerate`.) 4. **Configure**: Open [http://localhost:8000/admin/](http://localhost:8000/admin/) to connect your AI providers and set pricing. diff --git a/docs/provider/configuration.md b/docs/provider/configuration.md index 40930676..f548c990 100644 --- a/docs/provider/configuration.md +++ b/docs/provider/configuration.md @@ -15,6 +15,11 @@ Before running your node, you should create a `.env` file in the project root. T ```bash ADMIN_PASSWORD=your-secure-password +# Encrypts node secrets at rest. Optional — if unset, the node generates a key on +# first start and prints it once (back it up). Set it to manage the key yourself +# (recommended in production). See "Secrets at Rest" below. +ROUTSTR_SECRET_KEY= + # Node Identity NAME="My AI Node" DESCRIPTION="Fast access to models" @@ -124,6 +129,8 @@ Use environment variables for: | `UPSTREAM_BASE_URL` | Upstream API endpoint | — | | `UPSTREAM_API_KEY` | Upstream API key | — | | `ADMIN_PASSWORD` | Dashboard password | (none) | +| `ROUTSTR_SECRET_KEY` | Master key encrypting node secrets at rest. Auto-generated to a key file if unset | (auto-generated) | +| `ROUTSTR_SECRET_KEY_FILE` | Path to the generated key file (used when `ROUTSTR_SECRET_KEY` is unset) | `routstr_secret.key` beside the database | | `DATABASE_URL` | Database connection string | `sqlite+aiosqlite:///keys.db` | | `NAME` | Node display name | `ARoutstrNode` | | `DESCRIPTION` | Node description | `A Routstr Node` | @@ -142,6 +149,20 @@ Use environment variables for: Environment variables are read on startup. Dashboard settings override them and persist in the database. Once you change a setting in the dashboard, the env var is ignored for that setting. +### Secrets at Rest + +The node's Nostr private key (`nsec`) is encrypted in the database using +`ROUTSTR_SECRET_KEY`. You don't have to set it: if it's unset, the node generates a +key on first start, writes it **beside the database** (the file named by +`ROUTSTR_SECRET_KEY_FILE`, default `routstr_secret.key`) so it persists on the same +volume as your data, and prints it once. + +**Back up that key** — it lives on the same volume as your database, so include it +in your backups. If it is lost or changed, previously encrypted secrets can't be +decrypted and must be re-entered — there is no rotation. To keep the key off the +data volume, set `ROUTSTR_SECRET_KEY` explicitly (an env value always takes +precedence over the file). See also [Deployment](deployment.md). + --- ## Models diff --git a/docs/provider/deployment.md b/docs/provider/deployment.md index 5a3a2899..24c0591a 100644 --- a/docs/provider/deployment.md +++ b/docs/provider/deployment.md @@ -156,10 +156,21 @@ Example `.env`: UPSTREAM_BASE_URL=https://api.openai.com/v1 UPSTREAM_API_KEY=sk-proj-... ADMIN_PASSWORD=change-me +# Encrypts node secrets at rest. Optional — if unset, a key is generated next to +# your database (on the same volume) and printed once. Set it explicitly to +# manage the key yourself. +ROUTSTR_SECRET_KEY= NAME=My Provider Node RECEIVE_LN_ADDRESS=me@walletofsatoshi.com ``` +!!! note "Secret key persistence" + If you leave `ROUTSTR_SECRET_KEY` unset, the node generates one and stores it + as `routstr_secret.key` **next to your database**, so it persists on the same + volume as your data — just include that volume in your backups. For stronger + isolation (keeping the key off the data volume), set `ROUTSTR_SECRET_KEY` from + a secrets manager instead. + See [Configuration](configuration.md) for all available options. --- diff --git a/docs/provider/quickstart.md b/docs/provider/quickstart.md index 5b33c359..ef959f40 100644 --- a/docs/provider/quickstart.md +++ b/docs/provider/quickstart.md @@ -32,6 +32,10 @@ Create a `.env` file in the root of the project to store your secrets: # Initial Admin Password ADMIN_PASSWORD=mysecretpassword +# Encrypts node secrets at rest. Optional — if unset, the node generates a key on +# first start and prints it once (back it up). +ROUTSTR_SECRET_KEY= + # Node Identity NAME="My AI Node" DESCRIPTION="Fast access to models" From c2a2d76eae730b7af014f3fe2284566bea14ed5b Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 15 Jul 2026 14:13:07 +0200 Subject: [PATCH 18/26] fix(vault): harden the auto-generated master key file Three fixes to how a node provisions its own master key when the operator sets no ROUTSTR_SECRET_KEY: - Publish the key atomically. It is written to a same-directory temp file, fsynced, then os.link-ed into place and the directory fsynced. os.link publishes the complete file in one step, so a crash mid-write can no longer strand an empty key at the final path that a later boot would read as corrupt and then fail to decrypt every secret under. os.link also refuses to clobber, so a racing worker that generated first keeps ownership and the loser adopts its key. - Tighten loose permissions on read. A key file that is group/other-readable is repaired to 0600 rather than trusted, keeping an upgrading node booting. - Stop printing the key value. The one-time notice names the file to back up and shouts the backup imperative, but no longer echoes the key itself, which would leak it into captured stdout / aggregated container logs; the durable 0600 file is the recovery path. Co-Authored-By: Claude Opus 4.8 --- routstr/core/vault.py | 88 ++++++++++++++++++---- tests/integration/test_secret_bootstrap.py | 5 +- tests/unit/test_vault.py | 79 +++++++++++++++++-- 3 files changed, 149 insertions(+), 23 deletions(-) diff --git a/routstr/core/vault.py b/routstr/core/vault.py index 674d74c2..183b923b 100644 --- a/routstr/core/vault.py +++ b/routstr/core/vault.py @@ -27,6 +27,7 @@ import hashlib import hmac import os import secrets +import tempfile from pathlib import Path from cryptography.fernet import Fernet @@ -99,7 +100,27 @@ def _read_key_file(path: Path) -> str | None: stored = path.read_text().strip() except OSError: return None - return stored or None + if not stored: + return None + _repair_key_file_perms(path) + return stored + + +def _repair_key_file_perms(path: Path) -> None: + # A master key must never be group/other-readable. On POSIX, tighten loose + # permissions to owner-only (0600) rather than trust — or hard-fail on — a + # world-readable key; a friendlier repair keeps an upgrading node booting. + if os.name != "posix": + return + try: + mode = path.stat().st_mode + except OSError: + return + if mode & 0o077: + try: + os.chmod(path, 0o600) + except OSError: + pass def _load_secret_key() -> str | None: @@ -107,7 +128,7 @@ def _load_secret_key() -> str | None: return os.environ.get("ROUTSTR_SECRET_KEY") or _read_key_file(_key_file_path()) -def _warn_generated_key(path: Path, key: str) -> None: +def _warn_generated_key(path: Path) -> None: # stdout, not the logger: the operator must see this once (e.g. in # ``docker compose logs``), but it must never be persisted into the on-disk # log files the logger also writes. Mirrors the generated-admin-password @@ -117,31 +138,66 @@ def _warn_generated_key(path: Path, key: str) -> None: f"rest and saved it to {path}.\n" "!! BACK UP THIS FILE. If it is lost, the encrypted secrets cannot be " "recovered and will have to be re-entered.\n" - "To manage the key yourself (e.g. from a secrets manager) set it in the " - f"environment instead:\n ROUTSTR_SECRET_KEY={key}", + "To manage the key yourself (e.g. from a secrets manager) set " + "ROUTSTR_SECRET_KEY in the environment instead; the value is in the file " + "above.", flush=True, ) def _generate_and_persist_key(path: Path) -> str: - """Generate a Fernet key, persist it owner-only, and warn once.""" + """Generate a Fernet key, persist it owner-only and atomically, warn once. + + The key is written to a temp file in the same directory, flushed durably, + then ``os.link``-ed into place. ``os.link`` publishes the complete file in a + single atomic step — a crash mid-write leaves only the temp file (which is + removed), never a half-written or empty key at the final path that a later + boot would read as corrupt. It also refuses to overwrite an existing key, so + a racing worker that generated first keeps ownership (secrets may already be + encrypted under its key); the loser adopts that key instead of clobbering it. + """ key = Fernet.generate_key().decode() path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + dir=path.parent, prefix=".routstr_secret.", suffix=".tmp" + ) + tmp = Path(tmp_name) try: - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - except FileExistsError: - # Another worker won the race and wrote the key first; adopt theirs - # rather than clobber a key that secrets may already be encrypted under. - existing = _read_key_file(path) - if existing: - return existing - raise - with os.fdopen(fd, "w") as handle: - handle.write(key) - _warn_generated_key(path, key) + with os.fdopen(fd, "w") as handle: + handle.write(key) # mkstemp already created it 0600 + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(tmp, path) + except FileExistsError: + # A concurrent worker linked its key in first; adopt theirs rather + # than clobber a key that secrets may already be encrypted under. + existing = _read_key_file(path) + if existing: + return existing + raise + _fsync_dir(path.parent) + finally: + tmp.unlink(missing_ok=True) + _warn_generated_key(path) return key +def _fsync_dir(directory: Path) -> None: + # Persist the new directory entry so the linked key survives a crash right + # after publish. Best-effort: not every platform lets you fsync a directory. + try: + dir_fd = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + os.fsync(dir_fd) + except OSError: + pass + finally: + os.close(dir_fd) + + def ensure_secret_key() -> str: """Return the master key, provisioning one if the operator supplied none. diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 4a177ee2..9147ce1e 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -202,10 +202,11 @@ async def test_legacy_nsec_without_secret_key_generates_and_encrypts( assert settings.nsec == NSEC_HEX assert settings.npub == derive_npub_from_nsec(NSEC_HEX) # ...and the operator is loudly told a key was generated and must be backed up - # (path + value shown) so an upgrade cannot silently create an unbacked key. + # (path shown, but never the key value) so an upgrade cannot silently create + # an unbacked key nor leak the key into captured stdout / aggregated logs. out = capsys.readouterr().out assert str(key_file) in out - assert key_file.read_text().strip() in out + assert key_file.read_text().strip() not in out assert "BACK UP" in out.upper() diff --git a/tests/unit/test_vault.py b/tests/unit/test_vault.py index 52f1d9fd..2defd40a 100644 --- a/tests/unit/test_vault.py +++ b/tests/unit/test_vault.py @@ -197,19 +197,20 @@ def test_encrypt_without_key_generates_and_persists_key_file( assert vault.decrypt(token) == "nsec1secret" -def test_generated_key_warns_operator_with_path_and_value( +def test_generated_key_warns_operator_with_path_not_value( generated_key_file: Path, capsys: pytest.CaptureFixture[str] ) -> None: - # The whole point of auto-generating is that an upgrading operator MUST NOT - # miss it: the notice names the file to back up, prints the key so it can be - # promoted into a secrets manager, and shouts the back-up imperative. + # The notice names the file to back up and shouts the back-up imperative so an + # upgrading operator cannot miss it, but it MUST NOT echo the key value: the + # secret lives in the 0600 file, and printing it would leak it into captured + # stdout / aggregated container logs. key_file = generated_key_file vault.encrypt("x") out = capsys.readouterr().out assert "ROUTSTR_SECRET_KEY" in out assert str(key_file) in out - assert key_file.read_text().strip() in out + assert key_file.read_text().strip() not in out assert "BACK UP" in out.upper() @@ -229,6 +230,74 @@ def test_existing_key_file_is_reused_and_warns_only_once( assert capsys.readouterr().out == "" +def test_generated_key_is_published_atomically( + generated_key_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The key must appear at its final path only as a complete file: it is written + # to a temp file and atomically linked into place. If the publish (link) step + # fails — e.g. the process crashes — the destination must be absent, never a + # half-written or empty file that a later boot would read as a corrupt key and + # then refuse to decrypt every secret. No temp debris is left behind. + key_file = generated_key_file + + def boom(src: object, dst: object) -> None: + raise OSError("crash during atomic publish") + + monkeypatch.setattr(vault.os, "link", boom) + + with pytest.raises(OSError): + vault.encrypt("x") + + assert not key_file.exists() + assert list(key_file.parent.iterdir()) == [] + + +def test_racing_worker_adopts_winners_key_without_clobber( + generated_key_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Two workers auto-generate a key concurrently. The first to link "wins" and + # its key is the one on disk; a worker that loses the link race must adopt the + # winner's key — secrets may already be encrypted under it — rather than + # clobber it or crash. Force the race deterministically: the winner publishes + # its key at the final path just as this worker tries to link, so os.link + # raises FileExistsError and the loser reads the winner's key back. + key_file = generated_key_file + + def winner_links_first(src: object, dst: object) -> None: + key_file.write_text(KEY_A) # the winner's already-published key + raise FileExistsError + + monkeypatch.setattr(vault.os, "link", winner_links_first) + + token = vault.encrypt("secret") + + # The winner's key stays put and the loser encrypted under it, so the value + # round-trips under KEY_A even though this worker had generated its own key. + assert key_file.read_text().strip() == KEY_A + assert vault.decrypt(token) == "secret" + # The losing worker left no temp debris behind. + assert [p.name for p in key_file.parent.iterdir()] == [key_file.name] + + +def test_loose_key_file_perms_are_tightened_on_read( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # A key file left group/other-readable (e.g. written under a loose umask + # before this hardening, or by a careless operator) is a leaked-secret risk. + # Reading it repairs the permissions to owner-only rather than trusting a + # world-readable master key, while still using the key so boot is not broken. + monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False) + key_file = tmp_path / "routstr_secret.key" + key_file.write_text(KEY_A) + key_file.chmod(0o644) + monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file)) + + token = vault.encrypt("secret") # reads the loose file, repairs its perms + + assert key_file.stat().st_mode & 0o077 == 0 + assert vault.decrypt(token) == "secret" + + def test_env_key_takes_precedence_over_key_file( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From eb6a612e5af93dbe8248cf13e599ab508f83e42f Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 15 Jul 2026 14:13:23 +0200 Subject: [PATCH 19/26] fix(secrets): make nsec ownership explicit and claim the admin password atomically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bootstrap-correctness fixes on the encrypted Secret store: - Track nsec ownership with an explicit nsec_state (legacy | encrypted | cleared) instead of a nsec_managed bool. The bool could not tell "never migrated" apart from "intentionally cleared" — both leave encrypted_nsec empty — so a cleared identity could be resurrected on a fresh process from a stale legacy NSEC (env or old settings blob) and re-derive its npub. Bootstrap now branches purely on the state: encrypted decrypts (a missing ciphertext is a fail-fast inconsistency, never a silent fall-through to legacy), cleared actively empties the live nsec and npub, and legacy imports the plaintext once. - Claim a generated admin password atomically. When no password exists, the generated one is written via a conditional UPDATE (WHERE admin_password_hash IS NULL) and only the worker that wins the update (rowcount 1) prints it. A racing worker on a shared database adopts the winner's hash and stays silent, so the operator never sees a second password that was never stored. Co-Authored-By: Claude Opus 4.8 --- .../c6f8d2e4a1b3_add_secrets_table.py | 13 +-- routstr/core/db.py | 35 ++++++-- routstr/core/settings.py | 84 +++++++++++++------ tests/integration/test_secret_bootstrap.py | 75 +++++++++++++++-- 4 files changed, 159 insertions(+), 48 deletions(-) diff --git a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py index 77a5a8d9..73639fc0 100644 --- a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py +++ b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py @@ -6,9 +6,10 @@ Create Date: 2026-07-07 00:00:00.000000 Creates the node-level singleton secret store (issue #553). Schema only; moving any legacy plaintext into the encrypted/hashed columns happens at bootstrap, -where the live ROUTSTR_SECRET_KEY is available. ``nsec_managed`` records that the -vault has taken ownership of the nsec, so a cleared identity is never resurrected -from a stale legacy ``NSEC`` env var / settings blob on the next boot. +where the live ROUTSTR_SECRET_KEY is available. ``nsec_state`` records the vault's +ownership of the nsec (legacy | encrypted | cleared), so a cleared identity is +never resurrected from a stale legacy ``NSEC`` env var / settings blob on the next +boot. """ import sqlalchemy as sa @@ -36,10 +37,10 @@ def upgrade() -> None: nullable=True, ), sa.Column( - "nsec_managed", - sa.Boolean(), + "nsec_state", + sqlmodel.sql.sqltypes.AutoString(), nullable=False, - server_default=sa.false(), + server_default="legacy", ), sa.Column("updated_at", sa.Integer(), nullable=True), sa.PrimaryKeyConstraint("id"), diff --git a/routstr/core/db.py b/routstr/core/db.py index 5fe29858..b81cb56d 100644 --- a/routstr/core/db.py +++ b/routstr/core/db.py @@ -6,6 +6,7 @@ import sqlite3 import time import uuid from contextlib import asynccontextmanager +from enum import Enum from typing import AsyncGenerator from alembic import command @@ -443,6 +444,26 @@ class RoutstrFee(SQLModel, table=True): # type: ignore payout_started_at: int | None = Field(default=None) +class NsecState(str, Enum): + """Ownership state of the node's nsec — an explicit 3-state machine. + + The single ``encrypted_nsec`` column cannot distinguish "never migrated" from + "intentionally cleared" (both leave it empty), which let a cleared identity be + resurrected from a stale legacy ``NSEC``. This names the three states so the + bootstrap branches on ownership rather than inferring it: + + * ``legacy`` — the vault has not taken ownership; a plaintext ``NSEC`` (env or + old settings blob) may still exist and should be migrated in once. + * ``encrypted`` — the vault owns a ciphertext; decrypt it, never re-read env. + * ``cleared`` — the vault owns it but the operator emptied it; stay empty, + never re-import from a stale legacy copy. + """ + + legacy = "legacy" + encrypted = "encrypted" + cleared = "cleared" + + class Secret(SQLModel, table=True): # type: ignore """Node-level secrets, stored encrypted/hashed at rest (singleton, id=1). @@ -455,11 +476,7 @@ class Secret(SQLModel, table=True): # type: ignore id: int = Field(default=1, primary_key=True) admin_password_hash: str | None = Field(default=None) encrypted_nsec: str | None = Field(default=None) - # True once the vault owns the nsec (imported from legacy plaintext, or set - # via the admin API). A cleared nsec then stays cleared: bootstrap must not - # resurrect it from a stale legacy ``NSEC`` env var / settings blob, which an - # empty ``encrypted_nsec`` alone cannot distinguish from "never migrated". - nsec_managed: bool = Field(default=False) + nsec_state: NsecState = Field(default=NsecState.legacy) updated_at: int | None = Field(default=None) @@ -536,15 +553,15 @@ async def set_nsec(session: AsyncSession, nsec: str) -> None: """Store the node's nsec, Fernet-encrypted, on the Secret singleton. An empty string clears it (the node then holds no Nostr identity and signs - no events). Either way the vault now owns the nsec, so ``nsec_managed`` is - set: a cleared identity must not be resurrected from a stale legacy ``NSEC`` - on the next boot. + no events). Either way the vault now owns the nsec, so the state moves off + ``legacy``: a cleared identity (``cleared``) must not be resurrected from a + stale legacy ``NSEC`` on the next boot. """ from .vault import encrypt secret = await get_secret(session) secret.encrypted_nsec = encrypt(nsec) if nsec else None - secret.nsec_managed = True + secret.nsec_state = NsecState.encrypted if nsec else NsecState.cleared secret.updated_at = int(time.time()) session.add(secret) await session.commit() diff --git a/routstr/core/settings.py b/routstr/core/settings.py index 133496f6..a0b5d05c 100644 --- a/routstr/core/settings.py +++ b/routstr/core/settings.py @@ -467,9 +467,10 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: hash it, and log it once with the /admin URL. """ from cryptography.fernet import InvalidToken + from sqlmodel import col, update from . import vault - from .db import get_secret + from .db import NsecState, Secret, get_secret raw_blob = await _read_raw_settings_blob(db_session) secret = await get_secret(db_session) @@ -482,23 +483,52 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: ) if legacy_password: secret.admin_password_hash = vault.hash_password(legacy_password) + changed = True else: generated = secrets.token_urlsafe(24) - secret.admin_password_hash = vault.hash_password(generated) - admin_url = (settings.http_url or "http://localhost:8000").rstrip("/") - # Print to stdout rather than the logger: the operator must see this - # once (e.g. `docker compose logs`), but it must not be persisted - # into the on-disk log files the logger also writes to. - print( - "No admin password set; generated a temporary one (shown only " - f"now): {generated}\nLog in at {admin_url}/admin and change it " - "from the dashboard settings.", - flush=True, + # Claim the empty slot atomically: only the worker whose UPDATE flips + # NULL -> hash owns the generated password and announces it. On a + # shared DB a racing worker gets rowcount 0, so it neither clobbers + # the winner's hash (which the operator may already be using) nor + # prints a second password that would never work. + claim_stmt = ( + update(Secret) + .where(col(Secret.id) == 1) + .where(col(Secret.admin_password_hash).is_(None)) + .values( + admin_password_hash=vault.hash_password(generated), + updated_at=int(time.time()), + ) ) - changed = True + result = await db_session.exec(claim_stmt) # type: ignore[call-overload] + await db_session.commit() + await db_session.refresh(secret) + if result.rowcount == 1: + admin_url = (settings.http_url or "http://localhost:8000").rstrip("/") + # Print to stdout rather than the logger: the operator must see + # this once (e.g. `docker compose logs`), but it must not be + # persisted into the on-disk log files the logger also writes to. + print( + "No admin password set; generated a temporary one (shown " + f"only now): {generated}\nLog in at {admin_url}/admin and " + "change it from the dashboard settings.", + flush=True, + ) - # Nostr nsec — reversible Fernet encryption. - if secret.encrypted_nsec is not None: + # Nostr nsec — reversible Fernet encryption. ``nsec_state`` is the single + # source of truth for ownership, so "intentionally cleared" is never + # conflated with "never migrated" (the bug the old bool could not encode). + if secret.nsec_state == NsecState.encrypted: + # The vault owns the identity: decrypt the ciphertext, never re-read + # env/blob. A missing ciphertext here means the row is inconsistent (a + # failed write or manual edit); fail fast rather than silently dropping + # the identity and falling back to a stale legacy copy. + if secret.encrypted_nsec is None: + raise RuntimeError( + "nsec_state is 'encrypted' but no ciphertext is stored; the " + "secrets row is inconsistent. Refusing to boot rather than " + "silently resurrecting a stale legacy NSEC." + ) try: settings.nsec = vault.decrypt(secret.encrypted_nsec) except InvalidToken as exc: @@ -507,21 +537,23 @@ async def bootstrap_secrets(db_session: AsyncSession) -> None: "ROUTSTR_SECRET_KEY. The key changed, or this database came from " "another node. Restore the original ROUTSTR_SECRET_KEY to recover." ) from exc - elif not secret.nsec_managed: - # The vault has not taken ownership yet: import any legacy plaintext - # (env, or the old settings blob). Once managed, an empty encrypted_nsec - # means the identity was intentionally cleared via the admin API, so this - # branch is skipped and the nsec stays empty rather than being resurrected - # from a stale legacy copy. + elif secret.nsec_state == NsecState.cleared: + # The operator emptied the identity via the admin API. A fresh process + # has already reloaded a stale ``NSEC`` from env/blob into the live + # settings (and may have derived its npub); actively clear both so the + # cleared store wins rather than silently resurrecting the old identity. + settings.nsec = "" + settings.npub = "" + else: # NsecState.legacy — the vault has not taken ownership yet. + # Import any legacy plaintext (env, or the old settings blob) exactly + # once. Encryption at rest is mandatory, but a missing key is + # provisioned, not fatal: vault.encrypt generates and persists a master + # key (with a loud one-time operator notice) when none was supplied, so + # an upgrading node keeps running. The nsec is never stored in plaintext. legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec") if legacy_nsec: - # The node has a Nostr identity to protect. Encryption at rest is - # mandatory, but a missing key is provisioned, not fatal: - # vault.encrypt generates and persists a master key (with a loud - # one-time operator notice) when none was supplied, so an upgrading - # node keeps running. The nsec is never persisted in plaintext. secret.encrypted_nsec = vault.encrypt(legacy_nsec) - secret.nsec_managed = True + secret.nsec_state = NsecState.encrypted settings.nsec = legacy_nsec changed = True diff --git a/tests/integration/test_secret_bootstrap.py b/tests/integration/test_secret_bootstrap.py index 9147ce1e..12f0d9a1 100644 --- a/tests/integration/test_secret_bootstrap.py +++ b/tests/integration/test_secret_bootstrap.py @@ -18,7 +18,7 @@ from sqlmodel import text from sqlmodel.ext.asyncio.session import AsyncSession from routstr.core import vault -from routstr.core.db import get_secret, set_nsec +from routstr.core.db import NsecState, get_secret, set_nsec from routstr.core.settings import ( SettingsService, bootstrap_secrets, @@ -111,6 +111,57 @@ async def test_hashes_legacy_admin_password_from_blob( assert vault.verify_password("blobpw", secret.admin_password_hash or "") is True +@pytest.mark.asyncio +async def test_admin_password_race_adopts_winner_without_clobber( + clean_secret_env: None, + integration_engine: Any, + integration_session: AsyncSession, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + # Two workers boot against one shared DB and both read a null admin password. + # The first to commit "wins" and shows the operator its generated password. A + # worker that read null but lost the race must NOT overwrite the winner's hash + # (which the operator may already be logging in with) and must NOT print a + # second password that will never work. + # + # The race window is forced deterministically: a hook fires inside bootstrap's + # generate branch (so it only runs once this worker has committed to + # generating) and commits the winner's password on a separate connection + # before this worker writes its own. + import sqlite3 + + from routstr.core import settings as settings_mod + + db_file = integration_engine.url.database + winner_hash = vault.hash_password("winner-password-123") + real_token = settings_mod.secrets.token_urlsafe + + def commit_winner_then_generate(nbytes: int) -> str: + conn = sqlite3.connect(db_file) + conn.execute( + "UPDATE secrets SET admin_password_hash = ? WHERE id = 1", (winner_hash,) + ) + conn.commit() + conn.close() + return real_token(nbytes) + + monkeypatch.setattr( + settings_mod.secrets, "token_urlsafe", commit_winner_then_generate + ) + + await get_secret(integration_session) # row exists, password still null + capsys.readouterr() # drop anything emitted before the race resolves + await bootstrap_secrets(integration_session) + + secret = await get_secret(integration_session) + assert secret.admin_password_hash is not None + # The winner's password survives and still verifies — no clobber. + assert vault.verify_password("winner-password-123", secret.admin_password_hash) + # The losing worker stayed silent — no second generated password was leaked. + assert "generated a temporary" not in capsys.readouterr().out + + # --- nsec ------------------------------------------------------------------ @@ -137,6 +188,7 @@ async def test_decrypts_existing_nsec_column( ) -> None: secret = await get_secret(integration_session) secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + secret.nsec_state = NsecState.encrypted integration_session.add(secret) await integration_session.commit() stored = secret.encrypted_nsec @@ -159,6 +211,7 @@ async def test_fail_fast_when_nsec_encrypted_with_different_key( monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY_ALT) secret = await get_secret(integration_session) secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + secret.nsec_state = NsecState.encrypted integration_session.add(secret) await integration_session.commit() @@ -197,7 +250,7 @@ async def test_legacy_nsec_without_secret_key_generates_and_encrypts( assert secret.encrypted_nsec is not None assert vault.is_encrypted(secret.encrypted_nsec) is True assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX - assert secret.nsec_managed is True + assert secret.nsec_state == NsecState.encrypted # ...the node holds the live identity (npub derived from it)... assert settings.nsec == NSEC_HEX assert settings.npub == derive_npub_from_nsec(NSEC_HEX) @@ -253,6 +306,7 @@ async def test_initialize_does_not_clobber_store_only_nsec( await _create_settings_blob(integration_session, {"name": "LegacyNode"}) secret = await get_secret(integration_session) secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + secret.nsec_state = NsecState.encrypted integration_session.add(secret) await integration_session.commit() @@ -325,23 +379,29 @@ async def test_cleared_nsec_stays_cleared_across_reboot( monkeypatch: pytest.MonkeyPatch, ) -> None: # An identity was imported from env, then the operator cleared it via the - # admin API. The old NSEC is still in env. On the next boot the cleared + # admin API. The old NSEC is still in env. On the NEXT PROCESS the cleared # identity must stay cleared, not get resurrected from the stale env value. monkeypatch.setenv("NSEC", NSEC_HEX) await bootstrap_secrets(integration_session) assert settings.nsec == NSEC_HEX - # Clear via the admin path (mirrors the endpoint: store empty, live empty). + # Clear via the admin path (store empty, vault owns it). await set_nsec(integration_session, "") - monkeypatch.setattr(settings, "nsec", "") - # Reboot with the stale NSEC still present in env. + # Simulate a fresh process rather than pre-clearing the live singleton: the + # pydantic settings global reloads the (still-stale) NSEC from env and derives + # its npub, which is exactly the in-memory state a new boot starts from before + # bootstrap runs. The cleared store must win over this stale live value. + monkeypatch.setattr(settings, "nsec", NSEC_HEX) + monkeypatch.setattr(settings, "npub", derive_npub_from_nsec(NSEC_HEX)) + await bootstrap_secrets(integration_session) reloaded = await get_secret(integration_session) - assert reloaded.nsec_managed is True + assert reloaded.nsec_state == NsecState.cleared assert reloaded.encrypted_nsec is None # not re-imported assert settings.nsec == "" # stays cleared + assert settings.npub == "" # and no derived public identity survives @pytest.mark.asyncio @@ -360,6 +420,7 @@ async def test_initialize_keeps_npub_matching_store_only_nsec( await _create_settings_blob(integration_session, {"name": "LegacyNode"}) secret = await get_secret(integration_session) secret.encrypted_nsec = vault.encrypt(NSEC_HEX) + secret.nsec_state = NsecState.encrypted integration_session.add(secret) await integration_session.commit() From f47e16aa61e98f78c712a17efe45b08c6cd503c3 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 15 Jul 2026 14:13:35 +0200 Subject: [PATCH 20/26] refactor(admin): remove the unreachable /api/setup endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated-password bootstrap now sets an admin password on every fresh node, so the /admin/api/setup success path is unreachable — it 409s ("already set") on any booted node. No caller exists across the admin UI, routstr-cli, routstr-sdk, routstrd, or routstr-chat, so the endpoint (and its SetupRequest model) are dead surface. First-run is now: the generated password is printed once at boot, log in via /admin, and change it from the dashboard. Co-Authored-By: Claude Opus 4.8 --- routstr/core/admin.py | 23 ------------ tests/integration/test_admin_auth.py | 56 +++++++++------------------- 2 files changed, 18 insertions(+), 61 deletions(-) diff --git a/routstr/core/admin.py b/routstr/core/admin.py index 02b81611..66a1d288 100644 --- a/routstr/core/admin.py +++ b/routstr/core/admin.py @@ -310,29 +310,6 @@ async def update_nsec(request: Request, payload: NsecUpdate) -> dict[str, object return {"ok": True, "npub": npub} -class SetupRequest(BaseModel): - password: str - - -@admin_router.post("/api/setup") -async def initial_setup(request: Request, payload: SetupRequest) -> dict[str, object]: - async with create_session() as session: - secret = await get_secret(session) - if secret.admin_password_hash: - raise HTTPException(status_code=409, detail="Admin password already set") - pw = (payload.password or "").strip() - if len(pw) < vault.MIN_PASSWORD_LENGTH: - raise HTTPException( - status_code=400, - detail=( - "Password must be at least " - f"{vault.MIN_PASSWORD_LENGTH} characters" - ), - ) - await set_admin_password(session, pw) - return {"ok": True} - - class AdminLoginRequest(BaseModel): password: str diff --git a/tests/integration/test_admin_auth.py b/tests/integration/test_admin_auth.py index e7743d45..6f2b912a 100644 --- a/tests/integration/test_admin_auth.py +++ b/tests/integration/test_admin_auth.py @@ -1,11 +1,11 @@ """Tests for admin password auth backed by the hashed Secret store (issue #553). -Login, password change and first-run setup verify against the one-way -``Secret.admin_password_hash`` (scrypt) instead of a plaintext settings field, -which also closes the old ``!=`` timing-attack comparison. The flow is exercised -end-to-end through the public admin endpoints: setup writes the first hash, -login checks against it, and a password change re-hashes so the old password -stops working and the new one starts. +Login and password change verify against the one-way ``Secret.admin_password_hash`` +(scrypt) instead of a plaintext settings field, which also closes the old ``!=`` +timing-attack comparison. The first hash is written by ``bootstrap_secrets`` at +startup (generated, or migrated from a legacy password); here it is seeded +directly into the store, then login checks against it and a password change +re-hashes so the old password stops working and the new one starts. """ from __future__ import annotations @@ -13,10 +13,14 @@ from __future__ import annotations import pytest from httpx import AsyncClient, Response +from routstr.core.db import create_session, set_admin_password -async def _setup_password(client: AsyncClient, password: str) -> None: - resp = await client.post("/admin/api/setup", json={"password": password}) - assert resp.status_code == 200, resp.text + +async def _seed_password(password: str) -> None: + # bootstrap_secrets owns first-password creation at startup; tests seed the + # hash straight into the store the same way, then exercise login/change. + async with create_session() as session: + await set_admin_password(session, password) async def _login(client: AsyncClient, password: str) -> Response: @@ -40,7 +44,7 @@ async def test_login_500_when_no_password_configured( async def test_login_succeeds_with_correct_password( integration_client: AsyncClient, ) -> None: - await _setup_password(integration_client, "correct horse") + await _seed_password("correct horse") resp = await _login(integration_client, "correct horse") assert resp.status_code == 200 body = resp.json() @@ -53,35 +57,11 @@ async def test_login_succeeds_with_correct_password( async def test_login_rejects_wrong_password( integration_client: AsyncClient, ) -> None: - await _setup_password(integration_client, "correct horse") + await _seed_password("correct horse") resp = await _login(integration_client, "wrong horse") assert resp.status_code == 401 -# --- first-run setup ------------------------------------------------------- - - -@pytest.mark.integration -@pytest.mark.asyncio -async def test_setup_rejects_short_password( - integration_client: AsyncClient, -) -> None: - resp = await integration_client.post("/admin/api/setup", json={"password": "short"}) - assert resp.status_code == 400 - - -@pytest.mark.integration -@pytest.mark.asyncio -async def test_setup_409_when_password_already_set( - integration_client: AsyncClient, -) -> None: - await _setup_password(integration_client, "first password") - resp = await integration_client.post( - "/admin/api/setup", json={"password": "second password"} - ) - assert resp.status_code == 409 - - # --- password change ------------------------------------------------------- @@ -90,7 +70,7 @@ async def test_setup_409_when_password_already_set( async def test_update_password_rehashes_so_only_new_works( integration_client: AsyncClient, ) -> None: - await _setup_password(integration_client, "old password") + await _seed_password("old password") login = await _login(integration_client, "old password") token = login.json()["token"] integration_client.headers["Authorization"] = f"Bearer {token}" @@ -112,7 +92,7 @@ async def test_update_password_rehashes_so_only_new_works( async def test_update_password_rejects_wrong_current( integration_client: AsyncClient, ) -> None: - await _setup_password(integration_client, "old password") + await _seed_password("old password") login = await _login(integration_client, "old password") token = login.json()["token"] integration_client.headers["Authorization"] = f"Bearer {token}" @@ -129,7 +109,7 @@ async def test_update_password_rejects_wrong_current( async def test_update_password_rejects_short_new( integration_client: AsyncClient, ) -> None: - await _setup_password(integration_client, "old password") + await _seed_password("old password") login = await _login(integration_client, "old password") token = login.json()["token"] integration_client.headers["Authorization"] = f"Bearer {token}" From 030d8b61ce309a6720cee0cbcdff0866790c5c53 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 15 Jul 2026 14:13:35 +0200 Subject: [PATCH 21/26] docs(deployment): keep the database and key file on the mounted volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several compose/.env examples mounted /app/data but left DATABASE_URL at the relative default, so the database — and the master key file generated beside it — landed off the persisted volume and would be lost on a container recreate. Point DATABASE_URL inside /app/data in those examples and list routstr_secret.key in the persistence table. Co-Authored-By: Claude Opus 4.8 --- docs/provider/deployment.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/provider/deployment.md b/docs/provider/deployment.md index 24c0591a..3a6fcce3 100644 --- a/docs/provider/deployment.md +++ b/docs/provider/deployment.md @@ -87,6 +87,8 @@ services: - ./logs:/app/logs environment: - TOR_PROXY_URL=socks5://tor:9050 + # Keep the database (and the key file generated beside it) on the volume. + - DATABASE_URL=sqlite:////app/data/routstr.db depends_on: - tor @@ -134,6 +136,9 @@ services: # Lightning withdrawals - RECEIVE_LN_ADDRESS=me@walletofsatoshi.com + + # Keep the database (and the key file generated beside it) on the volume. + - DATABASE_URL=sqlite:////app/data/routstr.db volumes: - ./data:/app/data ``` @@ -156,9 +161,11 @@ Example `.env`: UPSTREAM_BASE_URL=https://api.openai.com/v1 UPSTREAM_API_KEY=sk-proj-... ADMIN_PASSWORD=change-me +# Keep the database (and the key file generated beside it) on the mounted volume. +DATABASE_URL=sqlite:////app/data/routstr.db # Encrypts node secrets at rest. Optional — if unset, a key is generated next to -# your database (on the same volume) and printed once. Set it explicitly to -# manage the key yourself. +# your database (on the same volume) and its file is named once for backup. Set +# it explicitly to manage the key yourself. ROUTSTR_SECRET_KEY= NAME=My Provider Node RECEIVE_LN_ADDRESS=me@walletofsatoshi.com @@ -177,11 +184,13 @@ See [Configuration](configuration.md) for all available options. ## Persistence -Routstr stores all data in `/app/data`: +Point `DATABASE_URL` inside `/app/data` (as the examples above do) so everything +Routstr persists lands on the mounted volume: | Path | Contents | |------|----------| -| `keys.db` | SQLite database (settings, API keys, sessions) | +| `routstr.db` | SQLite database (settings, API keys, sessions) | +| `routstr_secret.key` | Auto-generated master key, written beside the database when `ROUTSTR_SECRET_KEY` is unset | | `.wallet/` | Cashu wallet data (your Bitcoin!) | !!! warning "Back Up Your Data" From 52742a6a045a9a05f3ca81a34be9523b616e7b25 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 22 Jul 2026 16:34:50 +0200 Subject: [PATCH 22/26] docs: align onboarding with UI-managed secrets The admin password is generated and logged on first start and the nsec is set from the admin UI; ADMIN_PASSWORD/NSEC in .env are only a legacy seed. Update the README, quickstart, configuration, and deployment docs to match, and drop the unused ADMIN_KEY environment variable. --- README.md | 5 ++++- docs/provider/configuration.md | 12 +++++------- docs/provider/deployment.md | 6 ++---- docs/provider/quickstart.md | 12 +++++++----- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 023ccd18..0636c299 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,13 @@ If you are a node runner, start a Routstr Core instance using Docker Compose: ROUTSTR_SECRET_KEY= NAME="My AI Node" DESCRIPTION="Fast access to models" - NSEC=yournsec RECEIVE_LN_ADDRESS=yourname@wallet.com ``` + Your Nostr identity (`nsec`) is not set in `.env` — configure it from the admin + UI after first start, where it's stored encrypted in the database. (`NSEC` in + `.env` is still read once as a legacy seed for existing deployments.) + If you don't set one, a key is generated and printed on first start — save it somewhere safe (losing it makes previously encrypted secrets unreadable). To supply your own, generate it once and keep it stable: diff --git a/docs/provider/configuration.md b/docs/provider/configuration.md index f548c990..33efabb8 100644 --- a/docs/provider/configuration.md +++ b/docs/provider/configuration.md @@ -13,8 +13,6 @@ Before running your node, you should create a `.env` file in the project root. T ### Example .env ```bash -ADMIN_PASSWORD=your-secure-password - # Encrypts node secrets at rest. Optional — if unset, the node generates a key on # first start and prints it once (back it up). Set it to manage the key yourself # (recommended in production). See "Secrets at Rest" below. @@ -30,10 +28,10 @@ RECEIVE_LN_ADDRESS=yourname@wallet.com ### Setting the UI Password -There are two ways to set or change your Admin Dashboard password: +On first start the node generates an admin password and logs it once — read it from the container logs to sign in. You can then change it two ways: -1. **Via Environment Variable**: Set `ADMIN_PASSWORD` in your `.env` file before starting the container. This will be the password used for the first login. -2. **Via Dashboard**: Once logged in, go to **Settings** → **Security** to update your password. Dashboard settings override the `.env` file once saved. +1. **Via Dashboard**: Once logged in, go to **Settings** → **Security** to update your password. +2. **Via Environment Variable (legacy seed)**: Setting `ADMIN_PASSWORD` in `.env` before the first start seeds the initial password instead of generating one. It's read only once, for existing deployments; a value left in `.env` is ignored after the node has been configured. --- @@ -128,14 +126,14 @@ Use environment variables for: | -------------------- | --------------------------------- | ------------------------------------ | | `UPSTREAM_BASE_URL` | Upstream API endpoint | — | | `UPSTREAM_API_KEY` | Upstream API key | — | -| `ADMIN_PASSWORD` | Dashboard password | (none) | +| `ADMIN_PASSWORD` | Legacy seed for the dashboard password (otherwise generated + logged on first start) | (auto-generated) | | `ROUTSTR_SECRET_KEY` | Master key encrypting node secrets at rest. Auto-generated to a key file if unset | (auto-generated) | | `ROUTSTR_SECRET_KEY_FILE` | Path to the generated key file (used when `ROUTSTR_SECRET_KEY` is unset) | `routstr_secret.key` beside the database | | `DATABASE_URL` | Database connection string | `sqlite+aiosqlite:///keys.db` | | `NAME` | Node display name | `ARoutstrNode` | | `DESCRIPTION` | Node description | `A Routstr Node` | | `NPUB` | Nostr public key (bech32) | — | -| `NSEC` | Nostr private key | — | +| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — | | `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` | | `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` | | `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — | diff --git a/docs/provider/deployment.md b/docs/provider/deployment.md index 3a6fcce3..784a764d 100644 --- a/docs/provider/deployment.md +++ b/docs/provider/deployment.md @@ -38,7 +38,6 @@ services: - routstr-data:/app/data environment: DATABASE_URL: "sqlite:////app/data/routstr.db" - ADMIN_KEY: "your-secure-admin-key" LOG_LEVEL: "info" volumes: @@ -127,8 +126,8 @@ services: - UPSTREAM_BASE_URL=https://api.openai.com/v1 - UPSTREAM_API_KEY=sk-proj-... - # Secure the dashboard (recommended) - - ADMIN_PASSWORD=your-secure-password + # The admin password is generated and logged once on first start; set + # ADMIN_PASSWORD here only as a legacy seed for an existing deployment. # Node identity - NAME=My Provider Node @@ -160,7 +159,6 @@ Example `.env`: ```bash UPSTREAM_BASE_URL=https://api.openai.com/v1 UPSTREAM_API_KEY=sk-proj-... -ADMIN_PASSWORD=change-me # Keep the database (and the key file generated beside it) on the mounted volume. DATABASE_URL=sqlite:////app/data/routstr.db # Encrypts node secrets at rest. Optional — if unset, a key is generated next to diff --git a/docs/provider/quickstart.md b/docs/provider/quickstart.md index ef959f40..b528c803 100644 --- a/docs/provider/quickstart.md +++ b/docs/provider/quickstart.md @@ -29,9 +29,6 @@ In future versions, you'll be able to run a node that connects to other Routstr Create a `.env` file in the root of the project to store your secrets: ```bash -# Initial Admin Password -ADMIN_PASSWORD=mysecretpassword - # Encrypts node secrets at rest. Optional — if unset, the node generates a key on # first start and prints it once (back it up). ROUTSTR_SECRET_KEY= @@ -39,13 +36,18 @@ ROUTSTR_SECRET_KEY= # Node Identity NAME="My AI Node" DESCRIPTION="Fast access to models" -NSEC=yournsec # Lightning Payouts RECEIVE_LN_ADDRESS=yourname@wallet.com ``` +The admin password is generated and logged once on first start (read it from the +logs to sign in), and your Nostr identity (`nsec`) is configured afterwards from +the admin UI — both are stored encrypted in the database, not in `.env`. +(`ADMIN_PASSWORD` / `NSEC` are still read once as a legacy seed for existing +deployments.) + ## 2. Start the Node The recommended way to run Routstr is using Docker Compose, which handles the node, the UI, and optional services like Tor. @@ -76,7 +78,7 @@ docker compose up -d Open the **Admin Dashboard** at [http://localhost:8000/admin/](http://localhost:8000/admin/). !!! note "Login" -Use the `ADMIN_PASSWORD` you defined in your `.env` file to log in. If you didn't set one, the dashboard will prompt you to set one on first visit. +On first start the node generates an admin password and logs it once — read it from the container logs to sign in. You can change it afterwards from **Settings** → **Security**. ### Connect Your AI Providers From 7e8c2033ae4d77f050b1ab725b76b00aa0a2d874 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 22 Jul 2026 16:40:42 +0200 Subject: [PATCH 23/26] fix(migrations): chain secrets migration onto the fee-payout head Rebasing onto main picked up the fee-payout-checkpoint migration (d7e8f9a0b1c2), which forked from the same parent as the secrets migration. Re-point secrets onto it so alembic has a single head. --- migrations/versions/c6f8d2e4a1b3_add_secrets_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py index 73639fc0..af9af210 100644 --- a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py +++ b/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py @@ -1,7 +1,7 @@ """add secrets table Revision ID: c6f8d2e4a1b3 -Revises: c6d7e8f9a0b1 +Revises: d7e8f9a0b1c2 Create Date: 2026-07-07 00:00:00.000000 Creates the node-level singleton secret store (issue #553). Schema only; moving @@ -17,7 +17,7 @@ import sqlmodel from alembic import op revision = "c6f8d2e4a1b3" -down_revision = "c6d7e8f9a0b1" +down_revision = "d7e8f9a0b1c2" branch_labels = None depends_on = None From 03b00f3eb6a5847e68b16c70c4a81b7190c414e0 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Wed, 22 Jul 2026 17:01:28 +0200 Subject: [PATCH 24/26] test(wallet): isolate get_balance from the module wallet cache test_get_balance mocked Wallet.with_db but not the module-level _wallets cache, so a real wallet cached by an earlier unmocked path (e.g. an admin-withdraw amount-rejection test) could shadow the mock and fail the assertion depending on collection order. Reset the cache for the test, as the sibling wallet tests already do. --- tests/unit/test_wallet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_wallet.py b/tests/unit/test_wallet.py index 3bb36a28..ddb5aa54 100644 --- a/tests/unit/test_wallet.py +++ b/tests/unit/test_wallet.py @@ -26,7 +26,11 @@ async def test_get_balance() -> None: mock_wallet.load_mint = AsyncMock() mock_wallet.load_proofs = AsyncMock() - with patch("routstr.wallet.Wallet.with_db", return_value=mock_wallet): + # Reset the module-level wallet cache so a real wallet cached by an earlier + # test (e.g. an unmocked admin-withdraw path) can't shadow the mock here. + with patch("routstr.wallet._wallets", {}), patch( + "routstr.wallet.Wallet.with_db", return_value=mock_wallet + ): balance = await get_balance("sat") assert balance == 50000 From 88fe9758a3e5107f558029229d358ec978607154 Mon Sep 17 00:00:00 2001 From: Jeroen Ubbink Date: Thu, 23 Jul 2026 11:19:00 +0200 Subject: [PATCH 25/26] fix(migrations): recreate the secrets migration with a fresh revision id The add-secrets migration was amended in place across the review rounds (notably the nsec_state column), so its revision id no longer maps to a single schema step and any DB that ran an intermediate form would not re-migrate. Recreate it under a fresh id (fc4fa29630d2) chained onto the current head so the migration is one clean, unambiguous step. Co-Authored-By: Claude Opus 4.8 --- ...d_secrets_table.py => fc4fa29630d2_add_secrets_table.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename migrations/versions/{c6f8d2e4a1b3_add_secrets_table.py => fc4fa29630d2_add_secrets_table.py} (93%) diff --git a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py b/migrations/versions/fc4fa29630d2_add_secrets_table.py similarity index 93% rename from migrations/versions/c6f8d2e4a1b3_add_secrets_table.py rename to migrations/versions/fc4fa29630d2_add_secrets_table.py index af9af210..63447661 100644 --- a/migrations/versions/c6f8d2e4a1b3_add_secrets_table.py +++ b/migrations/versions/fc4fa29630d2_add_secrets_table.py @@ -1,8 +1,8 @@ """add secrets table -Revision ID: c6f8d2e4a1b3 +Revision ID: fc4fa29630d2 Revises: d7e8f9a0b1c2 -Create Date: 2026-07-07 00:00:00.000000 +Create Date: 2026-07-23 00:00:00.000000 Creates the node-level singleton secret store (issue #553). Schema only; moving any legacy plaintext into the encrypted/hashed columns happens at bootstrap, @@ -16,7 +16,7 @@ import sqlalchemy as sa import sqlmodel from alembic import op -revision = "c6f8d2e4a1b3" +revision = "fc4fa29630d2" down_revision = "d7e8f9a0b1c2" branch_labels = None depends_on = None From 65ea28cb8541b1579ec8378732efd0b3c777c81a Mon Sep 17 00:00:00 2001 From: 9qeklajc Date: Fri, 24 Jul 2026 01:10:57 +0200 Subject: [PATCH 26/26] add test --- routstr/wallet.py | 2 + scripts/reconcile_reserved_proofs.py | 235 ----------------------- scripts/retry_minibits_reconciliation.sh | 62 ------ tests/unit/test_wallet.py | 64 ++++++ 4 files changed, 66 insertions(+), 297 deletions(-) delete mode 100755 scripts/reconcile_reserved_proofs.py delete mode 100755 scripts/retry_minibits_reconciliation.sh diff --git a/routstr/wallet.py b/routstr/wallet.py index 5ea4025b..cdfa6914 100644 --- a/routstr/wallet.py +++ b/routstr/wallet.py @@ -221,6 +221,8 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int wallet: Wallet = await get_wallet(effective_mint_url, unit) all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit) proofs = [proof for proof in all_proofs if not proof.reserved] + # Fallback must compare the requested amount with liquid proofs only. Counting + # reserved proofs here can suppress fallback even though they cannot be sent. proofs_for_mint = sum(p.amount for p in proofs) reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved) diff --git a/scripts/reconcile_reserved_proofs.py b/scripts/reconcile_reserved_proofs.py deleted file mode 100755 index 8148b145..00000000 --- a/scripts/reconcile_reserved_proofs.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -"""Reconcile Routstr's reserved Cashu proofs against their mints. - -Safe default is dry-run. --apply mutates wallet.sqlite3 and keys.db. -Run only while no process is using these databases and after verified backups. -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import sqlite3 -import time -from collections import defaultdict -from pathlib import Path - -import httpx -from cashu.core.base import Proof -from cashu.wallet.helpers import deserialize_token_from_string - - -def proof_from_row(row: sqlite3.Row) -> Proof: - return Proof(amount=row["amount"], C=row["C"], secret=row["secret"], id=row["id"]) - - -async def fetch_states( - client: httpx.AsyncClient, - mint_url: str, - proofs: list[Proof], - batch_size: int, -) -> dict[str, str]: - states: dict[str, str] = {} - endpoint = mint_url.rstrip("/") + "/v1/checkstate" - for offset in range(0, len(proofs), batch_size): - batch = proofs[offset : offset + batch_size] - last_error: Exception | None = None - for attempt in range(4): - try: - response = await client.post(endpoint, json={"Ys": [proof.Y for proof in batch]}) - response.raise_for_status() - for item in response.json().get("states", []): - states[item["Y"]] = item["state"] - last_error = None - break - except (httpx.TimeoutException, httpx.TransportError) as exc: - last_error = exc - await asyncio.sleep(2**attempt) - if last_error is not None: - raise last_error - return states - - -def load_pending_refund_secrets(keys: sqlite3.Connection) -> tuple[set[str], dict[str, set[str]]]: - pending: set[str] = set() - transaction_secrets: dict[str, set[str]] = {} - rows = keys.execute( - """ - SELECT id, token - FROM cashu_transactions - WHERE type = 'out' AND collected = 0 AND swept = 0 - """ - ).fetchall() - for row in rows: - try: - token = deserialize_token_from_string(row["token"]) - except Exception: - continue - secrets = {proof.secret for proof in token.proofs} - transaction_secrets[row["id"]] = secrets - pending.update(secrets) - return pending, transaction_secrets - - -async def run(args: argparse.Namespace) -> int: - root = Path(args.root).resolve() - keys_path = root / "keys.db" - wallet_path = root / ".wallet" / "wallet.sqlite3" - if not keys_path.exists() or not wallet_path.exists(): - raise SystemExit("keys.db or .wallet/wallet.sqlite3 not found") - - keys = sqlite3.connect(keys_path) - wallet = sqlite3.connect(wallet_path) - keys.row_factory = sqlite3.Row - wallet.row_factory = sqlite3.Row - keys.execute("PRAGMA foreign_keys=ON") - wallet.execute("PRAGMA foreign_keys=ON") - - if keys.execute("PRAGMA integrity_check").fetchone()[0] != "ok": - raise SystemExit("keys.db integrity check failed") - if wallet.execute("PRAGMA integrity_check").fetchone()[0] != "ok": - raise SystemExit("wallet.sqlite3 integrity check failed") - - pending_secrets, transaction_secrets = load_pending_refund_secrets(keys) - rows = wallet.execute( - """ - SELECT p.rowid AS proof_rowid, p.*, k.mint_url, k.unit - FROM proofs p - JOIN keysets k ON k.id = p.id - WHERE COALESCE(p.reserved, 0) != 0 - ORDER BY k.mint_url, k.unit, p.time_reserved - """ - ).fetchall() - grouped: dict[tuple[str, str], list[sqlite3.Row]] = defaultdict(list) - for row in rows: - grouped[(row["mint_url"], row["unit"])].append(row) - - mint_reports: dict[str, object] = {} - errors: dict[str, str] = {} - report: dict[str, object] = { - "mode": "apply" if args.apply else "dry-run", - "root": str(root), - "started_at": int(time.time()), - "pending_refund_secrets": len(pending_secrets), - "mints": mint_reports, - "errors": errors, - } - state_by_secret: dict[str, str] = {} - - async with httpx.AsyncClient(timeout=args.timeout) as client: - for (mint_url, unit), mint_rows in grouped.items(): - proofs = [proof_from_row(row) for row in mint_rows] - try: - states = await fetch_states(client, mint_url, proofs, args.batch_size) - except Exception as exc: - errors[f"{mint_url}|{unit}"] = f"{type(exc).__name__}: {exc}" - continue - - summary: dict[str, dict[str, int]] = defaultdict(lambda: {"proofs": 0, "amount": 0}) - actions = {"delete_spent": 0, "release_untracked_unspent": 0, "preserve_pending": 0, "preserve_unknown": 0} - for row, proof in zip(mint_rows, proofs): - state = states.get(proof.Y, "MISSING") - state_by_secret[row["secret"]] = state - summary[state]["proofs"] += 1 - summary[state]["amount"] += row["amount"] - - if state == "SPENT": - actions["delete_spent"] += 1 - if args.apply: - wallet.execute( - """ - INSERT OR IGNORE INTO proofs_used - (amount, C, secret, time_used, id, derivation_path, mint_id, melt_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - row["amount"], row["C"], row["secret"], - row["time_reserved"] or int(time.time()), row["id"], - row["derivation_path"], row["mint_id"], row["melt_id"], - ), - ) - wallet.execute("DELETE FROM proofs WHERE rowid = ?", (row["proof_rowid"],)) - elif state == "UNSPENT" and row["secret"] not in pending_secrets: - actions["release_untracked_unspent"] += 1 - if args.apply: - wallet.execute( - "UPDATE proofs SET reserved = 0, send_id = NULL, time_reserved = NULL WHERE rowid = ?", - (row["proof_rowid"],), - ) - elif state == "UNSPENT": - actions["preserve_pending"] += 1 - else: - actions["preserve_unknown"] += 1 - - mint_reports[f"{mint_url}|{unit}"] = { - "states": dict(summary), - "actions": actions, - } - - # Mark pending outgoing tokens collected only when every proof the mint reported is SPENT. - collected_transactions: list[str] = [] - for transaction_id, secrets in transaction_secrets.items(): - known = [state_by_secret.get(secret) for secret in secrets] - if known and all(state == "SPENT" for state in known): - collected_transactions.append(transaction_id) - if args.apply: - keys.execute( - "UPDATE cashu_transactions SET collected = 1 WHERE id = ? AND collected = 0 AND swept = 0", - (transaction_id,), - ) - report["mark_collected_transactions"] = len(collected_transactions) - - malformed = keys.execute( - "SELECT COUNT(*), COALESCE(SUM(balance), 0) FROM api_keys WHERE refund_mint_url = ?", - ("https://mint.minibits.cash/Bi",), - ).fetchone() - report["canonicalize_minibits_url"] = {"keys": malformed[0], "balance_msat": malformed[1]} - if args.apply: - keys.execute( - "UPDATE api_keys SET refund_mint_url = ? WHERE refund_mint_url = ?", - ("https://mint.minibits.cash/Bitcoin", "https://mint.minibits.cash/Bi"), - ) - - negative_rows = keys.execute( - "SELECT hashed_key, balance FROM api_keys WHERE balance < 0 ORDER BY balance" - ).fetchall() - report["negative_balances"] = { - "keys": len(negative_rows), - "amount_msat": sum(row["balance"] for row in negative_rows), - "key_prefixes": [row["hashed_key"][:12] for row in negative_rows], - } - if args.apply: - keys.execute("UPDATE api_keys SET balance = 0 WHERE balance < 0") - - if args.apply: - wallet.commit() - keys.commit() - else: - wallet.rollback() - keys.rollback() - - report["finished_at"] = int(time.time()) - output = root / f"reconciliation-{'applied' if args.apply else 'dry-run'}-{report['finished_at']}.json" - output.write_text(json.dumps(report, indent=2, sort_keys=True)) - os.chmod(output, 0o600) - print(json.dumps(report, indent=2, sort_keys=True)) - print(f"report={output}") - - wallet.close() - keys.close() - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--root", default=".") - parser.add_argument("--apply", action="store_true") - parser.add_argument("--batch-size", type=int, default=300) - parser.add_argument("--timeout", type=float, default=45.0) - return asyncio.run(run(parser.parse_args())) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/retry_minibits_reconciliation.sh b/scripts/retry_minibits_reconciliation.sh deleted file mode 100755 index 8851ea14..00000000 --- a/scripts/retry_minibits_reconciliation.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="${HOME}/proxy" -MARKER="${ROOT}/.minibits-reconciliation-complete" -LOCK="${ROOT}/.minibits-reconciliation.lock" -LOG="${ROOT}/logs/minibits-reconciliation.log" -TAG="ROUTSTR-MINIBITS-RECONCILE" - -cd "$ROOT" -[[ -e "$MARKER" ]] && exit 0 -exec 9>"$LOCK" -flock -n 9 || exit 0 - -printf '%s starting reconciliation retry\n' "$(date -Is)" >>"$LOG" -if ! .venv/bin/python scripts/reconcile_reserved_proofs.py --root . --timeout 90 --apply >>"$LOG" 2>&1; then - printf '%s reconciliation command failed\n' "$(date -Is)" >>"$LOG" - exit 0 -fi - -latest=$(ls -1t reconciliation-applied-*.json 2>/dev/null | head -1 || true) -[[ -n "$latest" ]] || exit 0 -if .venv/bin/python - "$latest" <<'PY' -import json, sys -report = json.load(open(sys.argv[1])) -error_keys = report.get("errors", {}) -raise SystemExit(any(key.startswith("https://mint.minibits.cash/Bitcoin|") for key in error_keys)) -PY -then - if .venv/bin/python - <<'PY' >>"$LOG" 2>&1 -import sqlite3 -keys = sqlite3.connect("keys.db") -wallet = sqlite3.connect(".wallet/wallet.sqlite3") -positive_balances = keys.execute( - "SELECT COALESCE(SUM(CASE WHEN balance > 0 THEN balance ELSE 0 END), 0) FROM api_keys" -).fetchone()[0] -pending_refunds = keys.execute( - "SELECT COALESCE(SUM(CASE WHEN unit = 'sat' THEN amount * 1000 ELSE amount END), 0) " - "FROM cashu_transactions WHERE type = 'out' AND collected = 0 AND swept = 0" -).fetchone()[0] -fees = keys.execute("SELECT COALESCE(SUM(accumulated_msats), 0) FROM routstr_fees").fetchone()[0] -liquid_msats = wallet.execute( - "SELECT COALESCE(SUM(CASE WHEN k.unit = 'msat' THEN p.amount ELSE p.amount * 1000 END), 0) " - "FROM proofs p JOIN keysets k ON k.id = p.id WHERE COALESCE(p.reserved, 0) = 0" -).fetchone()[0] -obligations = positive_balances + pending_refunds + fees -print(f"liquid_msats={liquid_msats} obligations_msats={obligations} surplus_msats={liquid_msats-obligations}") -keys.close(); wallet.close() -raise SystemExit(0 if liquid_msats >= obligations else 1) -PY - then - touch "$MARKER" - chmod 600 "$MARKER" - printf '%s Minibits reconciliation completed and solvency verified; starting Routstr\n' "$(date -Is)" >>"$LOG" - docker compose up -d routstr >>"$LOG" 2>&1 - (crontab -l 2>/dev/null | grep -v "$TAG" || true) | crontab - - else - printf '%s reconciliation completed but liquid assets remain below obligations; node stays stopped\n' "$(date -Is)" >>"$LOG" - fi -else - printf '%s Minibits remains unavailable; retry retained\n' "$(date -Is)" >>"$LOG" -fi diff --git a/tests/unit/test_wallet.py b/tests/unit/test_wallet.py index 3bb36a28..b42e5278 100644 --- a/tests/unit/test_wallet.py +++ b/tests/unit/test_wallet.py @@ -15,6 +15,7 @@ from routstr.wallet import ( get_balance, is_mint_connection_error, recieve_token, + send, send_token, ) @@ -147,6 +148,69 @@ async def test_send_token() -> None: assert token == "test_token" +@pytest.mark.asyncio +async def test_send_falls_back_when_preferred_mint_has_only_reserved_balance() -> None: + from routstr.core.settings import settings + + preferred_wallet = Mock(keysets={}, proofs=[]) + preferred_wallet.select_to_send = AsyncMock() + primary_wallet = Mock(keysets={}, proofs=[]) + primary_wallet.select_to_send = AsyncMock() + primary_wallet.serialize_proofs = AsyncMock(return_value="primary-token") + primary_wallet.set_reserved_for_send = AsyncMock() + + preferred_liquid = Mock(amount=500, reserved=False) + preferred_reserved = Mock(amount=600, reserved=True) + primary_liquid = Mock(amount=1000, reserved=False) + primary_wallet.select_to_send.return_value = ([primary_liquid], None) + + async def get_wallet(mint_url: str, unit: str) -> Mock: + assert unit == "sat" + return primary_wallet if mint_url == "http://primary:3338" else preferred_wallet + + def get_proofs(wallet: Mock, mint_url: str, unit: str) -> list[Mock]: + assert unit == "sat" + if wallet is primary_wallet: + assert mint_url == "http://primary:3338" + return [primary_liquid] + assert mint_url == "http://preferred:3338" + return [preferred_liquid, preferred_reserved] + + with ( + patch.object(settings, "primary_mint", "http://primary:3338"), + patch("routstr.wallet.get_wallet", side_effect=get_wallet), + patch("routstr.wallet.get_proofs_per_mint_and_unit", side_effect=get_proofs), + ): + amount, token = await send(1000, "sat", "http://preferred:3338") + + assert (amount, token) == (1000, "primary-token") + preferred_wallet.select_to_send.assert_not_awaited() + primary_wallet.select_to_send.assert_awaited_once_with( + [primary_liquid], 1000, set_reserved=False, include_fees=False + ) + + +@pytest.mark.asyncio +async def test_send_primary_with_only_reserved_proofs_still_raises() -> None: + from routstr.core.settings import settings + + wallet = Mock(keysets={}, proofs=[]) + wallet.select_to_send = AsyncMock(side_effect=RuntimeError("balance too low")) + reserved = Mock(amount=1000, reserved=True) + + with ( + patch.object(settings, "primary_mint", "http://primary:3338"), + patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)), + patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[reserved]), + pytest.raises(RuntimeError, match="balance too low"), + ): + await send(1000, "sat", "http://primary:3338") + + wallet.select_to_send.assert_awaited_once_with( + [], 1000, set_reserved=False, include_fees=False + ) + + @pytest.mark.asyncio async def test_credit_balance() -> None: token_data = {