From 9a5beac80dca254cac50d78059032f0f52f7e10a Mon Sep 17 00:00:00 2001 From: Didactyl User Date: Wed, 25 Mar 2026 09:22:15 -0400 Subject: [PATCH] v0.2.16 - Add Python test harness and fix increment script commit identity handling --- README.md | 4 +- increment_and_push.sh | 34 + plans/automated_test_harness.md | 616 ++++++++++++++++++ src/main.h | 4 +- tests/__pycache__/run_tests.cpython-313.pyc | Bin 0 -> 5816 bytes tests/configs/test_genesis.jsonc | 53 ++ tests/harness/__init__.py | 26 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 660 bytes .../__pycache__/agent_process.cpython-313.pyc | Bin 0 -> 6178 bytes .../didactyl_client.cpython-313.pyc | Bin 0 -> 7743 bytes .../__pycache__/log_watcher.cpython-313.pyc | Bin 0 -> 6298 bytes .../__pycache__/reporter.cpython-313.pyc | Bin 0 -> 5814 bytes .../__pycache__/test_runner.cpython-313.pyc | Bin 0 -> 7106 bytes tests/harness/agent_process.py | 113 ++++ tests/harness/didactyl_client.py | 104 +++ tests/harness/log_watcher.py | 77 +++ tests/harness/reporter.py | 77 +++ tests/harness/test_runner.py | 141 ++++ tests/run_tests.py | 107 +++ tests/suites/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 176 bytes .../suites/__pycache__/common.cpython-313.pyc | Bin 0 -> 2341 bytes .../test_conversation.cpython-313.pyc | Bin 0 -> 3267 bytes .../__pycache__/test_errors.cpython-313.pyc | Bin 0 -> 2356 bytes .../__pycache__/test_health.cpython-313.pyc | Bin 0 -> 3006 bytes .../__pycache__/test_restart.cpython-313.pyc | Bin 0 -> 2784 bytes .../__pycache__/test_timeouts.cpython-313.pyc | Bin 0 -> 2638 bytes .../test_tools_blossom.cpython-313.pyc | Bin 0 -> 446 bytes .../test_tools_cashu.cpython-313.pyc | Bin 0 -> 481 bytes .../test_tools_identity.cpython-313.pyc | Bin 0 -> 937 bytes .../test_tools_memory.cpython-313.pyc | Bin 0 -> 787 bytes .../test_tools_nostr.cpython-313.pyc | Bin 0 -> 1282 bytes .../test_tools_skills.cpython-313.pyc | Bin 0 -> 763 bytes .../test_tools_system.cpython-313.pyc | Bin 0 -> 984 bytes tests/suites/common.py | 34 + tests/suites/test_conversation.py | 45 ++ tests/suites/test_errors.py | 38 ++ tests/suites/test_health.py | 47 ++ tests/suites/test_restart.py | 41 ++ tests/suites/test_timeouts.py | 42 ++ tests/suites/test_tools_blossom.py | 9 + tests/suites/test_tools_cashu.py | 9 + tests/suites/test_tools_identity.py | 13 + tests/suites/test_tools_memory.py | 23 + tests/suites/test_tools_nostr.py | 15 + tests/suites/test_tools_skills.py | 17 + tests/suites/test_tools_system.py | 19 + 47 files changed, 1705 insertions(+), 4 deletions(-) create mode 100644 plans/automated_test_harness.md create mode 100644 tests/__pycache__/run_tests.cpython-313.pyc create mode 100644 tests/configs/test_genesis.jsonc create mode 100644 tests/harness/__init__.py create mode 100644 tests/harness/__pycache__/__init__.cpython-313.pyc create mode 100644 tests/harness/__pycache__/agent_process.cpython-313.pyc create mode 100644 tests/harness/__pycache__/didactyl_client.cpython-313.pyc create mode 100644 tests/harness/__pycache__/log_watcher.cpython-313.pyc create mode 100644 tests/harness/__pycache__/reporter.cpython-313.pyc create mode 100644 tests/harness/__pycache__/test_runner.cpython-313.pyc create mode 100644 tests/harness/agent_process.py create mode 100644 tests/harness/didactyl_client.py create mode 100644 tests/harness/log_watcher.py create mode 100644 tests/harness/reporter.py create mode 100644 tests/harness/test_runner.py create mode 100755 tests/run_tests.py create mode 100644 tests/suites/__init__.py create mode 100644 tests/suites/__pycache__/__init__.cpython-313.pyc create mode 100644 tests/suites/__pycache__/common.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_conversation.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_errors.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_health.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_restart.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_timeouts.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_blossom.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_cashu.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_identity.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_memory.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_nostr.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_skills.cpython-313.pyc create mode 100644 tests/suites/__pycache__/test_tools_system.cpython-313.pyc create mode 100644 tests/suites/common.py create mode 100644 tests/suites/test_conversation.py create mode 100644 tests/suites/test_errors.py create mode 100644 tests/suites/test_health.py create mode 100644 tests/suites/test_restart.py create mode 100644 tests/suites/test_timeouts.py create mode 100644 tests/suites/test_tools_blossom.py create mode 100644 tests/suites/test_tools_cashu.py create mode 100644 tests/suites/test_tools_identity.py create mode 100644 tests/suites/test_tools_memory.py create mode 100644 tests/suites/test_tools_nostr.py create mode 100644 tests/suites/test_tools_skills.py create mode 100644 tests/suites/test_tools_system.py diff --git a/README.md b/README.md index e62c402..3316c62 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,11 @@ Skills compose by adoption-list order (`10123`) and trigger tags carry runtime e Didactyl will support local inference, which is very privacy preserving. Remote inference does however have it's advantages, and in those cases Didactyl supports using Bitcoin Lightning and eCash inference providers. -## Current Status — v0.2.14 +## Current Status — v0.2.16 **Active build — this project is barely working. Experiment at your own risk.** -> Last release update: v0.2.14 — Follow-up push after context verification and startup-skill refactor +> Last release update: v0.2.16 — Add Python test harness and fix increment script commit identity handling - Connects to configured relays with auto-reconnect and relay state transition logging - Publishes configured startup events per relay as each relay becomes connected diff --git a/increment_and_push.sh b/increment_and_push.sh index e728b6f..0087756 100755 --- a/increment_and_push.sh +++ b/increment_and_push.sh @@ -111,6 +111,37 @@ check_git_repo() { fi } +# Ensure git identity exists so commits do not fail on clean machines +ensure_git_identity() { + local current_name="" + local current_email="" + + current_name=$(git config --get user.name 2>/dev/null || true) + current_email=$(git config --get user.email 2>/dev/null || true) + + if [[ -n "$current_name" && -n "$current_email" ]]; then + return 0 + fi + + print_warning "Git user.name / user.email not fully configured for this repository" + + local fallback_name + local fallback_email + + fallback_name="${GIT_AUTHOR_NAME:-Didactyl User}" + fallback_email="${GIT_AUTHOR_EMAIL:-didactyl@local}" + + if [[ -z "$current_name" ]]; then + git config user.name "$fallback_name" + print_status "Set local git user.name to '$fallback_name'" + fi + + if [[ -z "$current_email" ]]; then + git config user.email "$fallback_email" + print_status "Set local git user.email to '$fallback_email'" + fi +} + # Function to get current version and increment appropriately increment_version() { local increment_type="$1" # "patch", "minor", or "major" @@ -293,6 +324,8 @@ git_commit_and_push_no_tag() { print_success "Committed changes" else print_error "Failed to commit changes" + print_error "git commit output:" + git commit -m "$NEW_VERSION - $COMMIT_MESSAGE" 2>&1 || true exit 1 fi fi @@ -498,6 +531,7 @@ main() { # Check prerequisites check_git_repo + ensure_git_identity if [[ "$RELEASE_MODE" == true ]]; then print_status "=== RELEASE MODE ===" diff --git a/plans/automated_test_harness.md b/plans/automated_test_harness.md new file mode 100644 index 0000000..a42564a --- /dev/null +++ b/plans/automated_test_harness.md @@ -0,0 +1,616 @@ +# Didactyl Automated Test Harness + +## Overview + +An automated testing system that starts a Didactyl agent locally (debug build), converses with it via the HTTP API, exercises all tools, monitors logs in real-time, handles agent crashes/restarts, and produces a structured test results report. + +**Language:** Python (stdlib only, no external dependencies) +**Location:** `tests/` +**Phase 1:** Scripted tests (no LLM driving the tester) +**Phase 2 (future):** LLM-driven test agent that generates prompts, evaluates responses, and adapts + +--- + +## Architecture + +```mermaid +flowchart TB + subgraph Test Harness - Python + RUNNER[test_runner.py
orchestrator] + PROC[agent_process.py
start/stop/restart] + CLIENT[didactyl_client.py
HTTP API wrapper] + LOG[log_watcher.py
tail debug.log] + REPORT[reporter.py
results output] + + RUNNER --> PROC + RUNNER --> CLIENT + RUNNER --> LOG + RUNNER --> REPORT + end + + subgraph Test Suites + TH[test_health] + TC[test_conversation] + TI[test_tools_identity] + TN[test_tools_nostr] + TS[test_tools_skills] + TSY[test_tools_system] + TM[test_tools_memory] + TCA[test_tools_cashu] + TB[test_tools_blossom] + TTO[test_timeouts] + TE[test_errors] + TR[test_restart] + end + + RUNNER --> TH & TC & TI & TN & TS & TSY & TM & TCA & TB & TTO & TE & TR + + subgraph Didactyl Agent - debug build + AGENT[didactyl process] + API[HTTP API :8484] + LOGFILE[debug.log] + end + + CLIENT -- HTTP --> API + PROC -- subprocess --> AGENT + LOG -- tail --> LOGFILE +``` + +--- + +## Design Decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Language | Python | Best subprocess/HTTP/threading support; already used in project | +| Dependencies | stdlib only | No pip install needed; `urllib`, `subprocess`, `threading`, `json` | +| Test framework | Standalone runner | Self-contained, no pytest dependency | +| Relay strategy | Flexible genesis config | User provides their own test_genesis.jsonc | +| LLM for agent | Real model | User provides API key in test genesis config | +| Tool scope | All tools | Disposable test identity; full coverage | +| Entry point | `tests/run_tests.py` | Single script to run everything | + +--- + +## Directory Structure + +``` +tests/ +├── harness/ +│ ├── __init__.py +│ ├── agent_process.py # start/stop/restart didactyl subprocess +│ ├── didactyl_client.py # HTTP API wrapper with timeouts +│ ├── log_watcher.py # real-time log tail + marker system +│ ├── test_runner.py # orchestrator +│ └── reporter.py # results formatting (JSON + text) +├── suites/ +│ ├── __init__.py +│ ├── test_health.py # status, context endpoints +│ ├── test_conversation.py # basic prompt/response +│ ├── test_tools_identity.py # identity and context tools +│ ├── test_tools_nostr.py # nostr event, messaging, relay tools +│ ├── test_tools_skills.py # skill and trigger tools +│ ├── test_tools_system.py # system, local, model, config tools +│ ├── test_tools_memory.py # task and memory tools +│ ├── test_tools_cashu.py # cashu wallet tools +│ ├── test_tools_blossom.py # blossom tools +│ ├── test_timeouts.py # response time assertions +│ ├── test_errors.py # API error handling paths +│ └── test_restart.py # crash recovery +├── configs/ +│ └── test_genesis.jsonc # example test config (user fills in secrets) +├── results/ # test run output (gitignored) +├── run_tests.py # entry point +├── test.sh # existing bash test (kept as-is) +├── blossom_tool_validation_test.c # existing C test (kept as-is) +└── blossom_tool_validation_test # existing compiled test (kept as-is) +``` + +--- + +## Component Specifications + +### 1. agent_process.py — Process Manager + +Manages the didactyl subprocess lifecycle. + +```python +class AgentProcess: + def __init__(self, binary_path, config_path, api_port=8484, + api_bind="127.0.0.1", debug_level=5, log_file=None): + """Configure but don't start yet.""" + + def start(self, timeout=30) -> bool: + """ + Spawn didactyl as subprocess: + ./didactyl --config --debug + --api-port --api-bind + + Set DIDACTYL_LOG_FILE env var to log_file path. + Wait for GET /api/status to return 200 (poll with timeout). + Returns True if agent started successfully. + """ + + def stop(self, timeout=10) -> bool: + """ + Send SIGTERM, wait for clean exit. + If still alive after timeout, send SIGKILL. + Returns True if stopped cleanly. + """ + + def restart(self, timeout=30) -> bool: + """stop() then start().""" + + def is_alive(self) -> bool: + """Check process.poll() and optionally /api/status.""" + + def pid(self) -> int | None: + """Return PID if running.""" + + def return_code(self) -> int | None: + """Return exit code if stopped.""" +``` + +Key details: +- Uses `subprocess.Popen` with `stdout=PIPE, stderr=PIPE` +- Captures stdout/stderr for crash diagnostics +- The health check polls `GET /api/status` every 500ms until success or timeout +- Sets `DIDACTYL_LOG_FILE` to a test-run-specific path like `tests/results//agent_debug.log` + +### 2. didactyl_client.py — HTTP API Client + +Thin wrapper around the Didactyl HTTP API using only `urllib`. + +```python +class DidactylClient: + def __init__(self, base_url="https://127.0.0.1:8484", timeout=60, + verify_tls=False): + """Configure base URL and default timeout.""" + + def status(self) -> dict: + """GET /api/status""" + + def prompt(self, message: str, max_turns: int = 4, + model: str = None) -> dict: + """POST /api/prompt/agent — full agent context conversation""" + + def prompt_raw(self, messages: list, max_turns: int = 4, + model: str = None) -> dict: + """POST /api/prompt/run — raw messages, no auto-context""" + + def prompt_simple(self, system: str, user: str, + model: str = None) -> dict: + """POST /api/prompt/run-simple — no tools""" + + def context_current(self) -> dict: + """GET /api/context/current""" + + def context_parts(self) -> dict: + """GET /api/context/parts""" + + def fire_webhook(self, d_tag: str, payload: dict = None) -> dict: + """POST /api/trigger/""" +``` + +Key details: +- All methods return parsed JSON dict +- Raises `TimeoutError` if response exceeds timeout +- Raises `ConnectionError` if agent is unreachable +- Raises `APIError(status_code, body)` for non-2xx responses +- Uses `ssl._create_unverified_context()` for local TLS (same as chat CLI) + +### 3. log_watcher.py — Real-time Log Monitor + +Background thread that tails the agent's debug log file. + +```python +class LogWatcher: + def __init__(self, log_path: str): + """Configure log file path.""" + + def start(self): + """ + Start background thread. + Open file, seek to end, poll for new lines. + Store all lines in memory with timestamps. + """ + + def stop(self): + """Stop background thread.""" + + def set_marker(self, name: str): + """Record current line count as a named marker.""" + + def get_lines_since(self, marker: str) -> list[str]: + """Return all lines captured since the named marker.""" + + def get_all_lines(self) -> list[str]: + """Return all captured lines.""" + + def search(self, pattern: str, since_marker: str = None) -> list[str]: + """Regex search through captured lines.""" + + def has_errors(self, since_marker: str = None) -> bool: + """Check for [ERROR] lines since marker.""" + + def has_warnings(self, since_marker: str = None) -> bool: + """Check for [WARN] lines since marker.""" + + def error_lines(self, since_marker: str = None) -> list[str]: + """Return all ERROR lines since marker.""" +``` + +Key details: +- Uses `threading.Thread(daemon=True)` for background polling +- Polls file every 100ms for new content +- Handles file rotation (agent restart creates new file) +- Thread-safe access to captured lines via `threading.Lock` + +### 4. test_runner.py — Orchestrator + +```python +class TestResult: + name: str + suite: str + status: str # "pass", "fail", "error", "skip", "timeout" + message: str + duration_seconds: float + agent_errors: list[str] # ERROR lines from log during this test + details: dict # arbitrary test-specific data + +class TestCase: + name: str + description: str + requires_restart: bool = False + + def run(self, client: DidactylClient, log: LogWatcher) -> TestResult: + """Execute the test and return result.""" + +class TestRunner: + def __init__(self, agent: AgentProcess, client: DidactylClient, + log: LogWatcher): + """Configure with harness components.""" + + def discover_suites(self, suites_dir: str) -> list: + """Import all test_*.py modules from suites directory.""" + + def run_all(self, suites: list = None) -> list[TestResult]: + """ + For each suite: + 1. If test requires restart, restart agent + 2. Set log marker for this test + 3. Run test with timeout wrapper + 4. Capture result + any agent errors from log + 5. If agent crashed, restart and record error + 6. Collect all results + Return list of TestResult. + """ + + def run_suite(self, suite_name: str) -> list[TestResult]: + """Run a single named suite.""" +``` + +Key details: +- Each test gets a fresh log marker so errors can be correlated +- If `client.prompt()` raises `TimeoutError`, the test is marked "timeout" and the agent is restarted +- If `agent.is_alive()` returns False mid-suite, the agent is restarted and remaining tests continue +- Supports filtering by suite name or test name via CLI args + +### 5. reporter.py — Results Output + +```python +class Reporter: + def __init__(self, results: list[TestResult], output_dir: str): + """Configure with results and output directory.""" + + def print_summary(self): + """Print pass/fail/error/skip/timeout counts to stdout.""" + + def print_details(self): + """Print each test result with details.""" + + def write_json(self, path: str): + """Write full results as JSON for programmatic consumption.""" + + def write_text(self, path: str): + """Write human-readable report.""" +``` + +Output format example: +``` +== Didactyl Test Results == +Run: 2026-03-25T09:50:00Z +Agent: v0.0.26 +Model: claude-haiku-4.5 + +Pass: 42 +Fail: 3 +Error: 1 +Timeout: 2 +Skip: 0 +Total: 48 + +-- Failures -- +[FAIL] test_tools_nostr::nostr_post_kind1 + Expected tool_calls to contain 'nostr_post', got: ['nostr_query'] + Agent errors during test: 0 + +[FAIL] test_conversation::multi_turn + Agent returned empty final_response + Agent errors during test: 2 + [ERROR] [llm.c:234] HTTP 429 rate limited + [ERROR] [llm.c:240] LLM call failed after 3 retries + +[TIMEOUT] test_tools_skills::skill_create + No response within 60s + Agent was restarted +``` + +--- + +## Test Suite Specifications + +### test_health.py + +| Test | Prompt/Action | Assertions | +|---|---|---| +| `status_returns_200` | `GET /api/status` | HTTP 200, `success=true` | +| `status_has_fields` | `GET /api/status` | Has `name`, `version`, `pubkey`, `relay_count` | +| `context_current_returns_messages` | `GET /api/context/current` | Has `messages` array, `total_chars > 0` | +| `context_parts_has_system_prompt` | `GET /api/context/parts` | Has part named `system_prompt` | + +### test_conversation.py + +| Test | Prompt | Assertions | +|---|---|---| +| `simple_greeting` | "Hello, what is your name?" | `final_response` is non-empty string | +| `agent_responds_about_itself` | "What are you? Describe yourself briefly." | `final_response` mentions agent/Didactyl/Nostr | +| `empty_message_handling` | "" (empty string) | Returns error or handles gracefully | +| `very_long_message` | 10000 char string | Returns response or graceful error, no crash | + +### test_tools_identity.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `get_pubkey` | "What is your public key in hex?" | `nostr_pubkey` or `my_pubkey` | Tool called, result has hex pubkey | +| `get_npub` | "What is your npub?" | `nostr_npub` or `my_npub` | Tool called, result has npub1... | +| `agent_identity` | "Tell me about your identity" | `agent_identity` | Tool called, success | +| `agent_version` | "What version are you?" | `agent_version` | Tool called, result has version string | +| `admin_identity` | "Who is your administrator?" | `admin_identity` | Tool called, success | + +### test_tools_nostr.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `nostr_post_kind1` | "Post a test note saying 'Automated test post'" | `nostr_post` | Tool called with kind=1, success, event_id returned | +| `nostr_query_recent` | "Query the 3 most recent kind 1 notes from any author" | `nostr_query` | Tool called, returns events array | +| `nostr_my_events` | "List your recent events" | `nostr_my_events` | Tool called, success | +| `nostr_relay_status` | "What is the status of your relay connections?" | `nostr_relay_status` | Tool called, returns relay info | +| `nostr_dm_send` | "Send a test DM to yourself" | `nostr_dm_send` | Tool called, success | +| `nostr_encode_npub` | "Encode your pubkey as an npub" | `nostr_encode` | Tool called, returns npub | +| `nostr_profile_get` | "Look up your own Nostr profile" | `nostr_profile_get` | Tool called, returns profile | + +### test_tools_skills.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `skill_list` | "List your available skills" | `skill_list` | Tool called, returns skills array | +| `trigger_list` | "List your active triggers" | `trigger_list` | Tool called, success | +| `skill_create_and_remove` | "Create a test skill called 'test-harness-probe' with content 'Test skill' then remove it" | `skill_create`, `skill_remove` | Both tools called, success | + +### test_tools_system.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `tool_list` | "List all your available tools" | `tool_list` | Tool called, returns tools array | +| `model_get` | "What model are you currently using?" | `model_get` | Tool called, returns model info | +| `model_list` | "List available models" | `model_list` | Tool called, success | +| `local_http_fetch` | "Fetch https://httpbin.org/get" | `local_http_fetch` | Tool called, returns HTTP response | +| `config_store_recall` | "Store a test config with d_tag 'test_harness_probe' containing 'hello', then recall it" | `config_store`, `config_recall` | Both called, recalled value matches | + +### test_tools_memory.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `task_list` | "Show me your current task list" | `task_list` or `task_manage` | Tool called, success | +| `task_manage_add_remove` | "Add a task 'test harness probe task' then remove it" | `task_manage` | Tool called with add then remove | +| `memory_save_recall` | "Save 'test harness probe' to memory, then recall your memory" | `memory_save`, `memory_recall` | Both called, recalled contains probe text | + +### test_tools_cashu.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `wallet_balance` | "Check your cashu wallet balance" | `cashu_wallet_balance` | Tool called, success (even if empty) | + +Note: Most cashu tools require a configured mint and funded wallet. Phase 1 tests only the read-only balance check. Full cashu testing requires a test mint setup. + +### test_tools_blossom.py + +| Test | Prompt | Expected Tool | Assertions | +|---|---|---|---| +| `blossom_list` | "List your blossom blobs" | `blossom_list` | Tool called, success (even if empty) | + +Note: Full blossom testing requires a configured blossom server. Phase 1 tests only the list operation. + +### test_timeouts.py + +| Test | Action | Assertions | +|---|---|---| +| `response_within_timeout` | Send simple prompt, measure time | Response received within 60s | +| `status_responds_fast` | `GET /api/status`, measure time | Response within 2s | +| `context_responds_fast` | `GET /api/context/current`, measure time | Response within 5s | + +### test_errors.py + +| Test | Action | Assertions | +|---|---|---| +| `invalid_json_body` | POST malformed JSON to `/api/prompt/agent` | Returns 400, `success=false` | +| `missing_message_field` | POST `{}` to `/api/prompt/agent` | Returns 400, `success=false` | +| `unknown_endpoint` | GET `/api/nonexistent` | Returns 404 | +| `webhook_nonexistent_dtag` | POST to `/api/trigger/nonexistent-dtag` | Returns 404 | + +### test_restart.py + +| Test | Action | Assertions | +|---|---|---| +| `clean_restart` | Stop agent, start agent | Agent comes back, `/api/status` works | +| `status_after_restart` | Restart, then `GET /api/status` | Same pubkey, version as before restart | +| `conversation_after_restart` | Restart, then send prompt | Agent responds normally | + +--- + +## Entry Point: run_tests.py + +``` +Usage: + python tests/run_tests.py [options] + +Options: + --config PATH Path to test genesis.jsonc (default: tests/configs/test_genesis.jsonc) + --binary PATH Path to didactyl binary (default: ./didactyl) + --suite NAME Run only this suite (can repeat) + --test NAME Run only this test (can repeat) + --api-port PORT API port (default: 8485) + --timeout SECS Default response timeout (default: 60) + --debug-level N Agent debug level 0-5 (default: 5) + --output-dir PATH Results output directory (default: tests/results/) + --verbose Print each test result as it runs + --no-restart Don't auto-restart agent on crash (fail remaining tests) +``` + +--- + +## Test Genesis Config + +`tests/configs/test_genesis.jsonc` — an example config the user fills in: + +```jsonc +{ + // TEST CONFIGURATION — fill in your test identity secrets + + "key": { + "nsec": "nsec1REPLACE_WITH_DISPOSABLE_TEST_NSEC" + }, + + "admin": { + "pubkey": "npub1REPLACE_WITH_TEST_ADMIN_PUBKEY" + }, + + "dm_protocol": "nip04", + + "llm": { + "provider": "openai", + "api_key": "sk-REPLACE_WITH_API_KEY", + "model": "claude-haiku-4.5", + "base_url": "https://api.anthropic.com/v1", + "max_tokens": 512, + "temperature": 0.3 + }, + + "api": { + "enabled": true, + "port": 8485, + "bind_address": "127.0.0.1" + }, + + "startup_events": [ + { + "kind": 0, + "content_fields": { + "name": "Didactyl Test Agent", + "about": "Automated test instance" + }, + "tags": [] + }, + { + "kind": 10002, + "content": "", + "tags": [ + ["r", "wss://relay.damus.io"], + ["r", "wss://relay.primal.net"] + ] + }, + { + "kind": 31124, + "content": "# Test Agent\n\nYou are a test agent. Respond to all requests. Use tools when asked.\n\n{{my_kind0_profile}}\n\nYour npub: {{my_npub}}", + "tags": [ + ["d", "identity_and_rules"], + ["app", "didactyl"], + ["scope", "private"], + ["trigger", "dm"], + ["filter", "{\"from\":\"admin\"}"] + ] + } + ] +} +``` + +--- + +## Execution Flow + +```mermaid +flowchart TD + START[run_tests.py] --> PARSE[Parse CLI args] + PARSE --> BUILD[Verify didactyl binary exists] + BUILD --> CONFIG[Load test genesis config] + CONFIG --> MKDIR[Create results output dir] + MKDIR --> LOG_START[Start LogWatcher] + LOG_START --> AGENT_START[Start AgentProcess] + AGENT_START --> HEALTH[Wait for /api/status 200] + + HEALTH -->|timeout| FAIL_STARTUP[Report startup failure and exit] + HEALTH -->|success| DISCOVER[Discover test suites] + + DISCOVER --> LOOP{Next test?} + + LOOP -->|yes| CHECK_ALIVE{Agent alive?} + CHECK_ALIVE -->|no| RESTART_MID[Restart agent] + RESTART_MID --> CHECK_ALIVE + CHECK_ALIVE -->|yes| MARKER[Set log marker] + MARKER --> RUN_TEST[Run test with timeout] + + RUN_TEST -->|pass| RECORD_PASS[Record PASS] + RUN_TEST -->|fail| RECORD_FAIL[Record FAIL] + RUN_TEST -->|timeout| RESTART_TIMEOUT[Restart agent] + RESTART_TIMEOUT --> RECORD_TIMEOUT[Record TIMEOUT] + RUN_TEST -->|error| RECORD_ERROR[Record ERROR] + + RECORD_PASS & RECORD_FAIL & RECORD_TIMEOUT & RECORD_ERROR --> COLLECT_LOGS[Collect agent errors since marker] + COLLECT_LOGS --> LOOP + + LOOP -->|no more| STOP_AGENT[Stop agent] + STOP_AGENT --> STOP_LOG[Stop LogWatcher] + STOP_LOG --> REPORT[Generate report] + REPORT --> EXIT[Exit with code 0 if all pass else 1] +``` + +--- + +## Implementation Order + +1. **Harness infrastructure** (agent_process, didactyl_client, log_watcher) +2. **Test runner + reporter** (orchestration layer) +3. **test_health.py** (validates the harness itself works) +4. **test_conversation.py** (validates basic agent interaction) +5. **test_errors.py** (validates error handling) +6. **test_timeouts.py** (validates timeout detection) +7. **test_restart.py** (validates process management) +8. **Tool test suites** (one at a time, identity → nostr → skills → system → memory → cashu → blossom) +9. **Entry point + config** (run_tests.py, test_genesis.jsonc) +10. **End-to-end validation** against a running agent + +--- + +## Future: Phase 2 — LLM-Driven Testing + +The architecture supports this naturally. In Phase 2: + +- Add an `LLMTestAgent` class that wraps an LLM API call +- The LLM receives: tool documentation, test objectives, previous results +- It generates test prompts, evaluates responses, decides next actions +- The `TestCase.run()` method delegates to the LLM agent instead of scripted logic +- The LLM can also analyze agent logs for anomalies +- Potentially: the LLM can generate code patches for bugs it finds + +No architectural changes needed — just a new test suite type that uses LLM reasoning instead of scripted assertions. diff --git a/src/main.h b/src/main.h index 142ba28..44aae13 100644 --- a/src/main.h +++ b/src/main.h @@ -12,8 +12,8 @@ // Using DIDACTYL_ prefix to avoid conflicts with nostr_core_lib VERSION macros #define DIDACTYL_VERSION_MAJOR 0 #define DIDACTYL_VERSION_MINOR 2 -#define DIDACTYL_VERSION_PATCH 14 -#define DIDACTYL_VERSION "v0.2.14" +#define DIDACTYL_VERSION_PATCH 16 +#define DIDACTYL_VERSION "v0.2.16" // Agent metadata #define DIDACTYL_NAME "Didactyl" diff --git a/tests/__pycache__/run_tests.cpython-313.pyc b/tests/__pycache__/run_tests.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f78d7124c913abc829b2ed95e3c264c9fd9b842 GIT binary patch literal 5816 zcmcH-TWlN0aY-I|$Ky+U=>4K)Nj7U!iW9%&SV;{zwjxEc%`+XPG%+$%OtKCSZA_SP=RU2}rPvq7e(Wgb8dN zwPLF=+Aw2`c5H{yHtHB*F&jn~*)DXEI|OUVFBUY%+UKFlBo?a0n(%fw?7HX%nJy~x zqCM8~^vdT!qtu1a_S9JW(<=nDRqi3H$z$x38TF0S;W`7hkNQUfIAFkz(cnl3hYXk< ztsiN?4F=4OHjXsmCV<_7vjx4*;AX)i@CMX!u~l##n*jSfU|$>TrKJl|FVJ1dmT7Cm z`+%4bvFI*IQ8JpOr80^-4s+(Lq+N#5ePUWxw6i!fDXS{Xy(d#iX;Pa_52aH;01W?1 zX8Ki0o4hPzm~k;Vo57j^^XFw%i{%tW#*?+)AO~(k$nxkjS}v{=O1bMh zQ%*l1fu46qqrj8RY0+c~lT~lZ_)M9*03HR4p94X zP7w{LFFT8yfMPsIUR4n=&Y=xYec99fcTacWK&f6S`I09|Sb)@VWxlV@+ z@gYhOZyG=P)+i{kne}1}rB=rnTZJi9vtlz_P#Tm#qtsUC1foel1_j~CpJ1U>#R55& zgwqqhLCf9>CJBKFCC3`Kc+qoK#hUO_*0_~Q*{%dbmWDh{a!kz%v|aI~QWRU<7Hg!^ zt0VQ>rbRx463vt@*yA>08I=-7(~d0~w<+Y5S7GAJ|DchLGf!#UEdDjRH*FHQC}h!1 zt#-i)UhkkgsE%GF@IVPyF%$xQjN(Y_rt`$6zPOFbt-&!Ax4pFOwF9=B%CC-bd)xt) zkX6{k9*Uj#S&SYAMZpMdiD$s3aEW~s8MjkA6ldHSi&A`bq-P72nB9&pcm*GH0(_jO z3X^SHPQQ+7013`me^oMtsv4&K9Vl^t;wP?7ED3=DMsdYmvF9kZItoE5S#ihRo7(qp z(H^2aVbno(u#JA9p6uYiu&x2@pvG3sKyf$Kp?GSt!2dsQ9H}v)0#fthj_qQt23L*A z0V*Z#q5BXT`|OH0@jRtW+>cv@CWstg+!q_9*y>2Nf)9PM7pr&*ZK0o_*y<=WQ_V^p zNEoV8C?vFOa`w8{9Ym580(&C;Vx)4yU{YbS2(J9D@IhMd4 zSt6a|d2i2aJu^MY^F6~oV?Dx8^4?0zAgj=@sr~1nH)5fufKEi!Ir(Y|+Pln^d@F6< z!e&1ghHFT8Dw9)^$HEUtWy}Z6jB>-3ayuDv|c&~;>4*`TGqKsk}8WioIVw?VUq52 zw$iuhJhWn|saa7=s}D%(hBk~GRKDYs1S_HA&4i6kEL`n|iNOy@`HwKV-7sHwOd6Lc z-3BUkhV)%|x7l48jjEatj%UoSIGjwWlNnGdCm)dV{=h~A*>#W6kU=vE+Get*kq&da zy2JE=&J&U1B)C+I*mM?pL~P075aa|Fz16qx3;Nu5(Q zJVj9WD6GRIm+AJID_{*iNMMH%E2?hKVrW(|$vu-z{0v{$EBoo8oRj%2(lPIr*Hk^_PS+H)qBqdeSn1{FdVbTWa%#4&$ z@M{1PPpY55u|Wgxs{h6Dxs%^`e2a;N#G<$${x0@W;%4F_@upbvM9ZGTMbF{Fk<%s5 zFf{l6`Wv0sI?MjS73I!_JHv(U!MTwQhMR9GF`+WkQe;|I4&Lc55a`G^9{*DJjeXbl zl{_7=)a73sTo_z@apA=c&Qqw{w|Z*zXu*4MjXSjNtJ|5kYvu6D-hy}E8n>VD`U1Z` zxXhP=y9>QXioqi#-_g0@b$|16_eas2(dEun%WC(>(Oc2g&Vp}XZulE_(?+Oq*>Qce zGLTKQUu&mBsHq(4E{3{60{s8Z$9WucFFg(*Z`;bmpWpuD+l907qUXZgsSQ4~G*RL^ z=7zpx*biQQ_vQCT*S*2TD+^avy5_GGyghTL@4EtIZ{EH&u790#EpiLoFL|)|A=73L zZunbQdhQMt{iE}a^}2>~U2n0jcjcW@-QjureP3w9?JorOthx8D2SX3{Bd2$fU*MOY zo#%^OM@81J_=o#Yu=z&cwZ4`5Qee+K``Cdx!t}Im$39NnN_;Hd5=-ra<@VFX_S2>I zm)D!Z8x76Nz1OeMp{0F!`nUVPb2&o%Jong(LTxLjKRs3qUYO_Bxw{U1+L>u&a>UF|Md?i z@}05#^P%H=-12XgG@YMt(J?3bv|(sB41XO6jWNt;eg}Crx&iYy#!jBjU1LA8>6|E% zmq8TC_W&$wIjrOZkdTc#aNjuG`_-&8DMzfBe67L_@QAeQyeLlPfL9hpOj02ZlE-oK zAVnRr)T~Nso_0B%x@0CcP(*5rC>n`4lYSGvRhUIlXJTj0oX020LdK|Dj+6oyc1>?J zYeMA%N}pN!vgRi_BlYR+TRCV+70vo$Bt^Y>D-Wr1Z<(L%u#qYow$f>WG1K4^iwUa; zNeyz7kyXu40VI6J7Q(H>D@NMkj?ZM0xwQNOo&g-Gi0UFd z9@;Dx%eScOYvlU^)qjEbd#LFv)-^x;-aBQsy~wtg*v>NBTV#97Y_!Nm=a|oJjydhU zfu*4v!`FsOwzhS64~O3!{@~2JXO>P}QLr3qSXYQeW@O

SUMkv mhb)%h!&j_`anI*Vw&u_I(6#f|FO>MU0@L=;hL{$^;Qs(*=h;>O literal 0 HcmV?d00001 diff --git a/tests/configs/test_genesis.jsonc b/tests/configs/test_genesis.jsonc new file mode 100644 index 0000000..105941a --- /dev/null +++ b/tests/configs/test_genesis.jsonc @@ -0,0 +1,53 @@ +{ + // TEST CONFIGURATION + // Use disposable keys/accounts only. + "key": { + "nsec": "nsec1REPLACE_WITH_DISPOSABLE_TEST_NSEC" + }, + "admin": { + "pubkey": "npub1REPLACE_WITH_TEST_ADMIN_PUBKEY" + }, + "dm_protocol": "nip04", + "llm": { + "provider": "openai", + "api_key": "sk-REPLACE_WITH_API_KEY", + "model": "claude-haiku-4.5", + "base_url": "https://api.anthropic.com/v1", + "max_tokens": 512, + "temperature": 0.3 + }, + "api": { + "enabled": true, + "port": 8485, + "bind_address": "127.0.0.1" + }, + "startup_events": [ + { + "kind": 0, + "content_fields": { + "name": "Didactyl Test Agent", + "about": "Automated test instance" + }, + "tags": [] + }, + { + "kind": 10002, + "content": "", + "tags": [ + ["r", "wss://relay.damus.io"], + ["r", "wss://relay.primal.net"] + ] + }, + { + "kind": 31124, + "content": "# Test Agent\n\nYou are a test agent. Respond to requests and use tools when needed.", + "tags": [ + ["d", "identity_and_rules"], + ["app", "didactyl"], + ["scope", "private"], + ["trigger", "dm"], + ["filter", "{\"from\":\"admin\"}"] + ] + } + ] +} diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py new file mode 100644 index 0000000..5f50206 --- /dev/null +++ b/tests/harness/__init__.py @@ -0,0 +1,26 @@ +"""Didactyl automated test harness package.""" + +from .agent_process import AgentProcess +from .didactyl_client import ( + APIError, + DidactylClient, + DidactylConnectionError, + DidactylTimeoutError, +) +from .log_watcher import LogWatcher +from .reporter import Reporter +from .test_runner import SkipTest, TestCase, TestResult, TestRunner + +__all__ = [ + "AgentProcess", + "APIError", + "DidactylClient", + "DidactylConnectionError", + "DidactylTimeoutError", + "LogWatcher", + "Reporter", + "SkipTest", + "TestCase", + "TestResult", + "TestRunner", +] diff --git a/tests/harness/__pycache__/__init__.cpython-313.pyc b/tests/harness/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ad3b7bc4b7e469177e9b487bb331d4299caf9db GIT binary patch literal 660 zcmZvZy^hmB5XW~NJ9c6_IS}HbK`xNM6v+!9q?3<=HA2VvB+8X#uYA@yUTeG-IyxSL zhoI#Zcmyg|LKKLC4iO<*W}SrUrucbgcRcf-ooyJ-3EIiWPjzAu^4$an!+5l9iJ$i* zBS12mSt(@}SdsB3>4e!}GY1?tg(-8v9nbC5V>6hIzLUujmnE>Zy%l1WnQSOCdvmTH@OmJq(rZxiyT_f_feM@mw6>tKPifesSb`S z(W+CB1>VNC_Kr`VKLe1U7l%?wQ)7z0^FNktD+*Q1c9(eTaB*I5gdEg(ilyf-Wc4~% z#f1Qjyfe{BQ0QZm?6U68QI_@$le`ytZoD(m4~^RU1Ga(KhY2bL+rTlH8n_0Y!OXxc z0epkNAVj>3Z)%tGQWl)+5Jzqm?{;I374@>*KNT;=0k3PJ{YF;&&2EUXpY~us$Jkl> z=5;xYTCbB~G~`Xg`3JH^RYc!2KlJPUKGvMqZLK(8bywPZvXSM`h^KHDGtGkf$B17O nO6eD}`avH4wH#_~3F1#c=!381{x_0*CdqF*@U3?)+{W~O^1-um literal 0 HcmV?d00001 diff --git a/tests/harness/__pycache__/agent_process.cpython-313.pyc b/tests/harness/__pycache__/agent_process.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5c75ad0a794e63bc362cc56c3090f92a910a00f GIT binary patch literal 6178 zcmbstTW}NC^{#d$tzMRF*;wG0aLhx*h7cgdsUMK=wlUOeGtC&Xm94c{kR|2rnt}(n z{6$XEF(ku~&U7ZuOlJaLq5bJEX*=n!Ri%}(ftk!C{ov0Tdq^fDB1m zbI(2ZaqfBCvv)ZV@DeCT{`@!fiJy>fuwfUciEzum0pvE3h{O$$Qyk@@h&u)x)WK?= z@~jr9z-lLTLd_4jPPwU@@d^XtDG&89+&SPq<)c2pU6Omie=0x&QF7iLA?-vGw-Cv5 zz+tX?z^n+ZNjhTQ$JmN-hA*B@XY{zPX42Xav^|Nq9-m0XH4PfV*|>fYYVk}K8RDtO zoH?rIh?hF^1lJOP0ka_Nd!dWD8CKFJTFN64(MT+z%Ufh z1#~xSiPXjVJgnuVZr0~xEk6~d0C3h=Ee~r2S!;_F+CnY~)GIYfwG8t~%~Fe0hxjFq z`lVKAg{@YA_0?Og8mUccU}J((RNBO_5MYfAtChA&5r)-Cn`t=dkF^{A!$}bQEX_

gqISVIe_$D&xNj1qD>b9-Oqo#F3jAvCjo1yxQXYb2D+x-&! z_L@W(oOq0OV)zou_;gZEDOZ%#&`iKqz!dVK&?7((s;a8gkbO>A&+<<& zO*oxYd6!8Na}Le4=!&Lmd#IvKr=X6b_GPA{-JlG^c~RH1+6@2do;?QlYRp4%0~BYZ z0R*@ztw+#+pcX+0fZ@)XCKPiRf~KS<4VN}?QJGRSJojkSXf@Zp7^i74G;3XJ%E)Sbe&9I!q^h0GUp#m;m&0 zvF{SwYvaI^g!mDs1@weHXfi_!J$~FK-40~LM zWB7%tA<(pUGme(yY9*um@V1S)(#}!mFsi+(NTO{QW)H;IG4L`fSN>YF6Jd>fhMV&0( zJab$=c5?72gnD;2PT>O43{knNLiEmDjtLO?^+aY`XSG79A?V6g-QbnQ@O;2DtVzGbSAKPSzswT%%>J`OsWPk%EQ)wn`2oss? zHS>~Uw-QED!)5Nua86xLsMPRk)8p1nV>r)dvPzn^Bb#vcsihPp3y~3Rgmxl$5y!eFSQcrB2_>FLscD!8vrr8~ zdXyD2KssSwsvC7{q9dcBMQeLO^*2b#7o6{%>z(hP>o0{HzpV+)Uz)o#pP9=PYT9!A z!xy$M=%37fJX?6-K#pJZx8=p?x1v8cGJj$2LP2aQitTx^y(sR?i#zYvKM)oXJNj+o z)?(w1e>U#0;^(_(9)?@)^xx{Ys#0x3DHQ(B=ZbjYzVokyNL}=9{iiMWS|0F=XGin3 z=W?!x;l?}N?{|Oj^M%HIcxO(0SW`EzzN5Z-`NI>1nvPO1Jb!)u`hxI@|6_kKx;G!) zTZrz*a%c9|><7QSDS%ja4&FNW;mKlndp^9q)YP`bllmPiA_>&a zpL*xiP3^riD`5gNTJ!&_remp|gc^QWag+L%Wx@p-z6{kb@lgM$VYK`Ef$hhFmWAY^AL`vO z;7TK;eF(@eTUtz_40r1 z;Q+3&Yyo(fWgi0MS;-F5+&UHvR>O63kRUpS=nxtYb_u0?T|nY4kQl?x_bOpXK~8+oWi2D!=y zQyf;=&=%K`(zaiig>;a6+z=Q`+3x68Sa($}OZPy#Dvh--A{kjRT39wScYR_1VsJ+> zc>L~>PY3S}7CU?MoxO$5{s;RPJC84l$A2m=iq6Cx0c0a_%K-@vM}jSH;Hr3ZRi%Sa zj9#Ee6wu(aYT_pXp2MEd8WjTDI8Zgr*g&?XwvT!~>dEPgV(;CNPtV^wU+n75clAQ^ zrvjDrcOJ(6p8`Eaca~sOf>o*|GJI6gr)e5KLK2D*StHr19t9(HWj=qW;ufc~B-B=9659H;T`cz)g4T8~3Y@cg5O*STJ7gRi)Z^)_L?M!*qNm-W6u zpJPAeYpgQg|F|E=FYER@AOP`0kXsUpM#Ml&FkKWkVF(nmD;fh7l1R+J3?36m9#TyW zZ3Yr8GuV5PQO>GaMKccoW!HKx@(0TB_T0r}I{4 zv2I(wZd$OnWti zp-frakUQWlS6nL5O%JNhuF`Yu(FUxOD$5U2MP=>cWs_B^d zrZdnDgjKJyvRrvJpm?dRdPl-g!;*6IvSHWG8KhzSr5K~=8wh>{z-W+VOw82@Ij-we z9iN7$C0V{l*oiXLQ~dB|0N<;$41l-E*R}Npp}u6o`pAtFxru@hDb=@`SX-$va%13T z7-_BTK$|KE?Gn>+k zcV;blVh3%+`{<Pb z-^ZJ4(Ew|8L8~+#EVWjRyIH#%+CAgdc6*;Wh|mzjdI9U}b67LgOgSR{M_7q)hBu|E zIX$Ika;i20b$2?YrzWx~O^Y}T{)jqbNN4hhO=TynNm<636>hy23OC6(p`1)$+Z=4! zL4_$kA{ezpnRIGGpUIxfOe?v9ew@-AedL1yi)3U`G^AuwO-(Dwq~S>>r*r8-7R$b5 z@^T@SwJ^bCaxz0TJ)2P#H3#UrWKu(VCz2^$rw{1LZRZjs zj_nmG0CWJx(z0@7fV?KOkqAFBE1o)c?reD0F&K_G=yvEsUjnem8G@!{CpBC(Op!B& z3&l)9-c>f$T~^CBKx<(D-zBTn_1E>4hW4d~_G10Z3yy`}Ph*R*g;$Hg{-V@xvJw?z z1a#R+6}>=J+6?uGi=ybMA3*>?5P$)?R(2XdsdgZ$B*HSeH6j85a#BNre?b1Dx^`Kp zUG@0?*WU7W7WJ+Ym~trFq}``Ci}2V{j#41;LtW1fs!h- z^YRDM`1##hF-=3G^zhv9%J# z@%&UI*?K;13km;G(nAC0);JgVAV2 z0wn^a%;XK}VoFnz1)4Qnx@p@C&pQguOwJ_rtOln_%fgXTb>&^%@W2tHx^^X_UxE{+ zWsRof1XWTxj91y1jFPrHM+6!M3hAo|IuLXM(2zA@NaoQ1vzbvdI-HFSvt!ye8_|M; z5&+EHyu0R4 zuXv+N-snGi)>?P2v>yDI)`QEfhv$ZV;cI@(lh`q?=#4VP9+^P9X(zO(Oegw*RM3eR z7>I-eOo2lc6N2hY@E3Q3bV~XVFUjAYl)yEZ;B5)VT^sFoX+z6Y=NQt!Qk-4fcFk(T zqh66lzZJ)0Uy(}s;FKLPUUem0aWu&lN#@4WKMRgmM>>m6FLQ-&=$Kb4*Y4p!*UMgI|tD*A*Kf5hV z+Q<&lL2Me*U{59;VbV&%@S7qB`BpN??I@Q)n?3VJ9^bQ3wo5LOf;&z1C49d<-!^su zv)Cls3qTIsuyj1(Q($MR?}%@bE7O;)Xnmz@0;>qLom1f$Y6DYGN zvh#9IOj`1b_(qEOV1o}0-l&#F@Sd^R(| z{MP8(T25VbFnejZY_sd4Nj~)WGnGT=@oq-e#nsuY~PKUuajWjc( zYX~@*A*Kt{c?|}t%rSBTIoZrb1LnM30PoUp#jS!^$mbPmcv9(f@{*EDL)itLb9qHI z?ROWfX}G2oolPXbx@p6iR#;6mFB37FOQkjDP8cF1)o`Pq#C$bFU|pp|{2Q#!{1-MM zy1xdDy#hldScN&0hOgY$js(o>WB`c}@+r81T=~=tu4hOzbtQ?u9^Hq7(KZ-7?9Tui zwY=dp9YaIFJB-E<%PI6IsEK`r5i6@56%m> z-zf(6%<-$ihB;x?8(Q+VtkpE!8vN1V?WyIOJu5YxOEsMfQ0$)@S_{-(Q?IHkfwrYU z+d|FXjsMm7-+cRFTQLBCXJ87+do6G^aQ!>Q_Sl!w;A5fM-M?1daI57;%Sv_oQg!KTo*5Mn6=l9)< zT|Zo`IS5+~HQnyH+kdBjzIQqF@`kQ8Pq%j8L+MaQ2LRZnm23JWq4ec|8gpRUOpr4`2nh|A=Us z*lBD~5#$gcUPBL=8A;PVI?XwrQ%JM$;OA7t6px3qI)&h>BJs@2xE>nE3d&9;u7 zFYX@rtbOi-V(@fPI{kZMkz}z*E8AcmD4h+KvxXK_dKp?bv88|wznLu_bN@-WkBn^< z+OmN{TV%5*x^HRc*%z^=|1%e|=TuQT^?PEEQ=8FrW!ex58s6TxW5%8O3hs1jndy91 zd7eo#IAj*V3w6hDal)M9xl^?Xd%lcSov$tJJo+M59sKMHvg%AxI%Bd5Lby^Y9kzF& z=_VN~A+~zqiI)e1R?h^7mK#C~9**Kx!5bb$9s|OJ;~-~y&S`L#!=#&qJnV=X_(~~g zsY%7;s-JNcqVTCHg+>`OA!ZvJqDh-$E)TJw; zY9Bl^D#l+~`;dwJn9(<`efq#ZX%@)H~-Y zo!9vf?!9^MjpFV@4_*K4FUkpI`1zuA{&|KYi$DO}`W|4xZ-}8e7DMx8QCPW+gh?=M z#qb0VPj?O$)q)L-yUm{E@VTfKC^6Gk(fL*{GNZ4@OYbfASZ0gSX{<5v?r69KDX;I+EPj_ZHE39N+XR62w2f5dSz6&N#$Dl?ht^ zAA4wpSXgx)akg3cBv*==+#~EzP>jPUgtCNUNJmjQDYA$jLU0%WgaYhQ2@eXzdzYR< zfVU1EM!;U1y0G*O1m8q3g5X;S;s|5_@N^o+@+blf8qDwk%>%`QL6ZP9)MYC?uwny@ zHu52r9)E>Lv{xAaJL$Uo_vEkfPzRY|7Puu%8eRmGlv7C3)QU7@0?%_L!7xvwc z-9B90e!N(7{1?9C%$GjFJ!1yrbAIp9L*(*|JI&)`7sK;IN#A&tC# zA|jb1o1tn3$>Jo0fI53y#P|MOmhJ+aA&lfyCE^$viFhc6j4akPCC$d7Q}e9R{v0%x z&P?cI5W;qZ;WN>N3-k{GJsT{cFV4V+m1Fg_uM4KzQJREa!|67b<#0uI^Rg9Z5z9*K z4zB3zq<>t(BNyn0ngxmDi&MUZ0K*oF z@}qAfpa|YU@E(HiA@~7;Dg+(`Xg}FOqkFJ~0Ljn>0eqi)<#39xb6-_$6Z@a^bK+}H zoSb+7!QfX;K^*43@(JP_44wt(5Z~mUN|JbjGpEJLBDMI5Zn6OGF?^zL7)xmxr@RmL@DjG%U8%}*DpHZhM`VrKL;7tVhu3-d^ZT_fc zK6JQ`zdNC@FO%%$#N7C1W*E`|^Q+<^dIu)La6tPu00?h6?pI{zuZj1U#Pds1!~W`i zMRxo<2?Jj^_w55u2xLzrEazDHu|yo+Ck~O@H#f6RAY135;W0Kmts3UI6Wo6hmNBmV EHx*zq_W%F@ literal 0 HcmV?d00001 diff --git a/tests/harness/__pycache__/log_watcher.cpython-313.pyc b/tests/harness/__pycache__/log_watcher.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29a50d3efbd7236e7f2c3ccf9cd593ec498d50d6 GIT binary patch literal 6298 zcmdT|U2qfE6~6l?t!2qJ{*i={v9JwB223#|1ji&}Fpz+45(z)-aY5s@1DE&TrLNJlKR7+<0q>L`4%f?5eDAF> zGNnbecq*w5LtT6+s!hUEKAOh1Xd-4v8B{PH+`V4{bCtYCC>J8I7!DRv8>$QO&E&Lr znuF!62?@wk2{@5)HsD|(&}I+Shir_KLk%H2>&)xeIdxcSswE_uq zPZjtuNm#9~_!9-Y5we{?kEQEJSSxsnkWuDM)D?7^8dsiej=r%({m(HFUi&I_FQd6->+JIeBETKdx zt!2ZGvG~))2%SpOM%Jx@Ms1O!xHc6X>8Ea>bR+iG6~zx5h%#yd$a^F&J7--puAJ=4 z$iAH1mXX_T?7G#lgxvF1=emr%ZsFLH49_lXbj&z%vNt1pA=>VE{5enOZBOS7?c?){ z=Ra%Acn(5rIdPnG#+j3wGjj9aWglG&6P5%SZafxbe*xWbn~jxR*0-j%tD;=lL__<2*cmu?tsIe|`9*}nOK z>x0(@bG{uJ-;SJbXU4ZP@2s00oEgkHTQbg;h3JyA<+gM4U4gXhn(lktD>hKgyfOgv zfAI=hASA#h5|Uh)s|W(MtjTUiNKux`0;4sm0!mtdAB06~(6epi`tsG7Qx zLRc_;J@yOc>&FL^Anv)cVrnp@qoGx$*OZAu)s(ITE@8eGASb$oy<5}jvi{Yew8CyF z_RA=u0xLnU$*nvYG(yyu7xQfQlcBHKn<9>t6`rs)tuJeaYH-8jZDAqU{&;K1Zq8&r zSw04No0)&HIeuWwjDc+qaYIJjJkhojWCPhsOz(~Xo{sZ=(h6_rC&T@Fg;p}oZNTVz zqLy)Dj10)bI!7-^Q`-2pXLLJN(urtHfnz{clCc!aBWwz#DXLP03>pBUbB;j0aVm(Z zw35UeLOEjqhz(!^FhmzMC6Ul&%tbIo(QWLYQ4uzxnr??PM~z=l@aRaX05J$dx}+u) zCC$5_?^!qZ?&WuLo-G;A zmYio>#(2PPZ>YCU{Poo(U-y#l#5{MW(K~ne%Heq}*SaIqx?{Pq=c`6vzG>~;$Oj{N=sR-x zNUm{Hrg77P_R-rPzI}67w)2_g#yt-O;`ZNdAh4d|wJ*B^cfG__Gka*}(8U)Y+F{Vu zk^kJaSxt{rwB5)1I}f_ZU));`^a)>Z$bZr2IOwo{X|n_WrK|p6s}L*BuPj&wq*@i0 zaTAVO5K#H(Gq+Y{@C|dFpdT(oC7bR^8)Bjm<`ffa2ptS7T}UWN7F!fEumEoal$ea3 zRaAVdLKx`|2;@Rytv3}PbXIc@j8?mVKuo#T%#QqQ|qu*|?Y{gqJWVOSs0A!cK)9jC(pD7$&zF zw-#N9kDW6w9xOG~YeB%c+W<}^V~XBfWdE|hr$M9I2gGthzTQ2z?eex!HNlG7E0Ov-;w${%Ya#YDfQCdr2;6h4%#* z^d!f8-|+rnrZs|CFzk91mSP0)Cg6%QGZ)$?3fDyAZs;wuoYnIz%C!Qq>}FXg+t5*n zSTqer%EhLw!tWft_TiE%w`G`R7;=@20yRpd+ZP<*)F_8nh35zFoyo0h)Xuqj)+_qR>+jUR7a+q9010&;5ZDsm%ffwV;TwUQTBhq+!@hk>av#{Y z?;f5wi;jLGeqsai|6*Qg^hAGfm_}jo^b}g7)QGOP;Cm2!T&jRAB|R^oXg`qeYsg+^ z$j+O`7PlLA>|K(3!H&ID0WFoShnNqbV+AFE#S;V21b`6N`(r{n2ps|iaQx$9Y0Tg; zX=ZU$nh8us)rgS>mt+Df&4VDPwg7>hT3dfJ}!faOXyifC@AzFelM4%2o9b%>hpAH?7$l27`R04TtB=W{oG*RfOjYP)dknba6 zCR33J^`P>NNV<`9AUTW#!O;K$giz*1I)U72AbL{-zIVbW@K^+ZnZ{2|!AH$VWRWmG z7|4sq3q^u&tv(0jC*%Qdm!7#_FG(%;YXm89-%%&I?|1Oh&ik?;ZMpB@rHv1Od+q^n zdmqTW^z1{KlXm};05Mdf!hi$caC{1U1aC+&H0diy0Nn^hXvgnx+GHYr%J|G{)6S=1 zZ&5^Fwj_EEIs7zbWJPkHVXx=tJJ5!4ueJlZD{vh54Qcz9RDDhAz9!Y*5brl+-9y`c Mj@$h&f|M!wFY-W!FaQ7m literal 0 HcmV?d00001 diff --git a/tests/harness/__pycache__/reporter.cpython-313.pyc b/tests/harness/__pycache__/reporter.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fdd8f661dcca0f891406aeccb3f5b380124bb12c GIT binary patch literal 5814 zcmb7I>r)%o6~DWBON$UMfiaK8k6_t6Y*G_UzzKdO0YhvpO6_1rF?TksMGohc7PXB?WqJi1ioy?^D;BR*9X(s;Eb9S{7sB)&g znmIc6-gEA2&*OLR)3P!rfpYVYe~q=c2>AvpMltIQ^Yj`pcZom*W{3yTs6Nu9vk1pAO{kfR(=>c}1VuJj&`b z(Xt8F=X!({AEUZLqL`3Fax5WABT%=UOr%6PN`bY8q)03*17|)LlCQzPX+WHM#OR97 z(WoT*qf#m^hxLJVYSRChrx$>^OD+-02n6=PP?O*k*m`mu_93_glg3#DPB3en6*!B= z*@QB|Em)Di&QQBh4mDe@<`C@EIpOemRJ%W#Oi)n|HD*@BdTx)>Da~mKeK>SuiqVVr`6nKLZ82GWrSe{G}pJ`;&$Evnu zh(<+OQti=OF-Z<4Zul&!jcVrwjWi_&C!=zR;tZ-Yk&=@sIT(pi)iQYl{(NTDEJfqv z)C2vr0!bwh3Af2ZQFqC3{8Hz&5EY}6)Jb!P(UF`|?ZIG7jLE^ET49_~htX7qgLDG< zkZjd9-Ql+C_&a3}>YFm@Z2iG$^PTc7J2z8)yL{8$xNdJ;imZ+*_Qt>2d!K>|3FnUv zD!3Bukb8l-OUCkj0#lDim$NqL%$F*1G{+mqy^r|VksZUL+ROOudVtwWco;fBs>qvg z4v7kXey9TTQ*zhBlVLa=7f4T#S4`uKk3mzmN?_M12|fnfaad)jgk$qOt${z)(Kitl zqqmauxOy;uN>XZaGDN3>Ve|p1qc5HahvL%lj=XF&s7m-D9>D=B-tK!1O9S(_es$|! z!}r>!`B_~(<6|kFjrQey4AoTdaB>s7cTS?#zF_t$oT?VchvY%Icir2)Tz$XcUPIQ~ zoh|QH?A^2u`gR;08m9I|K7r}**!(zcQ8B)N$c$|_bX;*!F}{(=j@gVZfiYTnvaOYc zx$K5a;olFjuDA#$!E8h#H+h*YIiBVJ7|;4YjAtv%BG|#0x(E?Xh4lwae)Nl?Vw{xV zD2M17U`p*KU=m#0c9RbyV30r34YDwczy;WyJBDCUHj-{|_V@FrVv$f-o{IArRe3$E zN_>C+bBjME_VQ{O1`Ihi3B94oq-slvH^jty;&aacn6b=X4M|Zx;6DqJcI1O88dnKK z9CI=xNve4~6pO2tD5VLi+He>MI1pfN#FA;YuP4p+@)VsT?Hq+%5Qq5E%lmAUhbGk& zkBXWttCl2Drl|*W=C6A5 zcE6+D3v)`VKp;SIo=vWKoomi=Ez`^cPu1+H`LlCpv!0e|%L99v;%?utcK|bMnT^a( z&P{H5Ti3m<%jIjkvfh(Q%_+rwdc%GOB|ZF7_3s*g+ql`%wcgUT+PgNDZFx)K2Nn0w zhW#u^Okcg@_^P6Rdf)+9F?)Xg(%hwAURgS!RJShC<(B*VSN5+mt4Dr+Xt`0TeC127 z=ZT3_9{(rI?93d^v?}gBf4A@b0e@UG{1LWQT&l=BjKp>li3Pia%|nnY5jh3sRZxsT z&oIQ30CSEV2KUE+C$b_FU=QSmjlfJ3q>$KzZT=O&8D_xpnQi?huU2tj~vH9Ne?19grp5fy3X0&&W|L-c66|qIKgZAm~U@a znX%{WW&Sza+xwj5`R5*<*L_s;J_#Jlp(pj-I2NX2)sjjkqmcQfR7{SlW-&AwRc+wv zQfMNoazI2msB6;>koTE1U!#qvYq}bWNVq_7+8urt0dx_IOaJ9(x%L)Nk(PmgpVc=l zwtd`|bqgQ2&7Qx1VC8_)?O#9WpSiivHf>Ve!avF@x425p<$N3VgI`rt>y9!o8=Aj1 zcWqvr6SLlzlp5VzT31?^cdhPOJ^$G|pT4vDrc&9z!5v4xIx%~5erj%N_J&fqXM@|j zRqmPjVCI9{A3X5XD0K(^P_=yi{*{$0Szn)WpkMJG|I+i?w+>QS^F7e=QAR?unyETj zYyQLvB)r{f(OTgQn5WJk@U#Ha4*5vLT(w;nRIxy7i5v*tY07&i*x=5rVOSA(Bft~% zMz9&vSRNq4m?bix;YWtHz;u{IbATlzlz7Rt5`b>8p0*O#sBx!1Q8`n5H4 z^EZx_zV+Ik2erF1r#2fp*Bd&s4M(!IT~BQ0D(AHAv6DEQGu&00~oG`DU)4Hh|VM{}z?rY&@(<^-|sFf@b-nxSQYY7h$=ipw@b1F7O-e16e{cZSkt zS1SA#11AWzB~7-ZZV^br3nGA?5V!y3>+ zM=C1N?HJX; zG}JX%U=;vUL#164)zCE5@GeV_A;HUXn&l6qO&wk1X;$)41a`C!h{`zWTUZ$c;`8VM zn8yY!f*(c>)4vYJts%e+VuWfaq92=&BY6#o>WQSN_M#J%qTz%Xk?8B#H-Kaq1-6AE{7%iWZb7vEoaf3x}Udh_Ac`n8MM<~Nk4Hx=%! zEzZ5=t_Ex#m_AAvdDO~G|T5ABDMCYQJHCiq|~-4T>A@0_$Zv&x7fDOR?z3%k88OK=d|Tg~cliSF(-mN<+u;)zym6YCo-Ab*xpc4Sas)&u7+pzUcZQw0Y*j`k4#L#Y@WR z%gW`Tvg;=sT*w%uD-&9bEyOZo$}ZmqcQ6N%wmdcS+#IJgcmBz=I`G-Zrz6?JCzZ}q zO5^Db&ly9v@=LDaiHlS=J?3&*ZgzD3;@rhd3QlX)hB>ZB%W>LW1)zbJ<30H4(Zoi4 zcFnfZo;bW%jTPSaE^FOsH)j7aNv3T)yaQg=v{e(-RV$6kDJrUF0K`#v|G11pBAhb# zE66ixF8BV`fw>*7jGF0PLSan<%?+enhLn6u_POvm3lT9qXE}qx$wVX-N1h7?Z>B== zTu*s0I3A;tjJ7K#fb znj&!12;NR43=uyk&sekN)n^=IdF7eaVcGZ0>#)4e{M&1{RD4%mYpHlbfapV@w$F_V z9#6r^z=A140XhOib%hi0cr>g%xoW4Rx*{Pt6plmO0ngKr762ysT0C}De;>5UQ%OK& z6ce`U!kb1A-UCFq6Cl>q*BU_%Ff YdPwRX67Rpr-tVjf3{&wfL8^`OA3Qg2$p8QV literal 0 HcmV?d00001 diff --git a/tests/harness/__pycache__/test_runner.cpython-313.pyc b/tests/harness/__pycache__/test_runner.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f942e297ae86ad50c84b76387f2b14c88743766a GIT binary patch literal 7106 zcmc&ZZEPDycC+Mi`TZf06!m3^resM>B+H58bL3u(Q!LvuCDR8nY;q^L1Wm4Fp6R7B zyYfXkG#9{Vjt0_NKERNT{IL9|R&XsMUk|YID=CVixkG=5Q4sEJl*^&l0yWSdawRnF z^+)^OESHjL`YtUBbRf;XnK$!h=FNL=-h143yB!3|$WQ+!H`74KZ?It$n+|Z>Jscr7 zNrEJ}F%sh_7e?4LW}+tcHB&SD@|1_KdCU?Ms1UPKD;vj;*R3vuD#vI+IV!vSyjiiTNyyahxq==knq! z^Rpt1Fb_-fn(ah7pHE-Niw}Y4NXIoiBTCA*X(1!ZGK}(!@U6l7+%;2cW)5rU>N{1TK<;OEgh_#vJ7}TjEk~ z_7zc99(rNWdn6vUXtq>JN?#UJDb0~eU1s|R*p*7XK9|l{Fy2&ZI!9$CpOZwX0O(*U zB_j_qskEZd+=V$slv61Rf=0ICpJ?sSp~!<5(^P`v4Pq-r=Ojs_2WRIguFfh1WW3k6 ziN@P#$mrLcxXTj-NfF;r9y?}5PPpxH82k7!4}k1W@dL+#Aq#^9z*-q(qn3mn zS`OB7Qenmwb!#@ZXj&E@HUSCEEGtyA$a6VG)Oh5C=FEz6hURAR@-+cRpOQ4wv~)`2 zFBA%S%}2%8=W-Ny0sl&xDvTrXsFgP2$^2O1_0S;F2d?gsC177SqHt+8P3J~4HrtE} zk#Yp8ACX@>Jy&C@!+rIP+7Q^_1F8+%`y%UnU@`j{AJM1d%*Xi0LkARYTY>t2$Uo$+ zz&K_F$cE6AB|@fzgnfvk)`SCkoz-3&>$N9b(CcQiJk&AcjruePa&AJD=km(KP9Q?} zVby^ZHZ=idYfjdzmqB*YGa@KfzK~WlUv`c%#Y@RzrXXcyPzP^ToK6Fw)HEnhfzB`3 z^Ep|WgsVHRxnQoOq(syldBKy-Wf-DA3<`N(vt~sloy*IWJ95^$BYFTgg#xPaNS7XD zAX1`(h^lv6cvgxP9!5R})nm8i@?ITN-M*{iYE#<=-}cLnuGNuJ$Ix5ji_r~!sM-I`5m{&1Mi=Zedx2Z`*2Xpj5F$@EkNYqGf+_{RGN@s7Kq{l zqFERZsT*N)zA!_r2=Qq;V=zv5I^*hb2Nz>2;({`YH*m+*y1}=q&d>%QQtke$=RmIC zKEuvl|JdJxX^$T}VM~}=$@lQ7vL_q~Gs55FK+T;Ap7jd}SHi*|YeJy587nBfE{M!5 zJ$xQWX7Hwsg0BkjncIE=fScqsU*5ow*TVUVxh3SaL;1@41Ms)%2Rxm zD3YMx!0ISeV0C#}%um4scLXkIUo^w)80h{Y*TBF-gqw_Efk&(-95jEKQjzeu~ zf5)l%o8Gahf#yH8>m00Z55@`b&kpq|d-^A#sh5epqbhw7lH`t54l0=jiDaIvm=8iG zkWx*7@urE2bFmj?(D0MdpAaJPNq&MkCws>LMU=6clBllVSHWaqpJzcUlVrkEGZ>l^ zz9;*Pndu&wRd?mDPwNkW8)@9ZG$FsAcOH0#>aXY_hs_38BummfflotCfnDJjg?3lv`N3z}O8E04TpoxL6KC7X4A?=c#fDYcCdF${(erm$Yht9X-$A4GNcZib8^#6-O!>)vcm&J?b|X4s=EYpr`kz6Li0bLBsW z>MgSA36(tuKJgq_p4spmESlADq-fa;w3P#Wp9K0={O`BF*Zx`H8^saTA6go^F;w>V zmi)ad*0TRl$$#jVzGl_yzdnC${`wEE|8V(*dqa1JR!1J3xqs%tx%=lfx=z4=&0t44 z*#Akef7QA&zs9X~e;EBBy4JQnaB?Gfs(4CuHC!LRHhwEmZaGwHIkeie(emPFuAxV6 z52T3>JCaMMo=2wd0`06^EG=ZKa znVyk}s=gXy>uavDPDMv-yQ2*Vr15Lgs_MuHoW(qWPfATm@_S_W)jhInifi^6QPG7e zDrnY=X&D_%Isp`z_-Q-}T*XaD&(4AnMlDR>XdK39P9`}z0nH3I|03Q_>2KI_&fHY z>RSMBz$V*co!g#3@4BY!z9M0==bq37uPM9tBw0J~{oG$O0Gi3_TQ&z$J9|xXByGmo z@wH4i&T$E+i%EO`o3}2Y4Vb}lT35bCJaAV6s-H}bUUEZAmYwr0H|(hCJy)VRq7)A{g%>DaLab9f*eZDpZk-BVV{ zYUUi|(|9~1OR;Fy%Ho2L&mYp{Ou=-ijqF z&?;+I@Kw+~(d;MR$cTDYh^OK#)trovDa@BaT5y))1}t;nuF$Ag_g2Ok0oVxSE@aJ| zQQpu@N(NkGePvVO5~Q9mg`^3{L@}$g6>T5mvc`#;&B%u^PRaQ4>h5&4)?-L3WZvMM zd987e(^KjDHcXQL1dKF9klj5+>!zn=IdHG-ZX5W(vHP))M*eQ}uSeGtua*;2CHV84 zFPi_cF;sk6ZEn3|yKP%sSZ-K;{@&5MN0*~3&;3Q>XMqPTKW+J|wl$^n+=-2Wk@fzQ z>&!*=Hk7>)_)~p>CC3d%*%vAKA}i0YZ{+o$gDx)-_|QoEi}yGPZ5Lyw$7 zOY4r=7HoRdNP_!*?ebk8y*65Qb(UP6%fhFw-pxSsQvAp9jX-ztruDbl!ZQHN5cc}*k)cpehJo4Ha8h6YN zw`ZrB_(F@y()^8i;9RooJ^ZQn@TM<#i@Vixr~h{Uy9d`>qAQInFTX$f-ssA))rmFJ zht?0QtMeb3KRW%<(0bF^vM>Iqz!87*7Uh6e(ev*+Cpi*m-zH}FKDDKDQP^zVcPssF z&!QFfbkq8uoBrNyVu91PL>|X2S~o)-x0E~cx93;-*DU4G@lxnGBoLNjH)7>LPbtu| zLf7&8{D1w&rXLnyM8Zg< zAG_B7aI$=4s&r)PDRj4-K=ijVcuM4Nh0f8a>Eozlyv6eIVec8M<>yv6z&{TR^RW)g zFFLUCi)V+s#sikW4_KgKq**YzvYTs(M;&^G1(OYWHwGiRAw(Bj4`^9JOQ51MMp@Lr2at5>B z1z&aA1XKL;QG4Cw*-JPL9#z6(6*`Yly)DumpFS?H2!6 zO)ptOEW%&|qn()MpPqxmLdu`Q8$jsGSV6Exs0bA#xs4w*M49r~4emK40u}f*LuNfUo=vcgd*cUvz$)IB&NQv|BAtKi}VsMgcb}I znw!mwvG`s6>dA%U^ld<3&?G~4glw5Pj{AZ{ zenWhp6VK None: + self.binary_path = str(Path(self.binary_path)) + self.config_path = str(Path(self.config_path)) + self.log_file = self.log_file or "tests/results/agent_debug.log" + scheme = "https" + self.base_url = self.base_url or f"{scheme}://{self.api_bind}:{self.api_port}" + self.process: Optional[subprocess.Popen[str]] = None + + def _command(self) -> list[str]: + return [ + self.binary_path, + "--config", + self.config_path, + "--debug", + str(self.debug_level), + "--api-port", + str(self.api_port), + "--api-bind", + self.api_bind, + ] + + def start(self, timeout: float = 30.0) -> bool: + if self.is_alive(): + return True + + env = os.environ.copy() + env["DIDACTYL_LOG_FILE"] = str(self.log_file) + + Path(self.log_file).parent.mkdir(parents=True, exist_ok=True) + + self.process = subprocess.Popen( + self._command(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + client = DidactylClient(base_url=self.base_url, timeout=2.0, verify_tls=False) + deadline = time.time() + timeout + while time.time() < deadline: + if self.process and self.process.poll() is not None: + return False + try: + data = client.status() + if data.get("success"): + return True + except Exception: + pass + time.sleep(0.5) + return False + + def stop(self, timeout: float = 10.0) -> bool: + if not self.process: + return True + if self.process.poll() is not None: + return True + + try: + self.process.send_signal(signal.SIGTERM) + self.process.wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + return False + + def restart(self, timeout: float = 30.0) -> bool: + self.stop() + return self.start(timeout=timeout) + + def is_alive(self) -> bool: + return self.process is not None and self.process.poll() is None + + def pid(self) -> Optional[int]: + return None if not self.process else self.process.pid + + def return_code(self) -> Optional[int]: + return None if not self.process else self.process.poll() + + def read_pipes(self) -> tuple[str, str]: + if not self.process: + return "", "" + out = "" + err = "" + if self.process.stdout: + out = self.process.stdout.read() or "" + if self.process.stderr: + err = self.process.stderr.read() or "" + return out, err diff --git a/tests/harness/didactyl_client.py b/tests/harness/didactyl_client.py new file mode 100644 index 0000000..fc17732 --- /dev/null +++ b/tests/harness/didactyl_client.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import json +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any, Optional + + +class DidactylTimeoutError(TimeoutError): + pass + + +class DidactylConnectionError(ConnectionError): + pass + + +@dataclass +class APIError(Exception): + status_code: int + body: str + + def __str__(self) -> str: + return f"HTTP {self.status_code}: {self.body}" + + +class DidactylClient: + def __init__(self, base_url: str = "https://127.0.0.1:8485", timeout: float = 60.0, verify_tls: bool = False) -> None: + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.verify_tls = verify_tls + self.ssl_context = None + if self.base_url.startswith("https://") and not verify_tls: + self.ssl_context = ssl._create_unverified_context() + + def _request(self, method: str, path: str, payload: Optional[dict[str, Any]] = None, raw_body: Optional[bytes] = None) -> dict[str, Any]: + url = f"{self.base_url}{path}" + body = raw_body + if payload is not None: + body = json.dumps(payload).encode("utf-8") + req = urllib.request.Request(url=url, method=method.upper(), data=body) + req.add_header("Content-Type", "application/json") + + try: + with urllib.request.urlopen(req, timeout=self.timeout, context=self.ssl_context) as resp: + status = resp.getcode() + text = resp.read().decode("utf-8", errors="replace") + data = json.loads(text) if text.strip() else {} + if status < 200 or status >= 300: + raise APIError(status, text) + return data + except urllib.error.HTTPError as e: + text = e.read().decode("utf-8", errors="replace") + raise APIError(e.code, text) from e + except urllib.error.URLError as e: + reason = str(getattr(e, "reason", e)) + if "timed out" in reason.lower(): + raise DidactylTimeoutError(reason) from e + raise DidactylConnectionError(reason) from e + except TimeoutError as e: + raise DidactylTimeoutError(str(e)) from e + + def status(self) -> dict[str, Any]: + return self._request("GET", "/api/status") + + def context_current(self) -> dict[str, Any]: + return self._request("GET", "/api/context/current") + + def context_parts(self) -> dict[str, Any]: + return self._request("GET", "/api/context/parts") + + def prompt(self, message: str, max_turns: int = 4, model: Optional[str] = None) -> dict[str, Any]: + payload: dict[str, Any] = {"message": message, "max_turns": max_turns} + if model: + payload["model"] = model + return self._request("POST", "/api/prompt/agent", payload=payload) + + def prompt_raw(self, messages: list[dict[str, str]], max_turns: int = 4, model: Optional[str] = None) -> dict[str, Any]: + payload: dict[str, Any] = {"messages": messages, "max_turns": max_turns} + if model: + payload["model"] = model + return self._request("POST", "/api/prompt/run", payload=payload) + + def prompt_simple(self, system: str, user: str, model: Optional[str] = None) -> dict[str, Any]: + payload: dict[str, Any] = {"system": system, "user": user} + if model: + payload["model"] = model + return self._request("POST", "/api/prompt/run-simple", payload=payload) + + def fire_webhook(self, d_tag: str, payload: Optional[dict[str, Any]] = None) -> dict[str, Any]: + encoded = urllib.parse.quote(d_tag, safe="") + return self._request("POST", f"/api/trigger/{encoded}", payload=payload or {}) + + def raw_post(self, path: str, body: bytes) -> tuple[int, str]: + url = f"{self.base_url}{path}" + req = urllib.request.Request(url=url, method="POST", data=body) + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=self.timeout, context=self.ssl_context) as resp: + return resp.getcode(), resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", errors="replace") diff --git a/tests/harness/log_watcher.py b/tests/harness/log_watcher.py new file mode 100644 index 0000000..caef7d4 --- /dev/null +++ b/tests/harness/log_watcher.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import re +import threading +import time +from pathlib import Path +from typing import Optional + + +class LogWatcher: + def __init__(self, log_path: str, poll_interval: float = 0.1) -> None: + self.log_path = Path(log_path) + self.poll_interval = poll_interval + self._lines: list[str] = [] + self._markers: dict[str, int] = {} + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + + def start(self) -> None: + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self.log_path.touch(exist_ok=True) + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=2) + + def _run(self) -> None: + f = self.log_path.open("r", encoding="utf-8", errors="replace") + f.seek(0, 2) + try: + while not self._stop.is_set(): + pos = f.tell() + line = f.readline() + if not line: + if self.log_path.exists() and self.log_path.stat().st_size < pos: + f.close() + f = self.log_path.open("r", encoding="utf-8", errors="replace") + time.sleep(self.poll_interval) + continue + with self._lock: + self._lines.append(line.rstrip("\n")) + finally: + f.close() + + def set_marker(self, name: str) -> None: + with self._lock: + self._markers[name] = len(self._lines) + + def get_lines_since(self, marker: str) -> list[str]: + with self._lock: + idx = self._markers.get(marker, 0) + return list(self._lines[idx:]) + + def get_all_lines(self) -> list[str]: + with self._lock: + return list(self._lines) + + def search(self, pattern: str, since_marker: Optional[str] = None) -> list[str]: + regex = re.compile(pattern) + lines = self.get_all_lines() if since_marker is None else self.get_lines_since(since_marker) + return [line for line in lines if regex.search(line)] + + def error_lines(self, since_marker: Optional[str] = None) -> list[str]: + lines = self.get_all_lines() if since_marker is None else self.get_lines_since(since_marker) + return [line for line in lines if "[ERROR]" in line] + + def warning_lines(self, since_marker: Optional[str] = None) -> list[str]: + lines = self.get_all_lines() if since_marker is None else self.get_lines_since(since_marker) + return [line for line in lines if "[WARN" in line] + + def has_errors(self, since_marker: Optional[str] = None) -> bool: + return len(self.error_lines(since_marker)) > 0 diff --git a/tests/harness/reporter.py b/tests/harness/reporter.py new file mode 100644 index 0000000..264a3be --- /dev/null +++ b/tests/harness/reporter.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from .test_runner import TestResult + + +class Reporter: + def __init__(self, results: list[TestResult], run_meta: dict[str, Any], output_dir: str) -> None: + self.results = results + self.run_meta = run_meta + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + + def summary_counts(self) -> Counter: + return Counter(r.status for r in self.results) + + def print_summary(self) -> None: + c = self.summary_counts() + total = len(self.results) + print("\n== Didactyl Test Results ==") + print(f"Run: {self.run_meta.get('run_timestamp', 'unknown')}") + print(f"Agent base URL: {self.run_meta.get('base_url', 'unknown')}") + print("") + for k in ["pass", "fail", "error", "timeout", "skip"]: + print(f"{k.capitalize():<8}: {c.get(k, 0)}") + print(f"Total : {total}") + + def print_details(self) -> None: + bad = [r for r in self.results if r.status in {"fail", "error", "timeout"}] + if not bad: + return + print("\n-- Non-passing tests --") + for r in bad: + print(f"[{r.status.upper()}] {r.suite}::{r.name}") + print(f" {r.message}") + if r.agent_errors: + print(f" Agent errors: {len(r.agent_errors)}") + + def write_json(self, path: str = "results.json") -> Path: + p = self.output_dir / path + payload = { + "run_meta": self.run_meta, + "results": [asdict(r) for r in self.results], + "summary": dict(self.summary_counts()), + } + p.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return p + + def write_text(self, path: str = "results.txt") -> Path: + p = self.output_dir / path + c = self.summary_counts() + lines = [ + "== Didactyl Test Results ==", + f"Run: {self.run_meta.get('run_timestamp', 'unknown')}", + f"Agent base URL: {self.run_meta.get('base_url', 'unknown')}", + "", + f"Pass: {c.get('pass', 0)}", + f"Fail: {c.get('fail', 0)}", + f"Error: {c.get('error', 0)}", + f"Timeout: {c.get('timeout', 0)}", + f"Skip: {c.get('skip', 0)}", + f"Total: {len(self.results)}", + "", + ] + for r in self.results: + lines.append(f"[{r.status.upper()}] {r.suite}::{r.name} ({r.duration_seconds:.2f}s)") + lines.append(f" {r.message}") + if r.agent_errors: + lines.append(f" Agent errors: {len(r.agent_errors)}") + lines.append("") + p.write_text("\n".join(lines), encoding="utf-8") + return p diff --git a/tests/harness/test_runner.py b/tests/harness/test_runner.py new file mode 100644 index 0000000..5d7ae31 --- /dev/null +++ b/tests/harness/test_runner.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import importlib +import pkgutil +import time +from dataclasses import dataclass, field +from types import ModuleType +from typing import Any, Callable + +from .agent_process import AgentProcess +from .didactyl_client import DidactylClient, DidactylTimeoutError +from .log_watcher import LogWatcher + + +class SkipTest(Exception): + pass + + +TestFn = Callable[["TestContext"], tuple[bool, str, dict[str, Any]]] + + +@dataclass +class TestCase: + suite: str + name: str + description: str + fn: TestFn + requires_restart: bool = False + + +@dataclass +class TestResult: + suite: str + name: str + status: str + message: str + duration_seconds: float + agent_errors: list[str] = field(default_factory=list) + details: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class TestContext: + client: DidactylClient + agent: AgentProcess + log: LogWatcher + args: Any + + +class TestRunner: + def __init__(self, agent: AgentProcess, client: DidactylClient, log: LogWatcher, args: Any) -> None: + self.agent = agent + self.client = client + self.log = log + self.args = args + + def discover_suites(self, package: str = "tests.suites") -> list[TestCase]: + mod = importlib.import_module(package) + tests: list[TestCase] = [] + for info in pkgutil.iter_modules(mod.__path__): + if not info.name.startswith("test_"): + continue + if self.args.suite and info.name not in self.args.suite: + continue + module = importlib.import_module(f"{package}.{info.name}") + tests.extend(self._tests_from_module(module)) + return tests + + def _tests_from_module(self, module: ModuleType) -> list[TestCase]: + if not hasattr(module, "get_tests"): + return [] + suite_tests = module.get_tests() + out: list[TestCase] = [] + for t in suite_tests: + if self.args.test and t.name not in self.args.test: + continue + out.append(t) + return out + + def run_all(self, tests: list[TestCase]) -> list[TestResult]: + results: list[TestResult] = [] + ctx = TestContext(client=self.client, agent=self.agent, log=self.log, args=self.args) + + for tc in tests: + marker = f"{tc.suite}.{tc.name}.{int(time.time() * 1000)}" + if tc.requires_restart: + self.agent.restart(timeout=30) + if not self.agent.is_alive(): + ok = self.agent.restart(timeout=30) + if not ok: + results.append( + TestResult( + suite=tc.suite, + name=tc.name, + status="error", + message="Agent not alive and restart failed", + duration_seconds=0.0, + ) + ) + continue + + self.log.set_marker(marker) + start = time.monotonic() + try: + passed, message, details = tc.fn(ctx) + status = "pass" if passed else "fail" + except SkipTest as e: + status = "skip" + message = str(e) + details = {} + except DidactylTimeoutError as e: + status = "timeout" + message = str(e) + details = {} + if not getattr(self.args, "no_restart", False): + self.agent.restart(timeout=30) + except Exception as e: + status = "error" + message = repr(e) + details = {} + if not self.agent.is_alive() and not getattr(self.args, "no_restart", False): + self.agent.restart(timeout=30) + + duration = time.monotonic() - start + agent_errors = self.log.error_lines(marker) + results.append( + TestResult( + suite=tc.suite, + name=tc.name, + status=status, + message=message, + duration_seconds=duration, + agent_errors=agent_errors, + details=details, + ) + ) + + if getattr(self.args, "verbose", False): + print(f"[{status.upper()}] {tc.suite}::{tc.name} - {message}") + + return results diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100755 index 0000000..028629d --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import datetime as dt +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from tests.harness.agent_process import AgentProcess +from tests.harness.didactyl_client import DidactylClient +from tests.harness.log_watcher import LogWatcher +from tests.harness.reporter import Reporter +from tests.harness.test_runner import TestRunner + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Didactyl automated test harness") + parser.add_argument("--config", default="tests/configs/test_genesis.jsonc") + parser.add_argument("--binary", default="./didactyl") + parser.add_argument("--suite", action="append", help="Run only this suite (repeatable)") + parser.add_argument("--test", action="append", help="Run only this test name (repeatable)") + parser.add_argument("--api-port", type=int, default=8485) + parser.add_argument("--api-bind", default="127.0.0.1") + parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument("--debug-level", type=int, default=5) + parser.add_argument("--output-dir", default=None) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--no-restart", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + run_ts = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + output_dir = args.output_dir or f"tests/results/{run_ts}" + Path(output_dir).mkdir(parents=True, exist_ok=True) + + binary = Path(args.binary) + if not binary.exists(): + print(f"didactyl binary not found: {binary}") + return 2 + + config = Path(args.config) + if not config.exists(): + print(f"config not found: {config}") + return 2 + + log_file = str(Path(output_dir) / "agent_debug.log") + base_url = f"https://{args.api_bind}:{args.api_port}" + + agent = AgentProcess( + binary_path=str(binary), + config_path=str(config), + api_port=args.api_port, + api_bind=args.api_bind, + debug_level=args.debug_level, + log_file=log_file, + base_url=base_url, + ) + client = DidactylClient(base_url=base_url, timeout=args.timeout, verify_tls=False) + log = LogWatcher(log_file) + + log.start() + started = agent.start(timeout=30) + if not started: + print("Failed to start Didactyl agent") + log.stop() + return 1 + + try: + runner = TestRunner(agent=agent, client=client, log=log, args=args) + tests = runner.discover_suites("tests.suites") + if not tests: + print("No tests discovered") + return 3 + + results = runner.run_all(tests) + + run_meta = { + "run_timestamp": dt.datetime.now(dt.timezone.utc).isoformat(), + "base_url": base_url, + "config": str(config), + "binary": str(binary), + "test_count": len(results), + } + reporter = Reporter(results=results, run_meta=run_meta, output_dir=output_dir) + reporter.print_summary() + reporter.print_details() + json_path = reporter.write_json("results.json") + txt_path = reporter.write_text("results.txt") + print(f"\nWrote: {json_path}") + print(f"Wrote: {txt_path}") + + bad = [r for r in results if r.status in {"fail", "error", "timeout"}] + return 1 if bad else 0 + finally: + agent.stop(timeout=10) + log.stop() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/suites/__init__.py b/tests/suites/__init__.py new file mode 100644 index 0000000..62e974f --- /dev/null +++ b/tests/suites/__init__.py @@ -0,0 +1 @@ +"""Test suite modules for Didactyl harness.""" diff --git a/tests/suites/__pycache__/__init__.cpython-313.pyc b/tests/suites/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e580da59ad2aa61f6fbd82a091a444f2b145ba41 GIT binary patch literal 176 zcmey&%ge<81TLEnXX*p##~=<2FhUuhIe?6*48aUV4C#!TOjQ~osl_D<#if}gsS3II zDWy57#R_TpMG7vNDT&D?l{pF-iA8y-#l?DlnvA#D7fK6rSB3|HSLq#?B9>{49{9S&}9y5R^g)Y6w*uu+?U5aAgtW+MCqWURyJ3 zDY2?5$8aK16(vZOnnMri4fT{GQZF1gxN59b%B89*Zd5^{?Wu3ZKY@VMCwcs4_PsZ6 zzW2?Wz4mqw0-5~j=iFfjLcg;~Q>3~g>_tIYMHu5?Kg2Ht1x;P~fiR;pJJmnyc zDJOBl+^yjw=q@HMtxa=q&8>MfS#vS{u0Y!6oT~R33$6;boHmT2nKpApgThq$&{$AK z+a1%Xc{NRSTjpBPF!gyeBQU3iVEBg&g}r%DR#CF`Esm06(`clj^*0u>E`++_93zdn zF2vC^oco|?;z%Q&Cfz_;zzDo>9Qn{Wao=|isg7u+ljyVr+3QN7OikqrUCHJ$rtPBT zOh%_PRynHAm-LLOXO+g%Ws{WkL1nIJUQ|@c_T*^Jpk~_0=r-oQ(wuGr7??$-ED^=- zG64}fOv^b?hBL)Np=b=17HnUdQk|HoMrh&#Yd8Gp6&M!LuH3nt{cdtcKDjNQwD3ji z^u;QS^2wU)zdw3!borC2e0-zliQH%5h;`{VKzG6d2OUFJ zt7sS*48Ruj%)Ty~FpQeuUlv;MLpc=i3Zai@5HiHL_*P%>e|*QlSBgt-^_7~1GjOs& zV1(j0$>_qRnboT~MFzk|mPz#GtmvEEe;r zOgf*ZHa5})UByI(c%%&mTcReho#|3ZH?pdOc-fkRS%ES0w?#8ULa<619Tg^|of!pq z^b`-y?MWDx*(Ykiu!w5i;l*pUK*ySU#l7CWajF{V-3bhA2A>25mT)cDwbsAVzdp7Z ztOf^nf@dF!Pl9KcCTgM1wF@g3*89F4Uz*(Yhu3?4kaqk%Rew({(7EPa@vg^y4V>C@ zq0qU%By?!-xd(X)RG zqE{0?z#O_axCHEpZl_tp_7b9um(>0Gl`Zj9o+M4{e3?e5y`9-)zoE=TpsAWTQ&= z>nlo*Dte)0E>wK2cEmWPRGhpMDKT3te)a4zi!72L{E_2KciGNNKBpU|?JSWZ1mGQp zbT9*kknI3s=SrNEs?(M-<~%1i_lBzhTTr{bxh1DDISqc4oq@4@ak0*GoFZkzR%otJ z%Im56UMbe}hQ{(mxK(s~h*=zFz7Jsd8<_78-ml!N?D&;!zp~Nw@O{g#SpJbEyzA~- zHh*y+-3=dIkA0tbkg)nLY>Cxlm#X2ho$$nVc%mAfT=G5jAG6$})``)odvxiW-GH*u zQ4RDhIiAXgYawOhS~a9v-o)n2R%GkS<38)No7Shdtbs(;o3KJ^%^zBhZT4EzH!ba^ zb@kTvI|<95u-plLM^p^JuwKpYv@K_KngN*DGiQsl20+{kXCU=L5=L6pNzSk()&mIi zBgA1M?=p=srwQ1dJB`K=d=o129vF5rV^+I=7ST)5?f6`Hj@bC319>~vd$;6mIReeT zktQ$JzdrWtB%zXn3OvBKZ70!rQ^L`r!TK0$S7e9{!!T6Z@;S=QZ>vI7_0|*Tl;GKO zjURz)tlV|Nu)|b5u#NJy!Xoo%_0xN7PbxKAhF$ekim*FwJI#es&X^;tb=zV7{tbPZ zCI&pJLu|(A05P~fN3u>$)*d+!KO%m{I=$Rug<`gx*T)IFUhKu9Ct!Fk34-t&I`s$Y Y`V(Dx;S`0a@WSH~4!!J7)R7Ov{)9{0HIvB!>s^8?#~2zvsagpCagM8PHoJY(&^x@RD}BhhNllD zfZ#$&^X64obyanJ-}m0z?(OYH(565B-2OC<(EoTRYEtOn?HoAQkbw+5i*lG^o%_

x@x%EW7BJ~L`?eiTCpZ+`{OHB>}% z(a4_vIhfJe3~We7>}kotB_$dw;)(X!Ix2PTKAlTM{_eja8_GocwxYM@wB7N@n`k?Z zRFp>Beg)Qk02%RUj9{b7P>n=Uf;;Y((HT4*JBiX-zIkMZRIBde`kPCZr`t?laT`>3 ztQvW4ONPa6;2=@6mQAlg9Tp@PY{#mal(0HnE(y?&he)mNt?2uo7n+9_DIuQiEb7r* z-F+iaqt4bBuyi^e#LHEiI9?F1Qy12u{H}u-^JoxTBwnC^XV-7Uf>_yG4rKmzAbVul zWBj^xT@Oxh?AU46u)#Tf50IJqO3-gIyH>9fGddDI1~2pbW|zRMqOY~&JI`Es=9YGR zQ#(cL;Ts5h1 zA99s?`M2>_l*WN-N6{q^=^%>e7tyzao{4B2=-F*(Mo)p$vv-G{$$WG4HO@CwL z^l8G%)SeeT%1CuVpQkoisIFuvN0jogLwmU4C!$F(`)-``<}4TUu#zbJqyn-qiYa!{tQynV0;lh6>4QF_g0jOz`yCuNURbh`rcY|y4hXpI$` zgW|gO90(Li`(O*&4<_A3Q`{fmhRcu+a&rhwpj6#A2}sg%R;V~~sJ_YYqf_^MDNVy$ zd^Pq@FyG&!u$O$JAS71&l@bFz-b`jUPtCNF*;`t6Q_K2t!=G)mz-n0`z(?1Pt`{~= zZk%2Jn?E|~Ym>e_Nsq&E_R(P{w!a4>zg>PMzMg1N-0mQ}yod0L5jVPu_`t~D6$z#t zr`5dR^A^trg5y6vv;#a_avRl(4w}drFXwf4{u1C{Xby#exds^y7^5#)PNhmJL03EL z-B5P%4!Uii4)ZLhOai%Lmpy@dTA~l|{f5CXzO$}}h$l_4w%J~rJ^^E#|Lheo9cW*l zTYExaJqB27Lsu*Rp1!4>*wjw=@&*5q3oWqP34z$?+SBWmPnJGf+E{5lne|WQd@bk8 zx%+_j0c-~a1fR9UuNOYK_|e4;yEQT6r?a3iU(SZGzz#c&e;J$$ zZ+{5RHB^F>7PeBmhCu?V5|3)4O|-oOX97HX;JDvR&{0YKMQZ|JC7vKei*`RPLmdPF z^>tJU%|Szy4f+zVkMyuW366>A+b)1A%(Uj+hG*IykFCwaQ#>^5opCleyiYEskRpWY z=HVAa?G%AOPnN99wo8K)6rwBpKy41c3XgQb72BydJdk@hIUT3`astIDW(zNf(nf?| zAxgmo`Ue;naJ7>ZR92=ao9HUsY^kr^Nhm$>J4jLDV#9p8C+IfKg$7iQ#55_V zZE!^FomsM|1B}gxnvFIbhtTj#;$^p1bDb~=@pOFPNAaU~ewli%=2jY2a-L4YDCZwL w1LjT)WBd(z_&byV`(O3gs&Y53;OFt({y3h(cL!p47T@iWaSm@M<2XF}PqXjSjsO4v literal 0 HcmV?d00001 diff --git a/tests/suites/__pycache__/test_errors.cpython-313.pyc b/tests/suites/__pycache__/test_errors.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3783458db570e1b0a59e1db499a948135273903 GIT binary patch literal 2356 zcmd5;&rcgi6rT02mo)~0T`XLZMym=F%rDa#Bz_c#RFWuAh?=#dNLIUYyo=ey-nC}d zg*d7Nw@SSvQc(T{xKyYoj+J`tVIw0YTD4N8wg+w~zH#cC@!ByKZFA@$Bl+!{+4tVe zeDm!$u61{JB52D${%q{}5&DC68qHUCxY{B(dq_bFw~AIc%*oXEt@>7Y%!AKkK@kQJ z_A7p{q9TIrP&&X$iUc;055zl9Xt6lw2D7GV*;(7LOafggr4#$}EYaPd4W6oFY+;h) zXtdX0S%^o7t6^~VP#WEBylHvfg^2d1aEfo*+e8_jrEr>4___NCY266*BSlDaJxKAV zeLW~COnd1%r-(|2BBi-WZi*j7@xc07D3L80i4wMoB|DML>!wWt5dH*qB0nUSdGZt% zd=}QXOM1@MHF;`sa!k&{%QI1aU}=vRQk95dev>U28jF{){*XnwA^|MQhJEKPR>T8GeG%ChWUreKi-DI4Gu-p8At-G2J$%Cxt1?Q2#cjr#{kW7W|0cZGCxfAdIi z#uuF6f+H;82$VessQ?3v{snOUC;9^#4f-d2EyVXz>Nklmtyg*(@lDIrw+&(gJ#jy* z;^cWd-KXkwPgVNL<|EVEGG*P=N|s^T@GVTeMfZTmy-)y$z_@<3mD6>sL#kImZ4|&) zA(nEO+yVpq4M$##JR5lxj=c=WoS~VcST%foceRDTNsqryG+7mr9&@kl4<9Z&@mVK0 z>j<;&8O$Fsm~Cw2^Eytnz&)55Vs6WiX&6EfKP&LdbcP@f8Y}rxkTv`_wJ{ z!KP(BQd@Gd3~{vX#n9gPsay@O{WrAdMWGHY6m2&80Wt>v0L~uDG{rsRi8F$mMe$`Q z(^;GLybB|FMaYQn6bWq3Pz7M^Uu%NgXV^VRLmQC^fr-BxYE@H4*-HOfwy6kXS^q;IEgu*-qT8T-r_Ar25b!a8pPdEHjoWrK$yO@<%217-*PDfK>& zIMGpO{N^v+Rq5WZ^?t(nKV*|nY-X_unmxg;I$SnQ_@hAgfrqG}^3b*mwp@IT zzMzs?-;MNH(bCEVeG!ks808^}gE{3nj{6OL{1=+|U5f1Z--tXn|E7!Q7O9z|X1>-X Ia`k2Y4f0nSi2wiq literal 0 HcmV?d00001 diff --git a/tests/suites/__pycache__/test_health.cpython-313.pyc b/tests/suites/__pycache__/test_health.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8597ec1061598d1554a3f0316b59c2c99b5e2276 GIT binary patch literal 3006 zcmc&$%}*Og6rcSvhBaTNDS<@E5(gzF2__*#DWSA{H9!PPS-Y+3Mk~i&j1_zB%&eo} zTsVTN77mTXp$F0vm7=O2hej%ws{g>EMGD=fQl++s+$fxS>zn;BHlb0KOP{oF-@bYC zX6MaselvUZ^>qkj^xK~@DIY?=(+^Kr$6)p*!FY&dBr`YAB!d}=+UyN>lEWO>oXp?g zCj~5kEnuJQYe%?7t^w_r{h({*TF`ZJ9cVEvh6B%O;xJ>02~ErE2|bh72z>gZD$z$0 zM19UU?F|V+RIDqcm`tjKSfUOA%AA_W>T^jN<4Fw|974?AA{Y-*49$2g74I2{I4FIt zLu2BC*A`>?+}to9MrLI$PV?ImV`geSd;jfa@%r^Jm$}m@%uh#44Fu*_B$77;{*F3N z!h+>XW;3d$TRx{9L+z2pXEQ{{bbv5xaY_BY#bcEe2wezCvX0T|lL#qhz=%}TqPs9} zwJ1&|1*>`yYec!!(}SC!BVAw84{8Bziw(>DOZ`v9ik4cjdbgOtYRYjDsQ?v*g$T2l77B}?!W^!rb^|Dj zqYFlnBS{i&q@JKx>A3_^?8F2gg$7z|G7RdlLjTfuZIRn<41V&!@b|9f*OTip<9M&> z?_GTG7l)b#exnP}wLWZ!-3H%{n;}hA!6`oMg0q0lS1b782nR%$(G6p`A~O-xM_~t| zvGFqIV{Dw3$ri8utoxmUJaqB)m+fD217CQ3Wxjs{A-AWn-&6LG(NOf+Cou+uEYVR&zUAnq5xyG7e2>jxamDJ~> zPsNT+vBTg8jL?7yTI_&LZU;`ST{Okc^=4D-GWZGO{DcWw?6NtabG>n+B+??Ux85pP|C!3hwTX~!5?YdEDG05TS6g$b%I-AK(}EIe4r0;Nc{ubbwD#b&?Ljr!q-BEbP-3K26#M(sj{hc*p{%3ao`T#SlQH0&9V$8(d&3u8uLXj^PwRRNm@KV)#ehENXPXdCAFAKQ+APQqtdC==N{_!SrrQ5+EHNZe&BWS`k_ z-t+gl)}fKWL~&Y5TV5IoYaRFhr%1|M`yA4#q;8bl^y{Bo8YtHQf7y6wvrE^3;Jor(C8`CKlq*|d*) zpvPfzvVR_5q^s=?fj#{dI(tbPh&Fh=_ ze&26qI~LoAK+b*rQ|_*e&~LmFNT^W|wx^(Q2kA%`E}?0G3K}nmE`_E=DnePLk}mZk zD(f<6MOQ$F^)P64S&c`Y@M+?Liw)B(*oK`em=;vkIbzxG7#4A(Hk8s7ltJx`z`Oe* z!Cyp3*!~U*chEeV4O({nXQ4-{=z<ej~punNoCCe3;iG5FW#f-h? zh6*=a2|5-nelnV-x&8cvtx>C#gTmOJBwFjK z_H0Ia>yh4#Q!LWUBEyxlzu*{?23hBzBMsg>wS_xsr>mzoaep26J9waSZVPwTrm9o7 zE<$Z%=pi0v(lEorU}oJz8;cB&Fll5PRzS~%Tmgd@^4oBvFB0-*kRa%@0OeNEg=UaK zJ3#^v@D*F6ZIDG>0!RS_S|B9{D|?V?nTc210W=t6iu2+Nc-wwwO0cs%=4!T|zJ`#W z2;sm30BT5Ab>*MNjtL%=BPohuPjiFR=MoIzZrSX#V6*312kzm{1s%w>|*j3t{; z&4+EduK`yLev5hXd}Cau9elb@uDZC2auov=m*{@@xymA0Do{dS;uYDO+TuA37}w}Q zZakQ7tdRB^n~?5@Ha?~GJt%;5tgCjtdVMoCP>&5bF|Bf83wLw;-pV;x+c@{o11CWA z-(f%6S(~m-Z$^*Sqeq?SVCDQ4?y4oL$y@I^cyMFODB$KYPsjCiXJn3af;{K6uCVotQn|yH1c|FC(W*DAf(hNNe{3U3IG1?L{}~kSpoQi`V8KTKT(rPd z3q0XH(K#c_ZTkTh7n;02;g4wBGpwr%d}U99o}>KIj-=e!a~ETIzit!%BksO*dZ`52LDFf;6UOcFR!giH)PyHZqVeD; znI@r>`%GWuia#@UJoa3pC%BDsT*GtqjnzW7lqYY|F|ct!SbTR+ML`gLM#ug@iC@)2 b75SMW3-M=Ns&GYkc1RIk+ukn=4e$N}{P{+W literal 0 HcmV?d00001 diff --git a/tests/suites/__pycache__/test_timeouts.cpython-313.pyc b/tests/suites/__pycache__/test_timeouts.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c11872f2b49846f574c14a36f8184b366b62ac6f GIT binary patch literal 2638 zcmeHJ%}*Og6rcU@dd(U_C=@~rT~Hu(1tw8&6|6LJMIs;|D`m0T%1Kw2Jt13}T{^o~ zz$YJiXwm~5619~ol_R-DJ>?jwmsX0@p{@j4QK?eeL!=&}@DKFOewYNJh*s*YBYF1C zdvD&nnfH6|d9Ss#1;M!V{SU^c5rm$wML2k$;P$dW+(R1DxLI_CBb>@;ewHUZe1%!z zibzD@i6o*$dJ&N{3FfF4g;~~Qm}854GWL|2OLDGKG)>DXI)-K1IatdJ*mfq1Hg;Pb zpmfJr!dBI>OB~w}4DbyJ2yuH?fqM@XLOz-gylgUn7)0q{X>xfqFTN=A>G|kkp2lgs zCQR@qe^l#vZWLb66biA>TAnX>@5$Ebz_YglVHwul2y=O%(Av~T^KBu$|DA8&r$Z_o zyhbL2Q28xpNfY62N8H_pw8%Ljd=<=Tk`~qEyl{pa7fzvMELZE*imU2PT&Y;9VXBsi zRU4ZQHcL45lwAY@6PB&;xNPHcGUkem2SzPGgJl{eS1MHuIMbDuiM6!sxGjcln6^^{A6&6)lpHs5qhb{u zS1gi6+l>Y`?~ATba#q|3s1gveqX~u8&aQhgr>@mA)05SFw%2La#ZjTwiRz@Yvg_;uZZxn6ELrgM4;3P!iY+_03O1_FKEMkXr z0;dTo(!=C&-B-{(8<3ai4W?9u={0a^DE+U13B-2o{O$AWGa%kL^@}o0#bK%p160;a zUtXl*1no<0-r6)Cd`3@BG{lK7F2auWzKs)`Q&b(N$~YCrNiUrIKN!5KCg(^3ULyoT zXSUW+vY@7|IERoB4+XD*!m&U{FI5SFIz{@y_^U9{s)w4MwLws0e!LD8LtDyFLpl8~ zK#`$+V-KR6s}DJPGSd(@~i?Dv3AJ0rg5vK@$toku)QkRA6U&BeXfjT2M6SHk-zQDqQQo>NNy4tkUX%rqUVY zpf4}mUNY9YH9zaISzfjb(@`r$*3VYmt}r|=>-S6s;5fJessg*c8`1JJ3zxl~Kv<>| z7TNpAdtJX8#Gp#(UauiZkPX4leh;QP`a^EN+eLfM(sSCc`K5+z?)r=rJu~z8LPP#2 zSdT&Ze|)_m7oJ7MxbzH(5y`WkY;jw3{YDkqFV=Meu|r;|a!+{2rEV692}LjE^)^y9 zO-wwe$!Vs^+StC!o^OQxEqTFp`NEP_u2%3xG6DjI2b*a;6F83h4Glg)%AY9pyWCxm XJeMTyeeQXO#HHEv?p{ZP^PT(~;uS1; literal 0 HcmV?d00001 diff --git a/tests/suites/__pycache__/test_tools_blossom.cpython-313.pyc b/tests/suites/__pycache__/test_tools_blossom.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48c6dff1715f8013961ff9aa6c83c4619b83e012 GIT binary patch literal 446 zcmZWmy-LJD5Z=w7IZc{0LJ$ceLM)Q%I%SNpd@LNE=0%po1*3UU#%01w;Yf^2N;4i$GRis4r#){QHUL%d z<`7_sB1zR+w3Y{10%8OY^=>D``^eYE!s_|rEal~cHR{;Cw%WdgHE&KU=Jj1NPPnfy zsU~!CeRg&Fj2cNP_T-Q}8`}t7MwA_`7Xc${h03K@F{o+R-c##5F zK zGe-$K^MWcttf^D)FO91)*^e;D)!F6A3o50kaO4oY!}AE7&i{#a)3K-zp%s}*zXmm!yZl&NJe*e;_PfO>!trh zZyr4AwMVc10|U7Y3W8p|1@aI0W;a``lsuR>^S%Av_h#R`MY&u+U^}PZ=xGL_UmBza zIJQLtwiARAHg?f7gBuRQ8J@w}2IARKu4(=;fY3BlEuh}SCEW>U-b8c-352R^-xZ9x zL6;7RFQ}LfV%eoD!k&e-_zLa>9cr!E4%7Nj7eh6i+cKA5F5;Nj!fV)G$;smaF5>)3 zP6?OsIxel`RB#n<;7ZDAnKw|gwxJuUcV`og>JIlz`gKB`kK~OC7%U?F9npX#eASeg+wYo*UpLPP+Jv}JPp)(Zv;I)>vU9qsIilF z-q8;cUEU|I>v+WJ^;syA`=kt2P85y^4=D4aS|YH7N&cny5^1Q%iI!+X?V(5ge`{;} zU+u=hn4L8tPN4A9J;8Xpnc;d6l!;$<_8)&i3;o*ax*njQ6@(NHtqZ2<_GGH^x>+%i zZ$Od-D{#Rbqu+Mp9TUkwp;{`~$trdRj5WW@w{Hrp%-(t(W~0 z3m!ap)T>{?FR zQ*cO>$55mLa85nh#a$D>h=tdXTcoJCOqsb`Rja&nJV8U~M^P4i{^a1%XH@FRtLwV4 z9LQM$#y~s&yzA}hT)AUby}J~fP-4T9Lr}-)m(%~S_wnYZcc=R=PG2V9oY!Yg|IF_$ z{oSR1b?M(<`d3QNcH4h8kh^0N#N-~Y6RXQxzP&pr$b8}eXLC~-Y qUt1a@Hb5z7KjZN%W%ooET>X1w57erOG5(Hj{BU-TtTWb?R6rN3+bkinHo7PqX${->H(NcO*K`KHoR(nyWMX%j}_M+xb*lO~6x1X7Xyxoi^oA16z4WPq4GIe9Wj zhDg3Ar$CBim=t<)N@RqLl2T7jnT(Nfa-k<@f=rT&WFqBE4O~OjOB>OM?vmApURh1s zZA=HHCtzc3W0y%9+|Z61R;|?@ENZ^(?&AlZWKK=eMjTQa(coXbs2tm`J<5HH(>ir! zP=2Zqj*(Uq!EEf42Ncg>M`cgKst7zCv+y3gW?XZ9T=V1s(XYfxE~ecQW}mWsM`nz2<&)qJ*a3SY2)HLmP|_s+S-do<2%%?vj(-?wznG$Eop!? zxSRl;6QWHDZ4l569gYg;ucu-#7JD{~$3B(8@RJze7Gq7X9b8$16MOCWA=r_NH=ykt zCwHnDuEr$FlI_*4rH`nqW+BpVSxgDfhA(02-$A?8_M<%H(sL;IINZp=Pke&y82vIU z@8`m+v*G+oxO#YokZ&wyKsLXe`4)%f!#iu>-vmD#E14}F-cG|a;oMTV@+#aF9aE-P zz6^&CkANK*8=Actvp-Co-VGOCgfE*Nvz2h?PTz#fG~D+(=3(jzZ$mM5*nHbD-<_4r zLhcNi*&HuJ#T(WBC|75W!(818J_1htP<&Kbio?fXi_CitYj`cXz$XBze+!|4XMKiY Rd`H)Rn3dyfHfgM7}y2NX82JPFZ z?;k18j1{F!$kKlSi0e7a&^X7%g5!7@vos#3bQ4p7nye{jR%nc>V8&?Vw$(9s1 zA+=kJC7i~RMsYrgIL(SBEq4zMg@M8~RZEq_CV=%x)+A2iG^G>6?9F1HQz7=?)W;Nm zU~-0&B9}Cmc=)G6&S>5=W>N)L=jgY%z5`{GS5JqJKch+?w7RAB#6T<l*jx z$ErJ}^)(BA4N4sN#S?%N^wSML9(=lY)>}OstzNTp_uZvo_Rj{Z!#At9`MFyFa~W)3 z1b56gz6ip~ce>ts%L|+}@@$6((Cet*QBG21S&^G5xqh%lj{AD2b9fIxx%abTvPkIz dejQx>O2iJpx@}q3H+1*Ad-KGu9N((@{syN;&B_1( literal 0 HcmV?d00001 diff --git a/tests/suites/__pycache__/test_tools_system.cpython-313.pyc b/tests/suites/__pycache__/test_tools_system.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80d3426ee3907217159562abe402a8e98981cef0 GIT binary patch literal 984 zcmaJ=O>fgM70Nwd5QCEj_NyK4-w_r56!z zKrqy=MAYMS2NO5IJZs$=QHm1|HIr=q^a}+U!x4*;;F@R@+qF~s#_!si zW!ag+`!8CwN$PvV4LKjWyO{f7*0|g6KY|5bx}zwlR@s$aPtAiUSxafqlTRrY{WRH) z0+;cWVwYlBiDZjNWh+Gn3q|%=lhA{z14)-wWrm8>0va(}6Hi{(Z{nLGG!?CE#gXr#L zv@N$-@@BCN`{0M&dAt5@<^Ah?V>^EqPVDHYWZvZ0aURf#%?f5UzjQO-l%i;2ztB{E z?(#>HZ~8gjo7k@;b>z$+I?MWce&}?jO{?BGZaR*23>?#nG6;J^e$)}cSLuRG)i1ylIIbFo@fBSBW}n?Rr&h}->iz<~mJT5R literal 0 HcmV?d00001 diff --git a/tests/suites/common.py b/tests/suites/common.py new file mode 100644 index 0000000..bde43cc --- /dev/null +++ b/tests/suites/common.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import Any + +from tests.harness.test_runner import TestCase, TestContext + + +def assert_success(resp: dict[str, Any]) -> None: + assert isinstance(resp, dict), "response must be dict" + assert resp.get("success") is True, f"expected success=true, got: {resp}" + + +def turns_tool_names(resp: dict[str, Any]) -> list[str]: + names: list[str] = [] + for turn in resp.get("turns", []) or []: + for tc in turn.get("tool_calls", []) or []: + name = tc.get("name") + if isinstance(name, str): + names.append(name) + return names + + +def simple_prompt_test(suite: str, name: str, description: str, prompt: str, expected_tool: str | None = None) -> TestCase: + def _run(ctx: TestContext): + resp = ctx.client.prompt(prompt, max_turns=6) + assert_success(resp) + final_response = str(resp.get("final_response", "")).strip() + assert final_response, "final_response is empty" + tools = turns_tool_names(resp) + if expected_tool: + assert expected_tool in tools, f"expected tool {expected_tool}, got {tools}" + return True, "ok", {"tool_calls": tools, "final_response": final_response[:200]} + + return TestCase(suite=suite, name=name, description=description, fn=_run) diff --git a/tests/suites/test_conversation.py b/tests/suites/test_conversation.py new file mode 100644 index 0000000..764f40d --- /dev/null +++ b/tests/suites/test_conversation.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from tests.harness.test_runner import TestCase +from .common import assert_success + +SUITE = "test_conversation" + + +def _simple_greeting(ctx): + resp = ctx.client.prompt("Hello, what is your name?", max_turns=4) + assert_success(resp) + text = str(resp.get("final_response", "")).strip() + assert text, "empty final_response" + return True, "greeting response ok", {"response": text[:200]} + + +def _self_description(ctx): + resp = ctx.client.prompt("What are you? Describe yourself briefly.", max_turns=4) + assert_success(resp) + text = str(resp.get("final_response", "")).lower() + assert text, "empty final_response" + assert any(token in text for token in ["didactyl", "agent", "nostr"]), "response missing expected identity terms" + return True, "self description ok", {"response": text[:200]} + + +def _empty_message(ctx): + resp = ctx.client.prompt("", max_turns=2) + assert isinstance(resp, dict), "response should be JSON object" + return True, "empty message handled", {"success": resp.get("success")} + + +def _long_message(ctx): + long_msg = "A" * 10000 + resp = ctx.client.prompt(long_msg, max_turns=4) + assert isinstance(resp, dict), "response should be JSON object" + return True, "long message handled", {"success": resp.get("success")} + + +def get_tests(): + return [ + TestCase(SUITE, "simple_greeting", "Simple hello prompt", _simple_greeting), + TestCase(SUITE, "agent_responds_about_itself", "Agent self description", _self_description), + TestCase(SUITE, "empty_message_handling", "Empty message behavior", _empty_message), + TestCase(SUITE, "very_long_message", "Very long input behavior", _long_message), + ] diff --git a/tests/suites/test_errors.py b/tests/suites/test_errors.py new file mode 100644 index 0000000..56b8323 --- /dev/null +++ b/tests/suites/test_errors.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from tests.harness.test_runner import TestCase + +SUITE = "test_errors" + + +def _invalid_json(ctx): + code, body = ctx.client.raw_post("/api/prompt/agent", b"{not json") + assert code == 400, f"expected 400, got {code} body={body}" + return True, "invalid json rejected", {"status": code} + + +def _missing_message(ctx): + code, body = ctx.client.raw_post("/api/prompt/agent", b"{}") + assert code == 400, f"expected 400, got {code} body={body}" + return True, "missing message rejected", {"status": code} + + +def _unknown_endpoint(ctx): + code, _body = ctx.client.raw_post("/api/nonexistent", b"{}") + assert code == 404, f"expected 404, got {code}" + return True, "unknown endpoint 404", {"status": code} + + +def _webhook_nonexistent(ctx): + code, _body = ctx.client.raw_post("/api/trigger/nonexistent-dtag", b"{}") + assert code == 404, f"expected 404, got {code}" + return True, "nonexistent d_tag 404", {"status": code} + + +def get_tests(): + return [ + TestCase(SUITE, "invalid_json_body", "Malformed JSON rejected", _invalid_json), + TestCase(SUITE, "missing_message_field", "Missing message field rejected", _missing_message), + TestCase(SUITE, "unknown_endpoint", "Unknown endpoint returns 404", _unknown_endpoint), + TestCase(SUITE, "webhook_nonexistent_dtag", "Unknown webhook dtag returns 404", _webhook_nonexistent), + ] diff --git a/tests/suites/test_health.py b/tests/suites/test_health.py new file mode 100644 index 0000000..3854a07 --- /dev/null +++ b/tests/suites/test_health.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from tests.harness.test_runner import TestCase +from .common import assert_success + + +SUITE = "test_health" + + +def _status_returns_200(ctx): + resp = ctx.client.status() + assert_success(resp) + return True, "status success", {"keys": list(resp.keys())} + + +def _status_has_fields(ctx): + resp = ctx.client.status() + for key in ["name", "version", "pubkey", "relay_count"]: + assert key in resp, f"missing field: {key}" + return True, "required fields present", {} + + +def _context_current(ctx): + resp = ctx.client.context_current() + assert_success(resp) + msgs = resp.get("messages", []) + assert isinstance(msgs, list), "messages must be a list" + assert resp.get("total_chars", 0) >= 0, "missing/invalid total_chars" + return True, "context current ok", {"message_count": len(msgs)} + + +def _context_parts(ctx): + resp = ctx.client.context_parts() + assert_success(resp) + parts = resp.get("parts", []) + names = [p.get("name") for p in parts if isinstance(p, dict)] + assert "system_prompt" in names, "system_prompt part missing" + return True, "context parts ok", {"parts": names} + + +def get_tests(): + return [ + TestCase(SUITE, "status_returns_200", "GET /api/status success", _status_returns_200), + TestCase(SUITE, "status_has_fields", "GET /api/status has required fields", _status_has_fields), + TestCase(SUITE, "context_current_returns_messages", "GET /api/context/current has messages", _context_current), + TestCase(SUITE, "context_parts_has_system_prompt", "GET /api/context/parts has system_prompt", _context_parts), + ] diff --git a/tests/suites/test_restart.py b/tests/suites/test_restart.py new file mode 100644 index 0000000..9989f31 --- /dev/null +++ b/tests/suites/test_restart.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from tests.harness.test_runner import TestCase + +SUITE = "test_restart" + + +def _clean_restart(ctx): + ok = ctx.agent.restart(timeout=30) + assert ok, "restart failed" + resp = ctx.client.status() + assert resp.get("success") is True + return True, "restart succeeded", {} + + +def _status_after_restart(ctx): + before = ctx.client.status() + assert before.get("success") is True + ok = ctx.agent.restart(timeout=30) + assert ok, "restart failed" + after = ctx.client.status() + assert after.get("success") is True + assert before.get("pubkey") == after.get("pubkey"), "pubkey changed after restart" + return True, "status stable after restart", {"pubkey": after.get("pubkey")} + + +def _conversation_after_restart(ctx): + ok = ctx.agent.restart(timeout=30) + assert ok, "restart failed" + resp = ctx.client.prompt("After restart, say hello.", max_turns=3) + assert resp.get("success") is True + assert str(resp.get("final_response", "")).strip(), "empty final response" + return True, "conversation works post-restart", {} + + +def get_tests(): + return [ + TestCase(SUITE, "clean_restart", "Stop/start restart", _clean_restart, requires_restart=True), + TestCase(SUITE, "status_after_restart", "Status stable after restart", _status_after_restart), + TestCase(SUITE, "conversation_after_restart", "Prompt after restart", _conversation_after_restart), + ] diff --git a/tests/suites/test_timeouts.py b/tests/suites/test_timeouts.py new file mode 100644 index 0000000..1e0a0ef --- /dev/null +++ b/tests/suites/test_timeouts.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import time + +from tests.harness.test_runner import TestCase + +SUITE = "test_timeouts" + + +def _response_within_timeout(ctx): + start = time.monotonic() + resp = ctx.client.prompt("Say hello in one sentence.", max_turns=2) + elapsed = time.monotonic() - start + assert isinstance(resp, dict) + assert elapsed <= float(ctx.args.timeout), f"response exceeded timeout {ctx.args.timeout}s" + return True, "response within timeout", {"elapsed": elapsed} + + +def _status_fast(ctx): + start = time.monotonic() + resp = ctx.client.status() + elapsed = time.monotonic() - start + assert resp.get("success") is True + assert elapsed <= 2.0, f"status too slow: {elapsed}s" + return True, "status fast", {"elapsed": elapsed} + + +def _context_fast(ctx): + start = time.monotonic() + resp = ctx.client.context_current() + elapsed = time.monotonic() - start + assert resp.get("success") is True + assert elapsed <= 5.0, f"context too slow: {elapsed}s" + return True, "context fast", {"elapsed": elapsed} + + +def get_tests(): + return [ + TestCase(SUITE, "response_within_timeout", "Prompt response within timeout", _response_within_timeout), + TestCase(SUITE, "status_responds_fast", "Status endpoint latency", _status_fast), + TestCase(SUITE, "context_responds_fast", "Context endpoint latency", _context_fast), + ] diff --git a/tests/suites/test_tools_blossom.py b/tests/suites/test_tools_blossom.py new file mode 100644 index 0000000..8dc0d96 --- /dev/null +++ b/tests/suites/test_tools_blossom.py @@ -0,0 +1,9 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_blossom" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "blossom_list", "Blossom list", "List your blossom blobs", "blossom_list"), + ] diff --git a/tests/suites/test_tools_cashu.py b/tests/suites/test_tools_cashu.py new file mode 100644 index 0000000..572989e --- /dev/null +++ b/tests/suites/test_tools_cashu.py @@ -0,0 +1,9 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_cashu" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "wallet_balance", "Wallet balance", "Check your cashu wallet balance", "cashu_wallet_balance"), + ] diff --git a/tests/suites/test_tools_identity.py b/tests/suites/test_tools_identity.py new file mode 100644 index 0000000..469f484 --- /dev/null +++ b/tests/suites/test_tools_identity.py @@ -0,0 +1,13 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_identity" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "get_pubkey", "Fetch hex pubkey", "What is your public key in hex?", "nostr_pubkey"), + simple_prompt_test(SUITE, "get_npub", "Fetch npub", "What is your npub?", "nostr_npub"), + simple_prompt_test(SUITE, "agent_identity", "Agent identity tool", "Tell me about your identity", "agent_identity"), + simple_prompt_test(SUITE, "agent_version", "Agent version tool", "What version are you?", "agent_version"), + simple_prompt_test(SUITE, "admin_identity", "Admin identity tool", "Who is your administrator?", "admin_identity"), + ] diff --git a/tests/suites/test_tools_memory.py b/tests/suites/test_tools_memory.py new file mode 100644 index 0000000..7e86b98 --- /dev/null +++ b/tests/suites/test_tools_memory.py @@ -0,0 +1,23 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_memory" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "task_list", "Task list", "Show me your current task list", "task_list"), + simple_prompt_test( + SUITE, + "task_manage_add_remove", + "Task add/remove", + "Add a task test harness probe task then remove it", + "task_manage", + ), + simple_prompt_test( + SUITE, + "memory_save_recall", + "Save and recall memory", + "Save test harness probe to memory, then recall your memory", + "memory_save", + ), + ] diff --git a/tests/suites/test_tools_nostr.py b/tests/suites/test_tools_nostr.py new file mode 100644 index 0000000..d245a8f --- /dev/null +++ b/tests/suites/test_tools_nostr.py @@ -0,0 +1,15 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_nostr" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "nostr_post_kind1", "Post kind1 note", "Post a test note saying Automated test post", "nostr_post"), + simple_prompt_test(SUITE, "nostr_query_recent", "Query recent notes", "Query the 3 most recent kind 1 notes from any author", "nostr_query"), + simple_prompt_test(SUITE, "nostr_my_events", "List my events", "List your recent events", "nostr_my_events"), + simple_prompt_test(SUITE, "nostr_relay_status", "Relay status", "What is the status of your relay connections?", "nostr_relay_status"), + simple_prompt_test(SUITE, "nostr_dm_send", "Send DM", "Send a test DM to yourself", "nostr_dm_send"), + simple_prompt_test(SUITE, "nostr_encode_npub", "Encode npub", "Encode your pubkey as an npub", "nostr_encode"), + simple_prompt_test(SUITE, "nostr_profile_get", "Profile lookup", "Look up your own Nostr profile", "nostr_profile_get"), + ] diff --git a/tests/suites/test_tools_skills.py b/tests/suites/test_tools_skills.py new file mode 100644 index 0000000..e4be9be --- /dev/null +++ b/tests/suites/test_tools_skills.py @@ -0,0 +1,17 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_skills" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "skill_list", "List skills", "List your available skills", "skill_list"), + simple_prompt_test(SUITE, "trigger_list", "List triggers", "List your active triggers", "trigger_list"), + simple_prompt_test( + SUITE, + "skill_create_and_remove", + "Create and remove skill", + "Create a test skill called test-harness-probe with content Test skill then remove it", + "skill_create", + ), + ] diff --git a/tests/suites/test_tools_system.py b/tests/suites/test_tools_system.py new file mode 100644 index 0000000..76ec666 --- /dev/null +++ b/tests/suites/test_tools_system.py @@ -0,0 +1,19 @@ +from .common import simple_prompt_test + +SUITE = "test_tools_system" + + +def get_tests(): + return [ + simple_prompt_test(SUITE, "tool_list", "Tool list", "List all your available tools", "tool_list"), + simple_prompt_test(SUITE, "model_get", "Current model", "What model are you currently using?", "model_get"), + simple_prompt_test(SUITE, "model_list", "Available models", "List available models", "model_list"), + simple_prompt_test(SUITE, "local_http_fetch", "HTTP fetch", "Fetch https://httpbin.org/get", "local_http_fetch"), + simple_prompt_test( + SUITE, + "config_store_recall", + "Store+recall config", + "Store a test config with d_tag test_harness_probe containing hello, then recall it", + "config_store", + ), + ]