From 7c92d90a1e0170890691ea9ee36ac4a4825b0401 Mon Sep 17 00:00:00 2001 From: Henrique Velloso Date: Thu, 12 Feb 2026 09:24:03 -0300 Subject: [PATCH] Adds new automated test flows. - Changed .cursorrules to point to AGENTS.md. - Updated .gitignore to include test runs and reports, ensuring they are not tracked. - Enhanced AGENTS.md with a detailed quick reference table and added sections for E2E testing. - Introduced multiple new YAML files in the maestro directory for testing app installation, updates, and sign-in flows. - Created a comprehensive testing document in test/TESTING.md outlining agent-orchestrated testing procedures. --- .cursorrules | 2 +- .gitignore | 8 +- AGENTS.md | 48 ++-- lib/router.dart | 25 ++- maestro/flows/install_app.yaml | 42 ++++ maestro/flows/install_older_version.yaml | 66 ++++++ maestro/flows/search_app.yaml | 46 ++++ maestro/flows/sign_in_amber.yaml | 33 +++ maestro/flows/uninstall_if_installed.yaml | 22 ++ maestro/install_app_test.yaml | 43 ++++ maestro/smoke_test.yaml | 82 +++++++ maestro/update_flow_test.yaml | 79 +++++++ test/TESTING.md | 258 ++++++++++++++++++++++ test/reports/.gitkeep | 0 test/runs/.gitkeep | 0 test/specs/TEST-001-update-flow.md | 112 ++++++++++ test/specs/_TEMPLATE.md | 156 +++++++++++++ 17 files changed, 994 insertions(+), 28 deletions(-) create mode 100644 maestro/flows/install_app.yaml create mode 100644 maestro/flows/install_older_version.yaml create mode 100644 maestro/flows/search_app.yaml create mode 100644 maestro/flows/sign_in_amber.yaml create mode 100644 maestro/flows/uninstall_if_installed.yaml create mode 100644 maestro/install_app_test.yaml create mode 100644 maestro/smoke_test.yaml create mode 100644 maestro/update_flow_test.yaml create mode 100644 test/TESTING.md create mode 100644 test/reports/.gitkeep create mode 100644 test/runs/.gitkeep create mode 100644 test/specs/TEST-001-update-flow.md create mode 100644 test/specs/_TEMPLATE.md diff --git a/.cursorrules b/.cursorrules index fa62d27..47dc3e3 120000 --- a/.cursorrules +++ b/.cursorrules @@ -1 +1 @@ -CONTEXT.md \ No newline at end of file +AGENTS.md \ No newline at end of file diff --git a/.gitignore b/.gitignore index d333c8e..2c460b6 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,10 @@ app.*.map.json .env -reference/* \ No newline at end of file +reference/* + +# Test runs and reports (agent-generated, not tracked) +test/runs/* +!test/runs/.gitkeep +test/reports/* +!test/reports/.gitkeep \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 6a3f7a4..5019c38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,14 +15,17 @@ Users can support developers directly via Lightning zaps. ## Quick Reference -| What | Where | -|------|-------| +| What | Where | +| ----------------------- | --------------------------------- | | Architecture & patterns | `spec/guidelines/ARCHITECTURE.md` | -| Non-negotiable rules | `spec/guidelines/INVARIANTS.md` | -| Quality standards | `spec/guidelines/QUALITY_BAR.md` | -| Product vision | `spec/guidelines/VISION.md` | -| Feature specs | `spec/features/` | -| Active work | `work/` | +| Non-negotiable rules | `spec/guidelines/INVARIANTS.md` | +| Quality standards | `spec/guidelines/QUALITY_BAR.md` | +| Product vision | `spec/guidelines/VISION.md` | +| Feature specs | `spec/features/` | +| Active work | `work/` | +| E2E testing workflow | `test/TESTING.md` | +| Test specs | `test/specs/` | +| Maestro flows | `maestro/` | ## Project Spec Structure @@ -55,14 +58,19 @@ See `spec/guidelines/QUALITY_BAR.md` for what qualifies as "non-trivial." **Never modify** files in `spec/guidelines/`. If a guideline seems wrong or incomplete, report it as a Spec Issue. -| Path | Owner | AI May Modify | -|------|-------|---------------| -| `spec/guidelines/*` | Human | No | -| `spec/features/*` | Human | No (unless explicitly asked) | -| `work/*.md` | AI | Yes | -| `lib/**` | Shared | Yes | -| `test/**` | Shared | Yes | -| `AGENTS.md` | Human | No | +| Path | Owner | AI May Modify | +| ------------------- | ------ | ---------------------------- | +| `spec/guidelines/*` | Human | No | +| `spec/features/*` | Human | No (unless explicitly asked) | +| `work/*.md` | AI | Yes | +| `lib/**` | Shared | Yes | +| `test/**` | Shared | Yes | +| `AGENTS.md` | Human | No | +| `test/TESTING.md` | Human | No | +| `test/specs/*` | Human | No (propose changes) | +| `test/runs/*` | Agent | Yes | +| `test/reports/*` | Agent | Yes | +| `maestro/**` | Shared | Yes | ## Working Rules @@ -70,3 +78,13 @@ If a guideline seems wrong or incomplete, report it as a Spec Issue. - After dependency changes, run: `fvm flutter pub get` - Fix any analyze/lint errors introduced by your changes. - Assume Android as default target unless instructed otherwise. + +## E2E Testing + +Tests are agent-orchestrated: the agent handles setup, stateful checks, and +reporting while Maestro handles UI automation. Read `test/TESTING.md` first. + +Key concept: **Agent-Verified Criteria** — Maestro is stateless, so cross-flow +checks (e.g., "badge count incremented by 1") are captured before/after by the +agent and reported alongside Maestro's own assertions. See the test spec template +at `test/specs/_TEMPLATE.md` for the format. diff --git a/lib/router.dart b/lib/router.dart index 0c1caed..787c009 100644 --- a/lib/router.dart +++ b/lib/router.dart @@ -12,6 +12,7 @@ import 'package:zapstore/screens/search_screen.dart'; import 'package:zapstore/screens/updates_screen.dart'; import 'package:zapstore/screens/profile_screen.dart'; import 'package:zapstore/services/package_manager/package_manager.dart'; + /// Root paths for each navigation branch (used for back navigation handling) const kBranchRoots = ['/search', '/updates', '/profile']; @@ -140,18 +141,20 @@ final routerProvider = Provider((ref) { final wasUpdatesRoute = previousPath?.startsWith('/updates') ?? false; previousPath = currentPath; - // Sync installed packages on every navigation to catch sideloads, - // external installs/uninstalls, and self-updating apps. - // This is a local-only platform channel call (~100-500ms, no network). - unawaited( - ref.read(packageManagerProvider.notifier).syncInstalledPackages(), - ); + Future.microtask(() { + // Sync installed packages on every navigation to catch sideloads, + // external installs/uninstalls, and self-updating apps. + // This is a local-only platform channel call (~100-500ms, no network). + unawaited( + ref.read(packageManagerProvider.notifier).syncInstalledPackages(), + ); - // Clear completed operations when navigating AWAY from updates - // This cleans up the "All done" state without affecting the count while visible - if (wasUpdatesRoute && !isUpdatesRoute) { - ref.read(packageManagerProvider.notifier).clearCompletedOperations(); - } + // Clear completed operations when navigating AWAY from updates + // This cleans up the "All done" state without affecting the count while visible + if (wasUpdatesRoute && !isUpdatesRoute) { + ref.read(packageManagerProvider.notifier).clearCompletedOperations(); + } + }); } router.routerDelegate.addListener(onRouteChange); diff --git a/maestro/flows/install_app.yaml b/maestro/flows/install_app.yaml new file mode 100644 index 0000000..c32c9b4 --- /dev/null +++ b/maestro/flows/install_app.yaml @@ -0,0 +1,42 @@ +appId: dev.zapstore.alpha +--- +# ============================================================= +# Sub-flow: Install app from its detail page +# +# Expects to be on the app detail page with the app NOT installed. +# Taps Install, handles system dialogs, waits for completion. +# +# Handles: +# - Amber signing request (if prompted) +# - Samsung "Trust and install app" security dialog (if prompted) +# - Android system "Instalar" dialog +# +# After completion, the app is installed ("Open" visible). +# ============================================================= + +- tapOn: "Install" + +# Amber may ask to sign a request during install +- runFlow: + when: + visible: "Accept" + commands: + - tapOn: "Accept" + +# Samsung security dialog (may not appear if source already trusted) +- runFlow: + when: + visible: "Trust and install app" + commands: + - tapOn: "Trust and install app" + +# Android system install dialog +- extendedWaitUntil: + visible: "Instalar" + timeout: 15000 +- tapOn: "Instalar" + +# Wait for installation to complete +- extendedWaitUntil: + visible: "Open" + timeout: 30000 diff --git a/maestro/flows/install_older_version.yaml b/maestro/flows/install_older_version.yaml new file mode 100644 index 0000000..a90ef95 --- /dev/null +++ b/maestro/flows/install_older_version.yaml @@ -0,0 +1,66 @@ +appId: dev.zapstore.alpha +--- +# ============================================================= +# Sub-flow: Install an older version of an app +# +# Expects to be on the app detail page with the app uninstalled. +# Scrolls to "Debug: All Versions", expands it, and installs +# the second version in the list (the first older version). +# +# Handles: +# - Amber signing request (if prompted) +# - Samsung "Trust and install app" security dialog (if prompted) +# - Android system "Instalar" dialog +# +# After completion, the older version is installed ("Open" visible). +# ============================================================= + +- scrollUntilVisible: + element: + text: ".*All Versions.*" + direction: DOWN + timeout: 10000 + +- tapOn: + text: ".*All Versions.*" + label: "Expand versions list" + +- waitForAnimationToEnd + +# Scroll to make more versions visible +- scroll + +# Use "below" selector relative to the "All Versions" header: +# index 0 = latest version's Install +# index 1 = second (older) version's Install +- tapOn: + text: "Install" + below: + text: ".*All Versions.*" + index: 1 + label: "Install older version" + +# Amber may ask to sign a request during install +- runFlow: + when: + visible: "Accept" + commands: + - tapOn: "Accept" + +# Samsung security dialog (may not appear if source already trusted) +- runFlow: + when: + visible: "Trust and install app" + commands: + - tapOn: "Trust and install app" + +# Android system install dialog +- extendedWaitUntil: + visible: "Instalar" + timeout: 15000 +- tapOn: "Instalar" + +# Wait for installation to complete +- extendedWaitUntil: + visible: "Open" + timeout: 30000 diff --git a/maestro/flows/search_app.yaml b/maestro/flows/search_app.yaml new file mode 100644 index 0000000..2aee8f2 --- /dev/null +++ b/maestro/flows/search_app.yaml @@ -0,0 +1,46 @@ +appId: dev.zapstore.alpha +--- +# ============================================================= +# Sub-flow: Search for an app and open its detail page +# +# Env vars: +# APP_SEARCH_TERM - Text to type in the search field +# APP_MATCH_TEXT - Regex to match the search result card +# (use ".*partial text.*" for multi-line) +# +# After completion, the app detail page is visible. +# ============================================================= + +# Ensure we're on the Search tab root +- tapOn: + text: ".*Tab 1 of 3" + label: "Tap Search tab" +- tapOn: + text: ".*Tab 1 of 3" + label: "Tap Search tab again to reset" + +# Clear any previous search state +- runFlow: + when: + visible: "Clear search" + commands: + - tapOn: "Clear search" + +- extendedWaitUntil: + visible: "Search apps" + timeout: 5000 + +- tapOn: "Search apps" +- inputText: "${APP_SEARCH_TERM}" +- pressKey: Enter + +- extendedWaitUntil: + visible: + text: "${APP_MATCH_TEXT}" + timeout: 10000 + +- tapOn: + text: "${APP_MATCH_TEXT}" + label: "Tap search result" + +- waitForAnimationToEnd diff --git a/maestro/flows/sign_in_amber.yaml b/maestro/flows/sign_in_amber.yaml new file mode 100644 index 0000000..b00d576 --- /dev/null +++ b/maestro/flows/sign_in_amber.yaml @@ -0,0 +1,33 @@ +appId: dev.zapstore.alpha +--- +# ============================================================= +# Sub-flow: Sign in with Amber +# +# Navigates to Profile tab and signs in via Amber if needed. +# After completion, the user is signed in and on the Profile tab. +# +# Prerequisites: +# - Amber signer app installed on the device +# ============================================================= + +- tapOn: + text: ".*Tab 3 of 3" + label: "Tap Profile tab" + +- waitForAnimationToEnd + +# If not signed in, sign in via Amber +- runFlow: + when: + visible: "Sign in with Amber" + commands: + - tapOn: "Sign in with Amber" + - extendedWaitUntil: + visible: "Connect" + timeout: 10000 + - tapOn: "Connect" + - extendedWaitUntil: + visible: "Sign Out" + timeout: 15000 + +- assertVisible: "Sign Out" diff --git a/maestro/flows/uninstall_if_installed.yaml b/maestro/flows/uninstall_if_installed.yaml new file mode 100644 index 0000000..7ac196a --- /dev/null +++ b/maestro/flows/uninstall_if_installed.yaml @@ -0,0 +1,22 @@ +appId: dev.zapstore.alpha +--- +# ============================================================= +# Sub-flow: Uninstall app if currently installed +# +# Expects to be on the app detail page. +# If "Uninstall" is visible, taps it and confirms. +# After completion, the app is uninstalled (or was already). +# ============================================================= + +- runFlow: + when: + visible: "Uninstall" + commands: + - tapOn: "Uninstall" + - extendedWaitUntil: + visible: "OK" + timeout: 5000 + - tapOn: "OK" + - extendedWaitUntil: + visible: "Install" + timeout: 10000 diff --git a/maestro/install_app_test.yaml b/maestro/install_app_test.yaml new file mode 100644 index 0000000..f664aee --- /dev/null +++ b/maestro/install_app_test.yaml @@ -0,0 +1,43 @@ +appId: dev.zapstore.alpha +name: "Install App Test" +tags: + - install + - android +onFlowStart: + # Setup: navigate to the app and ensure it's uninstalled + - runFlow: + file: flows/search_app.yaml + env: + APP_SEARCH_TERM: "Flotilla" + APP_MATCH_TEXT: ".*Self-hosted community.*" + - runFlow: + file: flows/uninstall_if_installed.yaml +--- +# ============================================================= +# Test: Install Flotilla and verify success. +# +# The onFlowStart hook navigates to the Flotilla detail page +# and uninstalls it if needed, so we always start clean. +# +# Prerequisites: +# - Device has internet connectivity +# - Fresh app start recommended: +# adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity +# ============================================================= + +# At this point we're on the detail page with Flotilla uninstalled + +- assertVisible: "Install" + +- takeScreenshot: "01_ready_to_install.png" + +# ====== Install the app ====== + +- runFlow: + file: flows/install_app.yaml + +- takeScreenshot: "02_installed.png" + +# ====== Verify installation ====== + +- assertVisible: "Open" diff --git a/maestro/smoke_test.yaml b/maestro/smoke_test.yaml new file mode 100644 index 0000000..d8ba046 --- /dev/null +++ b/maestro/smoke_test.yaml @@ -0,0 +1,82 @@ +appId: dev.zapstore.alpha +name: "Zapstore Smoke Test" +tags: + - smoke + - android +--- +# NOTE: Run with a fresh app start to avoid stale state: +# adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity + +# ====== Search Tab (Initial Screen) ====== + +- extendedWaitUntil: + visible: "Search apps" + timeout: 15000 + +- extendedWaitUntil: + visible: "LATEST RELEASES" + timeout: 10000 + +- takeScreenshot: "01_search_home.png" + +# ====== Tab Navigation (before search to avoid keyboard issues) ====== + +- tapOn: + text: ".*Tab 2 of 3" + label: "Tap Updates tab" + +- waitForAnimationToEnd + +- takeScreenshot: "02_updates_tab.png" + +- tapOn: + text: ".*Tab 3 of 3" + label: "Tap Profile tab" + +- waitForAnimationToEnd + +- takeScreenshot: "03_profile_tab.png" + +- tapOn: + text: ".*Tab 1 of 3" + label: "Tap Search tab" + +- waitForAnimationToEnd + +- takeScreenshot: "04_back_to_search.png" + +# ====== Search Functionality ====== + +- tapOn: "Search apps" +- inputText: "nostr" +- pressKey: Enter + +- extendedWaitUntil: + notVisible: "LATEST RELEASES" + timeout: 10000 + +- takeScreenshot: "05_search_results.png" + +# ====== Open App Detail (via reusable search flow) ====== + +- runFlow: + file: flows/search_app.yaml + env: + APP_SEARCH_TERM: "Zapstore" + APP_MATCH_TEXT: ".*open app store.*" + +- extendedWaitUntil: + visible: "Zapstore Alpha" + timeout: 5000 + +- takeScreenshot: "06_app_detail.png" + +# Go back to search results +- back + +- extendedWaitUntil: + visible: + text: ".*open app store.*" + timeout: 5000 + +- takeScreenshot: "07_back_to_search.png" diff --git a/maestro/update_flow_test.yaml b/maestro/update_flow_test.yaml new file mode 100644 index 0000000..36d247f --- /dev/null +++ b/maestro/update_flow_test.yaml @@ -0,0 +1,79 @@ +appId: dev.zapstore.alpha +name: "Flotilla Update Flow Test" +tags: + - update + - android +--- +# ============================================================= +# Test: Install an older version of Flotilla, then verify it +# appears in the Updates tab with a newer version available. +# +# Prerequisites: +# - Amber signer app installed on the device +# - Device has internet connectivity +# - Fresh app start recommended: +# adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity +# ============================================================= + +# ====== Step 1: Ensure user is signed in ====== + +- runFlow: + file: flows/sign_in_amber.yaml + +- takeScreenshot: "01_signed_in.png" + +# ====== Step 2: Baseline — verify Flotilla is NOT in Updates ====== + +- tapOn: + text: ".*Tab 2 of 3" + label: "Tap Updates tab" + +- waitForAnimationToEnd + +- assertNotVisible: + text: ".*Flotilla.*" + label: "Flotilla should NOT be in Updates before install" + +- takeScreenshot: "02_updates_baseline.png" + +# ====== Step 3: Search for Flotilla ====== + +- runFlow: + file: flows/search_app.yaml + env: + APP_SEARCH_TERM: "Flotilla" + APP_MATCH_TEXT: ".*Flotilla.*hodlbod.*" + +- takeScreenshot: "03_flotilla_detail.png" + +# ====== Step 4: Uninstall if already installed ====== + +- runFlow: + file: flows/uninstall_if_installed.yaml + +- takeScreenshot: "04_after_uninstall_check.png" + +# ====== Step 5: Install older version ====== + +- runFlow: + file: flows/install_older_version.yaml + +- takeScreenshot: "05_after_install.png" + +# ====== Step 6: Verify Flotilla appears in Updates ====== + +- tapOn: + text: ".*Tab 2 of 3" + label: "Tap Updates tab" + +- waitForAnimationToEnd + +- assertVisible: + text: ".*Flotilla.*" + label: "Flotilla should now appear in Updates after installing older version" + +- assertVisible: + text: ".*Update All.*" + label: "Updates available with Update All button" + +- takeScreenshot: "06_updates_with_flotilla.png" diff --git a/test/TESTING.md b/test/TESTING.md new file mode 100644 index 0000000..63aa89c --- /dev/null +++ b/test/TESTING.md @@ -0,0 +1,258 @@ +# Agent-Orchestrated Testing + +This document defines how AI agents run end-to-end tests for Zapstore. +Tests combine **agent intelligence** (setup, analysis, retries, reporting) +with **Maestro UI automation** (tapping, scrolling, asserting). + +## Architecture + +``` +test/ + specs/ # Test specifications (human-owned) + _TEMPLATE.md # Template with example + TEST-001-*.md # Actual test specs + + runs/ # Run logs per test (agent-written, NOT git-tracked) + TEST-001-update-flow.md + + reports/ # Run reports with screenshots (agent-written, NOT git-tracked) + TEST-001-update-flow-2026-02-10.md + + TESTING.md # This file + +maestro/ # Maestro flows (shared, git-tracked) + flows/ # Reusable sub-flows + *_test.yaml # Test flows +``` + +## Ownership + +| Path | Owner | AI May Modify | +| ----------------- | ------ | ----------------------------------- | +| `test/specs/*` | Human | No (propose changes via Spec Issue) | +| `test/runs/*` | Agent | Yes (append run results) | +| `test/reports/*` | Agent | Yes (generate reports) | +| `test/TESTING.md` | Human | No | +| `maestro/**` | Shared | Yes | + +## Agent Workflow + +When asked to run a test, the agent follows these steps **in order**. + +### 0. Discover Device + +Before anything else, identify the connected device: + +adb devices -l + +Extract the device ID (first column, e.g., RQCT1029N9J). If no device is +connected, abort and report as NO_DEVICE. +If multiple devices are listed, use the first non-emulator physical device. +Log the device ID and model in the run log. + +### 1. Read the Spec + +Parse the test spec from `test/specs/TEST-XXX-*.md`. Extract: + +- Parameters (defaults and alternatives) +- Setup commands +- Maestro flow path +- Success criteria +- Known flaky patterns +- Retry strategy + +### 2. Execute Setup + +Run each command in the **Setup** section via ADB/shell: + +```bash +adb shell pm uninstall # may fail, ok +adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity +sleep 3 +``` + +- Log each command's exit code. +- If a critical command fails (e.g., `am start`), retry once. +- If it fails again, abort and log as **SETUP_FAILURE**. + +### 2.5. Capture Baseline (Agent-Verified Criteria) + +If the spec has an **Agent-Verified Criteria** section, run the **Before** +captures now — after setup but before Maestro. These are checks that require +comparing state across the test run (e.g., badge counts, version numbers). + +Maestro is stateless, so only the agent can hold values across the flow. + +For each row in the table: +1. Execute the **Before (capture)** command or MCP action. +2. Store the captured value (e.g., badge count = 4). +3. Log the baseline value. + +These stored values will be compared in step 5.5. + +### 3. Run Maestro Flow + +Execute the Maestro flow via the MCP tool: + +``` +run_flow_files(device_id, maestro_flow_path) +``` + +Capture the result: success/failure, commands executed, error message. + +### 4. Analyze Result + +**If PASS:** + +- Proceed to Success Criteria verification (step 5). + +**If FAIL:** + +- Extract the error message. +- Match against **Known Flaky Patterns** from the spec. +- If a pattern matches: + - Log the flaky occurrence. + - Apply the **Recovery Action** from the spec. + - Re-run Setup and retry (up to max attempts). +- If no pattern matches: + - Take a screenshot and inspect the view hierarchy. + - Log as a **real failure** with context. + - Still retry (the spec's retry strategy applies to all failures). + +### 5. Verify Success Criteria + +After a passing Maestro run, execute any post-test verifications: + +```bash +adb shell dumpsys package | grep versionName +``` + +Compare output against expected values in the spec. + +### 5.5. Verify Agent-Verified Criteria + +If the spec has an **Agent-Verified Criteria** section, run the **After** +captures now and compare with the baseline values from step 2.5. + +For each row in the table: +1. Execute the **After (verify)** command or MCP action. +2. Compare with the stored **Before** value using the specified logic. +3. Mark the check as pass/fail. + +Example: +``` +Before: "Update All (4)" → badge_count = 4 +After: "Update All (5)" → badge_count = 5 +Check: 5 == 4 + 1 → PASS +``` + +Include all agent-verified results in the run log and report. + +### 6. Execute Teardown + +Run Teardown commands (if any) regardless of pass/fail. + +### 7. Log the Run + +Prepend the result to `test/runs/TEST-XXX-*.md`: + +```markdown +## Run #N — YYYY-MM-DD HH:MM UTC + +- **Result**: PASS | FAIL | PASS (retry) | SETUP_FAILURE +- **Duration**: Xm Ys +- **Attempts**: N/max +- **Parameters**: default | alternative +- **Setup**: [command outcomes] +- **Maestro**: N commands executed, error (if any) +- **Verification**: [post-test check results] +- **Success Criteria**: + - [x] Criterion from spec + - [x] Another criterion + - [ ] Failed criterion +- **Agent-Verified Criteria**: + - [x] Badge count: before=4, after=5 (expected +1) — PASS + - [ ] Version check: 1.6.4 (expected older than latest) — FAIL +- **Notes**: [agent observations, flaky pattern matches, etc.] +``` + +### 8. Generate Report (if requested) + +Create or update `test/reports/TEST-XXX-*.md` with: + +- Run summary with pass/fail status +- **Success Criteria checklist** (each criterion from the spec, marked pass/fail) +- **Agent-Verified Criteria** (before/after values and comparison results) +- Screenshots at each step (via MCP `take_screenshot`) +- Error details and recovery actions taken +- Comparison with previous runs (regression detection) + +## Running Multiple Tests + +When asked to run a suite (multiple tests or "all tests"): + +1. Discover all `test/specs/TEST-*.md` files. +2. Sort by test number (smoke/light tests first, heavy tests last). +3. Run each test sequentially following the workflow above. +4. Generate a **suite report** summarizing all results: + +```markdown +# Test Suite Report — YYYY-MM-DD HH:MM + +| Test | Result | Attempts | Duration | Notes | +| -------------------- | ------------ | -------- | -------- | --------------------------- | +| TEST-001 Update Flow | PASS | 1/3 | 2m 30s | Clean run | +| TEST-002 Install App | PASS (retry) | 2/3 | 3m 15s | Flaky: keyboard issue | +| TEST-003 Smoke | FAIL | 3/3 | 5m 00s | Real failure: search broken | +``` + +The user decides which tests to run. The agent does NOT run all tests unless +explicitly asked. Valid requests: + +- "Run TEST-001" — single test +- "Run TEST-001 and TEST-002" — specific subset +- "Run all tests" — full suite + +## Proposing Spec Changes + +When the agent discovers a **new flaky pattern** not listed in the spec: + +1. Log the discovery in the run notes. +2. At the end of the run, propose adding it: + > "I encountered a new error pattern: `Element not found: XYZ`. This appeared + > to be a timing issue (resolved on retry). Should I propose adding this to + > the Known Flaky Patterns in TEST-001?" +3. Wait for human approval before modifying any spec file. + +The agent **never** modifies files in `test/specs/` without explicit approval. + +## Maestro Limitations (Samsung-specific) + +These are known device-specific issues the agent must account for: + +| Issue | Workaround | +| -------------------------------------------------- | -------------------------------------------------------------------------------- | +| `hideKeyboard` exits the app | Never use `hideKeyboard`. Use `pressKey: Enter` or tab navigation before search. | +| `launchApp` fails via CLI/MCP | Use `adb shell am start` in the Setup section instead. | +| Keyboard covers bottom tabs | Structure tests to do tab navigation BEFORE search input. | +| `pressKey: back` exits app when keyboard is hidden | Only use `back` (Maestro command) for navigation, not keyboard dismissal. | +| App retains search state across restarts | Always use `--activity-clear-task` flag in Setup. | + +## Maestro MCP Quirks + +Known issues when running Maestro via the MCP tool (as opposed to CLI): + +| Issue | Workaround | +| -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `run_flow_files` prepends `~/` to the path | Pass paths **without** the home directory prefix. E.g., use `/Developer/codecode/Zapstore/zapstore/maestro/flow.yaml` instead of `/Users/hvmelo/Developer/...`. | +| `takeScreenshot` files are not saved to a discoverable disk location | Use the MCP `take_screenshot` tool for post-test evidence instead of relying on in-flow screenshots. | +| | + +## Device Prerequisites + +Before running any test, ensure: + +- Android device is connected and authorized (`adb devices` shows it) +- Amber signer app is installed (for tests requiring sign-in) +- Device has internet connectivity +- Zapstore (`dev.zapstore.alpha`) is installed diff --git a/test/reports/.gitkeep b/test/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/runs/.gitkeep b/test/runs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/specs/TEST-001-update-flow.md b/test/specs/TEST-001-update-flow.md new file mode 100644 index 0000000..81bf9f2 --- /dev/null +++ b/test/specs/TEST-001-update-flow.md @@ -0,0 +1,112 @@ +# TEST-001 — Update Flow + +## Purpose + +Verify that installing an older version of an app causes it to appear in the +Updates tab with a newer version available and an "Update All" action. + +This exercises the full update detection pipeline: sign-in, search, version +selection, install with system dialogs, and update discovery. + +## Parameters + +| Parameter | Default | Alternative | Description | +| ----------------- | ----------------------------- | ------------------------------- | --------------------------------- | +| `APP_PACKAGE` | `social.flotilla` | `com.duckduckgo.mobile.android` | Package under test | +| `APP_SEARCH_TERM` | `"Flotilla"` | `"DuckDuckGo"` | Search text | +| `APP_MATCH_TEXT` | `".*Self-hosted community.*"` | `".*DuckDuckGo.*"` | Regex to match search result card | + +If Flotilla is unavailable (removed from store, no older versions listed), +the agent should retry with the alternative parameters. + +## Maestro Flow + +- **File**: `maestro/update_flow_test.yaml` +- **Sub-flows used**: + - `flows/sign_in_amber.yaml` — Conditional Amber sign-in + - `flows/search_app.yaml` — Parameterized search + open detail + - `flows/uninstall_if_installed.yaml` — Conditional UI-based uninstall + - `flows/install_older_version.yaml` — Expand "All Versions", install second version +- **Expected duration**: ~3 minutes + +## Setup (agent-executed) + +```bash +# Uninstall the target app to ensure clean state +adb shell pm uninstall social.flotilla # ignore exit code if not installed + +# Restart Zapstore with a clean activity stack +adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity + +# Wait for the app to fully initialize +sleep 3 +``` + +If `am start` fails, retry once. If it fails again, abort and report setup failure. + +## Success Criteria + +- [ ] Maestro flow completes with 0 errors +- [ ] Flotilla is NOT visible in Updates tab before install (Maestro `assertNotVisible`) +- [ ] Flotilla IS visible in Updates tab after install (Maestro `assertVisible`) +- [ ] The Updates tab shows "Update All" (matched by `".*Update All.*"`) +- [ ] Post-test verification: + ```bash + adb shell dumpsys package social.flotilla | grep versionName + ``` + Should return a version older than the latest (e.g., `1.6.2` when latest is `1.6.4`) + +## Agent-Verified Criteria + +Checks that require state across the test run. The agent captures values before +and after the Maestro flow and compares them. + +| Check | Before (capture) | After (verify) | How to compare | +|-------|-------------------|-----------------|----------------| +| Updates tab badge increments by 1 | Navigate to Updates tab via MCP, read the badge number from "Update All (N)" | Same after Maestro flow completes | After N == Before N + 1 | +| Installed version is older than latest | N/A | `adb shell dumpsys package social.flotilla \| grep versionName` | Version returned is strictly less than latest available | + +**How to capture the badge count**: Before running the Maestro flow, the agent +should use the MCP `take_screenshot` or `inspect_view_hierarchy` tool on the +Updates tab to read the current "Update All (N)" value. After the Maestro flow, +capture it again and verify it incremented by exactly 1. + +## Known Flaky Patterns + +| Error Pattern | Likely Cause | Recovery Action | +| ---------------------------------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------- | +| `"Element not found: .*Tab N of 3"` | Samsung keyboard covering bottom tabs | Re-run setup (clean restart dismisses keyboard), retry | +| `"Assertion is false: \".*Self-hosted community.*\" is visible"` | Search results slow to load on poor network | Retry with same state | +| `"Element not found: Search apps"` | Previous search state persisted from earlier run | Restart with `--activity-clear-task`, retry | +| `"Assertion is false: \"Instalar\" is visible"` | Samsung "Trust and install app" dialog blocking | This is handled conditionally in the flow; if it still fails, retry | +| `"Assertion is false: \".*Update All.*\" is visible"` | Updates tab not refreshed yet after install | Wait 5 seconds, navigate away from Updates tab and back, re-check | +| `"Element not found: .*All Versions.*"` | Page didn't scroll far enough to reveal debug section | Retry (scroll behavior varies by content height) | + +## Retry Strategy + +- **Max attempts**: 3 +- **Between retries**: Re-run all Setup commands (uninstall + clean restart) +- **Escalation**: If all 3 attempts fail with the default app, retry once with + the alternative app parameters before reporting failure +- **Special case**: If the failure is at the "Install older version" step + (Amber/system dialogs), the agent should take a screenshot and inspect the + view hierarchy to determine the actual blocking dialog + +## Teardown (agent-executed, optional) + +No cleanup required. The older version can remain installed. +To fully clean up (optional): + +```bash +adb shell pm uninstall social.flotilla +``` + +## Related + +- **Feature specs**: + - `spec/features/FEAT-001-package-manager.md` — Install/uninstall behavior + - `spec/features/FEAT-003-updates-screen.md` — Updates detection and display +- **Acceptance criteria covered**: + - "User can install a specific app version" + - "Outdated apps appear in Updates tab" + - "Update All button is visible when updates are available" diff --git a/test/specs/_TEMPLATE.md b/test/specs/_TEMPLATE.md new file mode 100644 index 0000000..4f51b97 --- /dev/null +++ b/test/specs/_TEMPLATE.md @@ -0,0 +1,156 @@ +# TEST-XXX — Short Name + +## Purpose + +1–2 sentences describing what this test verifies and why it matters. + +## Parameters + +| Parameter | Default | Alternative | Description | +|-----------|---------|-------------|-------------| +| `APP_PACKAGE` | `com.example.app` | `com.example.other` | Package ID of the app under test | +| `APP_SEARCH_TERM` | `"Example"` | `"Other App"` | Text to type in the search field | +| `APP_MATCH_TEXT` | `".*example description.*"` | `".*other description.*"` | Regex to match the search result card | + +If the primary app is unavailable (not found, install fails), the agent should +retry the test with the alternative parameters before reporting a failure. + +## Maestro Flow + +- **File**: `maestro/xxx_test.yaml` +- **Sub-flows**: `search_app.yaml`, `install_app.yaml` (list all used) +- **Expected duration**: ~2 minutes + +## Setup (agent-executed) + +Commands the agent runs BEFORE the Maestro flow. These handle what Maestro +cannot do (ADB commands, app lifecycle, device state). + +```bash +# Ensure clean app state +adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity +# Wait for app to initialize +sleep 3 +``` + +Any step that fails should be logged and retried once. If setup fails after +retry, abort the test and report setup failure. + +## Success Criteria + +How the agent verifies the test passed (beyond Maestro's own assertions). + +- [ ] Maestro flow completes with 0 errors +- [ ] Specific post-test verification (e.g., `adb shell dumpsys package ... | grep versionName`) + +## Agent-Verified Criteria + +Checks that require state across the test run. Maestro is stateless, so only the +agent can capture a "before" value, run the flow, then compare the "after" value. +These checks are executed and reported by the agent, not by Maestro. + +| Check | Before (capture) | After (verify) | How to compare | +|-------|-------------------|-----------------|----------------| +| _Example: badge count_ | `adb shell ...` or MCP screenshot to read badge | Same command/screenshot after flow | After value == Before value + 1 | + +The agent must: +1. Run the **Before** capture commands during Setup (after standard setup, before Maestro). +2. Run the **After** capture commands during Verification (after Maestro completes). +3. Compare values using the specified logic. +4. Report each check as pass/fail in the run log and report. + +## Known Flaky Patterns + +Errors that indicate flakiness, not real failures. The agent should match +against these before declaring a test failure. + +| Error Pattern | Likely Cause | Recovery Action | +|---------------|-------------|-----------------| +| `"Element not found: .*Tab N of 3"` | Samsung keyboard covers bottom tabs | Re-run setup, retry | +| `"Assertion is false: X is visible"` | Content still loading / slow network | Retry (same state) | +| `"Element not found: Search apps"` | Previous search state persisted | Restart app with `--activity-clear-task`, retry | + +## Retry Strategy + +- **Max attempts**: 3 +- **Between retries**: Re-run all Setup commands +- **Escalation**: If primary parameters fail all attempts, retry once with + alternative parameters before reporting failure + +## Teardown (agent-executed, optional) + +Cleanup after the test. Runs regardless of pass/fail. + +```bash +# Example: remove test app +adb shell pm uninstall com.example.app +``` + +## Related + +- **Feature spec**: `spec/features/FEAT-XXX-*.md` +- **Acceptance criteria covered**: List specific criteria from the feature spec + +--- + +# Example: TEST-002 — Install App + +## Purpose + +Verify that the user can search for an app, install it, and see the "Open" +button confirming successful installation. + +## Parameters + +| Parameter | Default | Alternative | Description | +|-----------|---------|-------------|-------------| +| `APP_PACKAGE` | `social.flotilla` | `com.duckduckgo.mobile.android` | Package to install | +| `APP_SEARCH_TERM` | `"Flotilla"` | `"DuckDuckGo"` | Search text | +| `APP_MATCH_TEXT` | `".*Self-hosted community.*"` | `".*DuckDuckGo.*"` | Result card regex | + +## Maestro Flow + +- **File**: `maestro/install_app_test.yaml` +- **Sub-flows**: `search_app.yaml`, `uninstall_if_installed.yaml`, `install_app.yaml` +- **Expected duration**: ~2 minutes + +## Setup (agent-executed) + +```bash +adb shell pm uninstall social.flotilla # ignore exit code +adb shell am start -S --activity-clear-task -n dev.zapstore.alpha/.MainActivity +sleep 3 +``` + +## Success Criteria + +- [ ] Maestro flow completes with 0 errors +- [ ] `adb shell pm list packages | grep social.flotilla` returns the package + +## Agent-Verified Criteria + +| Check | Before (capture) | After (verify) | How to compare | +|-------|-------------------|-----------------|----------------| +| App not installed before test | `adb shell pm list packages \| grep social.flotilla` → empty | `adb shell pm list packages \| grep social.flotilla` → found | Before: absent, After: present | + +## Known Flaky Patterns + +| Error Pattern | Likely Cause | Recovery Action | +|---------------|-------------|-----------------| +| `"Assertion is false: \"Install\" is visible"` | App not fully uninstalled | Re-run `pm uninstall`, retry | +| `"Assertion is false: \"Instalar\" is visible"` | Samsung trust dialog appeared instead | Retry (conditional flow handles it) | + +## Retry Strategy + +- **Max attempts**: 3 +- **Between retries**: Re-run Setup +- **Escalation**: Switch to alternative app parameters + +## Teardown + +None required (installed app can stay). + +## Related + +- **Feature spec**: `spec/features/FEAT-001-package-manager.md` +- **Acceptance criteria**: "User can install app from store"