diff --git a/Makefile b/Makefile index 25dab0a..e25d331 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ NOSTR_LIB = ./nostr_core_lib/libnostr_core_x64.a NOSTR_DEPS = -lsecp256k1 -lssl -lcrypto -lcurl -lz -ldl -lpthread -lm BIN := sovereign_browser -SRC := src/main.c src/key_store.c src/login_dialog.c src/nostr_bridge.c src/nostr_inject.c src/nostr_url.c src/nostr_scheme.c src/history.c src/settings.c src/net_services.c src/tor_control.c src/tor_scheme.c src/fips_control.c src/tab_manager.c src/session.c src/agent_server.c src/agent_login.c src/agent_snapshot.c src/agent_tools.c src/agent_mcp.c src/cli.c src/qr.c src/db.c src/relay_fetch.c src/bookmarks.c src/shortcuts.c src/settings_sync.c src/search.c src/profile.c src/agent_fs_tools.c src/agent_llm.c src/agent_loop.c src/agent_chat_store.c src/agent_chat.c src/agent_conversations.c src/agent_skills.c src/embedded_web_content.c src/process_info.c src/perf_probe.c src/webkit_data.c src/web_context.c +SRC := src/main.c src/key_store.c src/login_dialog.c src/nostr_bridge.c src/nostr_inject.c src/nostr_url.c src/nostr_scheme.c src/history.c src/settings.c src/net_services.c src/tor_control.c src/tor_scheme.c src/fips_control.c src/tab_manager.c src/session.c src/agent_server.c src/agent_login.c src/agent_snapshot.c src/agent_tools.c src/agent_mcp.c src/cli.c src/qr.c src/db.c src/relay_fetch.c src/bookmarks.c src/shortcuts.c src/settings_sync.c src/search.c src/profile.c src/agent_fs_tools.c src/agent_llm.c src/agent_loop.c src/agent_chat_store.c src/agent_chat.c src/agent_conversations.c src/agent_skills.c src/embedded_web_content.c src/process_info.c src/perf_probe.c src/webkit_data.c src/web_context.c src/site_downloader.c # Web files embedded into the binary as C byte arrays. WEB_FILES := $(shell find www -type f \( -name '*.html' -o -name '*.css' -o -name '*.js' \) 2>/dev/null) diff --git a/VERSION b/VERSION index ea7f030..a758e3a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.56 +0.0.57 diff --git a/plans/download-website-offline.md b/plans/download-website-offline.md new file mode 100644 index 0000000..8d27339 --- /dev/null +++ b/plans/download-website-offline.md @@ -0,0 +1,156 @@ +# Plan: Download Website for Offline Viewing + +## Goal + +Add a "Download Website" option to the tab right-click context menu. It crawls +the current site (same-origin, bounded by a page cap), saves all pages and +assets to a per-profile directory, rewrites links to relative paths, and opens +the offline copy in a new tab via `file://`. + +## User decisions + +- **Depth:** Entire site — follow all same-origin links, no depth limit, bounded + by a max-page-count safety cap (default 500). +- **Storage:** Per-profile: + `~/.sovereign_browser/profiles//offline_sites//` + (falls back to `~/.sovereign_browser/offline_sites//` when not + logged in). +- **Feedback:** Small modal progress dialog with pages/files counts, a progress + bar, and a Cancel button. + +## Trigger + +Right-click on a tab → "Download Website" menu item. Added to **both** tab +context menus in [`src/tab_manager.c`](../src/tab_manager.c): + +- `on_tab_label_button_press` menu (line ~828) +- `on_notebook_button_press` menu (line ~751) + +The callback reads the right-clicked tab's `current_url` and is only active for +`http://` / `https://` URLs (greyed out / error dialog otherwise). + +## Architecture + +```mermaid +flowchart TD + A[Right-click tab] --> B[Download Website menu item] + B --> C{URL is http/https?} + C -- No --> D[Show error, abort] + C -- Yes --> E[Show progress dialog] + E --> F[Start worker thread BFS crawl] + F --> G[Fetch page via libsoup] + G --> H[Save HTML to offline_sites/host/path] + H --> I[Extract same-origin links + assets] + I --> J{Cancel clicked?} + J -- Yes --> K[Stop crawl] + J -- No --> L{More URLs and under cap?} + L -- Yes --> G + L -- No --> M[Close dialog] + K --> M + M --> N[Open file://offline_sites/host/index.html in new tab] +``` + +### New files + +#### `src/site_downloader.h` + +Public API (self-contained, called from the tab context menu): + +```c +/* Start downloading the website at start_url. Shows a modal progress + * dialog parented to `parent`. On completion (or cancel) opens the + * offline copy in a new tab. No-op for non-http(s) URLs. */ +void site_downloader_start(const char *start_url, GtkWindow *parent); +``` + +#### `src/site_downloader.c` + +Implementation: + +1. **Progress dialog** — `GtkDialog` with: + - `GtkLabel` showing hostname + "Pages: N / Files: M" + - `GtkProgressBar` (pulse mode; or fraction = pages / cap) + - Cancel `GtkButton` → sets `g_atomic_int_set(&cancel, 1)` + +2. **Worker thread** — `g_thread_new("site-dl", ...)` running a BFS crawl: + - `SoupSession` (libsoup-3.0; pattern from + [`src/agent_llm.c`](../src/agent_llm.c:285) and + [`src/search.c`](../src/search.c:158)). + - Queue (GQueue) of URLs to fetch + a `GHashTable` visited-set. + - Politeness delay between requests (default 100ms). + +3. **Fetch + save** — for each URL: + - `soup_session_send_and_read()` (sync; we're on a worker thread). + - Map URL → local path: mirror the URL path under the site root; + directory-style URLs → `index.html`; sanitize illegal filename chars. + - Write bytes to disk (`g_file_set_contents` for text, manual write for + binary). + - For HTML responses: extract links, rewrite them to relative paths, then + save the rewritten HTML. + +4. **Link extraction & rewriting** (no HTML parser dep in the project, so use + regex over `href=`, `src=`, `srcset=`, `poster=`, and CSS `url(...)`): + - Resolve each link against the page URL (`g_uri_resolve_relative`). + - **Same-origin filter:** keep only links whose host matches the start + URL's host (via `g_uri_parse`, pattern from + [`src/tor_scheme.c`](../src/tor_scheme.c:74)). Enqueue same-origin HTML + pages; download all assets (CSS/JS/images/fonts/etc.) regardless of + origin but don't crawl cross-origin pages. + - Rewrite kept links to relative file paths so the offline copy works from + `file://`. + +5. **Completion** — `g_idle_add` a callback on the main thread that: + - Destroys the progress dialog. + - Opens `file:///index.html` in a new tab via + `tab_manager_new_tab()`. + +### Modified files + +- [`src/tab_manager.c`](../src/tab_manager.c) — add "Download Website" item to + both context menus; new callback `on_tab_download_website` calling + `site_downloader_start(tab->current_url, g_window)`. +- [`Makefile`](../Makefile) — add `src/site_downloader.c` to `SRC`. +- [`src/settings.h`](../src/settings.h) / [`src/settings.c`](../src/settings.c) + — add to `browser_settings_t`: + - `offline_site_max_pages` (default 500) + - `offline_site_request_delay_ms` (default 100) + - load/save in `settings_load()` / `settings_save()`. + +## Edge cases + +- Non-http(s) URLs → menu item shows an error dialog, no crawl. +- Not logged in → fall back to global `~/.sovereign_browser/offline_sites/`. +- Redirects → follow libsoup redirects; use the final URL for same-origin + checks and path mapping. +- Binary assets → write raw bytes, never parse/rewrite. +- Duplicate URLs → visited-set prevents re-fetching. +- Very large sites → max-page cap stops the crawl; dialog shows "cap reached". +- Filename sanitization → replace illegal chars (`/`, `?`, `:`, etc.) with `_`. +- Cancel → worker checks the cancel flag between requests; partial downloads + are kept and the new tab opens to whatever was fetched. + +## Implementation order + +1. Add settings fields + load/save. +2. Create `site_downloader.h` / `site_downloader.c` skeleton: progress dialog + + thread stub + completion callback. +3. Implement libsoup fetch + file write + URL→path mapping. +4. Implement HTML link extraction, same-origin filter, link rewriting. +5. Implement BFS crawl loop: visited-set, cap, cancel flag, progress updates. +6. Implement completion callback: open new tab with `file://` URL. +7. Wire "Download Website" into both tab context menus in `tab_manager.c`. +8. Add `site_downloader.c` to `Makefile`. +9. Build (`make`) and test with `tests/local-site/` served via + `python3 -m http.server`. + +## Testing + +- Serve `tests/local-site/` with `python3 -m http.server 8000` from the + `tests/` directory. +- Open `http://localhost:8000/local-site/index.html` in the browser. +- Right-click the tab → Download Website. +- Verify the progress dialog appears, files are saved under + `~/.sovereign_browser/profiles//offline_sites/localhost/`, and a new + tab opens the offline `index.html` with working relative links and assets. +- Test Cancel mid-download. +- Test a non-http URL (e.g. `sovereign://profile`) → error dialog. diff --git a/src/cli.c b/src/cli.c index d96c26d..b186842 100644 --- a/src/cli.c +++ b/src/cli.c @@ -109,6 +109,7 @@ enum { OPT_VERBOSE, OPT_QUIET, OPT_HELP, + OPT_DOWNLOAD_URL, }; static struct option long_opts[] = { @@ -143,6 +144,7 @@ static struct option long_opts[] = { {"verbose", no_argument, 0, OPT_VERBOSE}, {"quiet", no_argument, 0, OPT_QUIET}, {"help", no_argument, 0, OPT_HELP}, + {"download-url", required_argument, 0, OPT_DOWNLOAD_URL}, {0, 0, 0, 0} }; @@ -198,6 +200,7 @@ void cli_args_free(cli_args_t *args) { g_free(args->nsigner_transport); g_free(args->nsigner_device); g_free(args->nsigner_service); + g_free(args->download_url); memset(args, 0, sizeof(*args)); } @@ -356,6 +359,11 @@ int cli_parse(int *argc, char ***argv, cli_args_t *out) { out->want_help = TRUE; return 0; + case OPT_DOWNLOAD_URL: + g_free(out->download_url); + out->download_url = xstrdup(optarg); + break; + default: fprintf(stderr, "[cli] unhandled option\n"); return -1; diff --git a/src/cli.h b/src/cli.h index 69397c4..e46ece7 100644 --- a/src/cli.h +++ b/src/cli.h @@ -68,6 +68,10 @@ typedef struct { int verbose; /* --verbose / -v (repeatable count) */ gboolean quiet; /* --quiet / -q */ + /* Offline site download (test/diagnostic hook — triggers + * site_downloader_start shortly after startup). */ + char *download_url; /* --download-url */ + /* Exit-request flags set by --help / --version. */ gboolean want_help; gboolean want_version; diff --git a/src/main.c b/src/main.c index ab0c1ed..fc08f11 100644 --- a/src/main.c +++ b/src/main.c @@ -48,6 +48,7 @@ #include "agent_server.h" #include "agent_login.h" #include "cli.h" +#include "site_downloader.h" #include "db.h" #include "profile.h" #include "relay_fetch.h" @@ -163,6 +164,18 @@ const char *app_get_pubkey_hex(void) { return g_state.pubkey_hex; } key_store_method_t app_get_method(void) { return g_state.method; } gboolean app_get_readonly(void) { return g_state.readonly; } +/* Idle callback for the --download-url diagnostic flag: triggers the + * offline site downloader on the main thread shortly after startup. */ +static gboolean dl_test_trigger(gpointer data) { + const char *url = (const char *)data; + GtkWindow *win = gtk_window_get_focus(GTK_WINDOW(g_window)) ? + g_window : NULL; + /* g_window may not be set yet in some early-trigger scenarios; fall + * back to NULL (unparented dialog). */ + site_downloader_start(url, win); + return G_SOURCE_REMOVE; +} + /* ---- Menu proxy functions ------------------------------------------- * * These wrap the app_state_t-aware callbacks so they match GTK signal * handler signatures and can be called from tab_manager.c's hamburger @@ -1234,6 +1247,15 @@ int main(int argc, char **argv) { } } + /* Diagnostic/test hook: --download-url triggers the offline site + * downloader shortly after startup (same code path as the tab + * context menu's "Download Website"). */ + if (cli.download_url != NULL && cli.download_url[0] != '\0') { + char *dl_url = g_strdup(cli.download_url); + g_idle_add_full(G_PRIORITY_DEFAULT_IDLE, + (GSourceFunc)dl_test_trigger, dl_url, g_free); + } + cli_args_free(&cli); gtk_widget_show_all(window); diff --git a/src/settings.c b/src/settings.c index 8a686fd..7468266 100644 --- a/src/settings.c +++ b/src/settings.c @@ -126,6 +126,9 @@ static void settings_set_defaults(browser_settings_t *s) { s->fips_control_socket[0] = '\0'; snprintf(s->fips_config_dir, sizeof(s->fips_config_dir), "~/.sovereign_browser/fips"); + + s->offline_site_max_pages = SETTINGS_OFFLINE_SITE_MAX_PAGES_DEFAULT; + s->offline_site_request_delay_ms = SETTINGS_OFFLINE_SITE_REQUEST_DELAY_MS_DEFAULT; } /* ── Parsing helpers ──────────────────────────────────────────────── */ @@ -317,6 +320,17 @@ void settings_load_user(void) { val = db_kv_get("fips.config_dir"); if (val) snprintf(g_settings.fips_config_dir, sizeof(g_settings.fips_config_dir), "%s", val); + val = db_kv_get("offline_site.max_pages"); + if (val) { + g_settings.offline_site_max_pages = parse_int(val, g_settings.offline_site_max_pages); + if (g_settings.offline_site_max_pages < 1) g_settings.offline_site_max_pages = 1; + } + val = db_kv_get("offline_site.request_delay_ms"); + if (val) { + g_settings.offline_site_request_delay_ms = parse_int(val, g_settings.offline_site_request_delay_ms); + if (g_settings.offline_site_request_delay_ms < 0) g_settings.offline_site_request_delay_ms = 0; + } + /* Agent LLM provider settings — these are the "resolved" values * (active provider's base_url + api_key + selected model). They are * populated from db_kv here for local persistence, and overwritten @@ -483,6 +497,11 @@ void settings_save_user(void) { db_kv_set("fips.control_socket", g_settings.fips_control_socket); db_kv_set("fips.config_dir", g_settings.fips_config_dir); + snprintf(buf, sizeof(buf), "%d", g_settings.offline_site_max_pages); + db_kv_set("offline_site.max_pages", buf); + snprintf(buf, sizeof(buf), "%d", g_settings.offline_site_request_delay_ms); + db_kv_set("offline_site.request_delay_ms", buf); + /* Agent LLM provider settings — resolved values (active provider). */ db_kv_set("agent.llm_base_url", g_settings.agent_llm_base_url); db_kv_set("agent.llm_api_key", g_settings.agent_llm_api_key); diff --git a/src/settings.h b/src/settings.h index 749b9c5..44053a9 100644 --- a/src/settings.h +++ b/src/settings.h @@ -34,6 +34,8 @@ extern "C" { #define SETTINGS_AGENT_LLM_BASE_URL_DEFAULT "https://api.ppq.ai" #define SETTINGS_AGENT_LLM_MODEL_DEFAULT "gpt-4o" #define SETTINGS_AGENT_MAX_ITERATIONS_DEFAULT 100 +#define SETTINGS_OFFLINE_SITE_MAX_PAGES_DEFAULT 500 +#define SETTINGS_OFFLINE_SITE_REQUEST_DELAY_MS_DEFAULT 100 #define SETTINGS_AGENT_SYSTEM_PROMPT_DEFAULT \ "You are an AI assistant embedded in a web browser. You have access to browser automation tools (navigate, snapshot, click, fill, etc.) and system tools (filesystem read/write, shell command execution). Use the snapshot tool to understand page content, then interact with elements using refs (e.g. @e1). You can read and write files and run shell commands. Be concise in your responses. When a task is complete, summarize what you did." @@ -121,6 +123,10 @@ typedef struct { char fips_binary_path[256]; char fips_control_socket[512]; char fips_config_dir[512]; + + /* Offline website downloader (right-click tab → Download Website). */ + int offline_site_max_pages; /* safety cap on pages crawled */ + int offline_site_request_delay_ms; /* politeness delay between requests */ } browser_settings_t; /* diff --git a/src/site_downloader.c b/src/site_downloader.c new file mode 100644 index 0000000..45f16e2 --- /dev/null +++ b/src/site_downloader.c @@ -0,0 +1,1550 @@ +/* + * site_downloader.c — download a website for offline viewing + * + * See site_downloader.h and plans/download-website-offline.md. + * + * Architecture: + * - site_downloader_start() is called from the tab context menu on the + * GTK main thread. It validates the URL, builds a progress dialog, + * and spawns a worker thread that runs a BFS crawl. + * - The worker thread uses libsoup-3.0 (synchronous) to fetch each + * page, saves it to disk, extracts same-origin links + assets, + * rewrites links in saved HTML to relative paths, and enqueues new + * pages. It periodically posts progress updates back to the main + * thread via g_idle_add. + * - On completion (or cancel), a main-thread idle callback closes the + * progress dialog and opens the offline index.html in a new tab. + */ + +#include "site_downloader.h" +#include "settings.h" +#include "profile.h" +#include "tab_manager.h" +#include "web_context.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "../nostr_core_lib/cjson/cJSON.h" + +/* Provided by main.c — the currently logged-in user's hex pubkey, or + * an empty string when not logged in. */ +extern const char *app_get_pubkey_hex(void); + +/* ── Defaults / limits ──────────────────────────────────────────────── */ + +#define DL_MAX_ASSETS 4000 /* safety cap on total assets fetched */ +#define DL_MAX_URL_LEN 4096 +#define DL_MAX_PATH_LEN 4096 +#define DL_PROGRESS_UPDATE_MS 150 /* min interval between UI updates (ms) */ + +/* ── Worker state ───────────────────────────────────────────────────── */ + +typedef struct { + char *start_url; /* absolute starting URL (normalized) */ + char *root_dir; /* on-disk root for this site (no trailing /) */ + char *host; /* same-origin host to restrict crawl to */ + char *path_prefix; /* path prefix to restrict page crawling to */ + SoupSession *session; + + GQueue *queue; /* URLs to fetch (char*) */ + GHashTable *visited; /* set of absolute URL strings */ + + volatile gint cancel; /* 1 when user clicked Cancel */ + volatile gint pages; /* pages fetched so far */ + volatile gint files; /* total files (pages + assets) saved */ + volatile gint failed; /* fetch failures */ + + GtkWindow *parent; /* for the progress dialog (main thread) */ + GtkWidget *dialog; /* progress dialog (main thread only) */ + GtkWidget *status_label; /* "Pages: N Files: M" */ + GtkWidget *progress_bar; /* fraction = pages / cap */ + GtkWidget *host_label; /* shows the hostname */ + + gint64 last_ui_update_ms; /* last time we posted an idle update */ +} dl_state_t; + +/* ── Forward declarations ───────────────────────────────────────────── */ + +static gpointer dl_worker(dl_state_t *st); +static void dl_post_progress(dl_state_t *st); +static gboolean dl_open_result_tab(gpointer data); + +/* Global pointer to the active download state, so the completion + * callback can find it. Only one download is active at a time. */ +static dl_state_t *g_active_dl = NULL; + +/* ── WebKit-based rendered-link extraction ──────────────────────────── * + * For SPA pages where links are generated by JavaScript, we load the + * page in a hidden WebKit webview, let JS render, then extract all + * , , ,