v0.0.57 - Added Download Website feature: right-click tab to crawl and save a site for offline viewing, with WebKit-based SPA link discovery and path-prefix filtering

Makefile

@@ -21,7 +21,7 @@ NOSTR_LIB  = ./nostr_core_lib/libnostr_core_x64.a
 NOSTR_DEPS = -lsecp256k1 -lssl -lcrypto -lcurl -lz -ldl -lpthread -lm
 
 BIN := sovereign_browser
-SRC := src/main.c src/key_store.c src/login_dialog.c src/nostr_bridge.c src/nostr_inject.c src/nostr_url.c src/nostr_scheme.c src/history.c src/settings.c src/net_services.c src/tor_control.c src/tor_scheme.c src/fips_control.c src/tab_manager.c src/session.c src/agent_server.c src/agent_login.c src/agent_snapshot.c src/agent_tools.c src/agent_mcp.c src/cli.c src/qr.c src/db.c src/relay_fetch.c src/bookmarks.c src/shortcuts.c src/settings_sync.c src/search.c src/profile.c src/agent_fs_tools.c src/agent_llm.c src/agent_loop.c src/agent_chat_store.c src/agent_chat.c src/agent_conversations.c src/agent_skills.c src/embedded_web_content.c src/process_info.c src/perf_probe.c src/webkit_data.c src/web_context.c
+SRC := src/main.c src/key_store.c src/login_dialog.c src/nostr_bridge.c src/nostr_inject.c src/nostr_url.c src/nostr_scheme.c src/history.c src/settings.c src/net_services.c src/tor_control.c src/tor_scheme.c src/fips_control.c src/tab_manager.c src/session.c src/agent_server.c src/agent_login.c src/agent_snapshot.c src/agent_tools.c src/agent_mcp.c src/cli.c src/qr.c src/db.c src/relay_fetch.c src/bookmarks.c src/shortcuts.c src/settings_sync.c src/search.c src/profile.c src/agent_fs_tools.c src/agent_llm.c src/agent_loop.c src/agent_chat_store.c src/agent_chat.c src/agent_conversations.c src/agent_skills.c src/embedded_web_content.c src/process_info.c src/perf_probe.c src/webkit_data.c src/web_context.c src/site_downloader.c
 
 # Web files embedded into the binary as C byte arrays.
 WEB_FILES := $(shell find www -type f \( -name '*.html' -o -name '*.css' -o -name '*.js' \) 2>/dev/null)

VERSION

@@ -1 +1 @@
-0.0.56
+0.0.57

plans/download-website-offline.md

@@ -0,0 +1,156 @@
+# Plan: Download Website for Offline Viewing
+
+## Goal
+
+Add a "Download Website" option to the tab right-click context menu. It crawls
+the current site (same-origin, bounded by a page cap), saves all pages and
+assets to a per-profile directory, rewrites links to relative paths, and opens
+the offline copy in a new tab via `file://`.
+
+## User decisions
+
+- **Depth:** Entire site — follow all same-origin links, no depth limit, bounded
+  by a max-page-count safety cap (default 500).
+- **Storage:** Per-profile:
+  `~/.sovereign_browser/profiles/<pubkey>/offline_sites/<hostname>/`
+  (falls back to `~/.sovereign_browser/offline_sites/<hostname>/` when not
+  logged in).
+- **Feedback:** Small modal progress dialog with pages/files counts, a progress
+  bar, and a Cancel button.
+
+## Trigger
+
+Right-click on a tab → "Download Website" menu item. Added to **both** tab
+context menus in [`src/tab_manager.c`](../src/tab_manager.c):
+
+- `on_tab_label_button_press` menu (line ~828)
+- `on_notebook_button_press` menu (line ~751)
+
+The callback reads the right-clicked tab's `current_url` and is only active for
+`http://` / `https://` URLs (greyed out / error dialog otherwise).
+
+## Architecture
+
+```mermaid
+flowchart TD
+    A[Right-click tab] --> B[Download Website menu item]
+    B --> C{URL is http/https?}
+    C -- No --> D[Show error, abort]
+    C -- Yes --> E[Show progress dialog]
+    E --> F[Start worker thread BFS crawl]
+    F --> G[Fetch page via libsoup]
+    G --> H[Save HTML to offline_sites/host/path]
+    H --> I[Extract same-origin links + assets]
+    I --> J{Cancel clicked?}
+    J -- Yes --> K[Stop crawl]
+    J -- No --> L{More URLs and under cap?}
+    L -- Yes --> G
+    L -- No --> M[Close dialog]
+    K --> M
+    M --> N[Open file://offline_sites/host/index.html in new tab]
+```
+
+### New files
+
+#### `src/site_downloader.h`
+
+Public API (self-contained, called from the tab context menu):
+
+```c
+/* Start downloading the website at start_url. Shows a modal progress
+ * dialog parented to `parent`. On completion (or cancel) opens the
+ * offline copy in a new tab. No-op for non-http(s) URLs. */
+void site_downloader_start(const char *start_url, GtkWindow *parent);
+```
+
+#### `src/site_downloader.c`
+
+Implementation:
+
+1. **Progress dialog** — `GtkDialog` with:
+   - `GtkLabel` showing hostname + "Pages: N / Files: M"
+   - `GtkProgressBar` (pulse mode; or fraction = pages / cap)
+   - Cancel `GtkButton` → sets `g_atomic_int_set(&cancel, 1)`
+
+2. **Worker thread** — `g_thread_new("site-dl", ...)` running a BFS crawl:
+   - `SoupSession` (libsoup-3.0; pattern from
+     [`src/agent_llm.c`](../src/agent_llm.c:285) and
+     [`src/search.c`](../src/search.c:158)).
+   - Queue (GQueue) of URLs to fetch + a `GHashTable` visited-set.
+   - Politeness delay between requests (default 100ms).
+
+3. **Fetch + save** — for each URL:
+   - `soup_session_send_and_read()` (sync; we're on a worker thread).
+   - Map URL → local path: mirror the URL path under the site root;
+     directory-style URLs → `index.html`; sanitize illegal filename chars.
+   - Write bytes to disk (`g_file_set_contents` for text, manual write for
+     binary).
+   - For HTML responses: extract links, rewrite them to relative paths, then
+     save the rewritten HTML.
+
+4. **Link extraction & rewriting** (no HTML parser dep in the project, so use
+   regex over `href=`, `src=`, `srcset=`, `poster=`, and CSS `url(...)`):
+   - Resolve each link against the page URL (`g_uri_resolve_relative`).
+   - **Same-origin filter:** keep only links whose host matches the start
+     URL's host (via `g_uri_parse`, pattern from
+     [`src/tor_scheme.c`](../src/tor_scheme.c:74)). Enqueue same-origin HTML
+     pages; download all assets (CSS/JS/images/fonts/etc.) regardless of
+     origin but don't crawl cross-origin pages.
+   - Rewrite kept links to relative file paths so the offline copy works from
+     `file://`.
+
+5. **Completion** — `g_idle_add` a callback on the main thread that:
+   - Destroys the progress dialog.
+   - Opens `file://<offline_dir>/index.html` in a new tab via
+     `tab_manager_new_tab()`.
+
+### Modified files
+
+- [`src/tab_manager.c`](../src/tab_manager.c) — add "Download Website" item to
+  both context menus; new callback `on_tab_download_website` calling
+  `site_downloader_start(tab->current_url, g_window)`.
+- [`Makefile`](../Makefile) — add `src/site_downloader.c` to `SRC`.
+- [`src/settings.h`](../src/settings.h) / [`src/settings.c`](../src/settings.c)
+  — add to `browser_settings_t`:
+  - `offline_site_max_pages` (default 500)
+  - `offline_site_request_delay_ms` (default 100)
+  - load/save in `settings_load()` / `settings_save()`.
+
+## Edge cases
+
+- Non-http(s) URLs → menu item shows an error dialog, no crawl.
+- Not logged in → fall back to global `~/.sovereign_browser/offline_sites/`.
+- Redirects → follow libsoup redirects; use the final URL for same-origin
+  checks and path mapping.
+- Binary assets → write raw bytes, never parse/rewrite.
+- Duplicate URLs → visited-set prevents re-fetching.
+- Very large sites → max-page cap stops the crawl; dialog shows "cap reached".
+- Filename sanitization → replace illegal chars (`/`, `?`, `:`, etc.) with `_`.
+- Cancel → worker checks the cancel flag between requests; partial downloads
+  are kept and the new tab opens to whatever was fetched.
+
+## Implementation order
+
+1. Add settings fields + load/save.
+2. Create `site_downloader.h` / `site_downloader.c` skeleton: progress dialog
+   + thread stub + completion callback.
+3. Implement libsoup fetch + file write + URL→path mapping.
+4. Implement HTML link extraction, same-origin filter, link rewriting.
+5. Implement BFS crawl loop: visited-set, cap, cancel flag, progress updates.
+6. Implement completion callback: open new tab with `file://` URL.
+7. Wire "Download Website" into both tab context menus in `tab_manager.c`.
+8. Add `site_downloader.c` to `Makefile`.
+9. Build (`make`) and test with `tests/local-site/` served via
+   `python3 -m http.server`.
+
+## Testing
+
+- Serve `tests/local-site/` with `python3 -m http.server 8000` from the
+  `tests/` directory.
+- Open `http://localhost:8000/local-site/index.html` in the browser.
+- Right-click the tab → Download Website.
+- Verify the progress dialog appears, files are saved under
+  `~/.sovereign_browser/profiles/<pubkey>/offline_sites/localhost/`, and a new
+  tab opens the offline `index.html` with working relative links and assets.
+- Test Cancel mid-download.
+- Test a non-http URL (e.g. `sovereign://profile`) → error dialog.

src/cli.c

@@ -109,6 +109,7 @@ enum {
     OPT_VERBOSE,
     OPT_QUIET,
     OPT_HELP,
+    OPT_DOWNLOAD_URL,
 };
 
 static struct option long_opts[] = {
@@ -143,6 +144,7 @@ static struct option long_opts[] = {
     {"verbose",            no_argument,       0, OPT_VERBOSE},
     {"quiet",              no_argument,       0, OPT_QUIET},
     {"help",               no_argument,       0, OPT_HELP},
+    {"download-url",       required_argument, 0, OPT_DOWNLOAD_URL},
     {0, 0, 0, 0}
 };
 
@@ -198,6 +200,7 @@ void cli_args_free(cli_args_t *args) {
     g_free(args->nsigner_transport);
     g_free(args->nsigner_device);
     g_free(args->nsigner_service);
+    g_free(args->download_url);
     memset(args, 0, sizeof(*args));
 }
 
@@ -356,6 +359,11 @@ int cli_parse(int *argc, char ***argv, cli_args_t *out) {
             out->want_help = TRUE;
             return 0;
 
+        case OPT_DOWNLOAD_URL:
+            g_free(out->download_url);
+            out->download_url = xstrdup(optarg);
+            break;
+
         default:
             fprintf(stderr, "[cli] unhandled option\n");
             return -1;

src/cli.h

@@ -68,6 +68,10 @@ typedef struct {
     int      verbose;              /* --verbose / -v (repeatable count) */
     gboolean quiet;                /* --quiet / -q */
 
+    /* Offline site download (test/diagnostic hook — triggers
+     * site_downloader_start shortly after startup). */
+    char    *download_url;         /* --download-url <url> */
+
     /* Exit-request flags set by --help / --version. */
     gboolean want_help;
     gboolean want_version;

src/main.c

@@ -48,6 +48,7 @@
 #include "agent_server.h"
 #include "agent_login.h"
 #include "cli.h"
+#include "site_downloader.h"
 #include "db.h"
 #include "profile.h"
 #include "relay_fetch.h"
@@ -163,6 +164,18 @@ const char *app_get_pubkey_hex(void) { return g_state.pubkey_hex; }
 key_store_method_t app_get_method(void) { return g_state.method; }
 gboolean app_get_readonly(void) { return g_state.readonly; }
 
+/* Idle callback for the --download-url diagnostic flag: triggers the
+ * offline site downloader on the main thread shortly after startup. */
+static gboolean dl_test_trigger(gpointer data) {
+    const char *url = (const char *)data;
+    GtkWindow *win = gtk_window_get_focus(GTK_WINDOW(g_window)) ?
+                     g_window : NULL;
+    /* g_window may not be set yet in some early-trigger scenarios; fall
+     * back to NULL (unparented dialog). */
+    site_downloader_start(url, win);
+    return G_SOURCE_REMOVE;
+}
+
 /* ---- Menu proxy functions ------------------------------------------- *
  * These wrap the app_state_t-aware callbacks so they match GTK signal
  * handler signatures and can be called from tab_manager.c's hamburger
@@ -1234,6 +1247,15 @@ int main(int argc, char **argv) {
         }
     }
 
+    /* Diagnostic/test hook: --download-url triggers the offline site
+     * downloader shortly after startup (same code path as the tab
+     * context menu's "Download Website"). */
+    if (cli.download_url != NULL && cli.download_url[0] != '\0') {
+        char *dl_url = g_strdup(cli.download_url);
+        g_idle_add_full(G_PRIORITY_DEFAULT_IDLE,
+            (GSourceFunc)dl_test_trigger, dl_url, g_free);
+    }
+
     cli_args_free(&cli);
 
     gtk_widget_show_all(window);

src/settings.c

@@ -126,6 +126,9 @@ static void settings_set_defaults(browser_settings_t *s) {
     s->fips_control_socket[0] = '\0';
     snprintf(s->fips_config_dir, sizeof(s->fips_config_dir),
              "~/.sovereign_browser/fips");
+
+    s->offline_site_max_pages        = SETTINGS_OFFLINE_SITE_MAX_PAGES_DEFAULT;
+    s->offline_site_request_delay_ms = SETTINGS_OFFLINE_SITE_REQUEST_DELAY_MS_DEFAULT;
 }
 
 /* ── Parsing helpers ──────────────────────────────────────────────── */
@@ -317,6 +320,17 @@ void settings_load_user(void) {
     val = db_kv_get("fips.config_dir");
     if (val) snprintf(g_settings.fips_config_dir, sizeof(g_settings.fips_config_dir), "%s", val);
 
+    val = db_kv_get("offline_site.max_pages");
+    if (val) {
+        g_settings.offline_site_max_pages = parse_int(val, g_settings.offline_site_max_pages);
+        if (g_settings.offline_site_max_pages < 1) g_settings.offline_site_max_pages = 1;
+    }
+    val = db_kv_get("offline_site.request_delay_ms");
+    if (val) {
+        g_settings.offline_site_request_delay_ms = parse_int(val, g_settings.offline_site_request_delay_ms);
+        if (g_settings.offline_site_request_delay_ms < 0) g_settings.offline_site_request_delay_ms = 0;
+    }
+
     /* Agent LLM provider settings — these are the "resolved" values
      * (active provider's base_url + api_key + selected model). They are
      * populated from db_kv here for local persistence, and overwritten
@@ -483,6 +497,11 @@ void settings_save_user(void) {
     db_kv_set("fips.control_socket", g_settings.fips_control_socket);
     db_kv_set("fips.config_dir", g_settings.fips_config_dir);
 
+    snprintf(buf, sizeof(buf), "%d", g_settings.offline_site_max_pages);
+    db_kv_set("offline_site.max_pages", buf);
+    snprintf(buf, sizeof(buf), "%d", g_settings.offline_site_request_delay_ms);
+    db_kv_set("offline_site.request_delay_ms", buf);
+
     /* Agent LLM provider settings — resolved values (active provider). */
     db_kv_set("agent.llm_base_url", g_settings.agent_llm_base_url);
     db_kv_set("agent.llm_api_key", g_settings.agent_llm_api_key);

src/settings.h

@@ -34,6 +34,8 @@ extern "C" {
 #define SETTINGS_AGENT_LLM_BASE_URL_DEFAULT "https://api.ppq.ai"
 #define SETTINGS_AGENT_LLM_MODEL_DEFAULT    "gpt-4o"
 #define SETTINGS_AGENT_MAX_ITERATIONS_DEFAULT 100
+#define SETTINGS_OFFLINE_SITE_MAX_PAGES_DEFAULT       500
+#define SETTINGS_OFFLINE_SITE_REQUEST_DELAY_MS_DEFAULT 100
 #define SETTINGS_AGENT_SYSTEM_PROMPT_DEFAULT \
     "You are an AI assistant embedded in a web browser. You have access to browser automation tools (navigate, snapshot, click, fill, etc.) and system tools (filesystem read/write, shell command execution). Use the snapshot tool to understand page content, then interact with elements using refs (e.g. @e1). You can read and write files and run shell commands. Be concise in your responses. When a task is complete, summarize what you did."
 
@@ -121,6 +123,10 @@ typedef struct {
     char     fips_binary_path[256];
     char     fips_control_socket[512];
     char     fips_config_dir[512];
+
+    /* Offline website downloader (right-click tab → Download Website). */
+    int      offline_site_max_pages;        /* safety cap on pages crawled */
+    int      offline_site_request_delay_ms; /* politeness delay between requests */
 } browser_settings_t;
 
 /*

src/site_downloader.h

@@ -0,0 +1,43 @@
+/*
+ * site_downloader.h — download a website for offline viewing
+ *
+ * Triggered from the tab right-click context menu ("Download Website").
+ * Crawls the current site (same-origin, bounded by a page-count cap),
+ * saves all pages and assets to a per-profile directory, rewrites links
+ * to relative paths, and opens the offline copy in a new tab via file://.
+ *
+ * See plans/download-website-offline.md for the full design.
+ */
+
+#ifndef SITE_DOWNLOADER_H
+#define SITE_DOWNLOADER_H
+
+#include <gtk/gtk.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Start downloading the website at start_url.
+ *
+ * Shows a modal progress dialog (pages/files counts + progress bar +
+ * Cancel button) parented to `parent`. A background thread runs a BFS
+ * crawl over same-origin links using libsoup, saving pages and assets
+ * under ~/.sovereign_browser/profiles/<pubkey>/offline_sites/<host>/
+ * (or the global ~/.sovereign_browser/offline_sites/<host>/ when not
+ * logged in). On completion or cancel, the progress dialog is closed
+ * and the offline index.html is opened in a new tab.
+ *
+ * No-op (shows an error dialog) for non-http(s) URLs.
+ *
+ *   start_url — absolute http:// or https:// URL of the page to download
+ *   parent    — the GtkWindow to parent the progress dialog to (may be NULL)
+ */
+void site_downloader_start(const char *start_url, GtkWindow *parent);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SITE_DOWNLOADER_H */

src/tab_manager.c

@@ -21,6 +21,7 @@
 #include "db.h"
 #include "net_services.h"
 #include "agent_chat.h"   /* agent_chat_route_input() for ";" URL-bar shortcut */
+#include "site_downloader.h"
 
 #include <string.h>
 #include <strings.h>   /* strcasecmp */
@@ -168,6 +169,7 @@ static int          tab_array_find(tab_info_t *tab);
 static GtkWidget   *build_hamburger_menu(tab_info_t *tab);
 static char        *normalize_url(const char *input);
 static void         on_tab_close_clicked_proxy_new(GtkMenuItem *item, gpointer data);
+static void         on_tab_download_website(GtkMenuItem *item, gpointer data);
 static void         on_avatar_clicked(GtkButton *btn, gpointer data);
 static GtkWidget   *tab_manager_new_window(const char *url,
                                             WebKitWebView *related_view);
@@ -797,6 +799,15 @@ static gboolean on_notebook_button_press(GtkWidget *widget,
                                          GINT_TO_POINTER(index));
                 gtk_menu_shell_append(GTK_MENU_SHELL(menu), item_reload);
 
+                gtk_menu_shell_append(GTK_MENU_SHELL(menu),
+                                      gtk_separator_menu_item_new());
+
+                GtkWidget *item_dl =
+                    gtk_menu_item_new_with_label("Download Website");
+                g_signal_connect(item_dl, "activate",
+                                 G_CALLBACK(on_tab_download_website), tab);
+                gtk_menu_shell_append(GTK_MENU_SHELL(menu), item_dl);
+
                 gtk_widget_show_all(menu);
                 gtk_menu_popup_at_pointer(GTK_MENU(menu), (GdkEvent *)event);
                 return TRUE;
@@ -875,6 +886,15 @@ static gboolean on_tab_label_button_press(GtkWidget *widget,
                                  GINT_TO_POINTER(index));
         gtk_menu_shell_append(GTK_MENU_SHELL(menu), item_reload);
 
+        gtk_menu_shell_append(GTK_MENU_SHELL(menu),
+                              gtk_separator_menu_item_new());
+
+        GtkWidget *item_dl =
+            gtk_menu_item_new_with_label("Download Website");
+        g_signal_connect(item_dl, "activate",
+                         G_CALLBACK(on_tab_download_website), tab);
+        gtk_menu_shell_append(GTK_MENU_SHELL(menu), item_dl);
+
         gtk_widget_show_all(menu);
         gtk_menu_popup_at_pointer(GTK_MENU(menu), (GdkEvent *)event);
         return TRUE;
@@ -890,6 +910,19 @@ static void on_tab_close_clicked_proxy_new(GtkMenuItem *item, gpointer data) {
     tab_manager_new_tab(NULL);
 }
 
+/* "Download Website" — crawl the tab's current URL for offline viewing.
+ * The tab_info_t is passed as data; we read its current_url and hand off
+ * to site_downloader_start, which shows a progress dialog and opens the
+ * offline copy in a new tab when done. */
+static void on_tab_download_website(GtkMenuItem *item, gpointer data) {
+    (void)item;
+    tab_info_t *tab = (tab_info_t *)data;
+    if (tab == NULL) return;
+    if (tab->current_url[0] == '\0') return;
+    GtkWindow *parent = g_active_window ? g_active_window : g_window;
+    site_downloader_start(tab->current_url, parent);
+}
+
 /* ── Favicon handling ─────────────────────────────────────────────── *
  * WebKitGTK has a built-in favicon database (WebKitFaviconDatabase) that
  * automatically fetches and caches favicons as pages load. It must be

src/version.h

@@ -11,9 +11,9 @@
 #ifndef SOVEREIGN_BROWSER_VERSION_H
 #define SOVEREIGN_BROWSER_VERSION_H
 
-#define SB_VERSION         "v0.0.56"
+#define SB_VERSION         "v0.0.57"
 #define SB_VERSION_MAJOR   0
 #define SB_VERSION_MINOR   0
-#define SB_VERSION_PATCH   56
+#define SB_VERSION_PATCH   57
 
 #endif /* SOVEREIGN_BROWSER_VERSION_H */