diff --git a/plans/gpu_enablement_plan.md b/plans/gpu_enablement_plan.md new file mode 100644 index 0000000..dd42143 --- /dev/null +++ b/plans/gpu_enablement_plan.md @@ -0,0 +1,253 @@ +# GPU Enablement Plan — voice_linux on NVIDIA GTX 1080 Ti + +## Summary + +Enable CUDA-accelerated whisper.cpp inference for the voice_linux project by passing the GTX 1080 Ti through to the Qubes `ai` AppVM, installing NVIDIA drivers and CUDA toolkit, and rebuilding the application with `WHISPER_CUDA=1`. + +## Current State + +The codebase is **already GPU-ready**: +- [`build.sh`](../build.sh) accepts `WHISPER_CUDA=1` to build whisper.cpp with CUDA +- [`src/transcribe.c:22`](../src/transcribe.c:22) sets `cparams.use_gpu` from the transcribe params +- [`src/main.c:77`](../src/main.c:77) supports `--gpu` / `--cpu` CLI flags +- [`src/transcribe.h:9`](../src/transcribe.h:9) has `use_gpu` in the params struct + +What's missing is the **hardware and driver stack** underneath. + +## Architecture + +```mermaid +graph TD + A[dom0 - Xen Hypervisor] -->|PCI passthrough| B[ai AppVM] + B --> C[NVIDIA Driver 550.x] + C --> D[CUDA Toolkit 12.6] + D --> E[whisper.cpp built with WHISPER_CUDA=ON] + E --> F[voice_linux --gpu] + + style A fill:#f9f,stroke:#333 + style F fill:#9f9,stroke:#333 +``` + +## Phase 0: GPU Passthrough in dom0 + +> **All commands in this phase run in a dom0 terminal.** + +### Step 0.1 — Hide the GTX 1080 Ti from dom0 + +Edit `/etc/default/grub` in dom0 and add `rd.qubes.hide_pci=65:00.0,65:00.1` to the end of the **first** `GRUB_CMDLINE_LINUX` line (inside the quotes): + +```bash +sudo nano /etc/default/grub +``` + +The line should look like: +``` +GRUB_CMDLINE_LINUX="...existing options... rd.qubes.hide_pci=65:00.0,65:00.1" +``` + +The BDF addresses `65:00.0` (GPU) and `65:00.1` (HDMI audio) come from the `lspci` output documented in [`plans/architecture.md:97`](../plans/architecture.md:97). + +### Step 0.2 — Regenerate GRUB and reboot + +```bash +sudo grub2-mkconfig -o /boot/efi/EFI/qubes/grub.cfg +# If that path doesn't exist: sudo grub2-mkconfig -o /boot/grub2/grub.cfg +sudo reboot +``` + +### Step 0.3 — Verify GPU is assignable + +After reboot, in dom0: + +```bash +xl pci-assignable-list +``` + +Expected output should include: +``` +0000:65:00.0 +0000:65:00.1 +``` + +If these don't appear, check IOMMU grouping with `xl pci-assignable-list -l` and verify the GRUB change took effect with `cat /proc/cmdline`. + +### Step 0.4 — Attach GPU to the ai AppVM + +```bash +qvm-pci attach ai dom0:65_00.0 --persistent -o permissive=true +qvm-pci attach ai dom0:65_00.1 --persistent -o permissive=true +``` + +The `--persistent` flag ensures the GPU is attached on every boot. The `permissive=true` option is needed for NVIDIA GPUs in Qubes. + +### Step 0.5 — Decide on driver persistence + +Since `ai` is a template-based AppVM using `debian-13-xfce`, anything installed outside `/home` is lost on reboot. Three options: + +| Option | Pros | Cons | +|--------|------|------| +| **A: Install in template** | Persists automatically, simple | Affects ALL AppVMs using that template | +| **B: Convert to StandaloneVM** | Full persistence, isolated | Uses more disk, no template updates | +| **C: bind-dirs** | Keeps template-based, selective persistence | More complex setup | + +**Recommendation**: Option B (StandaloneVM) is simplest for a dedicated AI workload. Run in dom0: + +```bash +qvm-clone --class StandaloneVM ai ai-standalone +# Then attach GPU to ai-standalone instead +qvm-pci attach ai-standalone dom0:65_00.0 --persistent -o permissive=true +qvm-pci attach ai-standalone dom0:65_00.1 --persistent -o permissive=true +``` + +## Phase 1: NVIDIA Driver Installation + +> **All commands from here run inside the ai AppVM (or ai-standalone).** + +### Step 1.1 — Install build prerequisites + +```bash +sudo apt update +sudo apt install -y build-essential linux-headers-$(uname -r) +``` + +### Step 1.2 — Install NVIDIA driver + +For the GTX 1080 Ti (Pascal/GP102, compute capability 6.1), use the 550.x branch: + +```bash +wget https://us.download.nvidia.com/XFree86/Linux-x86_64/550.127.05/NVIDIA-Linux-x86_64-550.127.05.run +chmod +x NVIDIA-Linux-x86_64-550.127.05.run +sudo ./NVIDIA-Linux-x86_64-550.127.05.run --no-opengl-files --dkms +``` + +> **Critical**: `--no-opengl-files` prevents replacing the VM's display GL stack. We only need CUDA compute, not display rendering. + +### Step 1.3 — Verify driver + +```bash +nvidia-smi +``` + +Expected: Shows GTX 1080 Ti with 11GB VRAM and driver version 550.127.05. + +## Phase 2: CUDA Toolkit Installation + +### Step 2.1 — Install CUDA toolkit + +```bash +wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb +sudo apt update +sudo apt install -y cuda-toolkit-12-6 +``` + +### Step 2.2 — Configure environment + +Add to `~/.bashrc`: + +```bash +echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc +echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc +source ~/.bashrc +``` + +### Step 2.3 — Verify CUDA + +```bash +nvcc --version +``` + +Expected: CUDA 12.6. + +## Phase 3: Rebuild with CUDA + +### Step 3.1 — Clean and rebuild + +From the project directory, run: + +```bash +WHISPER_CUDA=1 bash ./build.sh +``` + +This will: +1. Clone whisper.cpp into `vendor/whisper.cpp` (if not already present) +2. Run `cmake -DWHISPER_CUDA=ON` to configure whisper.cpp with CUDA support +3. Build whisper.cpp with CUDA kernels +4. Download the `ggml-base.en.bin` model (if not present) +5. Compile voice_linux linking against the CUDA-enabled whisper library + +### Step 3.2 — Verify CUDA linkage + +```bash +ldd ./voice_linux | grep -i cuda +``` + +Should show linkage to CUDA libraries (indirectly through libwhisper). + +Also check: +```bash +ldd ./vendor/whisper.cpp/build/src/libwhisper.so | grep -i cuda +``` + +Should show `libcudart.so`, `libcublas.so`, etc. + +## Phase 4: Run with GPU + +### Step 4.1 — Launch with GPU flag + +```bash +./voice_linux --gpu +``` + +The startup log at [`src/main.c:119`](../src/main.c:119) will print `gpu: on` confirming GPU mode. Watch for whisper.cpp log lines mentioning CUDA device initialization. + +### Step 4.2 — Verify GPU utilization + +In a separate terminal while transcribing: + +```bash +nvidia-smi +``` + +Should show `voice_linux` process using GPU memory. + +### Step 4.3 — Optional: Try a larger model + +With 11GB VRAM on the 1080 Ti, you can comfortably run `medium.en` for much better accuracy: + +```bash +cd models/ +wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin +``` + +Then update [`config.ini`](../config.ini) line 5: +```ini +model_path=./models/ggml-medium.en.bin +``` + +| Model | VRAM Usage | Speed (10s audio) | Accuracy | +|-------|-----------|-------------------|----------| +| base.en | ~1GB | <1s | Good | +| medium.en | ~5GB | ~2-3s | Excellent | +| large-v3 | ~10GB | ~4-6s | Best | + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| `xl pci-assignable-list` empty | Check GRUB cmdline with `cat /proc/cmdline`, verify `rd.qubes.hide_pci` is present | +| `nvidia-smi` not found | Driver install failed; check `dmesg` for NVIDIA errors | +| `nvcc` not found | CUDA PATH not set; run `source ~/.bashrc` or check install | +| whisper.cpp cmake fails with CUDA | Ensure `nvcc` is in PATH before running `build.sh` | +| `voice_linux` crashes on `--gpu` | Run without `--gpu` first to verify CPU mode works, then check CUDA libs with `ldd` | +| GPU passthrough causes VM crash | Try without `permissive=true` first, or check IOMMU groups in dom0 | + +## No Code Changes Required + +The existing codebase handles everything: +- [`build.sh:8`](../build.sh:8) — `WHISPER_CUDA` env var controls CUDA build +- [`build.sh:92-93`](../build.sh:92) — passes `-DWHISPER_CUDA=ON` to cmake +- [`src/transcribe.c:21-22`](../src/transcribe.c:21) — `whisper_context_default_params()` + `use_gpu` flag +- [`src/main.c:77-78`](../src/main.c:77) — `--gpu` / `--cpu` CLI argument parsing + +The only change is the build command: `WHISPER_CUDA=1 bash ./build.sh` instead of `bash ./build.sh`. diff --git a/src/config.c b/src/config.c index 7ad2bcc..feccd35 100644 --- a/src/config.c +++ b/src/config.c @@ -37,6 +37,7 @@ void config_set_defaults(voice_config_t *cfg) { cfg->vad_min_speech_ms = 250; cfg->vad_max_speech_ms = 15000; cfg->autotype_enabled = 1; + cfg->filter_bracketed_tags = 1; } int config_load_file(const char *path, voice_config_t *cfg) { @@ -91,6 +92,8 @@ int config_load_file(const char *path, voice_config_t *cfg) { cfg->vad_max_speech_ms = atoi(value); } else if (strcmp(key, "autotype_enabled") == 0) { cfg->autotype_enabled = atoi(value); + } else if (strcmp(key, "filter_bracketed_tags") == 0) { + cfg->filter_bracketed_tags = atoi(value); } } diff --git a/src/config.h b/src/config.h index 1a80a4c..1e29e3f 100644 --- a/src/config.h +++ b/src/config.h @@ -20,6 +20,7 @@ typedef struct { int vad_min_speech_ms; int vad_max_speech_ms; int autotype_enabled; + int filter_bracketed_tags; } voice_config_t; void config_set_defaults(voice_config_t *cfg); diff --git a/src/gui_main.c b/src/gui_main.c index 1cabac8..b7a4410 100644 --- a/src/gui_main.c +++ b/src/gui_main.c @@ -18,6 +18,7 @@ typedef struct { GtkWidget *record_btn; GtkWidget *always_on_btn; GtkWidget *type_check; + GtkWidget *filter_tags_check; GtkWidget *transcript_view; GtkWidget *record_light; GtkWidget *transcribe_light; @@ -424,6 +425,13 @@ static GtkWidget *add_scale_row(GtkWidget *grid, return scale; } +static void on_filter_tags_toggled(GtkToggleButton *btn, gpointer user_data) { + gui_state_t *s = (gui_state_t *)user_data; + int enabled = gtk_toggle_button_get_active(btn) ? 1 : 0; + s->cfg.filter_bracketed_tags = enabled; + transcribe_set_filter_bracketed_tags(enabled); +} + static void on_app_activate(GtkApplication *app, gpointer user_data) { gui_state_t *s = (gui_state_t *)user_data; @@ -489,6 +497,11 @@ static void on_app_activate(GtkApplication *app, gpointer user_data) { gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(s->type_check), s->cfg.autotype_enabled ? TRUE : FALSE); gtk_box_pack_start(GTK_BOX(outer), s->type_check, FALSE, FALSE, 0); + s->filter_tags_check = gtk_check_button_new_with_label("Filter bracketed tags like [laughs]"); + gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(s->filter_tags_check), s->cfg.filter_bracketed_tags ? TRUE : FALSE); + g_signal_connect(s->filter_tags_check, "toggled", G_CALLBACK(on_filter_tags_toggled), s); + gtk_box_pack_start(GTK_BOX(outer), s->filter_tags_check, FALSE, FALSE, 0); + GtkWidget *meter_frame = gtk_frame_new("Live Microphone Level"); gtk_box_pack_start(GTK_BOX(outer), meter_frame, FALSE, FALSE, 0); @@ -597,6 +610,7 @@ int main(int argc, char **argv) { snprintf(tp.language, sizeof(tp.language), "%s", st.cfg.language); tp.use_gpu = st.use_gpu; tp.n_threads = (int)sysconf(_SC_NPROCESSORS_ONLN); + tp.filter_bracketed_tags = st.cfg.filter_bracketed_tags; if (audio_init(&ap) != 0) { fprintf(stderr, "fatal: audio init failed\n"); diff --git a/src/main.c b/src/main.c index 29a240a..72ccb1d 100644 --- a/src/main.c +++ b/src/main.c @@ -87,6 +87,7 @@ int main(int argc, char **argv) { snprintf(tp.language, sizeof(tp.language), "%s", cfg.language); tp.use_gpu = use_gpu; tp.n_threads = (int)sysconf(_SC_NPROCESSORS_ONLN); + tp.filter_bracketed_tags = cfg.filter_bracketed_tags; hotkey_ctx_t hk; diff --git a/src/transcribe.c b/src/transcribe.c index c7c9b8d..47b4f3c 100644 --- a/src/transcribe.c +++ b/src/transcribe.c @@ -1,5 +1,6 @@ #include "transcribe.h" +#include #include #include #include @@ -8,6 +9,71 @@ #define WITH_WHISPER 0 #endif +static int g_filter_bracketed_tags = 1; + +static char *filter_bracketed_tags_copy(const char *input) { + if (!input) return NULL; + + const size_t len = strlen(input); + char *no_tags = (char *)malloc(len + 1); + if (!no_tags) return NULL; + + size_t w = 0; + int bracket_depth = 0; + + for (size_t i = 0; i < len; ++i) { + unsigned char ch = (unsigned char)input[i]; + + if (ch == '[') { + bracket_depth++; + continue; + } + if (ch == ']' && bracket_depth > 0) { + bracket_depth--; + continue; + } + if (bracket_depth > 0) { + continue; + } + + no_tags[w++] = (char)ch; + } + no_tags[w] = '\0'; + + char *out = (char *)malloc(w + 1); + if (!out) { + free(no_tags); + return NULL; + } + + size_t j = 0; + int in_space = 1; + for (size_t i = 0; i < w; ++i) { + unsigned char ch = (unsigned char)no_tags[i]; + if (isspace(ch)) { + if (!in_space) { + out[j++] = ' '; + in_space = 1; + } + } else { + out[j++] = (char)ch; + in_space = 0; + } + } + + if (j > 0 && out[j - 1] == ' ') { + j--; + } + out[j] = '\0'; + + free(no_tags); + return out; +} + +void transcribe_set_filter_bracketed_tags(int enabled) { + g_filter_bracketed_tags = enabled ? 1 : 0; +} + #if WITH_WHISPER #include @@ -17,6 +83,7 @@ static transcribe_params_t g_params; int transcribe_init(const transcribe_params_t *params) { if (!params) return -1; g_params = *params; + transcribe_set_filter_bracketed_tags(g_params.filter_bracketed_tags); struct whisper_context_params cparams = whisper_context_default_params(); cparams.use_gpu = g_params.use_gpu ? true : false; @@ -63,7 +130,17 @@ char *transcribe_buffer(const float *samples, size_t count) { if (i + 1 < nseg) strcat(out, " "); } - return out; + if (!g_filter_bracketed_tags) { + return out; + } + + char *filtered = filter_bracketed_tags_copy(out); + if (!filtered) { + return out; + } + + free(out); + return filtered; } void transcribe_cleanup(void) { diff --git a/src/transcribe.h b/src/transcribe.h index df66c2a..1e9faf5 100644 --- a/src/transcribe.h +++ b/src/transcribe.h @@ -8,9 +8,11 @@ typedef struct { char language[32]; int use_gpu; int n_threads; + int filter_bracketed_tags; } transcribe_params_t; int transcribe_init(const transcribe_params_t *params); +void transcribe_set_filter_bracketed_tags(int enabled); char *transcribe_buffer(const float *samples, size_t count); void transcribe_cleanup(void); diff --git a/voice_linux_gui b/voice_linux_gui index 4d37dec..6f29f6f 100755 Binary files a/voice_linux_gui and b/voice_linux_gui differ