commit 9996466363a7def4cd2e80ffc8cb37b2a6d8f439 Author: Laan Tungir Date: Thu Mar 26 16:08:16 2026 -0400 First diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..12a24fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.o +voice_linux +models/*.bin +vendor/whisper.cpp/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a1b8fe4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,14 @@ +# Agent Build Policy + +This repository uses [`build.sh`](build.sh) as the single supported top-level build entrypoint. + +## Rules + +1. Always build with: + - `bash ./build.sh` +2. Do **not** use raw `gcc` or `make` directly for top-level project builds unless the user explicitly asks for it. +3. If `WITH_WHISPER`, `WHISPER_CUDA`, or `BUILD_GUI` behavior is needed, pass them as environment variables to [`build.sh`](build.sh), for example: + - `WITH_WHISPER=1 bash ./build.sh` + - `WHISPER_CUDA=1 bash ./build.sh` + - `BUILD_GUI=0 bash ./build.sh` +4. For any user-facing build instructions, prefer referencing [`build.sh`](build.sh) over direct compiler or make commands. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7cd9ba6 --- /dev/null +++ b/Makefile @@ -0,0 +1,59 @@ +CC ?= gcc + +CFLAGS_COMMON := -std=c99 -Wall -Wextra -Wpedantic -O2 -g -D_POSIX_C_SOURCE=200809L +CFLAGS := $(CFLAGS_COMMON) +LDFLAGS := +LIBS := -lpulse-simple -lpulse -lX11 -lXtst -lpthread +GTK_CFLAGS := $(shell pkg-config --cflags gtk+-3.0 2>/dev/null) +GTK_LIBS := $(shell pkg-config --libs gtk+-3.0 2>/dev/null) + +WITH_WHISPER ?= 0 +WHISPER_DIR ?= ./vendor/whisper.cpp + +ifeq ($(WITH_WHISPER),1) +CFLAGS += -DWITH_WHISPER=1 -isystem $(WHISPER_DIR)/include -isystem $(WHISPER_DIR)/ggml/include +LDFLAGS += -L$(WHISPER_DIR)/build/src -L$(WHISPER_DIR)/build/ggml/src -Wl,-rpath,$(WHISPER_DIR)/build/src -Wl,-rpath,$(WHISPER_DIR)/build/ggml/src +LIBS += -lwhisper -lggml +endif + +SRC := \ + ./src/main.c \ + ./src/config.c \ + ./src/hotkey.c \ + ./src/audio.c \ + ./src/transcribe.c \ + ./src/typer.c + +OBJ := $(SRC:.c=.o) +TARGET := ./voice_linux +GUI_TARGET := ./voice_linux_gui + +.PHONY: all clean run run-gui cpu gpu + +all: $(TARGET) $(GUI_TARGET) + +$(TARGET): $(OBJ) + $(CC) $(OBJ) -o $@ $(LDFLAGS) $(LIBS) + +./src/%.o: ./src/%.c + $(CC) $(CFLAGS) -c $< -o $@ + +$(GUI_TARGET): ./src/gui_main.c ./src/config.c ./src/audio.c ./src/transcribe.c ./src/typer.c + $(CC) $(CFLAGS) $(GTK_CFLAGS) $^ -o $@ $(LDFLAGS) $(LIBS) $(GTK_LIBS) + +cpu: + $(MAKE) clean + $(MAKE) WITH_WHISPER=1 + +gpu: + $(MAKE) clean + $(MAKE) WITH_WHISPER=1 + +run: $(TARGET) + ./voice_linux + +run-gui: $(GUI_TARGET) + ./voice_linux_gui + +clean: + rm -f $(OBJ) $(TARGET) $(GUI_TARGET) diff --git a/README.md b/README.md new file mode 100644 index 0000000..6549b85 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# voice_linux (C99) + +Local hotkey dictation for Linux/X11 using: +- PulseAudio capture +- whisper.cpp transcription +- X11/XTest text injection + +## Build deps (Debian/Qubes AppVM) + +```bash +sudo apt update +sudo apt install -y build-essential cmake git libpulse-dev libx11-dev libxtst-dev +``` + +## Build whisper.cpp + +```bash +mkdir -p ./vendor +cd ./vendor +git clone https://github.com/ggerganov/whisper.cpp.git +cd ./whisper.cpp +cmake -B build -DWHISPER_CUDA=ON +cmake --build build -j"$(nproc)" +``` + +If CUDA is not ready yet, you can still build whisper.cpp CPU-only by omitting `-DWHISPER_CUDA=ON`. + +## Download a model + +```bash +mkdir -p ./models +wget -O ./models/ggml-base.en.bin \ + https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin +``` + +## Build app + +```bash +# builds CLI + GUI by default +bash ./build.sh + +# optional switches: +WHISPER_CUDA=1 bash ./build.sh # enable CUDA whisper.cpp build +BUILD_GUI=0 bash ./build.sh # build CLI only +WITH_WHISPER=0 bash ./build.sh # build without whisper integration +``` + +## Run + +```bash +# CLI hotkey mode (uses ./config.ini by default) +./voice_linux + +# GTK3 window mode (uses ./config.ini by default) +./voice_linux_gui +``` + +Default hotkey: `Ctrl+Alt+V` +- Press once to **start** recording +- Press again to **stop** recording, transcribe, and type into the focused window + +If you see `samples=0`, that was an immediate start/stop event. + +If you see `[stt] [BLANK_AUDIO]`, recording worked but audio energy was near silent. +Check your input source with: + +```bash +pactl list short sources +``` + +Then set `audio_device=` in `./config.ini` to the source you want. + +## Notes + +- This implementation targets **X11** (not native Wayland). +- In template-based AppVMs, system packages may not persist unless installed in the template. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..99a6958 --- /dev/null +++ b/build.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT_DIR" + +WITH_WHISPER="${WITH_WHISPER:-1}" +WHISPER_CUDA="${WHISPER_CUDA:-0}" +BUILD_GUI="${BUILD_GUI:-1}" +JOBS="${JOBS:-$(nproc)}" + +WHISPER_DIR="./vendor/whisper.cpp" +MODEL_PATH="./models/ggml-base.en.bin" + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "[build] error: required command '$1' not found" >&2 + exit 2 + fi +} + +have_cmd() { + command -v "$1" >/dev/null 2>&1 +} + +CFLAGS_COMMON=( + -std=c99 -Wall -Wextra -Wpedantic -O2 -g + -D_POSIX_C_SOURCE=200809L +) + +LDFLAGS_COMMON=( + -lpulse-simple -lpulse -lX11 -lXtst -lpthread +) + +echo "[build] root: $ROOT_DIR" +echo "[build] WITH_WHISPER=$WITH_WHISPER WHISPER_CUDA=$WHISPER_CUDA BUILD_GUI=$BUILD_GUI JOBS=$JOBS" + +need_cmd gcc + +if [[ "$WITH_WHISPER" == "1" ]]; then + if ! have_cmd cmake; then + echo "[build] warning: cmake not found; forcing WITH_WHISPER=0" >&2 + WITH_WHISPER=0 + fi +fi + +if [[ "$WITH_WHISPER" == "1" ]]; then + if [[ ! -d "$WHISPER_DIR" ]] && ! have_cmd git; then + echo "[build] warning: git not found and whisper.cpp is missing; forcing WITH_WHISPER=0" >&2 + WITH_WHISPER=0 + fi +fi + +if [[ "$WITH_WHISPER" == "1" ]]; then + if [[ ! -f "$MODEL_PATH" ]] && ! have_cmd wget; then + echo "[build] warning: wget not found and model is missing; forcing WITH_WHISPER=0" >&2 + WITH_WHISPER=0 + fi +fi + +if [[ "$BUILD_GUI" == "1" ]]; then + if ! have_cmd pkg-config; then + echo "[build] warning: pkg-config not found; forcing BUILD_GUI=0" >&2 + BUILD_GUI=0 + elif ! pkg-config --exists gtk+-3.0; then + echo "[build] warning: gtk+-3.0 dev files not found; forcing BUILD_GUI=0" >&2 + BUILD_GUI=0 + else + GTK_CFLAGS_STR="$(pkg-config --cflags gtk+-3.0 2>/dev/null || true)" + if [[ -z "$GTK_CFLAGS_STR" ]]; then + echo "[build] warning: gtk+-3.0 cflags are unavailable; forcing BUILD_GUI=0" >&2 + BUILD_GUI=0 + else + if ! printf '#include \n' | gcc -x c -fsyntax-only $GTK_CFLAGS_STR - >/dev/null 2>&1; then + echo "[build] warning: gtk headers are not usable (gtk/gtk.h check failed); forcing BUILD_GUI=0" >&2 + BUILD_GUI=0 + fi + fi + fi +fi + +echo "[build] effective: WITH_WHISPER=$WITH_WHISPER BUILD_GUI=$BUILD_GUI" + +if [[ "$WITH_WHISPER" == "1" ]]; then + if [[ ! -d "$WHISPER_DIR" ]]; then + echo "[build] cloning whisper.cpp..." + mkdir -p ./vendor + git clone https://github.com/ggerganov/whisper.cpp.git "$WHISPER_DIR" + fi + + echo "[build] configuring whisper.cpp..." + if [[ "$WHISPER_CUDA" == "1" ]]; then + cmake -S "$WHISPER_DIR" -B "$WHISPER_DIR/build" -DWHISPER_CUDA=ON + else + cmake -S "$WHISPER_DIR" -B "$WHISPER_DIR/build" + fi + + echo "[build] building whisper.cpp..." + cmake --build "$WHISPER_DIR/build" -j"$JOBS" + + mkdir -p ./models + if [[ ! -f "$MODEL_PATH" ]]; then + echo "[build] downloading model $MODEL_PATH..." + wget -O "$MODEL_PATH" \ + https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin + fi +fi + +mkdir -p ./src + +echo "[build] building CLI binary..." +CLI_CFLAGS=("${CFLAGS_COMMON[@]}") +CLI_LDFLAGS=("${LDFLAGS_COMMON[@]}") + +if [[ "$WITH_WHISPER" == "1" ]]; then + CLI_CFLAGS+=( + -DWITH_WHISPER=1 + -isystem "$WHISPER_DIR/include" + -isystem "$WHISPER_DIR/ggml/include" + ) + CLI_LDFLAGS+=( + -L"$WHISPER_DIR/build/src" + -L"$WHISPER_DIR/build/ggml/src" + -Wl,-rpath,"$WHISPER_DIR/build/src" + -Wl,-rpath,"$WHISPER_DIR/build/ggml/src" + -lwhisper -lggml + ) +fi + +gcc "${CLI_CFLAGS[@]}" \ + ./src/main.c ./src/config.c ./src/hotkey.c ./src/audio.c ./src/transcribe.c ./src/typer.c \ + -o ./voice_linux \ + "${CLI_LDFLAGS[@]}" + +if [[ "$BUILD_GUI" == "1" ]]; then + echo "[build] building GTK3 GUI binary..." + + read -r -a GTK_CFLAGS <<< "$(pkg-config --cflags gtk+-3.0)" + read -r -a GTK_LIBS <<< "$(pkg-config --libs gtk+-3.0)" + + GUI_CFLAGS=("${CFLAGS_COMMON[@]}" "${GTK_CFLAGS[@]}") + GUI_LDFLAGS=("${LDFLAGS_COMMON[@]}" "${GTK_LIBS[@]}") + + if [[ "$WITH_WHISPER" == "1" ]]; then + GUI_CFLAGS+=( + -DWITH_WHISPER=1 + -isystem "$WHISPER_DIR/include" + -isystem "$WHISPER_DIR/ggml/include" + ) + GUI_LDFLAGS+=( + -L"$WHISPER_DIR/build/src" + -L"$WHISPER_DIR/build/ggml/src" + -Wl,-rpath,"$WHISPER_DIR/build/src" + -Wl,-rpath,"$WHISPER_DIR/build/ggml/src" + -lwhisper -lggml + ) + fi + + gcc "${GUI_CFLAGS[@]}" \ + ./src/gui_main.c ./src/config.c ./src/audio.c ./src/transcribe.c ./src/typer.c \ + -o ./voice_linux_gui \ + "${GUI_LDFLAGS[@]}" +fi + +echo "[build] done: ./voice_linux (and ./voice_linux_gui if BUILD_GUI=1)" +echo "[build] run CLI: ./voice_linux" +echo "[build] run GUI: ./voice_linux_gui" diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..bb87898 --- /dev/null +++ b/config.ini @@ -0,0 +1,30 @@ +# voice_linux runtime configuration +# Hotkey format supports Ctrl, Alt, Shift, Super + final key symbol +# Example: Ctrl+Alt+V + +model_path=./models/ggml-base.en.bin +audio_device=alsa_input.usb-Shure_Inc_Shure_MV6_MV6_5-ed2d37fa14d3e65d8d477cacf910d93c-01.mono-fallback +sample_rate=16000 +language=en +hotkey=Ctrl+Alt+V +type_delay_us=8000 +max_record_seconds=30 +# always-on VAD controls +always_on=1 +# legacy window size (unused by new VAD pipeline, kept for compatibility) +always_on_window_ms=1400 + +# live level threshold (0.0 - 1.0) +vad_peak_threshold=0.006 +# level must stay above threshold this long before speech starts +vad_trigger_ms=100 +# level must stay below threshold this long before speech ends +vad_release_ms=300 +# include this much pre-trigger audio in transcript segment +vad_preroll_ms=500 +# discard segments shorter than this +vad_min_speech_ms=250 +# hard cap for one speech segment +vad_max_speech_ms=15000 + +autotype_enabled=0 diff --git a/plans/always_on_plan.md b/plans/always_on_plan.md new file mode 100644 index 0000000..f786b0c --- /dev/null +++ b/plans/always_on_plan.md @@ -0,0 +1,182 @@ +# Always-On Dictation Plan (Qubes + Linux/X11) + +## Goal +Build an **always-running** speech pipeline that is safe, low-friction, and works with your mic’s physical mute button. + +## Design Principles +1. **Mic hardware mute is king** (privacy first). +2. **Continuous capture, segmented transcription** (VAD/endpointer). +3. **Conservative text injection defaults** (avoid accidental typing). +4. **Dedicated StandaloneVM deployment** for long-term reliability. + +--- + +## Phase A — Stabilize Current UX (1–2 sessions) + +### A1. Finalize GUI feedback states +- Keep current button states: + - idle + - recording + - transcribing +- Add explicit status line values: + - `Mic active, waiting for speech` + - `Speech detected` + - `Segment ended, transcribing` + - `Mic muted or near-silent` + +### A2. Add explicit mode selector in GUI +- Modes: + - **Push-to-talk** (existing) + - **Always-on preview** (new) + - **Always-on auto-type** (advanced) + +### A3. Add per-mode safety defaults +- Default to **Always-on preview** (no typing). +- Require explicit checkbox to enable auto-typing. + +--- + +## Phase B — Always-On Audio Engine (core) + +### B1. Ring buffer capture loop +Refactor audio path to run continuously: +- Thread captures audio at fixed chunk size (e.g., 20 ms frames). +- Store in ring buffer (N seconds history, e.g., 30s). +- Expose non-blocking read windows for VAD and segment extraction. + +### B2. Voice activity detection (VAD) +Implement lightweight VAD first: +- RMS/energy threshold + zero crossing heuristic. +- Track states: + - silence + - speech_started + - speech_active + - speech_ended +- Tunables in config: + - `vad_energy_threshold` + - `vad_start_ms` + - `vad_end_silence_ms` + +(Optionally later: replace with Silero VAD or WebRTC VAD wrapper.) + +### B3. Segment endpointer +When speech ends: +- Cut a segment with small pre-roll/post-roll (e.g., 250 ms / 200 ms). +- Dispatch to transcription worker queue. +- Continue capturing while worker runs. + +### B4. Transcription worker thread +- Single worker initially (simple, deterministic). +- Queue segment jobs FIFO. +- Return: + - text + - confidence surrogate (optional) + - latency metrics + +--- + +## Phase C — Hardware Mute Integration + +### C1. Software mute inference (must-have) +Even without HID events: +- Detect sustained near-zero energy (e.g., >800ms). +- Mark state as `muted_or_silent`. +- In GUI show muted icon/state. + +### C2. HID/evdev mute button (nice-to-have) +If the mic exposes events: +- Open relevant `/dev/input/event*`. +- Parse key/switch event for mute toggle. +- Map to explicit app state `hardware_muted=true/false`. + +Fallback remains C1 inference so behavior is robust across devices. + +--- + +## Phase D — Output Policy for Continuous Mode + +### D1. Preview-first pipeline +- Continuous transcript appears in GUI text area. +- Nothing typed automatically by default. + +### D2. Commit model +Two safe options: +1. **Manual commit** button: insert selected transcript into focused app. +2. **Auto-commit finals**: type only finalized segments (not partials). + +### D3. Guard rails +- Minimum confidence/length filter before typing. +- Ignore common noise artifacts (`[BLANK_AUDIO]`, empty, punctuation-only). +- Cooldown between auto-typed segments (e.g., 300 ms). + +--- + +## Phase E — Observability and Tuning + +### E1. Metrics +Track and expose: +- capture RMS +- VAD state transitions +- segment durations +- transcription latency +- typed character count + +### E2. Debug panel in GUI +Small collapsible panel with: +- live level meter +- current VAD state +- current device name +- queue length / worker busy + +### E3. Config knobs +Add to config: +- `mode=push_to_talk|always_on_preview|always_on_autotype` +- `vad_energy_threshold` +- `vad_end_silence_ms` +- `autotype_enabled` +- `autotype_min_chars` +- `autotype_cooldown_ms` + +--- + +## Phase F — Qubes Deployment Model + +### F1. Dedicated StandaloneVM +Deploy final app in its own StandaloneVM: +- stable dependencies +- controlled attack surface +- predictable startup behavior + +### F2. Device routing +- Attach USB mic to this VM via sys-usb. +- Keep always-on transcription isolated from high-trust VMs. + +### F3. Startup behavior +- `systemd --user` service for auto-launch on login. +- GUI starts minimized/normal per config. + +--- + +## Implementation Order (recommended) +1. Always-on preview mode (B + D1) +2. VAD + endpoint tuning UI (E) +3. Manual commit + guarded auto-commit (D2/D3) +4. Hardware mute event support if available (C2) +5. StandaloneVM packaging + autostart (F) + +--- + +## Risks and Mitigations +- **False triggers / noise typing** → preview-first default, confidence/length filters. +- **Long transcription latency** → worker queue + clear transcribing state. +- **Mic mute not exposed via HID** → software mute inference fallback. +- **Qubes audio routing quirks** → device selector + live level meter + status diagnostics. + +--- + +## Definition of Done (MVP Always-On) +- App runs continuously with no manual start/stop needed. +- Speech automatically segmented and transcribed. +- GUI clearly indicates idle/speech/transcribing/muted. +- Preview mode stable for long sessions. +- Optional typed output only for finalized segments with safety guards. diff --git a/plans/architecture.md b/plans/architecture.md new file mode 100644 index 0000000..7486bd1 --- /dev/null +++ b/plans/architecture.md @@ -0,0 +1,322 @@ +# Voice Linux — Speech-to-Text Dictation System + +## Overview + +A local, GPU-accelerated speech-to-text dictation tool for Qubes OS. Written in C99, using `whisper.cpp` with CUDA for transcription, PulseAudio for audio capture, and Xlib/XTest for global hotkey and text injection. + +**Press a hotkey → speak → text appears in the focused window.** + +## System Environment + +| Property | Value | +|---|---| +| OS | Qubes OS (AppVM) | +| Session | X11 / XFCE | +| Audio | PipeWire 1.4.2 with PulseAudio compat | +| Audio source | `qubes-source` (virtual PipeWire device) | +| Python | 3.13.5 (not used for main app — C99 instead) | +| Display GPU | AMD (dom0) | +| Compute GPU | NVIDIA GeForce GTX (to be passed through) | + +## Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ voice_linux │ +│ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │ +│ │ hotkey │──▶│ audio │──▶│ transcribe │ │ +│ │ (Xlib │ │ (Pulse │ │ (whisper.cpp │ │ +│ │ XGrabKey)│ │ Simple │ │ CUDA-accel) │ │ +│ └──────────┘ │ API) │ └────────┬─────────┘ │ +│ └──────────┘ │ │ +│ ▼ │ +│ ┌──────────────────┐ │ +│ │ typer │ │ +│ │ (XTest fake │ │ +│ │ key events) │ │ +│ └──────────────────┘ │ +└─────────────────────────────────────────────────────┘ +``` + +### Components + +| Module | File | C API Used | Purpose | +|---|---|---|---| +| Hotkey | `src/hotkey.c` | Xlib `XGrabKey` | Global push-to-talk hotkey on X11 | +| Audio | `src/audio.c` | PulseAudio Simple API | Record from `qubes-source` into PCM buffer | +| Transcribe | `src/transcribe.c` | whisper.cpp C API | Feed PCM to Whisper, get text back | +| Typer | `src/typer.c` | XTest `XTestFakeKeyEvent` | Type transcribed text into focused window | +| Config | `src/config.c` | stdio | Load YAML/INI config for hotkey, model, device | +| Main | `src/main.c` | — | Wire everything together | + +### Data Flow + +1. `main()` initializes all modules, loads config +2. `hotkey_listen()` blocks on X11 event loop +3. On hotkey press → `audio_start_recording()` begins capturing from PulseAudio +4. On hotkey release (or second press) → `audio_stop_recording()` returns PCM buffer +5. `transcribe()` feeds PCM buffer to whisper.cpp → returns `char *text` +6. `typer_type_text(text)` sends fake key events via XTest to focused window +7. Loop back to step 2 + +## Project Structure + +``` +voice_linux/ +├── Makefile # Build system +├── README.md # Setup and usage +├── config.ini # User config (hotkey, model, audio device) +├── plans/ +│ └── architecture.md # This document +├── src/ +│ ├── main.c # Entry point and orchestrator +│ ├── hotkey.h / hotkey.c # Global X11 hotkey via XGrabKey +│ ├── audio.h / audio.c # PulseAudio Simple API recording +│ ├── transcribe.h / transcribe.c # whisper.cpp integration +│ ├── typer.h / typer.c # XTest text injection +│ └── config.h / config.c # Config file parser +├── models/ # Downloaded whisper models (gitignored) +│ └── ggml-base.en.bin +└── vendor/ + └── whisper.cpp/ # Git submodule or local clone +``` + +## Phase 0: GPU Passthrough (Qubes) + +### Prerequisites +- AMD GPU handles dom0 display (confirmed) +- NVIDIA GTX is idle and available for passthrough (confirmed) +- IOMMU/VT-d enabled in BIOS (likely, since Qubes requires it) + +### Detected Hardware + +``` +17:00.0 VGA compatible controller: NVIDIA Corporation GA107 [GeForce RTX 3050 6GB] (rev a1) +17:00.1 Audio device: NVIDIA Corporation GA107 High Definition Audio Controller (rev a1) +65:00.0 VGA compatible controller: NVIDIA Corporation GP102 [GeForce GTX 1080 Ti] (rev a1) +65:00.1 Audio device: NVIDIA Corporation GP102 HDMI Audio Controller (rev a1) +``` + +**Target GPU**: GTX 1080 Ti (11GB VRAM, CUDA compute 6.1) at BDF `65:00.0` / `65:00.1` + +### AppVM Details + +- **Name**: `ai` +- **Type**: AppVM (template: `debian-13-xfce`) +- **Network**: `sys-vpn-mullvad-app` + +> **Important**: Since `ai` is a template-based AppVM, NVIDIA drivers installed inside it +> will be **lost on reboot** (only `/home` persists). Two options: +> - **Option A**: Install drivers in the `debian-13-xfce` template (affects all AppVMs using it) +> - **Option B**: Convert `ai` to a StandaloneVM: `qvm-clone --class StandaloneVM ai ai-standalone` +> - **Option C**: Use a bind-dirs mechanism to persist `/usr/lib/nvidia` and related paths + +### Steps (run in dom0 terminal) + +#### 0.1 — Hide the GTX 1080 Ti from dom0 + +Edit `/etc/default/grub` in dom0: + +```bash +sudo nano /etc/default/grub +``` + +Find the **first** `GRUB_CMDLINE_LINUX` line: +``` +GRUB_CMDLINE_LINUX="rd.luks.uuid=luks-ebcd163d-dcf3-47bf-8da1-f7c13d82304b rd.lvm.lv=qubes_dom0/root rd.lvm.lv=qubes_dom0/swap plymouth.ignore-serial-consoles rhgb quiet" +``` + +Add `rd.qubes.hide_pci=65:00.0,65:00.1` to the end (inside the quotes): +``` +GRUB_CMDLINE_LINUX="rd.luks.uuid=luks-ebcd163d-dcf3-47bf-8da1-f7c13d82304b rd.lvm.lv=qubes_dom0/root rd.lvm.lv=qubes_dom0/swap plymouth.ignore-serial-consoles rhgb quiet rd.qubes.hide_pci=65:00.0,65:00.1" +``` + +#### 0.2 — Regenerate GRUB and reboot + +```bash +# Try EFI path first (most common on modern Qubes): +sudo grub2-mkconfig -o /boot/efi/EFI/qubes/grub.cfg + +# If that path doesn't exist, try: +# sudo grub2-mkconfig -o /boot/grub2/grub.cfg + +# Reboot dom0: +sudo reboot +``` + +#### 0.3 — Verify GPU is assignable (after reboot) + +```bash +xl pci-assignable-list +# Should show: +# 0000:65:00.0 +# 0000:65:00.1 +``` + +#### 0.4 — Attach GPU to the 'ai' AppVM + +```bash +qvm-pci attach ai dom0:65_00.0 --persistent -o permissive=true +qvm-pci attach ai dom0:65_00.1 --persistent -o permissive=true +``` + +#### 0.5 — Install NVIDIA drivers inside the AppVM + +Start the `ai` VM, then inside it: + +```bash +# Inside the ai AppVM: +sudo apt update +sudo apt install -y build-essential linux-headers-$(uname -r) + +# For GTX 1080 Ti (Pascal/GP102), use the 550.x driver branch: +wget https://us.download.nvidia.com/XFree86/Linux-x86_64/550.127.05/NVIDIA-Linux-x86_64-550.127.05.run +chmod +x NVIDIA-Linux-x86_64-550.127.05.run +sudo ./NVIDIA-Linux-x86_64-550.127.05.run --no-opengl-files --dkms +``` + +> `--no-opengl-files` is critical in Qubes — we don't want to replace the VM's display GL, just get CUDA compute. + +#### 0.6 — Install CUDA Toolkit + +```bash +# Inside the ai AppVM: +wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb +sudo apt update +sudo apt install -y cuda-toolkit-12-6 +echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc +echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc +source ~/.bashrc +``` + +#### 0.7 — Verify + +```bash +nvidia-smi +nvcc --version +``` + +Expected output: GTX 1080 Ti with 11GB VRAM, CUDA 12.6. + +## Phase 1: Build and Test Core Application + +### 1.1 — Install build dependencies + +```bash +sudo apt install -y \ + build-essential \ + cmake \ + git \ + libpulse-dev \ + libx11-dev \ + libxtst-dev \ + xdotool +``` + +### 1.2 — Clone and build whisper.cpp with CUDA + +```bash +cd vendor/ +git clone https://github.com/ggerganov/whisper.cpp.git +cd whisper.cpp +cmake -B build -DWHISPER_CUDA=ON +cmake --build build --config Release -j$(nproc) +``` + +### 1.3 — Download whisper model + +```bash +cd models/ +# base.en — fast, good for English dictation (~150MB) +wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin + +# Or medium.en for better accuracy (~1.5GB, still fast with GPU): +# wget https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin +``` + +### 1.4 — Implement C99 modules + +Each module is a small, focused C file with a clean header: + +**`src/audio.h`** — Audio capture +- `int audio_init(const char *device_name)` — connect to PulseAudio +- `int audio_start_recording(void)` — begin capturing PCM +- `int audio_stop_recording(float **samples, size_t *n_samples)` — stop and return buffer +- `void audio_cleanup(void)` — disconnect + +**`src/transcribe.h`** — Whisper integration +- `int transcribe_init(const char *model_path)` — load model (CUDA auto-detected) +- `char *transcribe_buffer(const float *samples, size_t n_samples)` — run inference +- `void transcribe_cleanup(void)` — free model + +**`src/typer.h`** — Text injection +- `int typer_init(void)` — open X11 display +- `int typer_type_text(const char *text)` — send fake key events +- `void typer_cleanup(void)` — close display + +**`src/hotkey.h`** — Global hotkey +- `int hotkey_init(unsigned int keycode, unsigned int modifiers)` — grab key +- `int hotkey_wait_press(void)` — block until hotkey pressed +- `int hotkey_wait_release(void)` — block until hotkey released +- `void hotkey_cleanup(void)` — ungrab and close + +**`src/config.h`** — Configuration +- `typedef struct { ... } config_t` +- `int config_load(const char *path, config_t *cfg)` — parse INI file +- Default values for hotkey (Ctrl+Alt+V), model path, audio device + +### 1.5 — Build with Makefile + +```makefile +CC = gcc +CFLAGS = -std=c99 -Wall -Wextra -O2 +LDFLAGS = -lpulse-simple -lpulse -lX11 -lXtst -lwhisper -L./vendor/whisper.cpp/build/src +INCLUDES = -I./vendor/whisper.cpp/include + +voice_linux: src/main.c src/audio.c src/transcribe.c src/typer.c src/hotkey.c src/config.c + $(CC) $(CFLAGS) $(INCLUDES) -o $@ $^ $(LDFLAGS) +``` + +### 1.6 — Test end-to-end + +```bash +./voice_linux --model models/ggml-base.en.bin --device qubes-source +# Press Ctrl+Alt+V, speak, press again, text appears in focused window +``` + +## Phase 2: Polish + +- **Audio feedback**: Play a short beep via PulseAudio when recording starts/stops +- **Config file**: `config.ini` with hotkey, model path, audio device, typing delay +- **Systemd user service**: `~/.config/systemd/user/voice-linux.service` for auto-start +- **Logging**: Optional debug log to `~/.local/share/voice_linux/voice.log` + +## Phase 3: Hardware + +- **USB microphone**: Attach via Qubes device manager (`qvm-usb attach`) from sys-usb +- **Physical button**: Read HID events via `/dev/input/eventN` using `evdev` in C +- **Model tuning**: Try `medium.en` or `large-v3` models with GPU — should still be fast + +## Key Risks and Mitigations + +| Risk | Mitigation | +|---|---| +| GPU passthrough fails (IOMMU grouping) | Fall back to CPU-only whisper.cpp (still works, just slower) | +| Older GTX lacks CUDA compute capability | whisper.cpp supports compute 5.0+; most GTX 900+ series work | +| PulseAudio `qubes-source` has no audio | Need mic attached + Qubes audio policy allowing input | +| XGrabKey conflicts with desktop shortcuts | Use an uncommon hotkey combo; make it configurable | +| whisper.cpp API changes | Pin to a specific release tag | + +## Model Size vs Speed Reference (GPU) + +| Model | Size | VRAM | Speed (10s audio) | Accuracy | +|---|---|---|---|---| +| tiny.en | 75MB | ~1GB | <0.5s | Fair | +| base.en | 150MB | ~1GB | <1s | Good | +| small.en | 500MB | ~2GB | ~1-2s | Very good | +| medium.en | 1.5GB | ~5GB | ~2-3s | Excellent | +| large-v3 | 3GB | ~10GB | ~4-6s | Best | + +For dictation, `base.en` is the sweet spot to start. Upgrade to `medium.en` if accuracy isn't sufficient. diff --git a/src/audio.c b/src/audio.c new file mode 100644 index 0000000..4c9e292 --- /dev/null +++ b/src/audio.c @@ -0,0 +1,532 @@ +#include "audio.h" + +#include +#include + +#include +#include +#include +#include +#include + +typedef enum { + VAD_IDLE = 0, + VAD_ONSET, + VAD_SPEECH, + VAD_TRAILING, +} vad_state_t; + +typedef struct { + pa_simple *pa; + audio_params_t params; + + pthread_t monitor_thread; + bool monitor_running; + bool stop_requested; + + pthread_mutex_t lock; + + float peak_level; + + bool manual_recording; + float *manual_buffer; + size_t manual_count; + size_t manual_cap; + + bool always_on_enabled; + audio_vad_params_t vad; + vad_state_t vad_state; + int onset_count; + int trailing_count; + int speech_count; + + float *ring_buffer; + size_t ring_cap; + size_t ring_head; + size_t ring_count; + + float *segment_buffer; + size_t segment_count; + size_t segment_cap; + + float *ready_segment; + size_t ready_count; +} audio_state_t; + +static audio_state_t g_audio; + +static const char *vad_state_name(vad_state_t s) { + switch (s) { + case VAD_IDLE: return "IDLE"; + case VAD_ONSET: return "ONSET"; + case VAD_SPEECH: return "SPEECH"; + case VAD_TRAILING: return "TRAILING"; + default: return "?"; + } +} + +static void set_vad_state_locked(vad_state_t next, const char *reason, float level) { + if (g_audio.vad_state == next) return; + fprintf(stderr, "[vad] %s -> %s (%s, level=%.4f thr=%.4f)\n", + vad_state_name(g_audio.vad_state), vad_state_name(next), reason, level, g_audio.vad.threshold); + g_audio.vad_state = next; +} + +static size_t ms_to_samples(int sample_rate, int ms) { + if (sample_rate <= 0 || ms <= 0) return 1; + long long n = (long long)sample_rate * (long long)ms / 1000LL; + if (n < 1) n = 1; + return (size_t)n; +} + +static int reserve_float_buffer(float **buffer, size_t *cap, size_t needed) { + if (needed <= *cap) return 0; + size_t new_cap = *cap ? (*cap * 2) : 4096; + while (new_cap < needed) { + if (new_cap > ((size_t)-1) / 2) { + return -1; + } + new_cap *= 2; + } + float *p = (float *)realloc(*buffer, new_cap * sizeof(float)); + if (!p) return -1; + *buffer = p; + *cap = new_cap; + return 0; +} + +static void ring_push_sample(float s) { + if (!g_audio.ring_buffer || g_audio.ring_cap == 0) return; + + if (g_audio.ring_count < g_audio.ring_cap) { + size_t idx = (g_audio.ring_head + g_audio.ring_count) % g_audio.ring_cap; + g_audio.ring_buffer[idx] = s; + g_audio.ring_count++; + } else { + g_audio.ring_buffer[g_audio.ring_head] = s; + g_audio.ring_head = (g_audio.ring_head + 1) % g_audio.ring_cap; + } +} + +static int ring_copy_last(size_t n, float **dst, size_t *dst_count) { + if (!dst || !dst_count) return -1; + *dst = NULL; + *dst_count = 0; + + if (!g_audio.ring_buffer || g_audio.ring_count == 0 || n == 0) { + return 0; + } + + if (n > g_audio.ring_count) n = g_audio.ring_count; + float *out = (float *)malloc(n * sizeof(float)); + if (!out) return -1; + + size_t start = (g_audio.ring_head + g_audio.ring_count - n) % g_audio.ring_cap; + for (size_t i = 0; i < n; ++i) { + out[i] = g_audio.ring_buffer[(start + i) % g_audio.ring_cap]; + } + + *dst = out; + *dst_count = n; + return 0; +} + +static void queue_ready_segment_locked(void) { + if (g_audio.segment_count == 0) return; + + float *copy = (float *)malloc(g_audio.segment_count * sizeof(float)); + if (!copy) return; + memcpy(copy, g_audio.segment_buffer, g_audio.segment_count * sizeof(float)); + + free(g_audio.ready_segment); + g_audio.ready_segment = copy; + g_audio.ready_count = g_audio.segment_count; +} + +static void reset_segment_locked(void) { + g_audio.segment_count = 0; + g_audio.speech_count = 0; + g_audio.onset_count = 0; + g_audio.trailing_count = 0; +} + +static int append_segment_block_locked(const float *samples, size_t n) { + if (!samples || n == 0) return 0; + if (reserve_float_buffer(&g_audio.segment_buffer, &g_audio.segment_cap, g_audio.segment_count + n) != 0) { + return -1; + } + memcpy(&g_audio.segment_buffer[g_audio.segment_count], samples, n * sizeof(float)); + g_audio.segment_count += n; + return 0; +} + +static void vad_begin_speech_locked(void) { + size_t preroll_samples = ms_to_samples(g_audio.params.sample_rate, g_audio.vad.preroll_ms); + float *pre = NULL; + size_t pre_n = 0; + if (ring_copy_last(preroll_samples, &pre, &pre_n) == 0 && pre && pre_n > 0) { + if (reserve_float_buffer(&g_audio.segment_buffer, &g_audio.segment_cap, pre_n) == 0) { + memcpy(g_audio.segment_buffer, pre, pre_n * sizeof(float)); + g_audio.segment_count = pre_n; + g_audio.speech_count = (int)pre_n; + } + free(pre); + } +} + +static void vad_finish_segment_locked(void) { + size_t min_speech = ms_to_samples(g_audio.params.sample_rate, g_audio.vad.min_speech_ms); + if (g_audio.segment_count >= min_speech) { + queue_ready_segment_locked(); + fprintf(stderr, "[vad] segment ready: %zu samples (%.2fs)\n", + g_audio.ready_count, + (double)g_audio.ready_count / (double)g_audio.params.sample_rate); + } else { + fprintf(stderr, "[vad] segment dropped: too short (%zu < %zu samples)\n", + g_audio.segment_count, min_speech); + } + + set_vad_state_locked(VAD_IDLE, "finish", 0.0f); + reset_segment_locked(); +} + +static void process_vad_chunk_locked(const float *chunk, size_t n, float chunk_peak) { + if (!chunk || n == 0) return; + + const float threshold = g_audio.vad.threshold; + const int above = chunk_peak >= threshold; + + size_t trigger_n = ms_to_samples(g_audio.params.sample_rate, g_audio.vad.trigger_ms); + size_t release_n = ms_to_samples(g_audio.params.sample_rate, g_audio.vad.release_ms); + size_t max_speech_n = ms_to_samples(g_audio.params.sample_rate, g_audio.vad.max_speech_ms); + + switch (g_audio.vad_state) { + case VAD_IDLE: + if (above) { + g_audio.onset_count = (int)n; + set_vad_state_locked(VAD_ONSET, "above threshold", chunk_peak); + } + break; + + case VAD_ONSET: + if (above) { + g_audio.onset_count += (int)n; + if ((size_t)g_audio.onset_count >= trigger_n) { + reset_segment_locked(); + vad_begin_speech_locked(); + if (append_segment_block_locked(chunk, n) != 0) { + set_vad_state_locked(VAD_IDLE, "segment alloc failure", chunk_peak); + reset_segment_locked(); + break; + } + g_audio.speech_count += (int)n; + g_audio.trailing_count = 0; + set_vad_state_locked(VAD_SPEECH, "trigger reached", chunk_peak); + } + } else { + set_vad_state_locked(VAD_IDLE, "onset fell below threshold", chunk_peak); + g_audio.onset_count = 0; + } + break; + + case VAD_SPEECH: + if (append_segment_block_locked(chunk, n) != 0) { + set_vad_state_locked(VAD_IDLE, "segment alloc failure", chunk_peak); + reset_segment_locked(); + break; + } + g_audio.speech_count += (int)n; + + if ((size_t)g_audio.speech_count >= max_speech_n) { + vad_finish_segment_locked(); + break; + } + + if (!above) { + g_audio.trailing_count = (int)n; + set_vad_state_locked(VAD_TRAILING, "below threshold", chunk_peak); + } + break; + + case VAD_TRAILING: + if (append_segment_block_locked(chunk, n) != 0) { + set_vad_state_locked(VAD_IDLE, "segment alloc failure", chunk_peak); + reset_segment_locked(); + break; + } + g_audio.speech_count += (int)n; + + if ((size_t)g_audio.speech_count >= max_speech_n) { + vad_finish_segment_locked(); + break; + } + + if (above) { + set_vad_state_locked(VAD_SPEECH, "voice resumed", chunk_peak); + g_audio.trailing_count = 0; + } else { + g_audio.trailing_count += (int)n; + if ((size_t)g_audio.trailing_count >= release_n) { + vad_finish_segment_locked(); + } + } + break; + } +} + +static void *monitor_thread_fn(void *unused) { + (void)unused; + + const size_t chunk = 1024; + float tmp[1024]; + int err = 0; + + while (1) { + pthread_mutex_lock(&g_audio.lock); + bool should_stop = g_audio.stop_requested; + pthread_mutex_unlock(&g_audio.lock); + if (should_stop) break; + + if (pa_simple_read(g_audio.pa, tmp, sizeof(tmp), &err) < 0) { + fprintf(stderr, "audio: pa_simple_read failed: %s\n", pa_strerror(err)); + break; + } + + float chunk_peak = 0.0f; + for (size_t i = 0; i < chunk; ++i) { + float a = tmp[i] < 0.0f ? -tmp[i] : tmp[i]; + if (a > chunk_peak) chunk_peak = a; + } + + pthread_mutex_lock(&g_audio.lock); + + g_audio.peak_level = chunk_peak; + + size_t max_manual = (size_t)g_audio.params.sample_rate * (size_t)g_audio.params.max_seconds; + for (size_t i = 0; i < chunk; ++i) { + float s = tmp[i]; + ring_push_sample(s); + + if (g_audio.manual_recording) { + if (g_audio.manual_count < max_manual && + reserve_float_buffer(&g_audio.manual_buffer, &g_audio.manual_cap, g_audio.manual_count + 1) == 0) { + g_audio.manual_buffer[g_audio.manual_count++] = s; + } + } + } + + if (g_audio.always_on_enabled) { + process_vad_chunk_locked(tmp, chunk, chunk_peak); + } + + pthread_mutex_unlock(&g_audio.lock); + } + + pthread_mutex_lock(&g_audio.lock); + g_audio.monitor_running = false; + pthread_mutex_unlock(&g_audio.lock); + + return NULL; +} + +int audio_init(const audio_params_t *params) { + if (!params) return -1; + + memset(&g_audio, 0, sizeof(g_audio)); + g_audio.params = *params; + + pthread_mutex_init(&g_audio.lock, NULL); + + g_audio.vad.threshold = 0.006f; + g_audio.vad.trigger_ms = 100; + g_audio.vad.release_ms = 300; + g_audio.vad.preroll_ms = 500; + g_audio.vad.min_speech_ms = 250; + g_audio.vad.max_speech_ms = 15000; + + pa_sample_spec ss = { + .format = PA_SAMPLE_FLOAT32LE, + .rate = (uint32_t)g_audio.params.sample_rate, + .channels = 1, + }; + + int err = 0; + g_audio.pa = pa_simple_new( + NULL, + "voice_linux", + PA_STREAM_RECORD, + g_audio.params.device[0] ? g_audio.params.device : NULL, + "dictation", + &ss, + NULL, + NULL, + &err); + + if (!g_audio.pa) { + fprintf(stderr, "audio: pa_simple_new failed: %s\n", pa_strerror(err)); + pthread_mutex_destroy(&g_audio.lock); + return -2; + } + + size_t ring_cap = ms_to_samples(g_audio.params.sample_rate, 6000); + g_audio.ring_buffer = (float *)malloc(ring_cap * sizeof(float)); + if (!g_audio.ring_buffer) { + pa_simple_free(g_audio.pa); + g_audio.pa = NULL; + pthread_mutex_destroy(&g_audio.lock); + return -3; + } + g_audio.ring_cap = ring_cap; + + if (pthread_create(&g_audio.monitor_thread, NULL, monitor_thread_fn, NULL) != 0) { + free(g_audio.ring_buffer); + g_audio.ring_buffer = NULL; + g_audio.ring_cap = 0; + pa_simple_free(g_audio.pa); + g_audio.pa = NULL; + pthread_mutex_destroy(&g_audio.lock); + return -4; + } + + g_audio.monitor_running = true; + return 0; +} + +int audio_start_recording(void) { + pthread_mutex_lock(&g_audio.lock); + g_audio.manual_count = 0; + g_audio.manual_recording = true; + pthread_mutex_unlock(&g_audio.lock); + return 0; +} + +int audio_stop_recording(float **samples_out, size_t *count_out) { + if (!samples_out || !count_out) return -1; + + pthread_mutex_lock(&g_audio.lock); + g_audio.manual_recording = false; + + size_t n = g_audio.manual_count; + float *out = NULL; + if (n > 0) { + out = (float *)malloc(n * sizeof(float)); + if (out) { + memcpy(out, g_audio.manual_buffer, n * sizeof(float)); + } + } + pthread_mutex_unlock(&g_audio.lock); + + *samples_out = out; + *count_out = n; + return (out || n == 0) ? 0 : -2; +} + +int audio_set_vad_params(const audio_vad_params_t *params) { + if (!params) return -1; + + pthread_mutex_lock(&g_audio.lock); + g_audio.vad = *params; + + if (g_audio.vad.threshold < 0.0f) g_audio.vad.threshold = 0.0f; + if (g_audio.vad.threshold > 1.0f) g_audio.vad.threshold = 1.0f; + if (g_audio.vad.trigger_ms < 10) g_audio.vad.trigger_ms = 10; + if (g_audio.vad.release_ms < 10) g_audio.vad.release_ms = 10; + if (g_audio.vad.preroll_ms < 0) g_audio.vad.preroll_ms = 0; + if (g_audio.vad.min_speech_ms < 50) g_audio.vad.min_speech_ms = 50; + if (g_audio.vad.max_speech_ms < g_audio.vad.min_speech_ms) g_audio.vad.max_speech_ms = g_audio.vad.min_speech_ms; + pthread_mutex_unlock(&g_audio.lock); + + return 0; +} + +int audio_set_always_on_enabled(int enabled) { + pthread_mutex_lock(&g_audio.lock); + g_audio.always_on_enabled = enabled ? true : false; + if (!g_audio.always_on_enabled) { + set_vad_state_locked(VAD_IDLE, "always-on disabled", 0.0f); + reset_segment_locked(); + } + pthread_mutex_unlock(&g_audio.lock); + return 0; +} + +float audio_get_peak_level(void) { + pthread_mutex_lock(&g_audio.lock); + float p = g_audio.peak_level; + pthread_mutex_unlock(&g_audio.lock); + return p; +} + +int audio_get_speech_segment(float **samples_out, size_t *count_out) { + if (!samples_out || !count_out) return -1; + + pthread_mutex_lock(&g_audio.lock); + if (!g_audio.ready_segment || g_audio.ready_count == 0) { + pthread_mutex_unlock(&g_audio.lock); + *samples_out = NULL; + *count_out = 0; + return 1; + } + + *samples_out = g_audio.ready_segment; + *count_out = g_audio.ready_count; + g_audio.ready_segment = NULL; + g_audio.ready_count = 0; + pthread_mutex_unlock(&g_audio.lock); + + return 0; +} + +int audio_get_debug_info(audio_debug_info_t *out) { + if (!out) return -1; + + pthread_mutex_lock(&g_audio.lock); + out->always_on_enabled = g_audio.always_on_enabled ? 1 : 0; + out->vad_state = (int)g_audio.vad_state; + out->onset_samples = g_audio.onset_count; + out->trailing_samples = g_audio.trailing_count; + out->speech_samples = g_audio.speech_count; + out->ready_samples = g_audio.ready_count; + out->peak_level = g_audio.peak_level; + out->threshold = g_audio.vad.threshold; + pthread_mutex_unlock(&g_audio.lock); + + return 0; +} + +void audio_cleanup(void) { + pthread_mutex_lock(&g_audio.lock); + g_audio.stop_requested = true; + pthread_mutex_unlock(&g_audio.lock); + + if (g_audio.monitor_running) { + pthread_join(g_audio.monitor_thread, NULL); + } + + if (g_audio.pa) { + pa_simple_free(g_audio.pa); + g_audio.pa = NULL; + } + + free(g_audio.manual_buffer); + g_audio.manual_buffer = NULL; + g_audio.manual_cap = 0; + g_audio.manual_count = 0; + + free(g_audio.ring_buffer); + g_audio.ring_buffer = NULL; + g_audio.ring_cap = 0; + g_audio.ring_count = 0; + g_audio.ring_head = 0; + + free(g_audio.segment_buffer); + g_audio.segment_buffer = NULL; + g_audio.segment_cap = 0; + g_audio.segment_count = 0; + + free(g_audio.ready_segment); + g_audio.ready_segment = NULL; + g_audio.ready_count = 0; + + pthread_mutex_destroy(&g_audio.lock); +} diff --git a/src/audio.h b/src/audio.h new file mode 100644 index 0000000..1a909a3 --- /dev/null +++ b/src/audio.h @@ -0,0 +1,44 @@ +#ifndef VOICE_AUDIO_H +#define VOICE_AUDIO_H + +#include + +typedef struct { + char device[256]; + int sample_rate; + int max_seconds; +} audio_params_t; + +typedef struct { + float threshold; + int trigger_ms; + int release_ms; + int preroll_ms; + int min_speech_ms; + int max_speech_ms; +} audio_vad_params_t; + +typedef struct { + int always_on_enabled; + int vad_state; /* 0=IDLE,1=ONSET,2=SPEECH,3=TRAILING */ + int onset_samples; + int trailing_samples; + int speech_samples; + size_t ready_samples; + float peak_level; + float threshold; +} audio_debug_info_t; + +int audio_init(const audio_params_t *params); +int audio_start_recording(void); +int audio_stop_recording(float **samples_out, size_t *count_out); + +int audio_set_vad_params(const audio_vad_params_t *params); +int audio_set_always_on_enabled(int enabled); +float audio_get_peak_level(void); +int audio_get_speech_segment(float **samples_out, size_t *count_out); +int audio_get_debug_info(audio_debug_info_t *out); + +void audio_cleanup(void); + +#endif diff --git a/src/config.c b/src/config.c new file mode 100644 index 0000000..12aa0f5 --- /dev/null +++ b/src/config.c @@ -0,0 +1,99 @@ +#include "config.h" + +#include +#include +#include +#include + +static void trim(char *s) { + size_t len = strlen(s); + while (len > 0 && isspace((unsigned char)s[len - 1])) { + s[--len] = '\0'; + } + char *p = s; + while (*p && isspace((unsigned char)*p)) { + p++; + } + if (p != s) { + memmove(s, p, strlen(p) + 1); + } +} + +void config_set_defaults(voice_config_t *cfg) { + if (!cfg) return; + snprintf(cfg->model_path, sizeof(cfg->model_path), "./models/ggml-base.en.bin"); + snprintf(cfg->audio_device, sizeof(cfg->audio_device), "qubes-source"); + snprintf(cfg->language, sizeof(cfg->language), "en"); + snprintf(cfg->hotkey, sizeof(cfg->hotkey), "Ctrl+Alt+V"); + cfg->sample_rate = 16000; + cfg->type_delay_us = 8000; + cfg->max_record_seconds = 30; + cfg->always_on = 1; + cfg->always_on_window_ms = 1400; + cfg->vad_peak_threshold = 0.006f; + cfg->vad_trigger_ms = 100; + cfg->vad_release_ms = 300; + cfg->vad_preroll_ms = 500; + cfg->vad_min_speech_ms = 250; + cfg->vad_max_speech_ms = 15000; + cfg->autotype_enabled = 0; +} + +int config_load_file(const char *path, voice_config_t *cfg) { + if (!path || !cfg) return -1; + + FILE *f = fopen(path, "r"); + if (!f) return -2; + + char line[1024]; + while (fgets(line, sizeof(line), f)) { + trim(line); + if (line[0] == '\0' || line[0] == '#') continue; + + char *eq = strchr(line, '='); + if (!eq) continue; + *eq = '\0'; + + char *key = line; + char *value = eq + 1; + trim(key); + trim(value); + + if (strcmp(key, "model_path") == 0) { + snprintf(cfg->model_path, sizeof(cfg->model_path), "%s", value); + } else if (strcmp(key, "audio_device") == 0) { + snprintf(cfg->audio_device, sizeof(cfg->audio_device), "%s", value); + } else if (strcmp(key, "language") == 0) { + snprintf(cfg->language, sizeof(cfg->language), "%s", value); + } else if (strcmp(key, "hotkey") == 0) { + snprintf(cfg->hotkey, sizeof(cfg->hotkey), "%s", value); + } else if (strcmp(key, "sample_rate") == 0) { + cfg->sample_rate = atoi(value); + } else if (strcmp(key, "type_delay_us") == 0) { + cfg->type_delay_us = atoi(value); + } else if (strcmp(key, "max_record_seconds") == 0) { + cfg->max_record_seconds = atoi(value); + } else if (strcmp(key, "always_on") == 0) { + cfg->always_on = atoi(value); + } else if (strcmp(key, "always_on_window_ms") == 0) { + cfg->always_on_window_ms = atoi(value); + } else if (strcmp(key, "vad_peak_threshold") == 0) { + cfg->vad_peak_threshold = (float)atof(value); + } else if (strcmp(key, "vad_trigger_ms") == 0) { + cfg->vad_trigger_ms = atoi(value); + } else if (strcmp(key, "vad_release_ms") == 0) { + cfg->vad_release_ms = atoi(value); + } else if (strcmp(key, "vad_preroll_ms") == 0) { + cfg->vad_preroll_ms = atoi(value); + } else if (strcmp(key, "vad_min_speech_ms") == 0) { + cfg->vad_min_speech_ms = atoi(value); + } else if (strcmp(key, "vad_max_speech_ms") == 0) { + cfg->vad_max_speech_ms = atoi(value); + } else if (strcmp(key, "autotype_enabled") == 0) { + cfg->autotype_enabled = atoi(value); + } + } + + fclose(f); + return 0; +} diff --git a/src/config.h b/src/config.h new file mode 100644 index 0000000..1a80a4c --- /dev/null +++ b/src/config.h @@ -0,0 +1,28 @@ +#ifndef VOICE_CONFIG_H +#define VOICE_CONFIG_H + +#include + +typedef struct { + char model_path[512]; + char audio_device[256]; + char language[32]; + char hotkey[64]; + int sample_rate; + int type_delay_us; + int max_record_seconds; + int always_on; + int always_on_window_ms; + float vad_peak_threshold; + int vad_trigger_ms; + int vad_release_ms; + int vad_preroll_ms; + int vad_min_speech_ms; + int vad_max_speech_ms; + int autotype_enabled; +} voice_config_t; + +void config_set_defaults(voice_config_t *cfg); +int config_load_file(const char *path, voice_config_t *cfg); + +#endif diff --git a/src/gui_main.c b/src/gui_main.c new file mode 100644 index 0000000..3fe0f99 --- /dev/null +++ b/src/gui_main.c @@ -0,0 +1,636 @@ +#include "audio.h" +#include "config.h" +#include "transcribe.h" +#include "typer.h" + +#include + +#include +#include +#include +#include + +typedef struct { + GtkApplication *app; + GtkWidget *window; + GtkWidget *status_label; + GtkWidget *record_btn; + GtkWidget *always_on_btn; + GtkWidget *type_check; + GtkWidget *transcript_view; + GtkWidget *record_light; + GtkWidget *transcribe_light; + + GtkWidget *level_area; + GtkWidget *threshold_scale; + GtkWidget *trigger_scale; + GtkWidget *release_scale; + GtkWidget *preroll_scale; + GtkWidget *min_speech_scale; + GtkWidget *max_speech_scale; + GtkWidget *debug_label; + + voice_config_t cfg; + int use_gpu; + int recording; + int transcribing; + int always_on_running; + + unsigned int meter_timer_id; + float live_peak; +} gui_state_t; + +static float clampf(float v, float lo, float hi) { + if (v < lo) return lo; + if (v > hi) return hi; + return v; +} + +static void set_status(gui_state_t *s, const char *text) { + gtk_label_set_text(GTK_LABEL(s->status_label), text); +} + +static void append_transcript(gui_state_t *s, const char *text) { + if (!text || text[0] == '\0') return; + + GtkTextBuffer *buf = gtk_text_view_get_buffer(GTK_TEXT_VIEW(s->transcript_view)); + GtkTextIter end; + gtk_text_buffer_get_end_iter(buf, &end); + + int has_text = gtk_text_buffer_get_char_count(buf) > 0; + if (has_text) { + gtk_text_buffer_insert(buf, &end, "\n", -1); + } + gtk_text_buffer_insert(buf, &end, text, -1); +} + +static void update_activity_lights(gui_state_t *s) { + if (!s->record_light || !s->transcribe_light) return; + + const int rec_on = (s->recording || s->always_on_running) ? 1 : 0; + const char *rec_markup = rec_on + ? "● Recording" + : "● Recording"; + + const char *tr_markup = s->transcribing + ? "● Transcribing" + : "● Transcribing"; + + gtk_label_set_markup(GTK_LABEL(s->record_light), rec_markup); + gtk_label_set_markup(GTK_LABEL(s->transcribe_light), tr_markup); +} + +static void update_record_button(gui_state_t *s) { + GtkStyleContext *ctx = gtk_widget_get_style_context(s->record_btn); + gtk_style_context_remove_class(ctx, "idle"); + gtk_style_context_remove_class(ctx, "recording"); + gtk_style_context_remove_class(ctx, "busy"); + + if (s->transcribing) { + gtk_widget_set_sensitive(s->record_btn, FALSE); + gtk_button_set_label(GTK_BUTTON(s->record_btn), "Transcribing..."); + gtk_style_context_add_class(ctx, "busy"); + update_activity_lights(s); + return; + } + + if (s->always_on_running) { + gtk_widget_set_sensitive(s->record_btn, FALSE); + gtk_button_set_label(GTK_BUTTON(s->record_btn), "Manual Disabled (Always-On)"); + gtk_style_context_add_class(ctx, "busy"); + update_activity_lights(s); + return; + } + + gtk_widget_set_sensitive(s->record_btn, TRUE); + if (s->recording) { + gtk_button_set_label(GTK_BUTTON(s->record_btn), "Stop Recording"); + gtk_style_context_add_class(ctx, "recording"); + } else { + gtk_button_set_label(GTK_BUTTON(s->record_btn), "Start Recording"); + gtk_style_context_add_class(ctx, "idle"); + } + + update_activity_lights(s); +} + +static void update_always_on_button(gui_state_t *s) { + if (!s->always_on_btn) return; + gtk_button_set_label(GTK_BUTTON(s->always_on_btn), s->always_on_running ? "Stop Always-On" : "Start Always-On"); +} + +static void log_audio_stats(const float *samples, size_t count, int sample_rate, const char *device_name, char *out, size_t out_sz) { + if (!samples || count == 0 || sample_rate <= 0) { + snprintf(out, out_sz, "Empty capture"); + return; + } + + float peak_abs = 0.0f; + double sum_abs = 0.0; + + for (size_t i = 0; i < count; ++i) { + float v = samples[i]; + float a = v < 0.0f ? -v : v; + if (a > peak_abs) peak_abs = a; + sum_abs += (double)a; + } + + double mean_abs = sum_abs / (double)count; + double seconds = (double)count / (double)sample_rate; + + if (peak_abs < 0.005f) { + snprintf(out, out_sz, + "Near-silent audio (%.2fs, peak %.6f) from '%.120s'", + seconds, peak_abs, device_name ? device_name : "(default)"); + } else { + snprintf(out, out_sz, + "Audio OK (%.2fs, peak %.6f, mean %.6f)", + seconds, peak_abs, mean_abs); + } +} + +static int peak_is_speech(const float *samples, size_t count, float threshold) { + float peak_abs = 0.0f; + for (size_t i = 0; i < count; ++i) { + float v = samples[i]; + float a = v < 0.0f ? -v : v; + if (a > peak_abs) peak_abs = a; + } + return peak_abs >= threshold; +} + +static void process_segment(gui_state_t *s, float *samples, size_t count) { + if (!samples || count == 0) { + free(samples); + return; + } + + char audio_msg[256]; + log_audio_stats(samples, count, s->cfg.sample_rate, s->cfg.audio_device, audio_msg, sizeof(audio_msg)); + + if (!peak_is_speech(samples, count, s->cfg.vad_peak_threshold)) { + set_status(s, "Mic muted or near-silent"); + free(samples); + return; + } + + s->transcribing = 1; + update_record_button(s); + set_status(s, "Transcribing... (please wait)"); + while (gtk_events_pending()) gtk_main_iteration(); + + char *text = transcribe_buffer(samples, count); + free(samples); + + s->transcribing = 0; + update_record_button(s); + + if (!text) { + set_status(s, "Transcription failed"); + return; + } + + append_transcript(s, text); + + if (gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(s->type_check))) { + typer_type_text(text); + } + + set_status(s, audio_msg); + free(text); +} + +static int read_scale_int(GtkWidget *scale) { + return (int)gtk_range_get_value(GTK_RANGE(scale)); +} + +static float read_scale_float(GtkWidget *scale) { + return (float)gtk_range_get_value(GTK_RANGE(scale)); +} + +static void apply_vad_params(gui_state_t *s) { + audio_vad_params_t v; + v.threshold = read_scale_float(s->threshold_scale); + v.trigger_ms = read_scale_int(s->trigger_scale); + v.release_ms = read_scale_int(s->release_scale); + v.preroll_ms = read_scale_int(s->preroll_scale); + v.min_speech_ms = read_scale_int(s->min_speech_scale); + v.max_speech_ms = read_scale_int(s->max_speech_scale); + + if (v.max_speech_ms < v.min_speech_ms) { + v.max_speech_ms = v.min_speech_ms; + gtk_range_set_value(GTK_RANGE(s->max_speech_scale), (double)v.max_speech_ms); + } + + s->cfg.vad_peak_threshold = v.threshold; + s->cfg.vad_trigger_ms = v.trigger_ms; + s->cfg.vad_release_ms = v.release_ms; + s->cfg.vad_preroll_ms = v.preroll_ms; + s->cfg.vad_min_speech_ms = v.min_speech_ms; + s->cfg.vad_max_speech_ms = v.max_speech_ms; + + audio_set_vad_params(&v); + gtk_widget_queue_draw(s->level_area); +} + +static void on_vad_value_changed(GtkRange *range, gpointer user_data) { + (void)range; + gui_state_t *s = (gui_state_t *)user_data; + apply_vad_params(s); +} + +static gboolean on_level_draw(GtkWidget *widget, cairo_t *cr, gpointer user_data) { + gui_state_t *s = (gui_state_t *)user_data; + + GtkAllocation a; + gtk_widget_get_allocation(widget, &a); + + const double w = (double)a.width; + const double h = (double)a.height; + + cairo_set_source_rgb(cr, 0.10, 0.10, 0.10); + cairo_rectangle(cr, 0.0, 0.0, w, h); + cairo_fill(cr); + + double peak = (double)clampf(s->live_peak, 0.0f, 1.0f); + double threshold = (double)clampf(s->cfg.vad_peak_threshold, 0.0f, 1.0f); + + double fill_w = peak * w; + if (fill_w > 0.0) { + if (peak >= threshold) { + cairo_set_source_rgb(cr, 0.18, 0.72, 0.22); + } else { + cairo_set_source_rgb(cr, 0.20, 0.52, 0.85); + } + cairo_rectangle(cr, 0.0, 0.0, fill_w, h); + cairo_fill(cr); + } + + double tx = threshold * w; + cairo_set_source_rgb(cr, 0.95, 0.25, 0.25); + cairo_set_line_width(cr, 2.0); + cairo_move_to(cr, tx, 0.0); + cairo_line_to(cr, tx, h); + cairo_stroke(cr); + + cairo_set_source_rgb(cr, 0.75, 0.75, 0.75); + cairo_set_line_width(cr, 1.0); + cairo_rectangle(cr, 0.5, 0.5, w - 1.0, h - 1.0); + cairo_stroke(cr); + + return FALSE; +} + +static const char *vad_state_text(int s) { + switch (s) { + case 0: return "IDLE"; + case 1: return "ONSET"; + case 2: return "SPEECH"; + case 3: return "TRAILING"; + default: return "?"; + } +} + +static gboolean meter_tick_cb(gpointer user_data) { + gui_state_t *s = (gui_state_t *)user_data; + + audio_debug_info_t di; + if (audio_get_debug_info(&di) == 0) { + s->live_peak = di.peak_level; + + char dbg[256]; + snprintf(dbg, sizeof(dbg), + "Debug: always_on=%d state=%s peak=%.4f thr=%.4f onset=%d trail=%d speech=%d ready=%zu", + di.always_on_enabled, + vad_state_text(di.vad_state), + di.peak_level, + di.threshold, + di.onset_samples, + di.trailing_samples, + di.speech_samples, + di.ready_samples); + gtk_label_set_text(GTK_LABEL(s->debug_label), dbg); + } else { + s->live_peak = audio_get_peak_level(); + } + + gtk_widget_queue_draw(s->level_area); + + if (s->always_on_running && !s->transcribing) { + float *samples = NULL; + size_t count = 0; + int rc = audio_get_speech_segment(&samples, &count); + if (rc == 0) { + set_status(s, "Speech segment detected, transcribing..."); + process_segment(s, samples, count); + } + } + + return G_SOURCE_CONTINUE; +} + +static void on_record_clicked(GtkButton *btn, gpointer user_data) { + (void)btn; + gui_state_t *s = (gui_state_t *)user_data; + + if (s->always_on_running || s->transcribing) { + return; + } + + if (!s->recording) { + if (audio_start_recording() != 0) { + set_status(s, "Failed to start recording"); + return; + } + + s->recording = 1; + update_record_button(s); + set_status(s, "Recording..."); + return; + } + + float *samples = NULL; + size_t count = 0; + if (audio_stop_recording(&samples, &count) != 0) { + set_status(s, "Failed to stop recording"); + return; + } + + s->recording = 0; + update_record_button(s); + + process_segment(s, samples, count); +} + +static void on_always_on_clicked(GtkButton *btn, gpointer user_data) { + (void)btn; + gui_state_t *s = (gui_state_t *)user_data; + + if (s->transcribing || s->recording) return; + + if (s->always_on_running) { + audio_set_always_on_enabled(0); + s->always_on_running = 0; + update_always_on_button(s); + update_record_button(s); + set_status(s, "Always-on stopped"); + return; + } + + apply_vad_params(s); + audio_set_always_on_enabled(1); + s->always_on_running = 1; + update_always_on_button(s); + update_record_button(s); + set_status(s, "Always-on active (continuous monitor)"); +} + +static GtkWidget *add_scale_row(GtkWidget *grid, + int row, + const char *label_text, + double min, + double max, + double step, + double value, + int digits, + gui_state_t *s, + GtkWidget **out_scale) { + GtkWidget *label = gtk_label_new(label_text); + gtk_label_set_xalign(GTK_LABEL(label), 0.0f); + gtk_grid_attach(GTK_GRID(grid), label, 0, row, 1, 1); + + GtkWidget *scale = gtk_scale_new_with_range(GTK_ORIENTATION_HORIZONTAL, min, max, step); + gtk_scale_set_digits(GTK_SCALE(scale), digits); + gtk_scale_set_draw_value(GTK_SCALE(scale), TRUE); + gtk_range_set_value(GTK_RANGE(scale), value); + gtk_widget_set_hexpand(scale, TRUE); + gtk_grid_attach(GTK_GRID(grid), scale, 1, row, 1, 1); + + g_signal_connect(scale, "value-changed", G_CALLBACK(on_vad_value_changed), s); + *out_scale = scale; + + return scale; +} + +static void on_app_activate(GtkApplication *app, gpointer user_data) { + gui_state_t *s = (gui_state_t *)user_data; + + s->window = gtk_application_window_new(app); + gtk_window_set_title(GTK_WINDOW(s->window), "voice_linux"); + gtk_window_set_default_size(GTK_WINDOW(s->window), 760, 640); + + GtkWidget *outer = gtk_box_new(GTK_ORIENTATION_VERTICAL, 8); + gtk_container_set_border_width(GTK_CONTAINER(outer), 12); + gtk_container_add(GTK_CONTAINER(s->window), outer); + + char info[1024]; + snprintf(info, sizeof(info), "Device: %s\nModel: %s\nLanguage: %s\nGPU: %s", + s->cfg.audio_device, s->cfg.model_path, s->cfg.language, s->use_gpu ? "on" : "off"); + GtkWidget *info_label = gtk_label_new(info); + gtk_label_set_xalign(GTK_LABEL(info_label), 0.0f); + gtk_box_pack_start(GTK_BOX(outer), info_label, FALSE, FALSE, 0); + + s->status_label = gtk_label_new("Idle"); + gtk_label_set_xalign(GTK_LABEL(s->status_label), 0.0f); + gtk_box_pack_start(GTK_BOX(outer), s->status_label, FALSE, FALSE, 0); + + GtkWidget *lights_row = gtk_box_new(GTK_ORIENTATION_HORIZONTAL, 14); + s->record_light = gtk_label_new(NULL); + s->transcribe_light = gtk_label_new(NULL); + gtk_label_set_use_markup(GTK_LABEL(s->record_light), TRUE); + gtk_label_set_use_markup(GTK_LABEL(s->transcribe_light), TRUE); + gtk_box_pack_start(GTK_BOX(lights_row), s->record_light, FALSE, FALSE, 0); + gtk_box_pack_start(GTK_BOX(lights_row), s->transcribe_light, FALSE, FALSE, 0); + gtk_box_pack_start(GTK_BOX(outer), lights_row, FALSE, FALSE, 0); + + s->record_btn = gtk_button_new_with_label("Start Recording"); + gtk_widget_set_name(s->record_btn, "record-btn"); + + GtkCssProvider *provider = gtk_css_provider_new(); + gtk_css_provider_load_from_data( + provider, + "#record-btn { font-weight: 700; color: #ffffff; }" + "#record-btn.idle { background-image: none; background-color: #c62828; }" + "#record-btn.recording { background-image: none; background-color: #2e7d32; }" + "#record-btn.busy { background-image: none; background-color: #ef6c00; }" + "#record-btn:backdrop { opacity: 0.92; }", + -1, + NULL); + gtk_style_context_add_provider_for_screen( + gdk_screen_get_default(), + GTK_STYLE_PROVIDER(provider), + GTK_STYLE_PROVIDER_PRIORITY_APPLICATION); + g_object_unref(provider); + + s->recording = 0; + update_record_button(s); + + g_signal_connect(s->record_btn, "clicked", G_CALLBACK(on_record_clicked), s); + gtk_box_pack_start(GTK_BOX(outer), s->record_btn, FALSE, FALSE, 0); + update_activity_lights(s); + + s->always_on_btn = gtk_button_new_with_label("Start Always-On"); + g_signal_connect(s->always_on_btn, "clicked", G_CALLBACK(on_always_on_clicked), s); + gtk_box_pack_start(GTK_BOX(outer), s->always_on_btn, FALSE, FALSE, 0); + + s->type_check = gtk_check_button_new_with_label("Type transcript into focused window"); + gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(s->type_check), s->cfg.autotype_enabled ? TRUE : FALSE); + gtk_box_pack_start(GTK_BOX(outer), s->type_check, FALSE, FALSE, 0); + + GtkWidget *meter_frame = gtk_frame_new("Live Microphone Level"); + gtk_box_pack_start(GTK_BOX(outer), meter_frame, FALSE, FALSE, 0); + + GtkWidget *meter_box = gtk_box_new(GTK_ORIENTATION_VERTICAL, 6); + gtk_container_set_border_width(GTK_CONTAINER(meter_box), 8); + gtk_container_add(GTK_CONTAINER(meter_frame), meter_box); + + s->level_area = gtk_drawing_area_new(); + gtk_widget_set_size_request(s->level_area, -1, 30); + g_signal_connect(s->level_area, "draw", G_CALLBACK(on_level_draw), s); + gtk_box_pack_start(GTK_BOX(meter_box), s->level_area, FALSE, FALSE, 0); + + GtkWidget *grid = gtk_grid_new(); + gtk_grid_set_row_spacing(GTK_GRID(grid), 6); + gtk_grid_set_column_spacing(GTK_GRID(grid), 10); + gtk_box_pack_start(GTK_BOX(meter_box), grid, FALSE, FALSE, 0); + + add_scale_row(grid, 0, "Threshold (0.0-1.0)", 0.0, 1.0, 0.001, + s->cfg.vad_peak_threshold, 3, s, &s->threshold_scale); + add_scale_row(grid, 1, "Trigger ms (above threshold)", 10.0, 2000.0, 10.0, + (double)s->cfg.vad_trigger_ms, 0, s, &s->trigger_scale); + add_scale_row(grid, 2, "Release ms (below threshold)", 10.0, 4000.0, 10.0, + (double)s->cfg.vad_release_ms, 0, s, &s->release_scale); + add_scale_row(grid, 3, "Preroll ms (audio before trigger)", 0.0, 3000.0, 10.0, + (double)s->cfg.vad_preroll_ms, 0, s, &s->preroll_scale); + add_scale_row(grid, 4, "Min speech ms", 50.0, 5000.0, 10.0, + (double)s->cfg.vad_min_speech_ms, 0, s, &s->min_speech_scale); + add_scale_row(grid, 5, "Max speech ms", 200.0, 60000.0, 100.0, + (double)s->cfg.vad_max_speech_ms, 0, s, &s->max_speech_scale); + + GtkWidget *hint = gtk_label_new( + "Green bar means level >= threshold. Red line is threshold.\n" + "Always-On uses continuous capture + preroll so word starts are not clipped."); + gtk_label_set_xalign(GTK_LABEL(hint), 0.0f); + gtk_box_pack_start(GTK_BOX(meter_box), hint, FALSE, FALSE, 0); + + s->debug_label = gtk_label_new("Debug: waiting for audio..."); + gtk_label_set_xalign(GTK_LABEL(s->debug_label), 0.0f); + gtk_box_pack_start(GTK_BOX(meter_box), s->debug_label, FALSE, FALSE, 0); + + GtkWidget *frame = gtk_frame_new("Transcript Log"); + gtk_box_pack_start(GTK_BOX(outer), frame, TRUE, TRUE, 0); + + GtkWidget *scroll = gtk_scrolled_window_new(NULL, NULL); + gtk_container_add(GTK_CONTAINER(frame), scroll); + + s->transcript_view = gtk_text_view_new(); + gtk_text_view_set_wrap_mode(GTK_TEXT_VIEW(s->transcript_view), GTK_WRAP_WORD_CHAR); + gtk_container_add(GTK_CONTAINER(scroll), s->transcript_view); + + apply_vad_params(s); + update_always_on_button(s); + gtk_widget_show_all(s->window); + + s->meter_timer_id = g_timeout_add(60, meter_tick_cb, s); + + if (s->cfg.always_on) { + audio_set_always_on_enabled(1); + s->always_on_running = 1; + update_always_on_button(s); + update_record_button(s); + set_status(s, "Always-on active (continuous monitor)"); + } +} + +static const char *arg_value(int argc, char **argv, const char *name) { + for (int i = 1; i + 1 < argc; ++i) { + if (strcmp(argv[i], name) == 0) { + return argv[i + 1]; + } + } + return NULL; +} + +static int arg_flag(int argc, char **argv, const char *name) { + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], name) == 0) { + return 1; + } + } + return 0; +} + +int main(int argc, char **argv) { + gui_state_t st; + memset(&st, 0, sizeof(st)); + + config_set_defaults(&st.cfg); + + const char *cfg_path = arg_value(argc, argv, "--config"); + if (!cfg_path) cfg_path = "./config.ini"; + if (config_load_file(cfg_path, &st.cfg) != 0) { + fprintf(stderr, "warning: failed to load %s, using defaults\n", cfg_path); + } + + st.use_gpu = arg_flag(argc, argv, "--gpu") ? 1 : 0; + if (arg_flag(argc, argv, "--cpu")) st.use_gpu = 0; + + audio_params_t ap = {0}; + snprintf(ap.device, sizeof(ap.device), "%s", st.cfg.audio_device); + ap.sample_rate = st.cfg.sample_rate; + ap.max_seconds = st.cfg.max_record_seconds; + + transcribe_params_t tp = {0}; + snprintf(tp.model_path, sizeof(tp.model_path), "%s", st.cfg.model_path); + snprintf(tp.language, sizeof(tp.language), "%s", st.cfg.language); + tp.use_gpu = st.use_gpu; + tp.n_threads = (int)sysconf(_SC_NPROCESSORS_ONLN); + + if (audio_init(&ap) != 0) { + fprintf(stderr, "fatal: audio init failed\n"); + return 1; + } + + audio_vad_params_t vp = { + .threshold = st.cfg.vad_peak_threshold, + .trigger_ms = st.cfg.vad_trigger_ms, + .release_ms = st.cfg.vad_release_ms, + .preroll_ms = st.cfg.vad_preroll_ms, + .min_speech_ms = st.cfg.vad_min_speech_ms, + .max_speech_ms = st.cfg.vad_max_speech_ms, + }; + audio_set_vad_params(&vp); + + if (transcribe_init(&tp) != 0) { + fprintf(stderr, "fatal: transcribe init failed\n"); + audio_cleanup(); + return 1; + } + if (typer_init(st.cfg.type_delay_us) != 0) { + fprintf(stderr, "fatal: typer init failed\n"); + transcribe_cleanup(); + audio_cleanup(); + return 1; + } + + st.app = gtk_application_new("local.voice_linux", G_APPLICATION_DEFAULT_FLAGS); + g_signal_connect(st.app, "activate", G_CALLBACK(on_app_activate), &st); + + int gtk_argc = 1; + char *gtk_argv[] = { argv[0], NULL }; + int rc = g_application_run(G_APPLICATION(st.app), gtk_argc, gtk_argv); + + if (st.meter_timer_id != 0) { + g_source_remove(st.meter_timer_id); + st.meter_timer_id = 0; + } + + audio_set_always_on_enabled(0); + + g_object_unref(st.app); + typer_cleanup(); + transcribe_cleanup(); + audio_cleanup(); + + return rc; +} diff --git a/src/hotkey.c b/src/hotkey.c new file mode 100644 index 0000000..c1d930d --- /dev/null +++ b/src/hotkey.c @@ -0,0 +1,107 @@ +#include "hotkey.h" + +#include +#include +#include +#include +#include + +static unsigned int parse_mod_token(const char *tok) { + if (strcasecmp(tok, "Ctrl") == 0 || strcasecmp(tok, "Control") == 0) return ControlMask; + if (strcasecmp(tok, "Alt") == 0) return Mod1Mask; + if (strcasecmp(tok, "Shift") == 0) return ShiftMask; + if (strcasecmp(tok, "Super") == 0 || strcasecmp(tok, "Win") == 0) return Mod4Mask; + return 0; +} + +static int parse_combo(Display *dpy, const char *combo, KeyCode *kc, unsigned int *mods) { + if (!combo || !kc || !mods) return -1; + + char buf[128]; + snprintf(buf, sizeof(buf), "%s", combo); + + char *save = NULL; + char *tok = strtok_r(buf, "+", &save); + char *parts[8]; + size_t n = 0; + while (tok && n < 8) { + parts[n++] = tok; + tok = strtok_r(NULL, "+", &save); + } + if (n == 0) return -2; + + *mods = 0; + for (size_t i = 0; i + 1 < n; ++i) { + *mods |= parse_mod_token(parts[i]); + } + + KeySym ks = XStringToKeysym(parts[n - 1]); + if (ks == NoSymbol && strlen(parts[n - 1]) == 1) { + char tmp[2] = {parts[n - 1][0], '\0'}; + ks = XStringToKeysym(tmp); + } + if (ks == NoSymbol) return -3; + + *kc = XKeysymToKeycode(dpy, ks); + return *kc ? 0 : -4; +} + +int hotkey_init(hotkey_ctx_t *ctx, const char *combo) { + if (!ctx) return -1; + memset(ctx, 0, sizeof(*ctx)); + + ctx->display = XOpenDisplay(NULL); + if (!ctx->display) return -2; + + ctx->root = DefaultRootWindow(ctx->display); + + if (parse_combo(ctx->display, combo, &ctx->keycode, &ctx->modifiers) != 0) { + return -3; + } + + const unsigned int extra_masks[] = {0, LockMask, Mod2Mask, LockMask | Mod2Mask}; + for (size_t i = 0; i < sizeof(extra_masks) / sizeof(extra_masks[0]); ++i) { + XGrabKey(ctx->display, (int)ctx->keycode, ctx->modifiers | extra_masks[i], ctx->root, + True, GrabModeAsync, GrabModeAsync); + } + + XSelectInput(ctx->display, ctx->root, KeyPressMask | KeyReleaseMask); + XSync(ctx->display, False); + return 0; +} + +int hotkey_wait_press(hotkey_ctx_t *ctx) { + if (!ctx || !ctx->display) return -1; + + for (;;) { + XEvent ev; + XNextEvent(ctx->display, &ev); + if (ev.type == KeyPress && ev.xkey.keycode == ctx->keycode) { + return 0; + } + } +} + +int hotkey_wait_release(hotkey_ctx_t *ctx) { + if (!ctx || !ctx->display) return -1; + + for (;;) { + XEvent ev; + XNextEvent(ctx->display, &ev); + if (ev.type == KeyRelease && ev.xkey.keycode == ctx->keycode) { + return 0; + } + } +} + +void hotkey_cleanup(hotkey_ctx_t *ctx) { + if (!ctx || !ctx->display) return; + + const unsigned int extra_masks[] = {0, LockMask, Mod2Mask, LockMask | Mod2Mask}; + for (size_t i = 0; i < sizeof(extra_masks) / sizeof(extra_masks[0]); ++i) { + XUngrabKey(ctx->display, (int)ctx->keycode, ctx->modifiers | extra_masks[i], ctx->root); + } + + XCloseDisplay(ctx->display); + ctx->display = NULL; +} diff --git a/src/hotkey.h b/src/hotkey.h new file mode 100644 index 0000000..a3520ae --- /dev/null +++ b/src/hotkey.h @@ -0,0 +1,18 @@ +#ifndef VOICE_HOTKEY_H +#define VOICE_HOTKEY_H + +#include + +typedef struct { + Display *display; + Window root; + KeyCode keycode; + unsigned int modifiers; +} hotkey_ctx_t; + +int hotkey_init(hotkey_ctx_t *ctx, const char *combo); +int hotkey_wait_press(hotkey_ctx_t *ctx); +int hotkey_wait_release(hotkey_ctx_t *ctx); +void hotkey_cleanup(hotkey_ctx_t *ctx); + +#endif diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..e479eec --- /dev/null +++ b/src/main.c @@ -0,0 +1,185 @@ +#include "audio.h" +#include "config.h" +#include "hotkey.h" +#include "transcribe.h" +#include "typer.h" + +#include +#include +#include +#include +#include + +static const char *arg_value(int argc, char **argv, const char *name) { + for (int i = 1; i + 1 < argc; ++i) { + if (strcmp(argv[i], name) == 0) { + return argv[i + 1]; + } + } + return NULL; +} + +static int arg_flag(int argc, char **argv, const char *name) { + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], name) == 0) { + return 1; + } + } + return 0; +} + +static void log_audio_stats(const float *samples, size_t count, int sample_rate, const char *device_name) { + if (!samples || count == 0 || sample_rate <= 0) { + fprintf(stderr, "[audio] empty capture\n"); + return; + } + + float peak_abs = 0.0f; + double sum_abs = 0.0; + + for (size_t i = 0; i < count; ++i) { + float v = samples[i]; + float a = v < 0.0f ? -v : v; + if (a > peak_abs) peak_abs = a; + sum_abs += (double)a; + } + + double mean_abs = sum_abs / (double)count; + double seconds = (double)count / (double)sample_rate; + + fprintf(stderr, "[audio] duration=%.2fs samples=%zu peak=%.6f mean_abs=%.6f\n", + seconds, count, peak_abs, mean_abs); + + if (peak_abs < 0.005f) { + fprintf(stderr, "[audio] warning: signal is extremely low/near-silent. Check mic routing and selected source '%s'.\n", + device_name ? device_name : "(default)"); + } +} + +static double monotonic_seconds(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0; +} + +int main(int argc, char **argv) { + voice_config_t cfg; + config_set_defaults(&cfg); + + const char *cfg_path = arg_value(argc, argv, "--config"); + if (!cfg_path) cfg_path = "./config.ini"; + + if (config_load_file(cfg_path, &cfg) != 0) { + fprintf(stderr, "warning: failed to load %s, using defaults\n", cfg_path); + } + + int use_gpu = arg_flag(argc, argv, "--gpu") ? 1 : 0; + if (arg_flag(argc, argv, "--cpu")) use_gpu = 0; + + audio_params_t ap = {0}; + snprintf(ap.device, sizeof(ap.device), "%s", cfg.audio_device); + ap.sample_rate = cfg.sample_rate; + ap.max_seconds = cfg.max_record_seconds; + + transcribe_params_t tp = {0}; + snprintf(tp.model_path, sizeof(tp.model_path), "%s", cfg.model_path); + snprintf(tp.language, sizeof(tp.language), "%s", cfg.language); + tp.use_gpu = use_gpu; + tp.n_threads = (int)sysconf(_SC_NPROCESSORS_ONLN); + + hotkey_ctx_t hk; + + if (audio_init(&ap) != 0) { + fprintf(stderr, "fatal: audio init failed\n"); + return 1; + } + if (transcribe_init(&tp) != 0) { + fprintf(stderr, "fatal: transcribe init failed\n"); + audio_cleanup(); + return 1; + } + if (typer_init(cfg.type_delay_us) != 0) { + fprintf(stderr, "fatal: typer init failed\n"); + transcribe_cleanup(); + audio_cleanup(); + return 1; + } + if (hotkey_init(&hk, cfg.hotkey) != 0) { + fprintf(stderr, "fatal: hotkey init failed for '%s'\n", cfg.hotkey); + typer_cleanup(); + transcribe_cleanup(); + audio_cleanup(); + return 1; + } + + fprintf(stderr, "voice_linux ready\n"); + fprintf(stderr, "hotkey toggle mode: press %s to START recording, press %s again to STOP and transcribe\n", cfg.hotkey, cfg.hotkey); + fprintf(stderr, "device: %s | model: %s | language: %s | gpu: %s\n", + cfg.audio_device, cfg.model_path, cfg.language, use_gpu ? "on" : "off"); + + for (;;) { + if (hotkey_wait_press(&hk) != 0) break; + if (hotkey_wait_release(&hk) != 0) break; + + fprintf(stderr, "[rec] start\n"); + if (audio_start_recording() != 0) { + fprintf(stderr, "audio_start_recording failed\n"); + continue; + } + + const double rec_start = monotonic_seconds(); + for (;;) { + if (hotkey_wait_press(&hk) != 0) { + fprintf(stderr, "hotkey second press wait failed\n"); + goto done; + } + if (hotkey_wait_release(&hk) != 0) { + fprintf(stderr, "hotkey second release wait failed\n"); + goto done; + } + + const double elapsed = monotonic_seconds() - rec_start; + if (elapsed < 0.35) { + fprintf(stderr, "[hotkey] ignoring early stop event (%.2f s)\n", elapsed); + continue; + } + break; + } + + float *samples = NULL; + size_t count = 0; + if (audio_stop_recording(&samples, &count) != 0) { + fprintf(stderr, "audio_stop_recording failed\n"); + continue; + } + + fprintf(stderr, "[rec] stop, samples=%zu\n", count); + if (!samples || count == 0) { + free(samples); + continue; + } + + log_audio_stats(samples, count, cfg.sample_rate, cfg.audio_device); + + fprintf(stderr, "[stt] transcribing...\n"); + char *text = transcribe_buffer(samples, count); + free(samples); + + if (!text) { + fprintf(stderr, "[stt] no transcription\n"); + continue; + } + + fprintf(stderr, "[stt] %s\n", text); + typer_type_text(text); + free(text); + } + +done: + hotkey_cleanup(&hk); + typer_cleanup(); + transcribe_cleanup(); + audio_cleanup(); + + return 0; +} diff --git a/src/transcribe.c b/src/transcribe.c new file mode 100644 index 0000000..c7c9b8d --- /dev/null +++ b/src/transcribe.c @@ -0,0 +1,102 @@ +#include "transcribe.h" + +#include +#include +#include + +#ifndef WITH_WHISPER +#define WITH_WHISPER 0 +#endif + +#if WITH_WHISPER +#include + +static struct whisper_context *g_ctx = NULL; +static transcribe_params_t g_params; + +int transcribe_init(const transcribe_params_t *params) { + if (!params) return -1; + g_params = *params; + + struct whisper_context_params cparams = whisper_context_default_params(); + cparams.use_gpu = g_params.use_gpu ? true : false; + g_ctx = whisper_init_from_file_with_params(g_params.model_path, cparams); + if (!g_ctx) { + fprintf(stderr, "transcribe: failed to load model: %s\n", g_params.model_path); + return -2; + } + return 0; +} + +char *transcribe_buffer(const float *samples, size_t count) { + if (!g_ctx || !samples || count == 0) return NULL; + + struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + params.print_progress = false; + params.print_special = false; + params.print_realtime = false; + params.print_timestamps = false; + params.translate = false; + params.n_threads = g_params.n_threads > 0 ? g_params.n_threads : 4; + params.language = g_params.language[0] ? g_params.language : "en"; + + if (whisper_full(g_ctx, params, samples, (int)count) != 0) { + fprintf(stderr, "transcribe: whisper_full failed\n"); + return NULL; + } + + int nseg = whisper_full_n_segments(g_ctx); + size_t total = 1; + for (int i = 0; i < nseg; ++i) { + const char *seg = whisper_full_get_segment_text(g_ctx, i); + if (seg) total += strlen(seg) + 1; + } + + char *out = (char *)malloc(total); + if (!out) return NULL; + out[0] = '\0'; + + for (int i = 0; i < nseg; ++i) { + const char *seg = whisper_full_get_segment_text(g_ctx, i); + if (!seg) continue; + strcat(out, seg); + if (i + 1 < nseg) strcat(out, " "); + } + + return out; +} + +void transcribe_cleanup(void) { + if (g_ctx) { + whisper_free(g_ctx); + g_ctx = NULL; + } +} + +#else + +static transcribe_params_t g_params; + +int transcribe_init(const transcribe_params_t *params) { + if (!params) return -1; + g_params = *params; + fprintf(stderr, "transcribe: built without whisper support. Rebuild with WITH_WHISPER=1 bash ./build.sh\n"); + return 0; +} + +char *transcribe_buffer(const float *samples, size_t count) { + (void)samples; + (void)count; + + const char *msg = "[transcription unavailable: rebuild with WITH_WHISPER=1 and install whisper.cpp]"; + char *out = (char *)malloc(strlen(msg) + 1); + if (!out) return NULL; + strcpy(out, msg); + return out; +} + +void transcribe_cleanup(void) { + memset(&g_params, 0, sizeof(g_params)); +} + +#endif diff --git a/src/transcribe.h b/src/transcribe.h new file mode 100644 index 0000000..df66c2a --- /dev/null +++ b/src/transcribe.h @@ -0,0 +1,17 @@ +#ifndef VOICE_TRANSCRIBE_H +#define VOICE_TRANSCRIBE_H + +#include + +typedef struct { + char model_path[512]; + char language[32]; + int use_gpu; + int n_threads; +} transcribe_params_t; + +int transcribe_init(const transcribe_params_t *params); +char *transcribe_buffer(const float *samples, size_t count); +void transcribe_cleanup(void); + +#endif diff --git a/src/typer.c b/src/typer.c new file mode 100644 index 0000000..ca25b85 --- /dev/null +++ b/src/typer.c @@ -0,0 +1,95 @@ +#include "typer.h" + +#include +#include +#include + +#include +#include +#include +#include + +static Display *g_dpy = NULL; +static int g_delay_us = 8000; + +static int type_keysym(KeySym ks, int need_shift) { + KeyCode kc = XKeysymToKeycode(g_dpy, ks); + if (!kc) return -1; + + KeyCode shift_kc = XKeysymToKeycode(g_dpy, XK_Shift_L); + + if (need_shift && shift_kc) { + XTestFakeKeyEvent(g_dpy, shift_kc, True, CurrentTime); + } + + XTestFakeKeyEvent(g_dpy, kc, True, CurrentTime); + XTestFakeKeyEvent(g_dpy, kc, False, CurrentTime); + + if (need_shift && shift_kc) { + XTestFakeKeyEvent(g_dpy, shift_kc, False, CurrentTime); + } + + XFlush(g_dpy); + struct timespec ts; + ts.tv_sec = g_delay_us / 1000000; + ts.tv_nsec = (long)(g_delay_us % 1000000) * 1000L; + nanosleep(&ts, NULL); + return 0; +} + +static int type_char(char c) { + if (c >= 'a' && c <= 'z') { + return type_keysym((KeySym)(XK_a + (c - 'a')), 0); + } + if (c >= 'A' && c <= 'Z') { + return type_keysym((KeySym)(XK_a + (c - 'A')), 1); + } + if (c >= '0' && c <= '9') { + return type_keysym((KeySym)(XK_0 + (c - '0')), 0); + } + + switch (c) { + case ' ': return type_keysym(XK_space, 0); + case '\n': return type_keysym(XK_Return, 0); + case '.': return type_keysym(XK_period, 0); + case ',': return type_keysym(XK_comma, 0); + case '!': return type_keysym(XK_1, 1); + case '?': return type_keysym(XK_slash, 1); + case ':': return type_keysym(XK_semicolon, 1); + case ';': return type_keysym(XK_semicolon, 0); + case '\'': return type_keysym(XK_apostrophe, 0); + case '"': return type_keysym(XK_apostrophe, 1); + case '-': return type_keysym(XK_minus, 0); + case '_': return type_keysym(XK_minus, 1); + case '/': return type_keysym(XK_slash, 0); + case '(': return type_keysym(XK_9, 1); + case ')': return type_keysym(XK_0, 1); + default: return 0; + } +} + +int typer_init(int type_delay_us) { + g_dpy = XOpenDisplay(NULL); + if (!g_dpy) return -1; + g_delay_us = type_delay_us > 0 ? type_delay_us : 8000; + return 0; +} + +int typer_type_text(const char *text) { + if (!g_dpy || !text) return -1; + + for (size_t i = 0; i < strlen(text); ++i) { + if (type_char(text[i]) != 0) { + fprintf(stderr, "typer: failed at char '%c'\n", text[i]); + } + } + + return 0; +} + +void typer_cleanup(void) { + if (g_dpy) { + XCloseDisplay(g_dpy); + g_dpy = NULL; + } +} diff --git a/src/typer.h b/src/typer.h new file mode 100644 index 0000000..a6eaf43 --- /dev/null +++ b/src/typer.h @@ -0,0 +1,8 @@ +#ifndef VOICE_TYPER_H +#define VOICE_TYPER_H + +int typer_init(int type_delay_us); +int typer_type_text(const char *text); +void typer_cleanup(void); + +#endif diff --git a/voice_linux_gui b/voice_linux_gui new file mode 100755 index 0000000..80edb42 Binary files /dev/null and b/voice_linux_gui differ