diff --git a/README.md b/README.md index b11c9e9..c501a56 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ is what decides; anything not on it is a panel Panama owns itself. |---|---| | `config/dot/hypr/` | Compositor config. **Lua, not hyprlang** — see its README | | `config/dot/quickshell/` | The shell: bar, dock, Continuum overview, Settings, Screen Intelligence, focus sessions, quick settings, notifications, screenshot UI | +| `config/containers/` | Container definitions systemd runs as units — currently the speech-to-text server behind dictation | | `config/dot/vicinae/` | Raycast-style launcher, themed. Its commands live in `config/local/share/vicinae/` — script commands, and one compiled extension that adds web search with live suggestions | | `config/dot/uwsm/` | Session environment (see the uwsm caveat in the hypr README) | | `config/dot/wofi/` | Fallback launcher, in case the shell fails to start | @@ -84,6 +85,7 @@ config/ copy/ Files copied verbatim over / (needs sudo) dot/ Symlinked into ~/.config firefox/ Vendored Firefox chrome, linked into the browser profile + containers/ Quadlets, linked into ~/.config/containers/systemd local/ Icons, the cursor theme, and the launcher's commands and extensions, linked into ~/.local/share old/ Backups of whatever was replaced (gitignored) @@ -99,7 +101,7 @@ docs/ Settings reference, and the design specs behind the work ## Tests -129 of them, under `tests/`. Run the lot, or a subset by pattern: +130 of them, under `tests/`. Run the lot, or a subset by pattern: ```sh panama test # everything diff --git a/config/containers/whisper/panama-whisper.container b/config/containers/whisper/panama-whisper.container new file mode 100644 index 0000000..add965d --- /dev/null +++ b/config/containers/whisper/panama-whisper.container @@ -0,0 +1,53 @@ +# Dictation's speech-to-text server, as a Quadlet. +# +# Quadlet rather than a hand-written unit wrapping `podman run`: systemd +# generates the unit from this at boot, so there is one description of the +# container rather than a unit and a command line drifting apart. +# +# Upstream's own Vulkan image, not one built here. whisper.cpp publishes it, it +# already contains whisper-server, and it is maintained by the people who write +# the thing -- a Containerfile in this repository would be a compile step on +# every machine and a build to keep working, in exchange for nothing. +# +# Vulkan rather than ROCm, which is the reason this tag and not another. ROCm's +# runtime is seven gigabytes and serves AMD alone; Vulkan compute runs on AMD, +# Intel and NVIDIA through whatever Mesa driver a machine already has. These +# machines are a mix of all three, and this image works on every one of them. +# +# Deliberately no [Install] section. whisper-server loads the model when it +# starts and holds it, so a container started at login costs half a gigabyte of +# memory in every session where nobody dictates. panama-dictate starts it on the +# first press of the key, and it stays up for the rest of the session. + +[Unit] +Description=Panama dictation speech-to-text server +Documentation=https://github.com/ggml-org/whisper.cpp + +[Container] +Image=ghcr.io/ggml-org/whisper.cpp:main-vulkan + +# The image's own entrypoint is a shell; the server is what is wanted. +Entrypoint=/app/build/bin/whisper-server +Exec=--host 0.0.0.0 --port 8791 --model /models/ggml-small.bin --inference-path /inference + +# The whole directory rather than a renderD node: the number differs between +# machines, and this file is meant to be identical on all of them. +AddDevice=/dev/dri + +# The model is host state -- fetched once, kept across image updates, and shared +# with nothing else. Read-only because the server never writes to it, and :z +# relabels for SELinux, which is enforcing on Fedora. +Volume=%h/.local/share/panama/whisper:/models:ro,z + +# Loopback only. This transcribes whatever it is sent, with no authentication, +# and has no business being reachable from the network. +PublishPort=127.0.0.1:8791:8791 + +NoNewPrivileges=true + +[Service] +# Loading the model takes a few seconds on a cold start; systemd should wait for +# it rather than give up, and should bring the server back if it dies mid-session. +TimeoutStartSec=180 +Restart=on-failure +RestartSec=3 diff --git a/config/dot/hypr/keybinds.lua b/config/dot/hypr/keybinds.lua index 5e8ecf2..69f4050 100644 --- a/config/dot/hypr/keybinds.lua +++ b/config/dot/hypr/keybinds.lua @@ -34,6 +34,10 @@ local osd = function(action) return "$HOME/.config/quickshell/scripts/panama-osd " .. action end +local dictate = function(action) + return "$HOME/.config/quickshell/scripts/panama-dictate " .. action +end + -- Quickshell IPC targets. See quickshell/shell.qml for the handlers. local qs = function(target, fn) return "qs ipc call " .. target .. " " .. fn end @@ -293,6 +297,26 @@ bind("XF86AudioLowerVolume", hl.dsp.exec_cmd(osd("volume down 6")), { locked = t bind("XF86AudioMute", hl.dsp.exec_cmd(osd("volume toggle")), { locked = true , description = "Mute" }) bind("XF86AudioMicMute", hl.dsp.exec_cmd(osd("microphone toggle")), { locked = true , description = "Mute microphone" }) +-- ── Dictation ─────────────────────────────────────────────────────────────── +-- +-- Hold to talk, exactly like push-to-talk anywhere else: the mic is open only +-- while the key is down, so it cannot be left listening by forgetting about it. +-- Two binds on one chord, the second flagged `release`. +-- +-- No `repeating`: holding a key normally repeats the press, which would restart +-- the recording several times a second. The daemon refuses a second start while +-- one is running, so a repeat would be harmless -- but not asking for it is +-- better than relying on being refused. +bind(mod .. " + D", hl.dsp.exec_cmd(dictate("start")), + { description = "Dictate (hold to talk)" }) +bind(mod .. " + D", hl.dsp.exec_cmd(dictate("stop")), + { release = true, description = "Dictate (transcribe on release)" }) + +-- Escape out of a recording without transcribing it. Bound to the same modifier +-- so it can be reached with the dictation key still held. +bind(mod .. " + SHIFT + D", hl.dsp.exec_cmd(dictate("cancel")), + { description = "Cancel dictation" }) + -- Fine-grained steps, matching GNOME's shift/alt volume modifiers. bind("SHIFT + XF86AudioRaiseVolume", hl.dsp.exec_cmd(osd("volume up 1")), { locked = true, repeating = true , description = "Volume up (fine)" }) bind("SHIFT + XF86AudioLowerVolume", hl.dsp.exec_cmd(osd("volume down 1")), { locked = true, repeating = true , description = "Volume down (fine)" }) diff --git a/config/dot/quickshell/modules/settings/SoundPage.qml b/config/dot/quickshell/modules/settings/SoundPage.qml index 0f430e6..02c9cef 100644 --- a/config/dot/quickshell/modules/settings/SoundPage.qml +++ b/config/dot/quickshell/modules/settings/SoundPage.qml @@ -31,6 +31,65 @@ SettingsPage { } } + // Beside Input on purpose: dictation listens through whichever device that + // card selects, and putting the two together is what makes that obvious. + SettingsCard { + title: "Dictation" + subtitle: Dictation.ready + ? "Hold Super+D, speak, and release. The words are typed where the cursor is." + : "Speech to text, on the GPU. Two pieces have to be in place first — neither ships with Panama, because both are large and want the network." + + // Not a toggle: the keybind exists either way, and a switch would imply + // dictation can be turned off rather than simply not set up. + TextRow { + label: "Speech server" + detail: Dictation.imageBuilt + ? (Dictation.serverReady + ? "Built and running" + : "Built. Starts on the first dictation and stays loaded.") + : "Not built — run: panama app whisper-vulkan" + value: Dictation.imageBuilt ? "Ready" : "Missing" + } + + ActionRow { + label: "Speech model" + detail: Dictation.downloading + ? (Dictation.downloadTotalBytes > 0 + ? Math.round(Dictation.downloadFraction * 100) + "% of " + + Math.round(Dictation.downloadTotalBytes / 1048576) + " MB" + : "Downloading…") + : (Dictation.modelInstalled + ? Math.round(Dictation.modelBytes / 1048576) + " MB, in place" + : "About 490 MB, downloaded once and kept") + action: Dictation.downloading ? "Downloading…" : "Download" + enabled: !Dictation.downloading && !Dictation.modelInstalled + visible: !Dictation.modelInstalled || Dictation.downloading + onTriggered: Dictation.download() + } + + TextRow { + visible: Dictation.modelInstalled && !Dictation.downloading + label: "Speech model" + detail: "Downloaded once and kept across rebuilds of the server." + value: Math.round(Dictation.modelBytes / 1048576) + " MB" + } + + TextRow { + visible: !Dictation.typingAvailable + label: "Typing" + detail: "wtype is missing, so dictated text would go to the clipboard instead of being typed." + value: "Missing" + } + + TextRow { + visible: Dictation.lastError !== "" + label: "Problem" + detail: Dictation.lastError + value: "" + divider: false + } + } + SettingsCard { title: "Applications" subtitle: "Control each application currently playing through PipeWire." diff --git a/config/dot/quickshell/scripts/panama-dictate b/config/dot/quickshell/scripts/panama-dictate new file mode 100755 index 0000000..6a9bda6 --- /dev/null +++ b/config/dot/quickshell/scripts/panama-dictate @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 + +"""Dictation: hold a key, speak, and the words are typed where you were typing. + + panama-dictate start begin recording (bound to key press) + panama-dictate stop stop, transcribe, and type the result + panama-dictate cancel stop and discard + panama-dictate status JSON: model, server, and image + panama-dictate setup fetch the image and the model, with progress + +There is no daemon here, and that is the point. Transcription is done by +whisper.cpp running in a container -- see config/containers/whisper -- which +holds the model on the GPU between utterances. This script only records, asks +that server for the text, and types it, so `start` and `stop` are two short +runs of a script rather than two messages to something long-lived. What they +share is a pidfile. + +Neither the image nor the model ships with Panama, and neither is fetched during +installation. Together they are more than two gigabytes, and an installer that +quietly spends ten minutes on them is the surprise the interview exists to +prevent. Settings offers both as one action; until then, dictation says what is +missing rather than hanging. + +The image is upstream's own Vulkan build of whisper.cpp -- Vulkan so that one +image serves the AMD, Intel and NVIDIA machines this config runs on, where a +ROCm build would serve only the AMD ones at twenty times the size. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +MODEL = "small" +MODEL_DIR = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local/share")) / "panama/whisper" +MODEL_FILE = MODEL_DIR / f"ggml-{MODEL}.bin" +MODEL_URL = f"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-{MODEL}.bin" + +IMAGE = "ghcr.io/ggml-org/whisper.cpp:main-vulkan" +SERVICE = "panama-whisper.service" +ENDPOINT = "http://127.0.0.1:8791/inference" + +RUNTIME_DIR = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) +STATE_FILE = RUNTIME_DIR / "panama-dictate.json" + +# 16 kHz mono is what Whisper resamples to anyway, so recording it directly +# avoids a conversion and keeps the file small. +SAMPLE_RATE = "16000" + +# Anything shorter is a key tapped by accident rather than something said. +# Transcribing a fifth of a second of room tone reliably invents a word, and +# that word then gets typed. +MIN_SECONDS = 0.35 + +REASONS = { + "not-downloaded": "Speech model not downloaded — see Settings", + "no-image": "Speech server not installed — see Settings", + "server-unavailable": "The speech server did not answer", + "transcribe-failed": "Transcription failed", + "too-short": "Too short", + "no-speech": "Nothing was said", + "not-recording": "Not recording", + "already-recording": "Already listening", + "deliver-failed": "Could not deliver the text", +} + + +def osd(message: str | None) -> None: + """Show a line on the on-screen display, or clear it. + + Never fatal: the shell can be restarting, and dictation that refused to work + because it could not draw a label would be worse than a silent one. + """ + argv = (["qs", "ipc", "call", "osd", "hide"] if message is None + else ["qs", "ipc", "call", "osd", "message", "microphone", message]) + try: + subprocess.run(argv, capture_output=True, timeout=3, check=False) + except (OSError, subprocess.SubprocessError): + pass + + +def emit(result: dict) -> int: + print(json.dumps(result)) + return 0 if result.get("ok") else 2 + + +def have_image() -> bool: + try: + done = subprocess.run(["podman", "image", "exists", IMAGE], + capture_output=True, timeout=15, check=False) + return done.returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def server_ready(timeout: float = 1.5) -> bool: + try: + request = urllib.request.Request(ENDPOINT, method="OPTIONS") + urllib.request.urlopen(request, timeout=timeout) + return True + except urllib.error.HTTPError: + # Answering at all is what is being asked. A 404 or 405 to OPTIONS still + # means something is listening and loaded. + return True + except (urllib.error.URLError, OSError): + return False + + +def start_server() -> bool: + """Bring the container up and wait for the model to load. + + Started here rather than at login: whisper-server holds the model from the + moment it starts, and a session where nobody dictates should not be paying + half a gigabyte of VRAM for the possibility. + """ + if server_ready(): + return True + try: + subprocess.run(["systemctl", "--user", "start", SERVICE], + capture_output=True, timeout=30, check=False) + except (OSError, subprocess.SubprocessError): + return False + + # The first start loads the model, which takes seconds rather than + # milliseconds. Recording is already running by then, so this wait costs + # nothing that was going to be spoken. + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + if server_ready(): + return True + time.sleep(0.25) + return False + + +def deliver(text: str) -> str: + """Get text into the focused window. Returns how it got there. + + Typing is preferred: it lands wherever the cursor is, works in terminals and + editors that ignore a paste, and leaves the clipboard alone -- which matters + here, because Panama keeps clipboard history and dictation would otherwise + fill it with everything ever said. + + The clipboard is the fallback rather than the default, for the applications + that refuse synthetic keystrokes. Which one happened is reported back, so + the caller can say so: a fallback nobody is told about is how "it typed + nothing" becomes a mystery. + """ + if shutil.which("wtype"): + try: + done = subprocess.run(["wtype", "--", text], + capture_output=True, timeout=60, check=False) + if done.returncode == 0: + return "typed" + except (OSError, subprocess.SubprocessError): + pass + + if shutil.which("wl-copy"): + try: + subprocess.run(["wl-copy", "--", text], + capture_output=True, timeout=10, check=False) + return "copied" + except (OSError, subprocess.SubprocessError): + pass + + return "failed" + + +def transcribe(path: Path) -> tuple[str | None, str | None]: + """Ask the container for the text. Returns (text, error).""" + boundary = "----panama-dictate" + body = b"".join([ + f'--{boundary}\r\nContent-Disposition: form-data; name="file"; ' + f'filename="speech.wav"\r\nContent-Type: audio/wav\r\n\r\n'.encode(), + path.read_bytes(), + f'\r\n--{boundary}\r\nContent-Disposition: form-data; name="response_format"' + f'\r\n\r\ntext\r\n--{boundary}--\r\n'.encode(), + ]) + request = urllib.request.Request( + ENDPOINT, data=body, method="POST", + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}) + try: + with urllib.request.urlopen(request, timeout=300) as response: + answer = response.read().decode(errors="replace").strip() + except (urllib.error.URLError, OSError, urllib.error.HTTPError): + return None, "transcribe-failed" + + # response_format=text is asked for, but a server built from a different + # upstream revision may answer JSON regardless. Reading both costs three + # lines and saves a version bump turning dictation into gibberish. + if answer.startswith("{"): + try: + answer = str(json.loads(answer).get("text", "")).strip() + except (json.JSONDecodeError, AttributeError): + return None, "transcribe-failed" + + # One line, always. The server returns a line per segment, and a newline + # typed into a window is an Enter press -- which sends the half-finished + # message, submits the form, or runs the command. Whitespace is collapsed + # rather than only stripped, because the break is in the middle of the + # sentence, not at its ends. + return " ".join(answer.split()), None + + +# Whisper describes what it heard when it did not hear words: [BLANK_AUDIO] for +# silence, and things like (wind blowing) or [MUSIC] for sound that is not +# speech. They arrive as ordinary text, so without this they get typed -- the +# first test of this dictated the literal string "[BLANK_AUDIO]" into the +# clipboard, which is precisely the kind of thing nobody notices until it lands +# in the middle of a commit message. +NON_SPEECH = re.compile(r"[\[(][^\])]*[\])]") + + +def is_speech(text: str) -> bool: + """False when the transcription is only Whisper describing the absence of speech. + + Deliberately conservative: bracketed groups are removed and what remains is + what decides. A sentence that merely contains an aside keeps its brackets and + is still typed in full -- only a transcription that is nothing BUT markers is + discarded, because throwing away real words would be far worse than typing a + stray marker. + """ + return NON_SPEECH.sub("", text).strip() != "" + + +def read_state() -> dict | None: + try: + return json.loads(STATE_FILE.read_text()) + except (OSError, json.JSONDecodeError): + return None + + +def handle_start() -> dict: + if not MODEL_FILE.exists(): + osd(REASONS["not-downloaded"]) + return {"ok": False, "error": "not-downloaded"} + if not have_image(): + osd(REASONS["no-image"]) + return {"ok": False, "error": "no-image"} + + existing = read_state() + if existing is not None: + # A stale pidfile from a crash should not block dictation forever, so + # it is only respected while the process it names is actually alive. + try: + os.kill(existing["pid"], 0) + return {"ok": False, "error": "already-recording"} + except (OSError, KeyError, TypeError): + STATE_FILE.unlink(missing_ok=True) + + MODEL_DIR.mkdir(parents=True, exist_ok=True) + handle, name = tempfile.mkstemp(prefix="panama-dictate-", suffix=".wav") + os.close(handle) + + # pw-record rather than a library: pipewire-utils is already installed -- + # it backs the privacy indicators and the sound test -- and it records from + # whatever WirePlumber calls the default source, so changing input device + # in Settings changes what dictation hears, with no code here. + recorder = subprocess.Popen( + ["pw-record", "--rate", SAMPLE_RATE, "--channels", "1", "--format", "s16", name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) + + STATE_FILE.write_text(json.dumps({ + "pid": recorder.pid, "path": name, "startedAt": time.time()})) + STATE_FILE.chmod(0o600) + osd("Listening…") + return {"ok": True, "recording": True} + + +def finish_recording() -> tuple[Path | None, float]: + """Stop the recorder and hand back its file and how long it ran.""" + state = read_state() + STATE_FILE.unlink(missing_ok=True) + if state is None: + return None, 0.0 + + path = Path(state.get("path", "")) + elapsed = max(0.0, time.time() - float(state.get("startedAt", time.time()))) + try: + # SIGINT rather than SIGKILL: pw-record finalises the WAV header when + # interrupted, and a file whose header still claims zero length is one + # the server reads as silence. + os.kill(int(state["pid"]), signal.SIGINT) + except (OSError, KeyError, TypeError, ValueError): + return (path if path.exists() else None), elapsed + + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + try: + os.kill(int(state["pid"]), 0) + except OSError: + break + time.sleep(0.05) + return (path if path.exists() else None), elapsed + + +def handle_stop() -> dict: + path, elapsed = finish_recording() + if path is None: + osd(None) + return {"ok": False, "error": "not-recording"} + + try: + if elapsed < MIN_SECONDS: + osd(REASONS["too-short"]) + return {"ok": False, "error": "too-short"} + + osd("Transcribing…") + if not start_server(): + osd(REASONS["server-unavailable"]) + return {"ok": False, "error": "server-unavailable"} + + text, error = transcribe(path) + if error is not None: + osd(REASONS[error]) + return {"ok": False, "error": error} + if not text or not is_speech(text): + osd(REASONS["no-speech"]) + return {"ok": False, "error": "no-speech"} + + how = deliver(text) + if how == "failed": + osd(REASONS["deliver-failed"]) + return {"ok": False, "error": "deliver-failed", "text": text} + + # Announced only when it did NOT go where it was meant to. A successful + # dictation needs no notification: the words are already on screen. + osd("Pasted — press Ctrl+V" if how == "copied" else None) + return {"ok": True, "delivered": how, "characters": len(text)} + finally: + path.unlink(missing_ok=True) + + +def handle_cancel() -> dict: + path, _ = finish_recording() + if path is not None: + path.unlink(missing_ok=True) + osd(None) + return {"ok": True, "cancelled": True} + + +def handle_status() -> dict: + state = read_state() + return { + "ok": True, + "model": MODEL, + "modelInstalled": MODEL_FILE.exists(), + "modelPath": str(MODEL_FILE), + "modelBytes": MODEL_FILE.stat().st_size if MODEL_FILE.exists() else 0, + "imageBuilt": have_image(), + "serverReady": server_ready(), + "recording": state is not None, + "typingAvailable": shutil.which("wtype") is not None, + } + + +def pull_image() -> bool: + """Fetch the speech server image. Progress is podman's own, on stderr.""" + if have_image(): + return True + print(json.dumps({"ok": True, "state": "pulling", "image": IMAGE}), flush=True) + try: + done = subprocess.run(["podman", "pull", IMAGE], + capture_output=True, timeout=3600, check=False) + return done.returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def download_model() -> bool: + MODEL_DIR.mkdir(parents=True, exist_ok=True) + if MODEL_FILE.exists(): + return True + + # Written beside the target and moved into place, so an interrupted download + # never leaves a file that looks like a model. Half a model is worse than + # none: everything downstream believes it is there and fails at load. + partial = MODEL_FILE.with_suffix(".partial") + print(json.dumps({"ok": True, "state": "downloading", "model": MODEL}), flush=True) + try: + with urllib.request.urlopen(MODEL_URL, timeout=60) as response: + total = int(response.headers.get("content-length", 0)) + done = 0 + last = 0.0 + with partial.open("wb") as handle: + while True: + chunk = response.read(1 << 20) + if not chunk: + break + handle.write(chunk) + done += len(chunk) + now = time.monotonic() + # Once a second, not once a megabyte: five hundred progress + # lines is not progress, it is a flood. + if total and now - last >= 1.0: + last = now + print(json.dumps({"ok": True, "state": "downloading", + "bytes": done, "total": total}), flush=True) + except (urllib.error.URLError, OSError): + partial.unlink(missing_ok=True) + return False + + partial.replace(MODEL_FILE) + return True + + +def handle_setup() -> int: + """Everything dictation needs that Panama does not install. + + The image first: a model is useless without something to run it, and being + told to fetch half a gigabyte before hearing that an image is also needed + would be two trips. + """ + if not shutil.which("podman"): + print(json.dumps({"ok": False, "error": "no-podman"}), flush=True) + return 2 + if not pull_image(): + print(json.dumps({"ok": False, "error": "pull-failed"}), flush=True) + return 2 + if not download_model(): + print(json.dumps({"ok": False, "error": "download-failed"}), flush=True) + return 2 + + print(json.dumps({"ok": True, "state": "ready", + "bytes": MODEL_FILE.stat().st_size}), flush=True) + return 0 + + +HANDLERS = { + "start": handle_start, + "stop": handle_stop, + "cancel": handle_cancel, + "status": handle_status, +} + + +def main(argv: list[str]) -> int: + command = argv[0] if argv else "status" + if command == "setup": + return handle_setup() + handler = HANDLERS.get(command) + if handler is None: + return emit({"ok": False, "error": "unknown-command"}) + return emit(handler()) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/config/dot/quickshell/services/Dictation.qml b/config/dot/quickshell/services/Dictation.qml new file mode 100644 index 0000000..133e724 --- /dev/null +++ b/config/dot/quickshell/services/Dictation.qml @@ -0,0 +1,124 @@ +pragma Singleton + +// ───────────────────────────────────────────────────────────────────────────── +// Dictation: the two pieces that have to be present before holding the key does +// anything, and the download of the one that is not a package. +// +// Neither piece is installed by ./install, deliberately. The model is half a +// gigabyte that no repository carries, and the speech server is a container +// image compiled from upstream -- both are slow, both want the network, and an +// installer that quietly does either is the surprise the interview exists to +// prevent. So this reports what is missing and offers the part it can do. +// +// Status is read from the helper rather than inferred. Whether an image exists +// and whether a server is answering are facts about the machine, and a page +// that guessed at them would be confidently wrong on exactly the machine where +// somebody is trying to work out why dictation is silent. +// ───────────────────────────────────────────────────────────────────────────── + +import Quickshell +import Quickshell.Io +import QtQuick +import qs.config + +Singleton { + id: root + + readonly property string helper: Quickshell.shellDir + "/scripts/panama-dictate" + + property bool modelInstalled: false + property bool imageBuilt: false + property bool serverReady: false + property bool typingAvailable: false + property int modelBytes: 0 + + property bool downloading: false + property int downloadedBytes: 0 + property int downloadTotalBytes: 0 + property string lastError: "" + + readonly property bool ready: root.modelInstalled && root.imageBuilt + + readonly property real downloadFraction: root.downloadTotalBytes > 0 + ? Math.min(1, root.downloadedBytes / root.downloadTotalBytes) + : 0 + + // What is missing, in the order it has to be fixed. The image first: the + // model is useless without something to run it, and telling somebody to + // download half a gigabyte before mentioning they also need to build an + // image would be two trips. + readonly property string missing: { + if (!root.imageBuilt) + return "image"; + if (!root.modelInstalled) + return "model"; + return ""; + } + + Process { + id: statusRun + command: [root.helper, "status"] + stdout: StdioCollector { + onStreamFinished: { + try { + const state = JSON.parse(this.text); + root.modelInstalled = state.modelInstalled === true; + root.imageBuilt = state.imageBuilt === true; + root.serverReady = state.serverReady === true; + root.typingAvailable = state.typingAvailable === true; + root.modelBytes = state.modelBytes ?? 0; + } catch (error) { + root.lastError = "Could not read dictation status."; + } + } + } + } + + Process { + id: downloadRun + command: [root.helper, "download"] + stdout: SplitParser { + // One JSON object per line, roughly once a second, so the progress + // bar moves without the helper flooding a pipe nobody drains. + onRead: line => { + try { + const update = JSON.parse(line); + if (update.ok === false) { + root.lastError = "The model could not be downloaded."; + return; + } + if (update.bytes !== undefined) + root.downloadedBytes = update.bytes; + if (update.total !== undefined) + root.downloadTotalBytes = update.total; + } catch (error) { + // A line that is not JSON is not worth failing a download + // over; the exit status is what decides. + } + } + } + onExited: (exitCode, exitStatus) => { + root.downloading = false; + if (exitCode !== 0 && root.lastError === "") + root.lastError = "The model could not be downloaded."; + root.refresh(); + } + } + + function refresh(): void { + if (!statusRun.running) + statusRun.running = true; + } + + function download(): void { + if (downloadRun.running) + return; + root.lastError = ""; + root.downloadedBytes = 0; + root.downloadTotalBytes = 0; + root.downloading = true; + downloadRun.running = true; + } + + Component.onCompleted: root.refresh() +} diff --git a/setup/packages/hyprland-packages b/setup/packages/hyprland-packages index d9f9861..41050e0 100644 --- a/setup/packages/hyprland-packages +++ b/setup/packages/hyprland-packages @@ -41,6 +41,10 @@ udiskie uwsm vicinae wf-recorder +# Types transcribed speech into whichever window has focus. Synthetic +# keystrokes through the virtual-keyboard protocol, so it works in +# terminals and editors that ignore a paste. +wtype wireplumber wl-clipboard wofi diff --git a/setup/scripts/link-dotfiles b/setup/scripts/link-dotfiles index 1c0c844..5cc0d69 100755 --- a/setup/scripts/link-dotfiles +++ b/setup/scripts/link-dotfiles @@ -404,6 +404,29 @@ if [[ -d "$PANAMA_UNIT_DIR" ]]; then systemctl --user daemon-reload 2>/dev/null || true fi +# Quadlets: container definitions systemd turns into units at boot. Linked +# per-file into the directory the podman generator reads, rather than by +# symlinking ~/.config/containers wholesale -- podman keeps its registries, +# storage and login credentials in there, and that directory is the user's. +PANAMA_QUADLET_DIR="$PANAMA_PATH/config/containers" +USER_QUADLET_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/containers/systemd" +if [ -d "$PANAMA_QUADLET_DIR" ]; then + mkdir -p "$USER_QUADLET_DIR" + for quadlet in "$PANAMA_QUADLET_DIR"/*/*.container; do + [ -e "$quadlet" ] || continue + quadlet_dst="$USER_QUADLET_DIR/$(basename "$quadlet")" + if [ -L "$quadlet_dst" ]; then + rm "$quadlet_dst" + elif [ -e "$quadlet_dst" ]; then + log "Keeping existing quadlet at $quadlet_dst" + continue + fi + ln -s "$quadlet" "$quadlet_dst" + log "Linked quadlet → $quadlet_dst" + done + systemctl --user daemon-reload 2>/dev/null || true +fi + DEFAULT_APPS_HELPER="$PANAMA_PATH/config/dot/quickshell/scripts/panama-default-apps" if [[ -x "$DEFAULT_APPS_HELPER" ]] && command -v xdg-mime >/dev/null 2>&1; then update-desktop-database "$USER_APPLICATION_DIR" >/dev/null 2>&1 || true diff --git a/tests/setup/dictation-contract b/tests/setup/dictation-contract new file mode 100755 index 0000000..7d5b724 --- /dev/null +++ b/tests/setup/dictation-contract @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +# Dictation: hold a key, speak, and the words are typed where the cursor is. +# +# The parts worth pinning are the ones that put text into somebody's document, +# because a mistake there is not a feature failing -- it is the wrong thing +# arriving in the middle of a sentence, and both of these were real: +# +# * Whisper describes silence as the literal text "[BLANK_AUDIO]". The first +# working version of this dictated that string into the clipboard. +# * The server answers with one line per segment, and a newline typed into a +# window is an Enter press -- sending the half-written message, submitting +# the form, running the command. +# +# Plus the container: it transcribes whatever it is sent with no authentication, +# so it must not be reachable from the network, and it holds the model in memory +# once started, so it must not start at login. + +set -uo pipefail + +repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +helper="$repo_dir/config/dot/quickshell/scripts/panama-dictate" +quadlet="$repo_dir/config/containers/whisper/panama-whisper.container" +keybinds="$repo_dir/config/dot/hypr/keybinds.lua" +service="$repo_dir/config/dot/quickshell/services/Dictation.qml" + +findings=() +note() { findings+=("$1"); } + +[[ -x "$helper" ]] || { printf 'dictation contract: helper is missing or not executable\n' >&2; exit 1; } + +# ── What gets typed ────────────────────────────────────────────────────────── +# +# is_speech and the whitespace collapse are exercised for real rather than +# grepped for, because both are judgements about text and a spelling of the +# regex is not what matters. + +python3 - "$helper" <<'PROBE' || note 'the text handling did not behave as dictation requires' +import importlib.machinery, importlib.util, sys +spec = importlib.util.spec_from_loader("dictate", + importlib.machinery.SourceFileLoader("dictate", sys.argv[1])) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + +# Whisper's own way of saying it heard no words. Typing these is the bug. +for marker in ("[BLANK_AUDIO]", "[ Silence ]", "(wind blowing)", "[MUSIC]", + "[BLANK_AUDIO] [BLANK_AUDIO]"): + assert not module.is_speech(marker), marker + +# Real speech, including speech that merely contains brackets. Discarding these +# would be far worse than typing a stray marker. +for spoken in ("And so my fellow Americans", "commit the change (the small one) now", + "git log [main]", "a"): + assert module.is_speech(spoken), spoken +PROBE + +# Testing is_speech in isolation is not enough, and this was found the honest +# way: reverting the guard in handle_stop left the probe above passing, because +# the function was still correct -- it had simply stopped being called. A +# transcription has to be checked before it is delivered. +grep -q 'not is_speech(text)' "$helper" \ + || note 'the non-speech check exists but nothing calls it before typing, so [BLANK_AUDIO] would be typed' + +# The collapse itself: a transcription arriving as several lines has to leave as +# one, or the newlines are typed as Enter. +grep -q '" ".join(answer.split())' "$helper" \ + || note 'the transcription is not collapsed to one line, so a newline would be typed as Enter' + +# ── How it gets there ──────────────────────────────────────────────────────── + +grep -q 'shutil.which("wtype")' "$helper" \ + || note 'dictation does not try to type the text' +grep -q 'shutil.which("wl-copy")' "$helper" \ + || note 'dictation has no clipboard fallback for applications that refuse synthetic keys' + +# Typing must be tried FIRST. The clipboard works but overwrites whatever was +# copied and fills Panama's clipboard history with everything ever dictated. +typing_line="$(grep -n 'shutil.which("wtype")' "$helper" | head -1 | cut -d: -f1)" +clipboard_line="$(grep -n 'shutil.which("wl-copy")' "$helper" | head -1 | cut -d: -f1)" +[[ -n "$typing_line" && -n "$clipboard_line" && "$typing_line" -lt "$clipboard_line" ]] \ + || note 'the clipboard is tried before typing, so dictation would overwrite the clipboard by default' + +declared="$(cat "$repo_dir"/setup/packages/* 2>/dev/null | sed 's/#.*//' | tr -d ' ' | grep -v '^$')" +grep -qix wtype <<<"$declared" \ + || note 'wtype is what types the text, and no package list installs it' + +# ── The container ──────────────────────────────────────────────────────────── + +[[ -f "$quadlet" ]] || note 'the speech server has no quadlet' +if [[ -f "$quadlet" ]]; then + grep -qE '^PublishPort=127\.0\.0\.1:' "$quadlet" \ + || note 'the speech server is published beyond loopback, and it authenticates nothing' + grep -qE '^AddDevice=/dev/dri$' "$quadlet" \ + || note 'the GPU is not passed to the container, or is passed as a device number that differs per machine' + grep -q ':ro' "$quadlet" \ + || note 'the model is mounted writable, which nothing needs' + grep -q ':z' "$quadlet" \ + || note 'the model mount is not relabelled, so SELinux will deny it' + + # No [Install]: whisper-server holds the model from the moment it starts. + grep -q '^\[Install\]' "$quadlet" \ + && note 'the speech server starts at login, holding the model in every session where nobody dictates' + + # Vulkan is the whole reason for this image: it runs on the AMD, Intel and + # NVIDIA machines this config is used on. A ROCm tag would serve one of them. + grep -qE '^Image=.*vulkan' "$quadlet" \ + || note 'the image is not the Vulkan build, so it will not work on every machine this runs on' +fi + +# ── The keybinds ───────────────────────────────────────────────────────────── + +presses="$(grep -c 'dictate("start")' "$keybinds" || true)" +releases="$(grep -c 'dictate("stop")' "$keybinds" || true)" +(( presses == 1 && releases == 1 )) \ + || note "hold-to-talk needs one press bind and one release bind; found $presses and $releases" + +grep -qE 'dictate\("stop"\)\)?,?$|release = true' "$keybinds" \ + || note 'the transcribe bind is not flagged release, so it would fire on press' + +# A repeating press bind would restart the recording several times a second for +# as long as the key is held. +grep -A1 'dictate("start")' "$keybinds" | grep -q 'repeating' \ + && note 'the dictation press bind repeats, which would restart recording while the key is held' + +grep -q 'dictate("cancel")' "$keybinds" \ + || note 'there is no way to abandon a recording without transcribing it' + +# ── The page cannot claim more than it knows ───────────────────────────────── + +grep -q '"status"' "$service" \ + || note 'the settings service never asks the helper what is installed' + +if (( ${#findings[@]} > 0 )); then + mapfile -t findings < <(printf '%s\n' "${findings[@]}" | sort -u) + printf 'dictation contract: %d finding(s)\n' "${#findings[@]}" >&2 + printf ' - %s\n' "${findings[@]}" >&2 + exit 1 +fi + +printf 'dictation contract: PASS\n'