Add benchmark harness and compare against 11 other STT backends

Measures median latency, strict exact-match accuracy, digit formatting and silence behaviour across 48 clips (16 Home Assistant commands x 3 macOS TTS voices), on an M4 Mac mini. Headline: Parakeet v2 at 102ms median is 5x faster than the best whisper.cpp configuration and lands within one clip of it on accuracy. mlx-whisper large-v3 is the only backend to score 48/48, at 11x the latency. Moonshine is 2x faster again but gives up real accuracy (35/48). Also quantifies the reason this defaults to v2 over v3: v3 returned digits for only 10 of 21 number-bearing commands, against 21/21 for v2, which is most of the gap between their exact-match scores. The harness feeds audio to every backend as an array rather than a path -- mlx-whisper and moonshine otherwise shell out to ffmpeg, which this project deliberately does not require. Clips are gitignored; bench/make_clips.sh regenerates them. Caveats are documented in the README: this is clean synthetic TTS, so it measures latency rigorously and accuracy only as a domain smoke test, and faster-whisper is CPU-only on Apple Silicon because CTranslate2 has no Metal backend. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-29 03:32:57 +01:00
parent c846bc245c
commit 4475f6d39c
6 changed files with 373 additions and 12 deletions
@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+"""Benchmark speech-to-text backends on the Home Assistant command corpus.
+
+Each backend is loaded once, warmed on every clip, then timed. Reported
+latency is the median of the timed pass, which is what a voice assistant
+actually experiences -- means are skewed by Metal kernel compilation on the
+first request.
+
+Usage:
+    ./benchmark.py --backend parakeet:mlx-community/parakeet-tdt-0.6b-v2
+    ./benchmark.py --backend mlx-whisper:mlx-community/whisper-large-v3-turbo
+    ./benchmark.py --backend faster-whisper:base.en
+    ./benchmark.py --backend moonshine:moonshine/base
+    ./benchmark.py --backend whispercpp:http://127.0.0.1:8910/inference
+"""
+import argparse
+import json
+import re
+import statistics
+import sys
+import time
+import unicodedata
+from pathlib import Path
+
+BENCH_DIR = Path(__file__).resolve().parent
+CLIPS = BENCH_DIR / "clips"
+
+# Spoken-form numbers a model might emit instead of digits. Home Assistant's
+# local intent matching wants digits, so we score this separately.
+WORD_NUMBERS = re.compile(
+    r"\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|"
+    r"thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|"
+    r"thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|percent)\b"
+)
+
+
+def normalise(text: str) -> str:
+    """Fold away differences that do not change the intent: case, smart
+    quotes, punctuation, and whitespace. '%' is kept -- it is semantic."""
+    text = unicodedata.normalize("NFKD", text).lower().strip()
+    text = text.replace("’", "'").replace("‘", "'")
+    text = re.sub(r"[^\w\s%']", " ", text)
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def load_corpus():
+    rows = []
+    for line in (BENCH_DIR / "corpus.tsv").read_text().splitlines():
+        if not line.strip() or line.startswith("#"):
+            continue
+        _tts, expected, has_number = line.split("\t")
+        rows.append((expected, has_number == "1"))
+    return rows
+
+
+# --- backends ---------------------------------------------------------------
+
+
+def read_wav(path):
+    """Read a 16kHz mono 16-bit WAV as float32 in [-1, 1).
+
+    Several of these libraries shell out to ffmpeg to load audio, which is an
+    unnecessary dependency when the clips are already in the right format --
+    and unavailable on the benchmark machine. Feed them arrays instead.
+    """
+    import wave
+
+    import numpy as np
+
+    with wave.open(str(path), "rb") as w:
+        assert w.getframerate() == 16000 and w.getnchannels() == 1
+        pcm = w.readframes(w.getnframes())
+    return np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0
+
+
+def backend_parakeet(model_id):
+    import mlx.core as mx
+    from parakeet_mlx import from_pretrained
+    from parakeet_mlx.audio import get_logmel
+
+    model = from_pretrained(model_id)
+
+    def transcribe(path):
+        mel = get_logmel(mx.array(read_wav(path)), model.preprocessor_config)
+        results = model.generate(mel)
+        return results[0].text if results else ""
+
+    return transcribe
+
+
+def backend_mlx_whisper(model_id):
+    import mlx_whisper
+
+    def transcribe(path):
+        return mlx_whisper.transcribe(
+            read_wav(path), path_or_hf_repo=model_id, language="en", fp16=True
+        )["text"]
+
+    return transcribe
+
+
+def backend_faster_whisper(model_id):
+    from faster_whisper import WhisperModel
+
+    # Metal is unsupported by CTranslate2; int8 on CPU is the fastest option
+    # available on Apple Silicon and is what the HA add-on uses by default.
+    model = WhisperModel(model_id, device="cpu", compute_type="int8")
+
+    def transcribe(path):
+        segments, _info = model.transcribe(str(path), language="en", beam_size=5)
+        return "".join(s.text for s in segments)
+
+    return transcribe
+
+
+def backend_moonshine(model_id):
+    import moonshine_onnx
+
+    model = moonshine_onnx.MoonshineOnnxModel(model_name=model_id)
+    tokenizer = moonshine_onnx.load_tokenizer()
+
+    def transcribe(path):
+        # Bypass moonshine_onnx.transcribe() so we can supply the audio as an
+        # array; it expects shape [batch, samples].
+        audio = read_wav(path).reshape(1, -1)
+        return " ".join(tokenizer.decode_batch(model.generate(audio)))
+
+    return transcribe
+
+
+def backend_whispercpp(url):
+    import requests
+
+    def transcribe(path):
+        with open(path, "rb") as fh:
+            r = requests.post(
+                url,
+                files={"file": fh},
+                data={"response_format": "json", "no_context": "true"},
+                timeout=120,
+            )
+        r.raise_for_status()
+        return r.json()["text"]
+
+    return transcribe
+
+
+BACKENDS = {
+    "parakeet": backend_parakeet,
+    "mlx-whisper": backend_mlx_whisper,
+    "faster-whisper": backend_faster_whisper,
+    "moonshine": backend_moonshine,
+    "whispercpp": backend_whispercpp,
+}
+
+
+# --- runner -----------------------------------------------------------------
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--backend", required=True, help="kind:model_or_url")
+    ap.add_argument("--label", help="Name to report (defaults to --backend)")
+    ap.add_argument("--json", action="store_true", help="Emit a JSON result line")
+    args = ap.parse_args()
+
+    kind, _, target = args.backend.partition(":")
+    if kind not in BACKENDS:
+        sys.exit(f"unknown backend {kind!r}; pick one of {', '.join(BACKENDS)}")
+
+    corpus = load_corpus()
+    clips = sorted(
+        (p for p in CLIPS.glob("*.wav") if p.name != "silence.wav"),
+        key=lambda p: (int(p.stem.split("_")[0]), p.stem),
+    )
+    if not clips:
+        sys.exit("no clips found -- run ./make_clips.sh first")
+
+    load_started = time.monotonic()
+    transcribe = BACKENDS[kind](target)
+    load_seconds = time.monotonic() - load_started
+
+    # Warm every clip first: the first inference compiles kernels, and clip
+    # length varies enough that a single warm-up does not cover all shapes.
+    for clip in clips:
+        transcribe(clip)
+
+    latencies, exact, number_ok, number_total, failures = [], 0, 0, 0, []
+    for clip in clips:
+        index = int(clip.stem.split("_")[0]) - 1
+        expected, has_number = corpus[index]
+
+        started = time.monotonic()
+        got = transcribe(clip)
+        latencies.append((time.monotonic() - started) * 1000)
+
+        if normalise(got) == normalise(expected):
+            exact += 1
+        else:
+            failures.append((clip.name, normalise(expected), normalise(got)))
+        if has_number:
+            number_total += 1
+            if not WORD_NUMBERS.search(normalise(got)):
+                number_ok += 1
+
+    silence = CLIPS / "silence.wav"
+    silence_out = normalise(transcribe(silence)) if silence.exists() else "n/a"
+
+    label = args.label or args.backend
+    result = {
+        "backend": label,
+        "clips": len(clips),
+        "median_ms": round(statistics.median(latencies), 1),
+        "p90_ms": round(sorted(latencies)[int(len(latencies) * 0.9)], 1),
+        "exact_match": f"{exact}/{len(clips)}",
+        "digits_ok": f"{number_ok}/{number_total}",
+        "silence": silence_out,
+        "load_s": round(load_seconds, 1),
+    }
+
+    if args.json:
+        print(json.dumps(result))
+    else:
+        print(f"\n=== {label} ===")
+        for key, value in result.items():
+            if key != "backend":
+                print(f"  {key:12s} {value}")
+        if failures:
+            print(f"  mismatches ({len(failures)}):")
+            for name, want, got in failures[:8]:
+                print(f"    {name}\n      want: {want}\n      got:  {got}")
+
+
+if __name__ == "__main__":
+    main()