diff --git a/README.md b/README.md index cfa0fee..4ebc132 100644 --- a/README.md +++ b/README.md @@ -74,13 +74,20 @@ re-run `--write` unless you intend to remap slots. C920 mics are 16 kHz **stereo**. Each hop (default 200 ms) is kept stereo through a SpeechBrain preprocessor, then published as mono: -1. `AudioNormalizer(mix="keep")` at 16 kHz -2. Delay-and-Sum beamform (GCC-PHAT TDOA) — the live path from the - SpeechBrain [multi-microphone beamforming tutorial](https://speechbrain.readthedocs.io/en/latest/tutorials/preprocessing/multi-microphone-beamforming.html) - (`speechbrain.lobes.beamform_multimic.DelaySum_Beamformer`) -3. `speechbrain/metricgan-plus-voicebank` neural enhancement on a 1 s +1. Delay-and-Sum beamform (GCC-PHAT TDOA, frozen for + `BEAMFORM_TDOA_EVERY=5` hops) — live path from the SpeechBrain + [multi-microphone beamforming tutorial](https://speechbrain.readthedocs.io/en/latest/tutorials/preprocessing/multi-microphone-beamforming.html) +2. `speechbrain/metricgan-plus-voicebank` neural enhancement on a 1 s rolling window (`ENHANCE_CHUNK_S=1.0`, `ENHANCE_HOP_S=0.2`) +The 1 s window is **past context**, not +1 s delay: only the latest hop +is published. Algorithmic delay ≈ hop + compute. + +Each worker caps Torch/BLAS to `TORCH_NUM_THREADS=1` (default) so 14–20 +processes do not oversubscribe the i7-14700. This host has an Intel Arc +B580; the venv Torch is NVIDIA CUDA and **cannot** use it. Do not set +`run_opts={"device":"cuda"}`. + One worker process per camera loads the modules once and reuses them. `enhance_batch` must be called with `lengths=torch.tensor([1.0])` (relative, not sample count). `ENHANCE_MODE=off` (or a failed model @@ -89,7 +96,8 @@ instead of Delay-and-Sum. DSP high-pass + soft-clip is the last-resort fallback so streams still work. Until the 1 s window is full, hops use the DSP fallback so LiveKit is -not silent on startup. +not silent on startup. Workers log `process_ms` / `capture_ms` every 50 +hops. ## Appliance boot / power button diff --git a/scripts/bench_enhance.py b/scripts/bench_enhance.py index 1598c7b..6422da8 100644 --- a/scripts/bench_enhance.py +++ b/scripts/bench_enhance.py @@ -1,5 +1,12 @@ #!/usr/bin/env python3 -"""RTF bench for hop/window speechbrain enhance.""" +"""RTF bench for hop/window speechbrain enhance. + +Stop livekit-cameras.service first or the numbers fight the fleet. + + .venv/bin/python scripts/bench_enhance.py + .venv/bin/python scripts/bench_enhance.py --stereo --hop 0.1 +""" +import argparse import sys import time from pathlib import Path @@ -10,18 +17,49 @@ import numpy as np from audio_cleanup import AudioCleaner -c = AudioCleaner(mode="force", chunk_s=1.0, hop_s=0.2) -hop = 3200 -pcm = (np.random.randn(hop) * 800).astype(np.int16) -for _ in range(8): - c.process(pcm) -n = 50 -t0 = time.perf_counter() -for _ in range(n): - c.process(pcm) -dt = time.perf_counter() - t0 -audio_s = n * 0.2 -rtf = dt / audio_s -print(f"hops={n} wall={dt:.3f}s audio={audio_s:.1f}s rtf={rtf:.3f}") -if rtf > 0.8: - raise SystemExit("RTF too high for 20-way realtime") + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--stereo", action="store_true") + ap.add_argument("--hop", type=float, default=0.2) + ap.add_argument("--chunk", type=float, default=1.0) + ap.add_argument("--n", type=int, default=50) + args = ap.parse_args() + + c = AudioCleaner( + mode="force", + beamform="force" if args.stereo else "off", + chunk_s=args.chunk, + hop_s=args.hop, + torch_num_threads=1, + ) + hop_n = int(args.hop * 16000) + if args.stereo: + t = np.arange(hop_n) + ch0 = (np.sin(2 * np.pi * 440 * t / 16000) * 800).astype(np.int16) + pcm = np.stack([ch0, np.roll(ch0, 2)], axis=1) + else: + pcm = (np.random.randn(hop_n) * 800).astype(np.int16) + warmup = max(8, int(args.chunk / args.hop) + 2) + for _ in range(warmup): + c.process(pcm) + n = args.n + t0 = time.perf_counter() + for _ in range(n): + c.process(pcm) + dt = time.perf_counter() - t0 + audio_s = n * args.hop + rtf = dt / audio_s + print( + f"backend={c.backend} stereo={args.stereo} hops={n} " + f"wall={dt:.3f}s audio={audio_s:.1f}s rtf={rtf:.3f} " + f"last_ms={c.last_dt_ms:.1f}" + ) + limit = 0.5 if args.stereo else 0.8 + if rtf > limit: + raise SystemExit(f"RTF too high for 20-way realtime (limit {limit})") + return 0 + + +if __name__ == "__main__": + sys.exit(main())