docs: latency budget, thread cap, and Arc B580 CUDA mismatch
Also extend bench_enhance.py with --stereo --hop for isolated RTF.
This commit is contained in:
@@ -74,13 +74,20 @@ re-run `--write` unless you intend to remap slots.
|
||||
C920 mics are 16 kHz **stereo**. Each hop (default 200 ms) is kept
|
||||
stereo through a SpeechBrain preprocessor, then published as mono:
|
||||
|
||||
1. `AudioNormalizer(mix="keep")` at 16 kHz
|
||||
2. Delay-and-Sum beamform (GCC-PHAT TDOA) — the live path from the
|
||||
SpeechBrain [multi-microphone beamforming tutorial](https://speechbrain.readthedocs.io/en/latest/tutorials/preprocessing/multi-microphone-beamforming.html)
|
||||
(`speechbrain.lobes.beamform_multimic.DelaySum_Beamformer`)
|
||||
3. `speechbrain/metricgan-plus-voicebank` neural enhancement on a 1 s
|
||||
1. Delay-and-Sum beamform (GCC-PHAT TDOA, frozen for
|
||||
`BEAMFORM_TDOA_EVERY=5` hops) — live path from the SpeechBrain
|
||||
[multi-microphone beamforming tutorial](https://speechbrain.readthedocs.io/en/latest/tutorials/preprocessing/multi-microphone-beamforming.html)
|
||||
2. `speechbrain/metricgan-plus-voicebank` neural enhancement on a 1 s
|
||||
rolling window (`ENHANCE_CHUNK_S=1.0`, `ENHANCE_HOP_S=0.2`)
|
||||
|
||||
The 1 s window is **past context**, not +1 s delay: only the latest hop
|
||||
is published. Algorithmic delay ≈ hop + compute.
|
||||
|
||||
Each worker caps Torch/BLAS to `TORCH_NUM_THREADS=1` (default) so 14–20
|
||||
processes do not oversubscribe the i7-14700. This host has an Intel Arc
|
||||
B580; the venv Torch is NVIDIA CUDA and **cannot** use it. Do not set
|
||||
`run_opts={"device":"cuda"}`.
|
||||
|
||||
One worker process per camera loads the modules once and reuses them.
|
||||
`enhance_batch` must be called with `lengths=torch.tensor([1.0])`
|
||||
(relative, not sample count). `ENHANCE_MODE=off` (or a failed model
|
||||
@@ -89,7 +96,8 @@ instead of Delay-and-Sum. DSP high-pass + soft-clip is the last-resort
|
||||
fallback so streams still work.
|
||||
|
||||
Until the 1 s window is full, hops use the DSP fallback so LiveKit is
|
||||
not silent on startup.
|
||||
not silent on startup. Workers log `process_ms` / `capture_ms` every 50
|
||||
hops.
|
||||
|
||||
## Appliance boot / power button
|
||||
|
||||
|
||||
+54
-16
@@ -1,5 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""RTF bench for hop/window speechbrain enhance."""
|
||||
"""RTF bench for hop/window speechbrain enhance.
|
||||
|
||||
Stop livekit-cameras.service first or the numbers fight the fleet.
|
||||
|
||||
.venv/bin/python scripts/bench_enhance.py
|
||||
.venv/bin/python scripts/bench_enhance.py --stereo --hop 0.1
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -10,18 +17,49 @@ import numpy as np
|
||||
|
||||
from audio_cleanup import AudioCleaner
|
||||
|
||||
c = AudioCleaner(mode="force", chunk_s=1.0, hop_s=0.2)
|
||||
hop = 3200
|
||||
pcm = (np.random.randn(hop) * 800).astype(np.int16)
|
||||
for _ in range(8):
|
||||
c.process(pcm)
|
||||
n = 50
|
||||
t0 = time.perf_counter()
|
||||
for _ in range(n):
|
||||
c.process(pcm)
|
||||
dt = time.perf_counter() - t0
|
||||
audio_s = n * 0.2
|
||||
rtf = dt / audio_s
|
||||
print(f"hops={n} wall={dt:.3f}s audio={audio_s:.1f}s rtf={rtf:.3f}")
|
||||
if rtf > 0.8:
|
||||
raise SystemExit("RTF too high for 20-way realtime")
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--stereo", action="store_true")
|
||||
ap.add_argument("--hop", type=float, default=0.2)
|
||||
ap.add_argument("--chunk", type=float, default=1.0)
|
||||
ap.add_argument("--n", type=int, default=50)
|
||||
args = ap.parse_args()
|
||||
|
||||
c = AudioCleaner(
|
||||
mode="force",
|
||||
beamform="force" if args.stereo else "off",
|
||||
chunk_s=args.chunk,
|
||||
hop_s=args.hop,
|
||||
torch_num_threads=1,
|
||||
)
|
||||
hop_n = int(args.hop * 16000)
|
||||
if args.stereo:
|
||||
t = np.arange(hop_n)
|
||||
ch0 = (np.sin(2 * np.pi * 440 * t / 16000) * 800).astype(np.int16)
|
||||
pcm = np.stack([ch0, np.roll(ch0, 2)], axis=1)
|
||||
else:
|
||||
pcm = (np.random.randn(hop_n) * 800).astype(np.int16)
|
||||
warmup = max(8, int(args.chunk / args.hop) + 2)
|
||||
for _ in range(warmup):
|
||||
c.process(pcm)
|
||||
n = args.n
|
||||
t0 = time.perf_counter()
|
||||
for _ in range(n):
|
||||
c.process(pcm)
|
||||
dt = time.perf_counter() - t0
|
||||
audio_s = n * args.hop
|
||||
rtf = dt / audio_s
|
||||
print(
|
||||
f"backend={c.backend} stereo={args.stereo} hops={n} "
|
||||
f"wall={dt:.3f}s audio={audio_s:.1f}s rtf={rtf:.3f} "
|
||||
f"last_ms={c.last_dt_ms:.1f}"
|
||||
)
|
||||
limit = 0.5 if args.stereo else 0.8
|
||||
if rtf > limit:
|
||||
raise SystemExit(f"RTF too high for 20-way realtime (limit {limit})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
Reference in New Issue
Block a user