docs: latency budget, thread cap, and Arc B580 CUDA mismatch
Also extend bench_enhance.py with --stereo --hop for isolated RTF.
This commit is contained in:
@@ -74,13 +74,20 @@ re-run `--write` unless you intend to remap slots.
|
|||||||
C920 mics are 16 kHz **stereo**. Each hop (default 200 ms) is kept
|
C920 mics are 16 kHz **stereo**. Each hop (default 200 ms) is kept
|
||||||
stereo through a SpeechBrain preprocessor, then published as mono:
|
stereo through a SpeechBrain preprocessor, then published as mono:
|
||||||
|
|
||||||
1. `AudioNormalizer(mix="keep")` at 16 kHz
|
1. Delay-and-Sum beamform (GCC-PHAT TDOA, frozen for
|
||||||
2. Delay-and-Sum beamform (GCC-PHAT TDOA) — the live path from the
|
`BEAMFORM_TDOA_EVERY=5` hops) — live path from the SpeechBrain
|
||||||
SpeechBrain [multi-microphone beamforming tutorial](https://speechbrain.readthedocs.io/en/latest/tutorials/preprocessing/multi-microphone-beamforming.html)
|
[multi-microphone beamforming tutorial](https://speechbrain.readthedocs.io/en/latest/tutorials/preprocessing/multi-microphone-beamforming.html)
|
||||||
(`speechbrain.lobes.beamform_multimic.DelaySum_Beamformer`)
|
2. `speechbrain/metricgan-plus-voicebank` neural enhancement on a 1 s
|
||||||
3. `speechbrain/metricgan-plus-voicebank` neural enhancement on a 1 s
|
|
||||||
rolling window (`ENHANCE_CHUNK_S=1.0`, `ENHANCE_HOP_S=0.2`)
|
rolling window (`ENHANCE_CHUNK_S=1.0`, `ENHANCE_HOP_S=0.2`)
|
||||||
|
|
||||||
|
The 1 s window is **past context**, not +1 s delay: only the latest hop
|
||||||
|
is published. Algorithmic delay ≈ hop + compute.
|
||||||
|
|
||||||
|
Each worker caps Torch/BLAS to `TORCH_NUM_THREADS=1` (default) so 14–20
|
||||||
|
processes do not oversubscribe the i7-14700. This host has an Intel Arc
|
||||||
|
B580; the venv Torch is NVIDIA CUDA and **cannot** use it. Do not set
|
||||||
|
`run_opts={"device":"cuda"}`.
|
||||||
|
|
||||||
One worker process per camera loads the modules once and reuses them.
|
One worker process per camera loads the modules once and reuses them.
|
||||||
`enhance_batch` must be called with `lengths=torch.tensor([1.0])`
|
`enhance_batch` must be called with `lengths=torch.tensor([1.0])`
|
||||||
(relative, not sample count). `ENHANCE_MODE=off` (or a failed model
|
(relative, not sample count). `ENHANCE_MODE=off` (or a failed model
|
||||||
@@ -89,7 +96,8 @@ instead of Delay-and-Sum. DSP high-pass + soft-clip is the last-resort
|
|||||||
fallback so streams still work.
|
fallback so streams still work.
|
||||||
|
|
||||||
Until the 1 s window is full, hops use the DSP fallback so LiveKit is
|
Until the 1 s window is full, hops use the DSP fallback so LiveKit is
|
||||||
not silent on startup.
|
not silent on startup. Workers log `process_ms` / `capture_ms` every 50
|
||||||
|
hops.
|
||||||
|
|
||||||
## Appliance boot / power button
|
## Appliance boot / power button
|
||||||
|
|
||||||
|
|||||||
+48
-10
@@ -1,5 +1,12 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""RTF bench for hop/window speechbrain enhance."""
|
"""RTF bench for hop/window speechbrain enhance.
|
||||||
|
|
||||||
|
Stop livekit-cameras.service first or the numbers fight the fleet.
|
||||||
|
|
||||||
|
.venv/bin/python scripts/bench_enhance.py
|
||||||
|
.venv/bin/python scripts/bench_enhance.py --stereo --hop 0.1
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -10,18 +17,49 @@ import numpy as np
|
|||||||
|
|
||||||
from audio_cleanup import AudioCleaner
|
from audio_cleanup import AudioCleaner
|
||||||
|
|
||||||
c = AudioCleaner(mode="force", chunk_s=1.0, hop_s=0.2)
|
|
||||||
hop = 3200
|
def main() -> int:
|
||||||
pcm = (np.random.randn(hop) * 800).astype(np.int16)
|
ap = argparse.ArgumentParser()
|
||||||
for _ in range(8):
|
ap.add_argument("--stereo", action="store_true")
|
||||||
|
ap.add_argument("--hop", type=float, default=0.2)
|
||||||
|
ap.add_argument("--chunk", type=float, default=1.0)
|
||||||
|
ap.add_argument("--n", type=int, default=50)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
c = AudioCleaner(
|
||||||
|
mode="force",
|
||||||
|
beamform="force" if args.stereo else "off",
|
||||||
|
chunk_s=args.chunk,
|
||||||
|
hop_s=args.hop,
|
||||||
|
torch_num_threads=1,
|
||||||
|
)
|
||||||
|
hop_n = int(args.hop * 16000)
|
||||||
|
if args.stereo:
|
||||||
|
t = np.arange(hop_n)
|
||||||
|
ch0 = (np.sin(2 * np.pi * 440 * t / 16000) * 800).astype(np.int16)
|
||||||
|
pcm = np.stack([ch0, np.roll(ch0, 2)], axis=1)
|
||||||
|
else:
|
||||||
|
pcm = (np.random.randn(hop_n) * 800).astype(np.int16)
|
||||||
|
warmup = max(8, int(args.chunk / args.hop) + 2)
|
||||||
|
for _ in range(warmup):
|
||||||
c.process(pcm)
|
c.process(pcm)
|
||||||
n = 50
|
n = args.n
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
for _ in range(n):
|
for _ in range(n):
|
||||||
c.process(pcm)
|
c.process(pcm)
|
||||||
dt = time.perf_counter() - t0
|
dt = time.perf_counter() - t0
|
||||||
audio_s = n * 0.2
|
audio_s = n * args.hop
|
||||||
rtf = dt / audio_s
|
rtf = dt / audio_s
|
||||||
print(f"hops={n} wall={dt:.3f}s audio={audio_s:.1f}s rtf={rtf:.3f}")
|
print(
|
||||||
if rtf > 0.8:
|
f"backend={c.backend} stereo={args.stereo} hops={n} "
|
||||||
raise SystemExit("RTF too high for 20-way realtime")
|
f"wall={dt:.3f}s audio={audio_s:.1f}s rtf={rtf:.3f} "
|
||||||
|
f"last_ms={c.last_dt_ms:.1f}"
|
||||||
|
)
|
||||||
|
limit = 0.5 if args.stereo else 0.8
|
||||||
|
if rtf > limit:
|
||||||
|
raise SystemExit(f"RTF too high for 20-way realtime (limit {limit})")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
|
|||||||
Reference in New Issue
Block a user