feat: DelaySum beamform plus hop process latency

Keep C920 stereo through SpeechBrain GCC-PHAT Delay-and-Sum, then
MetricGAN, and publish mono to LiveKit. Record last_dt_ms and log
process_ms/capture_ms every 50 hops for the latency plan.
This commit is contained in:
root
2026-08-28 06:16:25 +00:00
parent abeea6256c
commit 950128deec
8 changed files with 252 additions and 56 deletions
+16 -6
View File
@@ -37,6 +37,7 @@ def main() -> int:
default="speechbrain/metricgan-plus-voicebank")
ap.add_argument("--enhance-chunk-s", type=float, default=1.0)
ap.add_argument("--enhance-hop-s", type=float, default=0.2)
ap.add_argument("--beamform-mode", default="auto")
args = ap.parse_args()
cam = json.loads(args.camera_json)
@@ -60,6 +61,7 @@ def main() -> int:
sample_rate=args.audio_rate,
chunk_s=args.enhance_chunk_s,
hop_s=args.enhance_hop_s,
beamform=args.beamform_mode,
)
log.info("audio backend: %s", cleaner.backend)
@@ -138,6 +140,7 @@ def main() -> int:
# chunk * 2 ch * 2 bytes
need = chunk * 2 * 2
buf = b""
hops = 0
while aproc.poll() is None:
data = await loop.run_in_executor(
None, aproc.stdout.read, 4096)
@@ -146,21 +149,28 @@ def main() -> int:
buf += data
while len(buf) >= need:
piece, buf = buf[:need], buf[need:]
pcm_stereo = np.frombuffer(piece, dtype=np.int16)
pcm_mono = pcm_stereo.reshape(-1, 2).mean(
axis=1).astype(np.int16)
# align to full chunks
usable = (len(pcm_mono) // chunk) * chunk
pcm_stereo = np.frombuffer(
piece, dtype=np.int16).reshape(-1, 2)
n_frames = pcm_stereo.shape[0]
usable = (n_frames // chunk) * chunk
if usable:
cleaned = await loop.run_in_executor(
None, cleaner.process,
pcm_mono[:usable])
pcm_stereo[:usable])
t_cap = time.perf_counter()
await audio_source.capture_frame(
rtc.AudioFrame(
cleaned.tobytes(),
args.audio_rate,
1,
len(cleaned)))
hops += 1
if hops % 50 == 0:
log.info(
"audio hop process_ms=%.1f capture_ms=%.1f",
cleaner.last_dt_ms,
(time.perf_counter() - t_cap) * 1000.0,
)
aproc = subprocess.Popen(
["ffmpeg", "-hide_banner", "-loglevel", "error",