feat: DelaySum beamform plus hop process latency
Keep C920 stereo through SpeechBrain GCC-PHAT Delay-and-Sum, then MetricGAN, and publish mono to LiveKit. Record last_dt_ms and log process_ms/capture_ms every 50 hops for the latency plan.
This commit is contained in:
+16
-6
@@ -37,6 +37,7 @@ def main() -> int:
|
||||
default="speechbrain/metricgan-plus-voicebank")
|
||||
ap.add_argument("--enhance-chunk-s", type=float, default=1.0)
|
||||
ap.add_argument("--enhance-hop-s", type=float, default=0.2)
|
||||
ap.add_argument("--beamform-mode", default="auto")
|
||||
args = ap.parse_args()
|
||||
|
||||
cam = json.loads(args.camera_json)
|
||||
@@ -60,6 +61,7 @@ def main() -> int:
|
||||
sample_rate=args.audio_rate,
|
||||
chunk_s=args.enhance_chunk_s,
|
||||
hop_s=args.enhance_hop_s,
|
||||
beamform=args.beamform_mode,
|
||||
)
|
||||
log.info("audio backend: %s", cleaner.backend)
|
||||
|
||||
@@ -138,6 +140,7 @@ def main() -> int:
|
||||
# chunk * 2 ch * 2 bytes
|
||||
need = chunk * 2 * 2
|
||||
buf = b""
|
||||
hops = 0
|
||||
while aproc.poll() is None:
|
||||
data = await loop.run_in_executor(
|
||||
None, aproc.stdout.read, 4096)
|
||||
@@ -146,21 +149,28 @@ def main() -> int:
|
||||
buf += data
|
||||
while len(buf) >= need:
|
||||
piece, buf = buf[:need], buf[need:]
|
||||
pcm_stereo = np.frombuffer(piece, dtype=np.int16)
|
||||
pcm_mono = pcm_stereo.reshape(-1, 2).mean(
|
||||
axis=1).astype(np.int16)
|
||||
# align to full chunks
|
||||
usable = (len(pcm_mono) // chunk) * chunk
|
||||
pcm_stereo = np.frombuffer(
|
||||
piece, dtype=np.int16).reshape(-1, 2)
|
||||
n_frames = pcm_stereo.shape[0]
|
||||
usable = (n_frames // chunk) * chunk
|
||||
if usable:
|
||||
cleaned = await loop.run_in_executor(
|
||||
None, cleaner.process,
|
||||
pcm_mono[:usable])
|
||||
pcm_stereo[:usable])
|
||||
t_cap = time.perf_counter()
|
||||
await audio_source.capture_frame(
|
||||
rtc.AudioFrame(
|
||||
cleaned.tobytes(),
|
||||
args.audio_rate,
|
||||
1,
|
||||
len(cleaned)))
|
||||
hops += 1
|
||||
if hops % 50 == 0:
|
||||
log.info(
|
||||
"audio hop process_ms=%.1f capture_ms=%.1f",
|
||||
cleaner.last_dt_ms,
|
||||
(time.perf_counter() - t_cap) * 1000.0,
|
||||
)
|
||||
|
||||
aproc = subprocess.Popen(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error",
|
||||
|
||||
Reference in New Issue
Block a user