root
736eccb932
Working snapshot before the OpenVINO MetricGAN GPU spike: thread-capped SpeechBrain DelaySum+MetricGAN on CPU, hop 200 ms, video hold for A/V sync, Arc VAAPI H.264, headless kmssink wall. 32 tests passing. Do not treat this as a GPU audio path.
313 lines
12 KiB
Python
313 lines
12 KiB
Python
"""One camera -> one LiveKit participant.
|
|
|
|
Run as a subprocess per camera: python -m cameras.run_worker --index 0 ...
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import contextlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections import deque
|
|
|
|
import numpy as np
|
|
|
|
log = logging.getLogger("cameras.worker")
|
|
|
|
|
|
def drain_held_frames(pending: deque, now: float, hold_s: float):
|
|
"""Yield frame payloads captured at least hold_s seconds ago."""
|
|
while pending and (now - pending[0][0]) >= hold_s:
|
|
yield pending.popleft()[1]
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--camera-json", required=True,
|
|
help="JSON of one discovered camera dict")
|
|
ap.add_argument("--room", required=True)
|
|
ap.add_argument("--url", required=True)
|
|
ap.add_argument("--token", required=True)
|
|
ap.add_argument("--width", type=int, default=640)
|
|
ap.add_argument("--height", type=int, default=360)
|
|
ap.add_argument("--fps", type=int, default=30)
|
|
ap.add_argument("--bitrate", type=int, default=800_000)
|
|
ap.add_argument("--video-codec", default="h264")
|
|
ap.add_argument("--video-encoder", default="vaapi")
|
|
ap.add_argument("--vaapi-device", default="/dev/dri/renderD129")
|
|
ap.add_argument("--audio-rate", type=int, default=16000)
|
|
ap.add_argument("--enhance-mode", default="auto")
|
|
ap.add_argument("--enhance-model",
|
|
default="speechbrain/metricgan-plus-voicebank")
|
|
ap.add_argument("--enhance-chunk-s", type=float, default=1.0)
|
|
ap.add_argument("--enhance-hop-s", type=float, default=0.2)
|
|
ap.add_argument("--beamform-mode", default="auto")
|
|
ap.add_argument("--torch-num-threads", type=int, default=1)
|
|
ap.add_argument("--tdoa-every", type=int, default=5)
|
|
args = ap.parse_args()
|
|
|
|
cam = json.loads(args.camera_json)
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format=f"%(asctime)s [{cam['identity']}] %(levelname)s %(message)s",
|
|
)
|
|
|
|
from config import Config # noqa: F401 (ensures .env is loaded)
|
|
from audio_cleanup import AudioCleaner, apply_torch_thread_limits
|
|
apply_torch_thread_limits(args.torch_num_threads)
|
|
import livekit.rtc as rtc
|
|
|
|
identity = cam["identity"]
|
|
video_dev = cam["video_device"]
|
|
audio_card = cam.get("audio_card")
|
|
if args.vaapi_device:
|
|
os.environ.setdefault("LIBVA_DRM_DEVICE", args.vaapi_device)
|
|
os.environ.setdefault("LIBVA_DRIVER_NAME", "iHD")
|
|
|
|
async def run() -> int:
|
|
cleaner = AudioCleaner(
|
|
mode=args.enhance_mode,
|
|
model_source=args.enhance_model,
|
|
sample_rate=args.audio_rate,
|
|
chunk_s=args.enhance_chunk_s,
|
|
hop_s=args.enhance_hop_s,
|
|
beamform=args.beamform_mode,
|
|
torch_num_threads=args.torch_num_threads,
|
|
tdoa_every=args.tdoa_every,
|
|
)
|
|
log.info("audio backend: %s", cleaner.backend)
|
|
|
|
room = rtc.Room()
|
|
await room.connect(args.url, args.token)
|
|
log.info("connected to room %r as %r", room.name, identity)
|
|
|
|
# ---------------- video ----------------
|
|
video_source = rtc.VideoSource(args.width, args.height)
|
|
video_track = rtc.LocalVideoTrack.create_video_track(
|
|
f"{identity}-video", video_source)
|
|
|
|
vproc = subprocess.Popen(
|
|
["ffmpeg", "-hide_banner", "-loglevel", "error",
|
|
"-fflags", "nobuffer", "-flags", "low_delay",
|
|
"-thread_queue_size", "4",
|
|
"-f", "v4l2",
|
|
"-input_format", "mjpeg",
|
|
"-video_size", f"{args.width}x{args.height}",
|
|
"-framerate", str(args.fps),
|
|
"-i", video_dev,
|
|
"-f", "rawvideo", "-pix_fmt", "yuv420p",
|
|
"pipe:1"],
|
|
stdout=subprocess.PIPE,
|
|
)
|
|
frame_bytes = args.width * args.height * 3 // 2
|
|
video_frames = 0
|
|
# Audio is published in enhance hops (default 200 ms). Hold video by
|
|
# that same amount so lipsync at the subscriber: a frame captured at
|
|
# T is sent when the audio hop covering T is sent.
|
|
video_hold_s = float(args.enhance_hop_s) if audio_card is not None else 0.0
|
|
pending: deque[tuple[float, bytes]] = deque()
|
|
max_pending = max(8, int(args.fps * (video_hold_s + 0.15)) + 4)
|
|
if video_hold_s > 0:
|
|
log.info("av sync: holding video %.0f ms to match audio hop",
|
|
video_hold_s * 1000.0)
|
|
|
|
async def video_loop():
|
|
nonlocal video_frames
|
|
loop = asyncio.get_running_loop()
|
|
while vproc.poll() is None:
|
|
data = await loop.run_in_executor(
|
|
None, vproc.stdout.read, frame_bytes)
|
|
if not data or len(data) < frame_bytes:
|
|
if not data:
|
|
break
|
|
keep = b""
|
|
while len(keep) < frame_bytes:
|
|
more = await loop.run_in_executor(
|
|
None, vproc.stdout.read,
|
|
frame_bytes - len(keep))
|
|
if not more:
|
|
return
|
|
keep += more
|
|
data = keep
|
|
now = time.monotonic()
|
|
pending.append((now, data))
|
|
while len(pending) > max_pending:
|
|
pending.popleft()
|
|
for payload in drain_held_frames(pending, now, video_hold_s):
|
|
try:
|
|
video_source.capture_frame(
|
|
rtc.VideoFrame(
|
|
args.width, args.height,
|
|
rtc.VideoBufferType.I420, payload),
|
|
timestamp_us=int(time.monotonic() * 1_000_000),
|
|
)
|
|
video_frames += 1
|
|
except Exception: # noqa: BLE001
|
|
log.exception("video capture_frame failed")
|
|
|
|
# ---------------- audio ----------------
|
|
aproc: subprocess.Popen | None = None
|
|
atrack: rtc.LocalAudioTrack | None = None
|
|
if audio_card is not None:
|
|
audio_source = rtc.AudioSource(
|
|
sample_rate=args.audio_rate, num_channels=1,
|
|
queue_size_ms=max(250, int(args.enhance_hop_s * 1000) + 50))
|
|
atrack = rtc.LocalAudioTrack.create_audio_track(
|
|
f"{identity}-audio", audio_source)
|
|
|
|
hop_s = args.enhance_hop_s
|
|
chunk = int(hop_s * args.audio_rate) # samples per hop
|
|
|
|
async def audio_loop():
|
|
assert aproc is not None and atrack is not None
|
|
loop = asyncio.get_running_loop()
|
|
# mic delivers stereo s16le: bytes per chunk =
|
|
# chunk * 2 ch * 2 bytes
|
|
need = chunk * 2 * 2
|
|
buf = b""
|
|
hops = 0
|
|
while aproc.poll() is None:
|
|
data = await loop.run_in_executor(
|
|
None, aproc.stdout.read, 4096)
|
|
if not data:
|
|
break
|
|
buf += data
|
|
while len(buf) >= need:
|
|
piece, buf = buf[:need], buf[need:]
|
|
pcm_stereo = np.frombuffer(
|
|
piece, dtype=np.int16).reshape(-1, 2)
|
|
n_frames = pcm_stereo.shape[0]
|
|
usable = (n_frames // chunk) * chunk
|
|
if usable:
|
|
cleaned = await loop.run_in_executor(
|
|
None, cleaner.process,
|
|
pcm_stereo[:usable])
|
|
t_cap = time.perf_counter()
|
|
await audio_source.capture_frame(
|
|
rtc.AudioFrame(
|
|
cleaned.tobytes(),
|
|
args.audio_rate,
|
|
1,
|
|
len(cleaned)))
|
|
hops += 1
|
|
if hops % 50 == 0:
|
|
log.info(
|
|
"audio hop process_ms=%.1f capture_ms=%.1f",
|
|
cleaner.last_dt_ms,
|
|
(time.perf_counter() - t_cap) * 1000.0,
|
|
)
|
|
|
|
aproc = subprocess.Popen(
|
|
["ffmpeg", "-hide_banner", "-loglevel", "error",
|
|
"-fflags", "nobuffer", "-flags", "low_delay",
|
|
"-thread_queue_size", "8",
|
|
"-f", "alsa", "-i", f"hw:{audio_card},0",
|
|
"-ar", str(args.audio_rate), "-ac", "2",
|
|
"-f", "s16le", "pipe:1"],
|
|
stdout=subprocess.PIPE,
|
|
)
|
|
|
|
# ---------------- publish ----------------
|
|
from livekit.rtc import TrackSource, TrackPublishOptions, VideoEncoding, VideoCodec
|
|
from livekit.rtc._proto.room_pb2 import (
|
|
ENCODER_BACKEND_AUTO,
|
|
ENCODER_BACKEND_SOFTWARE,
|
|
ENCODER_BACKEND_HARDWARE,
|
|
ENCODER_BACKEND_NVENC,
|
|
ENCODER_BACKEND_VAAPI,
|
|
ENCODER_BACKEND_VIDEOTOOLBOX,
|
|
DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION,
|
|
)
|
|
encoder_map = {
|
|
"auto": ENCODER_BACKEND_AUTO,
|
|
"software": ENCODER_BACKEND_SOFTWARE,
|
|
"hardware": ENCODER_BACKEND_HARDWARE,
|
|
"nvenc": ENCODER_BACKEND_NVENC,
|
|
"vaapi": ENCODER_BACKEND_VAAPI,
|
|
"videotoolbox": ENCODER_BACKEND_VIDEOTOOLBOX,
|
|
}
|
|
codec_map = {
|
|
"vp8": VideoCodec.VP8,
|
|
"h264": VideoCodec.H264,
|
|
"av1": VideoCodec.AV1,
|
|
"vp9": VideoCodec.VP9,
|
|
"h265": VideoCodec.H265,
|
|
}
|
|
encoder = encoder_map.get(args.video_encoder, ENCODER_BACKEND_VAAPI)
|
|
codec = codec_map.get(args.video_codec, VideoCodec.H264)
|
|
pub_opts = TrackPublishOptions(
|
|
source=TrackSource.SOURCE_CAMERA,
|
|
simulcast=False,
|
|
video_codec=codec,
|
|
video_encoding=VideoEncoding(
|
|
max_bitrate=int(args.bitrate),
|
|
max_framerate=int(args.fps),
|
|
),
|
|
video_encoder=encoder,
|
|
degradation_preference=DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION,
|
|
)
|
|
log.info(
|
|
"publish video %dx%d@%d %s encoder=%s bitrate=%d vaapi=%s",
|
|
args.width, args.height, args.fps, args.video_codec,
|
|
args.video_encoder, args.bitrate, args.vaapi_device,
|
|
)
|
|
await room.local_participant.publish_track(video_track, pub_opts)
|
|
if atrack is not None:
|
|
await room.local_participant.publish_track(
|
|
atrack,
|
|
TrackPublishOptions(
|
|
source=TrackSource.SOURCE_MICROPHONE,
|
|
dtx=False))
|
|
log.info("published video+audio (audio backend: %s)",
|
|
cleaner.backend)
|
|
else:
|
|
log.info("published video only (no audio card matched)")
|
|
|
|
log.info("status: UP identity=%s video=%s audio=%s",
|
|
identity, video_dev,
|
|
f"hw:{audio_card},0" if audio_card is not None else "none")
|
|
|
|
# ---------------- run ----------------
|
|
tasks = [asyncio.create_task(video_loop(), name="video")]
|
|
if aproc is not None:
|
|
tasks.append(asyncio.create_task(audio_loop(), name="audio"))
|
|
|
|
stop = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
with contextlib.suppress(NotImplementedError):
|
|
loop.add_signal_handler(sig, stop.set)
|
|
|
|
await stop.wait()
|
|
|
|
for t in tasks:
|
|
t.cancel()
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
for p in (vproc, aproc):
|
|
if p is not None and p.poll() is None:
|
|
with contextlib.suppress(Exception):
|
|
p.terminate()
|
|
with contextlib.suppress(Exception):
|
|
p.wait(timeout=3)
|
|
log.info("stopped; delivered %d video frames", video_frames)
|
|
with contextlib.suppress(Exception):
|
|
await room.disconnect()
|
|
return 0
|
|
|
|
try:
|
|
return asyncio.run(run())
|
|
except Exception: # noqa: BLE001
|
|
log.exception("worker fatal")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|