"""One camera -> one LiveKit participant. Run as a subprocess per camera: python -m cameras.run_worker --index 0 ... """ from __future__ import annotations import argparse import asyncio import contextlib import json import logging import os import signal import subprocess import sys import time from collections import deque import numpy as np log = logging.getLogger("cameras.worker") def drain_held_frames(pending: deque, now: float, hold_s: float): """Yield frame payloads captured at least hold_s seconds ago.""" while pending and (now - pending[0][0]) >= hold_s: yield pending.popleft()[1] def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--camera-json", required=True, help="JSON of one discovered camera dict") ap.add_argument("--room", required=True) ap.add_argument("--url", required=True) ap.add_argument("--token", required=True) ap.add_argument("--width", type=int, default=640) ap.add_argument("--height", type=int, default=360) ap.add_argument("--fps", type=int, default=30) ap.add_argument("--bitrate", type=int, default=800_000) ap.add_argument("--video-codec", default="h264") ap.add_argument("--video-encoder", default="vaapi") ap.add_argument("--vaapi-device", default="/dev/dri/renderD129") ap.add_argument("--audio-rate", type=int, default=16000) ap.add_argument("--enhance-mode", default="auto") ap.add_argument("--enhance-model", default="speechbrain/metricgan-plus-voicebank") ap.add_argument("--enhance-chunk-s", type=float, default=1.0) ap.add_argument("--enhance-hop-s", type=float, default=0.2) ap.add_argument("--beamform-mode", default="auto") ap.add_argument("--torch-num-threads", type=int, default=1) ap.add_argument("--tdoa-every", type=int, default=5) args = ap.parse_args() cam = json.loads(args.camera_json) logging.basicConfig( level=logging.INFO, format=f"%(asctime)s [{cam['identity']}] %(levelname)s %(message)s", ) from config import Config # noqa: F401 (ensures .env is loaded) from audio_cleanup import AudioCleaner, apply_torch_thread_limits apply_torch_thread_limits(args.torch_num_threads) import livekit.rtc as rtc identity = cam["identity"] video_dev = cam["video_device"] audio_card = cam.get("audio_card") if args.vaapi_device: os.environ.setdefault("LIBVA_DRM_DEVICE", args.vaapi_device) os.environ.setdefault("LIBVA_DRIVER_NAME", "iHD") async def run() -> int: cleaner = AudioCleaner( mode=args.enhance_mode, model_source=args.enhance_model, sample_rate=args.audio_rate, chunk_s=args.enhance_chunk_s, hop_s=args.enhance_hop_s, beamform=args.beamform_mode, torch_num_threads=args.torch_num_threads, tdoa_every=args.tdoa_every, ) log.info("audio backend: %s", cleaner.backend) room = rtc.Room() await room.connect(args.url, args.token) log.info("connected to room %r as %r", room.name, identity) # ---------------- video ---------------- video_source = rtc.VideoSource(args.width, args.height) video_track = rtc.LocalVideoTrack.create_video_track( f"{identity}-video", video_source) vproc = subprocess.Popen( ["ffmpeg", "-hide_banner", "-loglevel", "error", "-fflags", "nobuffer", "-flags", "low_delay", "-thread_queue_size", "4", "-f", "v4l2", "-input_format", "mjpeg", "-video_size", f"{args.width}x{args.height}", "-framerate", str(args.fps), "-i", video_dev, "-f", "rawvideo", "-pix_fmt", "yuv420p", "pipe:1"], stdout=subprocess.PIPE, ) frame_bytes = args.width * args.height * 3 // 2 video_frames = 0 # Audio is published in enhance hops (default 200 ms). Hold video by # that same amount so lipsync at the subscriber: a frame captured at # T is sent when the audio hop covering T is sent. video_hold_s = float(args.enhance_hop_s) if audio_card is not None else 0.0 pending: deque[tuple[float, bytes]] = deque() max_pending = max(8, int(args.fps * (video_hold_s + 0.15)) + 4) if video_hold_s > 0: log.info("av sync: holding video %.0f ms to match audio hop", video_hold_s * 1000.0) async def video_loop(): nonlocal video_frames loop = asyncio.get_running_loop() while vproc.poll() is None: data = await loop.run_in_executor( None, vproc.stdout.read, frame_bytes) if not data or len(data) < frame_bytes: if not data: break keep = b"" while len(keep) < frame_bytes: more = await loop.run_in_executor( None, vproc.stdout.read, frame_bytes - len(keep)) if not more: return keep += more data = keep now = time.monotonic() pending.append((now, data)) while len(pending) > max_pending: pending.popleft() for payload in drain_held_frames(pending, now, video_hold_s): try: video_source.capture_frame( rtc.VideoFrame( args.width, args.height, rtc.VideoBufferType.I420, payload), timestamp_us=int(time.monotonic() * 1_000_000), ) video_frames += 1 except Exception: # noqa: BLE001 log.exception("video capture_frame failed") # ---------------- audio ---------------- aproc: subprocess.Popen | None = None atrack: rtc.LocalAudioTrack | None = None if audio_card is not None: audio_source = rtc.AudioSource( sample_rate=args.audio_rate, num_channels=1, queue_size_ms=max(250, int(args.enhance_hop_s * 1000) + 50)) atrack = rtc.LocalAudioTrack.create_audio_track( f"{identity}-audio", audio_source) hop_s = args.enhance_hop_s chunk = int(hop_s * args.audio_rate) # samples per hop async def audio_loop(): assert aproc is not None and atrack is not None loop = asyncio.get_running_loop() # mic delivers stereo s16le: bytes per chunk = # chunk * 2 ch * 2 bytes need = chunk * 2 * 2 buf = b"" hops = 0 while aproc.poll() is None: data = await loop.run_in_executor( None, aproc.stdout.read, 4096) if not data: break buf += data while len(buf) >= need: piece, buf = buf[:need], buf[need:] pcm_stereo = np.frombuffer( piece, dtype=np.int16).reshape(-1, 2) n_frames = pcm_stereo.shape[0] usable = (n_frames // chunk) * chunk if usable: cleaned = await loop.run_in_executor( None, cleaner.process, pcm_stereo[:usable]) t_cap = time.perf_counter() await audio_source.capture_frame( rtc.AudioFrame( cleaned.tobytes(), args.audio_rate, 1, len(cleaned))) hops += 1 if hops % 50 == 0: log.info( "audio hop process_ms=%.1f capture_ms=%.1f", cleaner.last_dt_ms, (time.perf_counter() - t_cap) * 1000.0, ) aproc = subprocess.Popen( ["ffmpeg", "-hide_banner", "-loglevel", "error", "-fflags", "nobuffer", "-flags", "low_delay", "-thread_queue_size", "8", "-f", "alsa", "-i", f"hw:{audio_card},0", "-ar", str(args.audio_rate), "-ac", "2", "-f", "s16le", "pipe:1"], stdout=subprocess.PIPE, ) # ---------------- publish ---------------- from livekit.rtc import TrackSource, TrackPublishOptions, VideoEncoding, VideoCodec from livekit.rtc._proto.room_pb2 import ( ENCODER_BACKEND_AUTO, ENCODER_BACKEND_SOFTWARE, ENCODER_BACKEND_HARDWARE, ENCODER_BACKEND_NVENC, ENCODER_BACKEND_VAAPI, ENCODER_BACKEND_VIDEOTOOLBOX, DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION, ) encoder_map = { "auto": ENCODER_BACKEND_AUTO, "software": ENCODER_BACKEND_SOFTWARE, "hardware": ENCODER_BACKEND_HARDWARE, "nvenc": ENCODER_BACKEND_NVENC, "vaapi": ENCODER_BACKEND_VAAPI, "videotoolbox": ENCODER_BACKEND_VIDEOTOOLBOX, } codec_map = { "vp8": VideoCodec.VP8, "h264": VideoCodec.H264, "av1": VideoCodec.AV1, "vp9": VideoCodec.VP9, "h265": VideoCodec.H265, } encoder = encoder_map.get(args.video_encoder, ENCODER_BACKEND_VAAPI) codec = codec_map.get(args.video_codec, VideoCodec.H264) pub_opts = TrackPublishOptions( source=TrackSource.SOURCE_CAMERA, simulcast=False, video_codec=codec, video_encoding=VideoEncoding( max_bitrate=int(args.bitrate), max_framerate=int(args.fps), ), video_encoder=encoder, degradation_preference=DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION, ) log.info( "publish video %dx%d@%d %s encoder=%s bitrate=%d vaapi=%s", args.width, args.height, args.fps, args.video_codec, args.video_encoder, args.bitrate, args.vaapi_device, ) await room.local_participant.publish_track(video_track, pub_opts) if atrack is not None: await room.local_participant.publish_track( atrack, TrackPublishOptions( source=TrackSource.SOURCE_MICROPHONE, dtx=False)) log.info("published video+audio (audio backend: %s)", cleaner.backend) else: log.info("published video only (no audio card matched)") log.info("status: UP identity=%s video=%s audio=%s", identity, video_dev, f"hw:{audio_card},0" if audio_card is not None else "none") # ---------------- run ---------------- tasks = [asyncio.create_task(video_loop(), name="video")] if aproc is not None: tasks.append(asyncio.create_task(audio_loop(), name="audio")) stop = asyncio.Event() loop = asyncio.get_running_loop() for sig in (signal.SIGTERM, signal.SIGINT): with contextlib.suppress(NotImplementedError): loop.add_signal_handler(sig, stop.set) await stop.wait() for t in tasks: t.cancel() await asyncio.gather(*tasks, return_exceptions=True) for p in (vproc, aproc): if p is not None and p.poll() is None: with contextlib.suppress(Exception): p.terminate() with contextlib.suppress(Exception): p.wait(timeout=3) log.info("stopped; delivered %d video frames", video_frames) with contextlib.suppress(Exception): await room.disconnect() return 0 try: return asyncio.run(run()) except Exception: # noqa: BLE001 log.exception("worker fatal") return 1 if __name__ == "__main__": sys.exit(main())