feat: known-good CPU audio + 1080p VAAPI + KMS display wall
Working snapshot before the OpenVINO MetricGAN GPU spike: thread-capped SpeechBrain DelaySum+MetricGAN on CPU, hop 200 ms, video hold for A/V sync, Arc VAAPI H.264, headless kmssink wall. 32 tests passing. Do not treat this as a GPU audio path.
This commit is contained in:
root
+88
-21
@@ -15,12 +15,19 @@ import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
import numpy as np
|
||||
|
||||
log = logging.getLogger("cameras.worker")
|
||||
|
||||
|
||||
def drain_held_frames(pending: deque, now: float, hold_s: float):
|
||||
"""Yield frame payloads captured at least hold_s seconds ago."""
|
||||
while pending and (now - pending[0][0]) >= hold_s:
|
||||
yield pending.popleft()[1]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--camera-json", required=True,
|
||||
@@ -28,9 +35,13 @@ def main() -> int:
|
||||
ap.add_argument("--room", required=True)
|
||||
ap.add_argument("--url", required=True)
|
||||
ap.add_argument("--token", required=True)
|
||||
ap.add_argument("--width", type=int, default=1280)
|
||||
ap.add_argument("--height", type=int, default=720)
|
||||
ap.add_argument("--width", type=int, default=640)
|
||||
ap.add_argument("--height", type=int, default=360)
|
||||
ap.add_argument("--fps", type=int, default=30)
|
||||
ap.add_argument("--bitrate", type=int, default=800_000)
|
||||
ap.add_argument("--video-codec", default="h264")
|
||||
ap.add_argument("--video-encoder", default="vaapi")
|
||||
ap.add_argument("--vaapi-device", default="/dev/dri/renderD129")
|
||||
ap.add_argument("--audio-rate", type=int, default=16000)
|
||||
ap.add_argument("--enhance-mode", default="auto")
|
||||
ap.add_argument("--enhance-model",
|
||||
@@ -56,6 +67,9 @@ def main() -> int:
|
||||
identity = cam["identity"]
|
||||
video_dev = cam["video_device"]
|
||||
audio_card = cam.get("audio_card")
|
||||
if args.vaapi_device:
|
||||
os.environ.setdefault("LIBVA_DRM_DEVICE", args.vaapi_device)
|
||||
os.environ.setdefault("LIBVA_DRIVER_NAME", "iHD")
|
||||
|
||||
async def run() -> int:
|
||||
cleaner = AudioCleaner(
|
||||
@@ -81,29 +95,38 @@ def main() -> int:
|
||||
|
||||
vproc = subprocess.Popen(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error",
|
||||
"-fflags", "nobuffer", "-flags", "low_delay",
|
||||
"-thread_queue_size", "4",
|
||||
"-f", "v4l2",
|
||||
"-input_format", "mjpeg",
|
||||
"-video_size", f"{args.width}x{args.height}",
|
||||
"-framerate", str(args.fps),
|
||||
"-i", video_dev,
|
||||
"-f", "rawvideo", "-pix_fmt", "bgra",
|
||||
"-f", "rawvideo", "-pix_fmt", "yuv420p",
|
||||
"pipe:1"],
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
frame_bytes = args.width * args.height * 4
|
||||
frame_bytes = args.width * args.height * 3 // 2
|
||||
video_frames = 0
|
||||
# Audio is published in enhance hops (default 200 ms). Hold video by
|
||||
# that same amount so lipsync at the subscriber: a frame captured at
|
||||
# T is sent when the audio hop covering T is sent.
|
||||
video_hold_s = float(args.enhance_hop_s) if audio_card is not None else 0.0
|
||||
pending: deque[tuple[float, bytes]] = deque()
|
||||
max_pending = max(8, int(args.fps * (video_hold_s + 0.15)) + 4)
|
||||
if video_hold_s > 0:
|
||||
log.info("av sync: holding video %.0f ms to match audio hop",
|
||||
video_hold_s * 1000.0)
|
||||
|
||||
async def video_loop():
|
||||
nonlocal video_frames
|
||||
loop = asyncio.get_running_loop()
|
||||
last = time.monotonic()
|
||||
while vproc.poll() is None:
|
||||
data = await loop.run_in_executor(
|
||||
None, vproc.stdout.read, frame_bytes)
|
||||
if not data or len(data) < frame_bytes:
|
||||
if not data:
|
||||
break
|
||||
# partial read: accumulate (v4l2/ffmpeg shouldn't do this)
|
||||
keep = b""
|
||||
while len(keep) < frame_bytes:
|
||||
more = await loop.run_in_executor(
|
||||
@@ -114,24 +137,28 @@ def main() -> int:
|
||||
keep += more
|
||||
data = keep
|
||||
now = time.monotonic()
|
||||
ts_us = int((now - last) * 1_000_000)
|
||||
try:
|
||||
video_source.capture_frame(
|
||||
rtc.VideoFrame(
|
||||
args.width, args.height,
|
||||
rtc.VideoBufferType.BGRA, data),
|
||||
timestamp_us=ts_us,
|
||||
)
|
||||
video_frames += 1
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("video capture_frame failed")
|
||||
pending.append((now, data))
|
||||
while len(pending) > max_pending:
|
||||
pending.popleft()
|
||||
for payload in drain_held_frames(pending, now, video_hold_s):
|
||||
try:
|
||||
video_source.capture_frame(
|
||||
rtc.VideoFrame(
|
||||
args.width, args.height,
|
||||
rtc.VideoBufferType.I420, payload),
|
||||
timestamp_us=int(time.monotonic() * 1_000_000),
|
||||
)
|
||||
video_frames += 1
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("video capture_frame failed")
|
||||
|
||||
# ---------------- audio ----------------
|
||||
aproc: subprocess.Popen | None = None
|
||||
atrack: rtc.LocalAudioTrack | None = None
|
||||
if audio_card is not None:
|
||||
audio_source = rtc.AudioSource(
|
||||
sample_rate=args.audio_rate, num_channels=1)
|
||||
sample_rate=args.audio_rate, num_channels=1,
|
||||
queue_size_ms=max(250, int(args.enhance_hop_s * 1000) + 50))
|
||||
atrack = rtc.LocalAudioTrack.create_audio_track(
|
||||
f"{identity}-audio", audio_source)
|
||||
|
||||
@@ -180,7 +207,7 @@ def main() -> int:
|
||||
aproc = subprocess.Popen(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error",
|
||||
"-fflags", "nobuffer", "-flags", "low_delay",
|
||||
"-thread_queue_size", "64",
|
||||
"-thread_queue_size", "8",
|
||||
"-f", "alsa", "-i", f"hw:{audio_card},0",
|
||||
"-ar", str(args.audio_rate), "-ac", "2",
|
||||
"-f", "s16le", "pipe:1"],
|
||||
@@ -188,9 +215,49 @@ def main() -> int:
|
||||
)
|
||||
|
||||
# ---------------- publish ----------------
|
||||
from livekit.rtc import TrackSource, TrackPublishOptions
|
||||
from livekit.rtc import TrackSource, TrackPublishOptions, VideoEncoding, VideoCodec
|
||||
from livekit.rtc._proto.room_pb2 import (
|
||||
ENCODER_BACKEND_AUTO,
|
||||
ENCODER_BACKEND_SOFTWARE,
|
||||
ENCODER_BACKEND_HARDWARE,
|
||||
ENCODER_BACKEND_NVENC,
|
||||
ENCODER_BACKEND_VAAPI,
|
||||
ENCODER_BACKEND_VIDEOTOOLBOX,
|
||||
DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION,
|
||||
)
|
||||
encoder_map = {
|
||||
"auto": ENCODER_BACKEND_AUTO,
|
||||
"software": ENCODER_BACKEND_SOFTWARE,
|
||||
"hardware": ENCODER_BACKEND_HARDWARE,
|
||||
"nvenc": ENCODER_BACKEND_NVENC,
|
||||
"vaapi": ENCODER_BACKEND_VAAPI,
|
||||
"videotoolbox": ENCODER_BACKEND_VIDEOTOOLBOX,
|
||||
}
|
||||
codec_map = {
|
||||
"vp8": VideoCodec.VP8,
|
||||
"h264": VideoCodec.H264,
|
||||
"av1": VideoCodec.AV1,
|
||||
"vp9": VideoCodec.VP9,
|
||||
"h265": VideoCodec.H265,
|
||||
}
|
||||
encoder = encoder_map.get(args.video_encoder, ENCODER_BACKEND_VAAPI)
|
||||
codec = codec_map.get(args.video_codec, VideoCodec.H264)
|
||||
pub_opts = TrackPublishOptions(
|
||||
source=TrackSource.SOURCE_CAMERA, simulcast=False)
|
||||
source=TrackSource.SOURCE_CAMERA,
|
||||
simulcast=False,
|
||||
video_codec=codec,
|
||||
video_encoding=VideoEncoding(
|
||||
max_bitrate=int(args.bitrate),
|
||||
max_framerate=int(args.fps),
|
||||
),
|
||||
video_encoder=encoder,
|
||||
degradation_preference=DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION,
|
||||
)
|
||||
log.info(
|
||||
"publish video %dx%d@%d %s encoder=%s bitrate=%d vaapi=%s",
|
||||
args.width, args.height, args.fps, args.video_codec,
|
||||
args.video_encoder, args.bitrate, args.vaapi_device,
|
||||
)
|
||||
await room.local_participant.publish_track(video_track, pub_opts)
|
||||
if atrack is not None:
|
||||
await room.local_participant.publish_track(
|
||||
|
||||
Reference in New Issue
Block a user