feat: point the fleet at wss LiveKit room uwh-telhai
Publish and subscribe over wss://livekit.uni-wh.de:7800 and refuse cleartext ws://. Conference room is uwh-telhai. Includes the uncommitted encoded H.264 publish path, Rally hairpin, and KMS wall overlay.
This commit is contained in:
root
+155
-67
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
@@ -35,10 +36,10 @@ def main() -> int:
|
||||
ap.add_argument("--room", required=True)
|
||||
ap.add_argument("--url", required=True)
|
||||
ap.add_argument("--token", required=True)
|
||||
ap.add_argument("--width", type=int, default=640)
|
||||
ap.add_argument("--height", type=int, default=360)
|
||||
ap.add_argument("--fps", type=int, default=30)
|
||||
ap.add_argument("--bitrate", type=int, default=800_000)
|
||||
ap.add_argument("--width", type=int, default=320)
|
||||
ap.add_argument("--height", type=int, default=180)
|
||||
ap.add_argument("--fps", type=int, default=15)
|
||||
ap.add_argument("--bitrate", type=int, default=400_000)
|
||||
ap.add_argument("--video-codec", default="h264")
|
||||
ap.add_argument("--video-encoder", default="vaapi")
|
||||
ap.add_argument("--vaapi-device", default="/dev/dri/renderD129")
|
||||
@@ -52,6 +53,17 @@ def main() -> int:
|
||||
ap.add_argument("--beamform-mode", default="auto")
|
||||
ap.add_argument("--torch-num-threads", type=int, default=1)
|
||||
ap.add_argument("--tdoa-every", type=int, default=5)
|
||||
ap.add_argument("--audio-queue-ms", type=int, default=None)
|
||||
ap.add_argument("--video-hold-s", type=float, default=None)
|
||||
ap.add_argument("--split-executor", action="store_true")
|
||||
ap.add_argument("--enhance-socket", default="")
|
||||
ap.add_argument("--speaker-camera", default="rally")
|
||||
ap.add_argument("--audio-gate", action="store_true")
|
||||
ap.add_argument("--audio-gate-open-db", type=float, default=-28.0)
|
||||
ap.add_argument("--audio-gate-close-db", type=float, default=-34.0)
|
||||
ap.add_argument("--audio-gate-hold-s", type=float, default=0.3)
|
||||
ap.add_argument("--video-shm", default="",
|
||||
help="latest-frame I420 mmap from capture_daemon")
|
||||
args = ap.parse_args()
|
||||
|
||||
cam = json.loads(args.camera_json)
|
||||
@@ -62,15 +74,16 @@ def main() -> int:
|
||||
|
||||
from config import Config # noqa: F401 (ensures .env is loaded)
|
||||
from audio_cleanup import AudioCleaner, apply_torch_thread_limits
|
||||
apply_torch_thread_limits(args.torch_num_threads)
|
||||
import livekit.rtc as rtc
|
||||
|
||||
from audio_gate import make_gate
|
||||
from latency import audio_queue_size_ms, video_hold_seconds
|
||||
identity = cam["identity"]
|
||||
os.environ["LIBVA_DRIVER_NAME"] = "iHD"
|
||||
os.environ.pop("LIBVA_DRM_DEVICE", None)
|
||||
if not args.enhance_socket:
|
||||
apply_torch_thread_limits(args.torch_num_threads)
|
||||
import livekit.rtc as rtc
|
||||
video_dev = cam["video_device"]
|
||||
audio_card = cam.get("audio_card")
|
||||
if args.vaapi_device:
|
||||
os.environ.setdefault("LIBVA_DRM_DEVICE", args.vaapi_device)
|
||||
os.environ.setdefault("LIBVA_DRIVER_NAME", "iHD")
|
||||
|
||||
async def run() -> int:
|
||||
cleaner = AudioCleaner(
|
||||
@@ -83,76 +96,145 @@ def main() -> int:
|
||||
torch_num_threads=args.torch_num_threads,
|
||||
tdoa_every=args.tdoa_every,
|
||||
enhance_infer=args.enhance_infer,
|
||||
enhance_socket=args.enhance_socket,
|
||||
identity=identity,
|
||||
gate=make_gate(
|
||||
identity,
|
||||
args.speaker_camera,
|
||||
bool(args.audio_gate),
|
||||
args.audio_gate_open_db,
|
||||
args.audio_gate_close_db,
|
||||
args.audio_gate_hold_s,
|
||||
args.enhance_hop_s,
|
||||
),
|
||||
)
|
||||
log.info("audio backend: %s", cleaner.backend)
|
||||
log.info("audio backend: %s gate=%s", cleaner.backend,
|
||||
"on" if cleaner._gate is not None else "off")
|
||||
|
||||
io_ex = None
|
||||
enhance_ex = None
|
||||
split_owned: list[concurrent.futures.Executor] = []
|
||||
if args.split_executor:
|
||||
io_ex = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=2, thread_name_prefix=f"{identity}-io")
|
||||
enhance_ex = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix=f"{identity}-enh")
|
||||
split_owned.extend([io_ex, enhance_ex])
|
||||
log.info("split executor: ffmpeg io vs enhance")
|
||||
|
||||
room = rtc.Room()
|
||||
await room.connect(args.url, args.token)
|
||||
log.info("connected to room %r as %r", room.name, identity)
|
||||
await room.connect(
|
||||
args.url, args.token, rtc.RoomOptions(auto_subscribe=False))
|
||||
log.info("connected to room %r as %r (auto_subscribe=False)",
|
||||
room.name, identity)
|
||||
|
||||
# ---------------- video ----------------
|
||||
video_source = rtc.VideoSource(args.width, args.height)
|
||||
video_track = rtc.LocalVideoTrack.create_video_track(
|
||||
f"{identity}-video", video_source)
|
||||
|
||||
vproc = subprocess.Popen(
|
||||
["ffmpeg", "-hide_banner", "-loglevel", "error",
|
||||
"-fflags", "nobuffer", "-flags", "low_delay",
|
||||
"-thread_queue_size", "4",
|
||||
"-f", "v4l2",
|
||||
"-input_format", "mjpeg",
|
||||
"-video_size", f"{args.width}x{args.height}",
|
||||
"-framerate", str(args.fps),
|
||||
"-i", video_dev,
|
||||
"-f", "rawvideo", "-pix_fmt", "yuv420p",
|
||||
"pipe:1"],
|
||||
stdout=subprocess.PIPE,
|
||||
)
|
||||
frame_bytes = args.width * args.height * 3 // 2
|
||||
video_frames = 0
|
||||
# Audio is published in enhance hops (default 200 ms). Hold video by
|
||||
# that same amount so lipsync at the subscriber: a frame captured at
|
||||
# T is sent when the audio hop covering T is sent.
|
||||
video_hold_s = float(args.enhance_hop_s) if audio_card is not None else 0.0
|
||||
vproc: subprocess.Popen | None = None
|
||||
shm_path = (args.video_shm or "").strip()
|
||||
video_hold_s = video_hold_seconds(
|
||||
args.enhance_hop_s,
|
||||
audio_card is not None,
|
||||
0.0 if shm_path else args.video_hold_s,
|
||||
)
|
||||
pending: deque[tuple[float, bytes]] = deque()
|
||||
max_pending = max(8, int(args.fps * (video_hold_s + 0.15)) + 4)
|
||||
if video_hold_s > 0:
|
||||
log.info("av sync: holding video %.0f ms to match audio hop",
|
||||
video_hold_s * 1000.0)
|
||||
|
||||
async def video_loop():
|
||||
nonlocal video_frames
|
||||
loop = asyncio.get_running_loop()
|
||||
while vproc.poll() is None:
|
||||
data = await loop.run_in_executor(
|
||||
None, vproc.stdout.read, frame_bytes)
|
||||
if not data or len(data) < frame_bytes:
|
||||
if not data:
|
||||
break
|
||||
keep = b""
|
||||
while len(keep) < frame_bytes:
|
||||
more = await loop.run_in_executor(
|
||||
None, vproc.stdout.read,
|
||||
frame_bytes - len(keep))
|
||||
if not more:
|
||||
return
|
||||
keep += more
|
||||
data = keep
|
||||
now = time.monotonic()
|
||||
pending.append((now, data))
|
||||
while len(pending) > max_pending:
|
||||
pending.popleft()
|
||||
for payload in drain_held_frames(pending, now, video_hold_s):
|
||||
if shm_path:
|
||||
from frame_shm import LatestFrameReader
|
||||
shm_reader = LatestFrameReader(shm_path)
|
||||
log.info("video from shared capture shm=%s hold=0 (latest frame)",
|
||||
shm_path)
|
||||
|
||||
async def video_loop():
|
||||
nonlocal video_frames
|
||||
period = 1.0 / max(1, int(args.fps))
|
||||
last_seq = -1
|
||||
next_t = time.monotonic()
|
||||
scratch = bytearray(frame_bytes)
|
||||
while True:
|
||||
seq = shm_reader.copy_into(scratch)
|
||||
if seq is not None and seq != last_seq:
|
||||
last_seq = seq
|
||||
try:
|
||||
video_source.capture_frame(
|
||||
rtc.VideoFrame(
|
||||
args.width, args.height,
|
||||
rtc.VideoBufferType.I420, scratch),
|
||||
timestamp_us=int(time.monotonic() * 1_000_000),
|
||||
)
|
||||
video_frames += 1
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("video capture_frame failed")
|
||||
next_t += period
|
||||
delay = next_t - time.monotonic()
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
next_t = time.monotonic()
|
||||
else:
|
||||
from capture_va import video_capture_cmd
|
||||
vproc = subprocess.Popen(
|
||||
video_capture_cmd(
|
||||
video_dev, args.width, args.height, args.fps, fd=1),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
log.info("capture gst jpegdec Arc (no sidecar h264)")
|
||||
if video_hold_s > 0:
|
||||
log.info("av sync: holding video %.0f ms to match audio hop",
|
||||
video_hold_s * 1000.0)
|
||||
|
||||
async def video_loop():
|
||||
nonlocal video_frames
|
||||
assert vproc is not None
|
||||
loop = asyncio.get_running_loop()
|
||||
stdout = vproc.stdout
|
||||
assert stdout is not None
|
||||
while vproc.poll() is None:
|
||||
data = await loop.run_in_executor(
|
||||
io_ex, stdout.read, frame_bytes)
|
||||
if not data or len(data) < frame_bytes:
|
||||
if not data:
|
||||
break
|
||||
keep = b""
|
||||
while len(keep) < frame_bytes:
|
||||
more = await loop.run_in_executor(
|
||||
io_ex, stdout.read,
|
||||
frame_bytes - len(keep))
|
||||
if not more:
|
||||
return
|
||||
keep += more
|
||||
data = keep
|
||||
now = time.monotonic()
|
||||
pending.append((now, data))
|
||||
while len(pending) > max_pending:
|
||||
pending.popleft()
|
||||
for payload in drain_held_frames(pending, now, video_hold_s):
|
||||
try:
|
||||
video_source.capture_frame(
|
||||
rtc.VideoFrame(
|
||||
args.width, args.height,
|
||||
rtc.VideoBufferType.I420, payload),
|
||||
timestamp_us=int(time.monotonic() * 1_000_000),
|
||||
)
|
||||
video_frames += 1
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("video capture_frame failed")
|
||||
err = b""
|
||||
if vproc.stderr:
|
||||
try:
|
||||
video_source.capture_frame(
|
||||
rtc.VideoFrame(
|
||||
args.width, args.height,
|
||||
rtc.VideoBufferType.I420, payload),
|
||||
timestamp_us=int(time.monotonic() * 1_000_000),
|
||||
)
|
||||
video_frames += 1
|
||||
err = vproc.stderr.read() or b""
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("video capture_frame failed")
|
||||
pass
|
||||
log.error("capture gst exited rc=%s stderr=%s",
|
||||
vproc.returncode,
|
||||
err.decode("utf-8", "replace")[-800:])
|
||||
|
||||
# ---------------- audio ----------------
|
||||
aproc: subprocess.Popen | None = None
|
||||
@@ -160,9 +242,12 @@ def main() -> int:
|
||||
if audio_card is not None:
|
||||
audio_source = rtc.AudioSource(
|
||||
sample_rate=args.audio_rate, num_channels=1,
|
||||
queue_size_ms=max(250, int(args.enhance_hop_s * 1000) + 50))
|
||||
queue_size_ms=audio_queue_size_ms(
|
||||
args.enhance_hop_s, args.audio_queue_ms))
|
||||
atrack = rtc.LocalAudioTrack.create_audio_track(
|
||||
f"{identity}-audio", audio_source)
|
||||
log.info("audio queue_size_ms=%d", audio_queue_size_ms(
|
||||
args.enhance_hop_s, args.audio_queue_ms))
|
||||
|
||||
hop_s = args.enhance_hop_s
|
||||
chunk = int(hop_s * args.audio_rate) # samples per hop
|
||||
@@ -177,7 +262,7 @@ def main() -> int:
|
||||
hops = 0
|
||||
while aproc.poll() is None:
|
||||
data = await loop.run_in_executor(
|
||||
None, aproc.stdout.read, 4096)
|
||||
io_ex, aproc.stdout.read, 4096)
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
@@ -189,7 +274,7 @@ def main() -> int:
|
||||
usable = (n_frames // chunk) * chunk
|
||||
if usable:
|
||||
cleaned = await loop.run_in_executor(
|
||||
None, cleaner.process,
|
||||
enhance_ex, cleaner.process,
|
||||
pcm_stereo[:usable])
|
||||
t_cap = time.perf_counter()
|
||||
await audio_source.capture_frame(
|
||||
@@ -292,6 +377,9 @@ def main() -> int:
|
||||
for t in tasks:
|
||||
t.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for ex in split_owned:
|
||||
with contextlib.suppress(Exception):
|
||||
ex.shutdown(wait=False, cancel_futures=True)
|
||||
for p in (vproc, aproc):
|
||||
if p is not None and p.poll() is None:
|
||||
with contextlib.suppress(Exception):
|
||||
|
||||
Reference in New Issue
Block a user