livekit-cameras/tests/test_hand_raise.py

"""Hand-raise rule, latch, and yellow I420 chrome."""
import time

import numpy as np

from display_grid import FRAME_RAISE, CameraGrid, bgra_to_i420, i420_nbytes
from hand_raise import RaiseLatch, letterbox_rgb, wrist_is_raised


def _kpts() -> np.ndarray:
    k = np.zeros((17, 3), dtype=np.float32)
    k[:, 2] = 0.8
    k[0] = [0.35, 0.50, 0.9]  # nose
    k[5] = [0.50, 0.35, 0.9]  # L shoulder
    k[6] = [0.50, 0.65, 0.9]  # R shoulder
    k[7] = [0.62, 0.30, 0.9]  # L elbow
    k[8] = [0.62, 0.70, 0.9]
    k[9] = [0.70, 0.28, 0.9]  # L wrist down
    k[10] = [0.70, 0.72, 0.9]
    return k


def test_wrist_down_is_not_raised():
    assert wrist_is_raised(_kpts()) is False


def test_left_wrist_above_shoulder_is_raised():
    k = _kpts()
    k[7] = [0.32, 0.30, 0.9]  # elbow up
    k[9] = [0.18, 0.32, 0.9]  # wrist above shoulder (y 0.50)
    assert wrist_is_raised(k) is True


def test_modest_wrist_lift_is_raised():
    """Webcam framing: hand just above the shoulder still counts."""
    k = _kpts()
    k[7] = [0.46, 0.30, 0.9]
    k[9] = [0.43, 0.32, 0.9]  # 0.07 above shoulder y=0.50
    assert wrist_is_raised(k) is True


def test_wrist_on_face_rejected():
    k = _kpts()
    k[7] = [0.38, 0.48, 0.9]
    k[9] = [0.28, 0.50, 0.9]  # up, but next to the nose
    assert wrist_is_raised(k) is False


def test_wrist_above_head_without_shoulders():
    """Close-up webcam: shoulders off-frame, hand still clearly up."""
    k = np.zeros((17, 3), dtype=np.float32)
    k[0] = [0.55, 0.50, 0.85]  # nose
    k[9] = [0.18, 0.62, 0.7]   # wrist high, not on the face
    assert wrist_is_raised(k) is True


def test_low_score_not_raised():
    k = _kpts()
    k[:, 2] = 0.05
    assert wrist_is_raised(k) is False
    assert wrist_is_raised(np.zeros((17, 3), dtype=np.float32)) is False


def test_raise_latch_needs_hits_then_holds():
    latch = RaiseLatch(hits=2, hold_s=0.4, window=2)
    t = 10.0
    assert latch.update(True, t) is False
    assert latch.update(True, t + 0.1) is True
    assert latch.update(False, t + 0.2) is True
    assert latch.update(False, t + 0.55) is False


def test_raise_latch_survives_one_miss():
    latch = RaiseLatch(hits=3, hold_s=1.0, window=5)
    t = 10.0
    assert latch.update(True, t) is False
    assert latch.update(True, t + 0.1) is False
    assert latch.update(False, t + 0.2) is False
    assert latch.update(True, t + 0.3) is True
    assert latch.update(True, t + 0.4) is True
    assert latch.update(False, t + 0.5) is True


def test_letterbox_rgb_pads_16x9():
    rgb = np.full((360, 640, 3), 200, dtype=np.uint8)
    out = letterbox_rgb(rgb, 192)
    assert out.shape == (192, 192, 3)
    assert int(out[0, 96, 0]) == 0
    assert int(out[96, 96, 0]) == 200
    thunder = letterbox_rgb(rgb, 256)
    assert thunder.shape == (256, 256, 3)


def test_resolve_movenet_thunder_is_256():
    from pathlib import Path
    from hand_raise import resolve_movenet
    path, size, name = resolve_movenet("thunder")
    assert name == "thunder"
    assert size == 256
    assert path.name == "movenet_singlepose_thunder.onnx"
    path, size, name = resolve_movenet("lightning")
    assert name == "lightning" and size == 192
    assert Path(path).name == "movenet_singlepose_lightning.onnx"


def test_set_highlight_paints_yellow_i420_border():
    g = CameraGrid(width=1920, height=1080, cols=5, rows=4)
    src = np.zeros((360, 640, 4), dtype=np.uint8)
    src[:] = (40, 40, 40, 255)
    g.set_frame_i420("cam-07", bgra_to_i420(src), 640, 360)
    before = np.frombuffer(g.render_i420(), dtype=np.uint8).copy()
    g.set_highlight("cam-07", True)
    assert g.highlighted("cam-07") is True
    after = np.frombuffer(g.render_i420(), dtype=np.uint8)
    assert not np.array_equal(before, after)
    idx = g.identities.index("cam-07")
    ox, oy = g.tile_origin(idx)
    yplane = after[: g.width * g.height].reshape(g.height, g.width)
    yv, _, _ = g._yuv_raise
    # Top edge of the tile chrome should be yellow-luma, not gray live.
    assert int(yplane[oy, ox + g.tile_w // 2]) == yv
    assert int(yplane[oy - 2, ox + g.tile_w // 2]) == yv
    ix, iy, iw, _ih = g.inner_rect(idx)
    assert int(yplane[iy, ix + iw // 2]) != yv
    g.set_highlight("cam-07", False)
    assert g.highlighted("cam-07") is False
    restored = np.frombuffer(g.render_i420(), dtype=np.uint8)
    assert int(restored[: g.width * g.height].reshape(g.height, g.width)[oy, ox + g.tile_w // 2]) != yv


def test_clear_drops_yellow():
    g = CameraGrid(width=1920, height=1080, cols=5, rows=4)
    src = np.zeros((360, 640, 4), dtype=np.uint8)
    g.set_frame_i420("cam-01", bgra_to_i420(src), 640, 360)
    g.set_highlight("cam-01", True)
    g.clear("cam-01")
    assert g.highlighted("cam-01") is False
    assert len(g.render_i420()) == i420_nbytes(1920, 1080)


def test_fp32_movenet_sees_person_not_raised():
    """INT8 Xenova weights scored ~0.05 on a real person; FP32 must lock on."""
    from pathlib import Path
    from hand_raise import HandRaiseMonitor, MODEL_PATH

    fixture = Path(__file__).parent / "fixtures" / "movenet_warrior_192.npy"
    assert MODEL_PATH.name == "movenet_singlepose_lightning.onnx"
    rgb = np.load(fixture)
    mon = HandRaiseMonitor(enabled=True, model="lightning")
    assert mon.enabled
    k = mon.infer_rgb(rgb)
    assert float(k[:, 2].max()) > 0.4
    assert float(k[0, 2]) > 0.3  # nose
    assert wrist_is_raised(k) is False  # warrior: arms out, not up


def test_fp32_thunder_sees_person_not_raised():
    from pathlib import Path
    from hand_raise import HandRaiseMonitor, resolve_movenet

    path, size, name = resolve_movenet("thunder")
    assert path.is_file()
    assert size == 256 and name == "thunder"
    rgb = np.load(Path(__file__).parent / "fixtures" / "movenet_warrior_192.npy")
    mon = HandRaiseMonitor(enabled=True, model="thunder")
    assert mon.enabled
    assert mon._input_size == 256
    k = mon.infer_rgb(rgb)
    assert float(k[:, 2].max()) > 0.4
    assert float(k[0, 2]) > 0.3
    assert wrist_is_raised(k) is False