diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs index ee1410a..b2ff9ca 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -88,7 +88,9 @@ impl FfiVideoSource { buffer, }; - source.capture_frame(&frame); + // false = adapter / in-flight cap dropped the frame (do not + // treat as an FFI error; the source is still valid). + let _forwarded = source.capture_frame(&frame); } _ => {} } diff --git a/webrtc-sys/include/livekit/video_track.h b/webrtc-sys/include/livekit/video_track.h index 4146c45..733d110 100644 --- a/webrtc-sys/include/livekit/video_track.h +++ b/webrtc-sys/include/livekit/video_track.h @@ -18,6 +18,7 @@ #include #include +#include #include "api/media_stream_interface.h" #include "api/video/video_frame.h" @@ -127,6 +128,10 @@ class VideoTrackSource { std::shared_ptr rate_control_state_ = std::make_shared(); bool is_screencast_; + int64_t last_forwarded_us_ = 0; + // Latest-wins: at most one OnFrame in flight; new captures overwrite. + bool forwarding_ = false; + std::optional pending_frame_; }; public: diff --git a/webrtc-sys/src/vaapi/h264_encoder_impl.cpp b/webrtc-sys/src/vaapi/h264_encoder_impl.cpp index 5a3b76e..8c7ce11 100644 --- a/webrtc-sys/src/vaapi/h264_encoder_impl.cpp +++ b/webrtc-sys/src/vaapi/h264_encoder_impl.cpp @@ -189,18 +189,6 @@ int32_t VAAPIH264EncoderWrapper::Encode( return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - webrtc::scoped_refptr frame_buffer = - input_frame.video_frame_buffer()->ToI420(); - if (!frame_buffer) { - RTC_LOG(LS_ERROR) << "Failed to convert " - << VideoFrameBufferTypeToString( - input_frame.video_frame_buffer()->type()) - << " image to I420. Can't encode frame."; - return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; - } - RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 || - frame_buffer->type() == VideoFrameBuffer::Type::kI420A); - bool is_keyframe_needed = false; if (configuration_.key_frame_request && configuration_.sending) { is_keyframe_needed = true; @@ -214,20 +202,67 @@ int32_t VAAPIH264EncoderWrapper::Encode( configuration_.key_frame_request = false; } - RTC_DCHECK_EQ(configuration_.width, frame_buffer->width()); - RTC_DCHECK_EQ(configuration_.height, frame_buffer->height()); - if (!configuration_.sending) { return WEBRTC_VIDEO_CODEC_NO_OUTPUT; } if (frame_types != nullptr) { - // Skip frame? if ((*frame_types)[0] == VideoFrameType::kEmptyFrame) { return WEBRTC_VIDEO_CODEC_NO_OUTPUT; } } + // Drop stale/burst delta frames BEFORE ToI420. + // 1) Age: capture older than 1 frame period (was 3; that never fired). + // 2) Burst: Encode called closer than 8 ms = EncoderQueue draining. + // Mirror: livekit-cameras latency.should_drop_stale_vaapi_frame / + // should_drop_encoder_queue_depth. + const int fps = std::max( + 1, static_cast(configuration_.max_frame_rate > 0.0f + ? configuration_.max_frame_rate + : codec_.maxFramerate)); + const int64_t now_us = webrtc::TimeMicros(); + const int64_t capture_us = input_frame.timestamp_us(); + const int64_t delay_us = (capture_us > 0) ? (now_us - capture_us) : 0; + encode_count_++; + if (encode_count_ % 90 == 1) { + RTC_LOG(LS_WARNING) << "VAAPI encode sample delay_ms=" << delay_us / 1000 + << " now_us=" << now_us << " capture_us=" << capture_us + << " fps=" << fps << " n=" << encode_count_; + } + if (!send_key_frame) { + const int64_t max_delay_us = 1000000 / fps; // 1 frame + const bool stale = + capture_us > 0 && now_us - capture_us > max_delay_us; + const bool burst = + last_encode_us_ != 0 && now_us - last_encode_us_ < 8000; + if (stale || burst) { + RTC_LOG(LS_WARNING) << "VAAPI H264 drop " + << (stale ? "stale" : "burst") + << " frame delay_ms=" << delay_us / 1000 + << " dt_ms=" << (now_us - last_encode_us_) / 1000 + << " fps=" << fps; + last_encode_us_ = now_us; + return WEBRTC_VIDEO_CODEC_OK; + } + } + last_encode_us_ = now_us; + + webrtc::scoped_refptr frame_buffer = + input_frame.video_frame_buffer()->ToI420(); + if (!frame_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString( + input_frame.video_frame_buffer()->type()) + << " image to I420. Can't encode frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 || + frame_buffer->type() == VideoFrameBuffer::Type::kI420A); + + RTC_DCHECK_EQ(configuration_.width, frame_buffer->width()); + RTC_DCHECK_EQ(configuration_.height, frame_buffer->height()); + std::vector output; encoder_->Encode(VA_FOURCC_I420, frame_buffer->DataY(), frame_buffer->DataU(), frame_buffer->DataV(), send_key_frame, output); @@ -267,6 +302,7 @@ VideoEncoder::EncoderInfo VAAPIH264EncoderWrapper::GetEncoderInfo() const { info.implementation_name = "VAAPI H264 Encoder"; info.scaling_settings = VideoEncoder::ScalingSettings::kOff; info.is_hardware_accelerated = true; + info.has_trusted_rate_controller = false; info.supports_simulcast = false; info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; return info; diff --git a/webrtc-sys/src/vaapi/h264_encoder_impl.h b/webrtc-sys/src/vaapi/h264_encoder_impl.h index dc26355..bce1681 100644 --- a/webrtc-sys/src/vaapi/h264_encoder_impl.h +++ b/webrtc-sys/src/vaapi/h264_encoder_impl.h @@ -76,6 +76,8 @@ class VAAPIH264EncoderWrapper : public VideoEncoder { const SdpVideoFormat format_; H264Profile profile_ = H264Profile::kProfileConstrainedBaseline; H264Level level_ = H264Level::kLevel1_b; + int64_t last_encode_us_ = 0; + int encode_count_ = 0; }; } // namespace webrtc diff --git a/webrtc-sys/src/video_track.cpp b/webrtc-sys/src/video_track.cpp index 92410b9..89e6791 100644 --- a/webrtc-sys/src/video_track.cpp +++ b/webrtc-sys/src/video_track.cpp @@ -165,6 +165,7 @@ VideoResolution VideoTrackSource::InternalSource::video_resolution() const { bool VideoTrackSource::InternalSource::on_captured_frame( const webrtc::VideoFrame& frame, const FrameMetadata& frame_metadata) { + { webrtc::MutexLock lock(&mutex_); int64_t aligned_timestamp_us = timestamp_aligner_.TranslateTimestamp( @@ -235,13 +236,37 @@ bool VideoTrackSource::InternalSource::on_captured_frame( frame_metadata.has_packet_trailer ? frame_metadata.frame_id : 0); } - OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_rotation(rotation) - .set_timestamp_us(aligned_timestamp_us) - .build()); + webrtc::VideoFrame built = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(rotation) + .set_timestamp_us(aligned_timestamp_us) + .build(); + + // Latest-wins depth cap (max_queued=1). A capture that arrives while + // OnFrame is in flight overwrites pending_frame_ instead of pushing + // another I420 onto EncoderQueue. + // Mirror: livekit-cameras latency.should_drop_encoder_queue_depth. + pending_frame_ = std::move(built); + if (forwarding_) { + return true; + } + forwarding_ = true; + last_forwarded_us_ = aligned_timestamp_us; + } - return true; + for (;;) { + std::optional to_send; + { + webrtc::MutexLock lock(&mutex_); + if (!pending_frame_) { + forwarding_ = false; + return true; + } + to_send = std::move(pending_frame_); + pending_frame_.reset(); + } + OnFrame(*to_send); + } } void VideoTrackSource::InternalSource::set_packet_trailer_handler(