Remove VCMEncodedFrame from webrtc::EncodedFrame inheritance

Remove VCMEncodedFrame from the inheritance chain of EncodedFrames by
- moving getters for EncodedImage fields up to EncodedImage
- copying other non-deprecated fields & Methods from VCMEncodedFrame over to EncodedFrame
- Removing EncodedFrame's inheritance of VCMEncodedFrame

We leave VCMEncodedFrame as part of the (near) deprecated
VideoCodingModule code. The only place which needs to accept either is
in the generic decoder.

Bug: webrtc:9378, b:296992877
Change-Id: I60706aebbb6eacc7fd4b35ec90cc903efdbe14c8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/317160
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Auto-Submit: Tony Herre <herre@google.com>
Commit-Queue: Tony Herre <herre@google.com>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#40639}
This commit is contained in:
Tony Herre 2023-08-25 14:53:44 +02:00 committed by WebRTC LUCI CQ
parent 4bf853c7e8
commit 5f14f9e6ed
15 changed files with 195 additions and 56 deletions

View file

@ -170,7 +170,10 @@ rtc_library("encoded_frame") {
]
deps = [
"../../modules/video_coding:encoded_frame",
":encoded_image",
":video_frame",
"../../modules/rtp_rtcp:rtp_video_header",
"../../modules/video_coding:video_codec_interface",
"../units:timestamp",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]

View file

@ -1,11 +1,9 @@
specific_include_rules = {
# Until the new VideoStreamDecoder is implemented the current decoding
# pipeline will be used, and therefore EncodedFrame needs to inherit
# VCMEncodedFrame.
"encoded_frame.h": [
"+modules/video_coding/encoded_frame.h",
"encoded_frame.h" : [
"+modules/rtp_rtcp/source/rtp_video_header.h",
"+modules/video_coding/include/video_codec_interface.h",
"+modules/video_coding/include/video_coding_defines.h",
],
"encoded_image\.h" : [
"+rtc_base/ref_count.h",
],

View file

@ -11,6 +11,7 @@
#include "api/video/encoded_frame.h"
#include "absl/types/optional.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
namespace webrtc {
@ -30,4 +31,100 @@ bool EncodedFrame::delayed_by_retransmission() const {
return false;
}
void EncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) {
if (header) {
switch (header->codec) {
case kVideoCodecVP8: {
const auto& vp8_header =
absl::get<RTPVideoHeaderVP8>(header->video_type_header);
if (_codecSpecificInfo.codecType != kVideoCodecVP8) {
// This is the first packet for this frame.
_codecSpecificInfo.codecSpecific.VP8.temporalIdx = 0;
_codecSpecificInfo.codecSpecific.VP8.layerSync = false;
_codecSpecificInfo.codecSpecific.VP8.keyIdx = -1;
_codecSpecificInfo.codecType = kVideoCodecVP8;
}
_codecSpecificInfo.codecSpecific.VP8.nonReference =
vp8_header.nonReference;
if (vp8_header.temporalIdx != kNoTemporalIdx) {
_codecSpecificInfo.codecSpecific.VP8.temporalIdx =
vp8_header.temporalIdx;
_codecSpecificInfo.codecSpecific.VP8.layerSync = vp8_header.layerSync;
}
if (vp8_header.keyIdx != kNoKeyIdx) {
_codecSpecificInfo.codecSpecific.VP8.keyIdx = vp8_header.keyIdx;
}
break;
}
case kVideoCodecVP9: {
const auto& vp9_header =
absl::get<RTPVideoHeaderVP9>(header->video_type_header);
if (_codecSpecificInfo.codecType != kVideoCodecVP9) {
// This is the first packet for this frame.
_codecSpecificInfo.codecSpecific.VP9.temporal_idx = 0;
_codecSpecificInfo.codecSpecific.VP9.gof_idx = 0;
_codecSpecificInfo.codecSpecific.VP9.inter_layer_predicted = false;
_codecSpecificInfo.codecType = kVideoCodecVP9;
}
_codecSpecificInfo.codecSpecific.VP9.inter_pic_predicted =
vp9_header.inter_pic_predicted;
_codecSpecificInfo.codecSpecific.VP9.flexible_mode =
vp9_header.flexible_mode;
_codecSpecificInfo.codecSpecific.VP9.num_ref_pics =
vp9_header.num_ref_pics;
for (uint8_t r = 0; r < vp9_header.num_ref_pics; ++r) {
_codecSpecificInfo.codecSpecific.VP9.p_diff[r] =
vp9_header.pid_diff[r];
}
_codecSpecificInfo.codecSpecific.VP9.ss_data_available =
vp9_header.ss_data_available;
if (vp9_header.temporal_idx != kNoTemporalIdx) {
_codecSpecificInfo.codecSpecific.VP9.temporal_idx =
vp9_header.temporal_idx;
_codecSpecificInfo.codecSpecific.VP9.temporal_up_switch =
vp9_header.temporal_up_switch;
}
if (vp9_header.spatial_idx != kNoSpatialIdx) {
_codecSpecificInfo.codecSpecific.VP9.inter_layer_predicted =
vp9_header.inter_layer_predicted;
SetSpatialIndex(vp9_header.spatial_idx);
}
if (vp9_header.gof_idx != kNoGofIdx) {
_codecSpecificInfo.codecSpecific.VP9.gof_idx = vp9_header.gof_idx;
}
if (vp9_header.ss_data_available) {
_codecSpecificInfo.codecSpecific.VP9.num_spatial_layers =
vp9_header.num_spatial_layers;
_codecSpecificInfo.codecSpecific.VP9
.spatial_layer_resolution_present =
vp9_header.spatial_layer_resolution_present;
if (vp9_header.spatial_layer_resolution_present) {
for (size_t i = 0; i < vp9_header.num_spatial_layers; ++i) {
_codecSpecificInfo.codecSpecific.VP9.width[i] =
vp9_header.width[i];
_codecSpecificInfo.codecSpecific.VP9.height[i] =
vp9_header.height[i];
}
}
_codecSpecificInfo.codecSpecific.VP9.gof.CopyGofInfoVP9(
vp9_header.gof);
}
break;
}
case kVideoCodecH264: {
_codecSpecificInfo.codecType = kVideoCodecH264;
break;
}
case kVideoCodecAV1: {
_codecSpecificInfo.codecType = kVideoCodecAV1;
break;
}
default: {
_codecSpecificInfo.codecType = kVideoCodecGeneric;
break;
}
}
}
}
} // namespace webrtc

View file

@ -16,14 +16,17 @@
#include "absl/types/optional.h"
#include "api/units/timestamp.h"
#include "modules/video_coding/encoded_frame.h"
#include "api/video/encoded_image.h"
#include "api/video/video_codec_type.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_coding_defines.h"
namespace webrtc {
// TODO(philipel): Remove webrtc::VCMEncodedFrame inheritance.
// TODO(philipel): Move transport specific info out of EncodedFrame.
// NOTE: This class is still under development and may change without notice.
class EncodedFrame : public webrtc::VCMEncodedFrame {
class EncodedFrame : public EncodedImage {
public:
static const uint8_t kMaxFrameReferences = 5;
@ -33,14 +36,16 @@ class EncodedFrame : public webrtc::VCMEncodedFrame {
// When this frame was received.
// TODO(bugs.webrtc.org/13756): Use Timestamp instead of int.
virtual int64_t ReceivedTime() const = 0;
virtual int64_t ReceivedTime() const { return -1; }
// Returns a Timestamp from `ReceivedTime`, or nullopt if there is no receive
// time.
absl::optional<webrtc::Timestamp> ReceivedTimestamp() const;
// When this frame should be rendered.
// TODO(bugs.webrtc.org/13756): Use Timestamp instead of int.
virtual int64_t RenderTime() const = 0;
virtual int64_t RenderTime() const { return _renderTimeMs; }
// TODO(bugs.webrtc.org/13756): Migrate to ReceivedTimestamp.
int64_t RenderTimeMs() const { return _renderTimeMs; }
// Returns a Timestamp from `RenderTime`, or nullopt if there is no
// render time.
absl::optional<webrtc::Timestamp> RenderTimestamp() const;
@ -55,6 +60,23 @@ class EncodedFrame : public webrtc::VCMEncodedFrame {
void SetId(int64_t id) { id_ = id; }
int64_t Id() const { return id_; }
uint8_t PayloadType() const { return _payloadType; }
bool MissingFrame() const { return _missingFrame; }
void SetRenderTime(const int64_t renderTimeMs) {
_renderTimeMs = renderTimeMs;
}
const webrtc::EncodedImage& EncodedImage() const {
return static_cast<const webrtc::EncodedImage&>(*this);
}
const CodecSpecificInfo* CodecSpecific() const { return &_codecSpecificInfo; }
void SetCodecSpecific(const CodecSpecificInfo* codec_specific) {
_codecSpecificInfo = *codec_specific;
}
// TODO(philipel): Add simple modify/access functions to prevent adding too
// many `references`.
size_t num_references = 0;
@ -63,6 +85,19 @@ class EncodedFrame : public webrtc::VCMEncodedFrame {
// mean that the last packet has a marker bit set).
bool is_last_spatial_layer = true;
protected:
// TODO(https://bugs.webrtc.org/9378): Move RTP specifics down into a
// transport-aware subclass, eg RtpFrameObject.
void CopyCodecSpecific(const RTPVideoHeader* header);
// TODO(https://bugs.webrtc.org/9378): Make fields private with
// getters/setters as needed.
int64_t _renderTimeMs = -1;
uint8_t _payloadType = 0;
bool _missingFrame = false;
CodecSpecificInfo _codecSpecificInfo;
VideoCodecType _codec = kVideoCodecGeneric;
private:
// The ID of the frame is determined from RTP level information. The IDs are
// used to describe order and dependencies between frames.

View file

@ -201,6 +201,14 @@ class RTC_EXPORT EncodedImage {
at_target_quality_ = at_target_quality;
}
webrtc::VideoFrameType FrameType() const { return _frameType; }
void SetFrameType(webrtc::VideoFrameType frame_type) {
_frameType = frame_type;
}
VideoContentType contentType() const { return content_type_; }
VideoRotation rotation() const { return rotation_; }
uint32_t _encodedWidth = 0;
uint32_t _encodedHeight = 0;
// NTP time of the capture time in local timebase in milliseconds.
@ -223,6 +231,8 @@ class RTC_EXPORT EncodedImage {
int64_t receive_start_ms = 0;
int64_t receive_finish_ms = 0;
} timing_;
EncodedImage::Timing video_timing() const { return timing_; }
EncodedImage::Timing* video_timing_mutable() { return &timing_; }
private:
size_t capacity() const { return encoded_data_ ? encoded_data_->size() : 0; }

View file

@ -86,7 +86,7 @@ void VCMDecoderDatabase::DeregisterReceiveCodecs() {
}
VCMGenericDecoder* VCMDecoderDatabase::GetDecoder(
const VCMEncodedFrame& frame,
const EncodedFrame& frame,
VCMDecodedFrameCallback* decoded_frame_callback) {
RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
RTC_DCHECK(decoded_frame_callback->UserReceiveCallback());
@ -117,7 +117,7 @@ VCMGenericDecoder* VCMDecoderDatabase::GetDecoder(
return &*current_decoder_;
}
void VCMDecoderDatabase::CreateAndInitDecoder(const VCMEncodedFrame& frame) {
void VCMDecoderDatabase::CreateAndInitDecoder(const EncodedFrame& frame) {
uint8_t payload_type = frame.PayloadType();
RTC_DLOG(LS_INFO) << "Initializing decoder with payload type '"
<< int{payload_type} << "'.";

View file

@ -18,8 +18,8 @@
#include "absl/types/optional.h"
#include "api/sequence_checker.h"
#include "api/video/encoded_frame.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/encoded_frame.h"
#include "modules/video_coding/generic_decoder.h"
namespace webrtc {
@ -49,11 +49,11 @@ class VCMDecoderDatabase {
// nullptr is returned if no decoder with the specified payload type was found
// and the function failed to create one.
VCMGenericDecoder* GetDecoder(
const VCMEncodedFrame& frame,
const EncodedFrame& frame,
VCMDecodedFrameCallback* decoded_frame_callback);
private:
void CreateAndInitDecoder(const VCMEncodedFrame& frame)
void CreateAndInitDecoder(const EncodedFrame& frame)
RTC_RUN_ON(decoder_sequence_checker_);
SequenceChecker decoder_sequence_checker_;

View file

@ -62,29 +62,6 @@ class RTC_EXPORT VCMEncodedFrame : public EncodedImage {
* Get render time in milliseconds
*/
int64_t RenderTimeMs() const { return _renderTimeMs; }
/**
* Get frame type
*/
webrtc::VideoFrameType FrameType() const { return _frameType; }
/**
* Set frame type
*/
void SetFrameType(webrtc::VideoFrameType frame_type) {
_frameType = frame_type;
}
/**
* Get frame rotation
*/
VideoRotation rotation() const { return rotation_; }
/**
* Get video content type
*/
VideoContentType contentType() const { return content_type_; }
/**
* Get video timing
*/
EncodedImage::Timing video_timing() const { return timing_; }
EncodedImage::Timing* video_timing_mutable() { return &timing_; }
/**
* True if there's a frame missing before this frame
*/

View file

@ -281,19 +281,30 @@ bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
return ok;
}
int32_t VCMGenericDecoder::Decode(const EncodedFrame& frame, Timestamp now) {
return Decode(frame, now, frame.RenderTimeMs(), frame.MissingFrame());
}
int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
return Decode(frame, now, frame.RenderTimeMs(), frame.MissingFrame());
}
int32_t VCMGenericDecoder::Decode(const EncodedImage& frame,
Timestamp now,
int64_t render_time_ms,
int64_t missing_frame) {
TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
frame.Timestamp());
FrameInfo frame_info;
frame_info.rtp_timestamp = frame.Timestamp();
frame_info.decode_start = now;
frame_info.render_time =
frame.RenderTimeMs() >= 0
? absl::make_optional(Timestamp::Millis(frame.RenderTimeMs()))
render_time_ms >= 0
? absl::make_optional(Timestamp::Millis(render_time_ms))
: absl::nullopt;
frame_info.rotation = frame.rotation();
frame_info.timing = frame.video_timing();
frame_info.ntp_time_ms = frame.EncodedImage().ntp_time_ms_;
frame_info.ntp_time_ms = frame.ntp_time_ms_;
frame_info.packet_infos = frame.PacketInfos();
// Set correctly only for key frames. Thus, use latest key frame
@ -308,8 +319,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
frame_info.frame_type = frame.FrameType();
_callback->Map(std::move(frame_info));
int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
frame.RenderTimeMs());
int32_t ret = decoder_->Decode(frame, missing_frame, render_time_ms);
VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo();
if (decoder_info != decoder_info_) {
RTC_LOG(LS_INFO) << "Changed decoder implementation to: "

View file

@ -18,6 +18,7 @@
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/video/encoded_frame.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/encoded_frame.h"
#include "modules/video_coding/timing/timing.h"
@ -102,7 +103,10 @@ class VCMGenericDecoder {
*
* inputVideoBuffer reference to encoded video frame
*/
// TODO(https://bugs.webrtc.org/9378): Remove VCMEncodedFrame variant
// once the usage from code in deprecated/ is gone.
int32_t Decode(const VCMEncodedFrame& inputFrame, Timestamp now);
int32_t Decode(const EncodedFrame& inputFrame, Timestamp now);
/**
* Set decode callback. Deregistering while decoding is illegal.
@ -114,6 +118,10 @@ class VCMGenericDecoder {
}
private:
int32_t Decode(const EncodedImage& frame,
Timestamp now,
int64_t render_time_ms,
int64_t missing_frame);
VCMDecodedFrameCallback* _callback = nullptr;
VideoDecoder* const decoder_;
VideoContentType _last_keyframe_content_type;

View file

@ -94,7 +94,7 @@ class GenericDecoderTest : public ::testing::Test {
TEST_F(GenericDecoderTest, PassesPacketInfos) {
RtpPacketInfos packet_infos = CreatePacketInfos(3);
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
encoded_frame.SetPacketInfos(packet_infos);
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
time_controller_.AdvanceTime(TimeDelta::Millis(10));
@ -107,7 +107,7 @@ TEST_F(GenericDecoderTest, FrameDroppedIfTooManyFramesInFlight) {
constexpr int kMaxFramesInFlight = 10;
decoder_.SetDelayedDecoding(10);
for (int i = 0; i < kMaxFramesInFlight + 1; ++i) {
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
encoded_frame.SetTimestamp(90000 * i);
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
}
@ -128,7 +128,7 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) {
{
// Ensure the original frame is destroyed before the decoding is completed.
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
encoded_frame.SetPacketInfos(packet_infos);
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
}
@ -140,7 +140,7 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) {
}
TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) {
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
time_controller_.AdvanceTime(TimeDelta::Millis(10));
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
@ -151,7 +151,7 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) {
}
TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
// VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay
// is specified as X,Y, where X=0, Y>0.
constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
@ -167,7 +167,7 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
}
TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) {
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
generic_decoder_.Decode(encoded_frame, clock_->CurrentTime());
time_controller_.AdvanceTime(TimeDelta::Millis(10));
absl::optional<VideoFrame> decoded_frame = user_callback_.PopLastFrame();
@ -176,7 +176,7 @@ TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) {
}
TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) {
VCMEncodedFrame encoded_frame;
EncodedFrame encoded_frame;
const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Zero(),
TimeDelta::Millis(50));
timing_.set_min_playout_delay(kPlayoutDelay.min());

View file

@ -20,7 +20,6 @@
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/decoder_database.h"
#include "modules/video_coding/encoded_frame.h"
#include "modules/video_coding/generic_decoder.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "modules/video_coding/timing/timing.h"
@ -74,7 +73,7 @@ bool VideoReceiver2::IsExternalDecoderRegistered(uint8_t payload_type) const {
}
// Must be called from inside the receive side critical section.
int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) {
int32_t VideoReceiver2::Decode(const EncodedFrame* frame) {
RTC_DCHECK_RUN_ON(&decoder_sequence_checker_);
TRACE_EVENT0("webrtc", "VideoReceiver2::Decode");
// Change decoder if payload type has changed.

View file

@ -16,9 +16,9 @@
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/video/encoded_frame.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/decoder_database.h"
#include "modules/video_coding/encoded_frame.h"
#include "modules/video_coding/generic_decoder.h"
#include "modules/video_coding/timing/timing.h"
#include "rtc_base/system/no_unique_address.h"
@ -49,7 +49,7 @@ class VideoReceiver2 {
bool IsExternalDecoderRegistered(uint8_t payload_type) const;
int32_t RegisterReceiveCallback(VCMReceiveCallback* receive_callback);
int32_t Decode(const VCMEncodedFrame* frame);
int32_t Decode(const EncodedFrame* frame);
private:
RTC_NO_UNIQUE_ADDRESS SequenceChecker construction_sequence_checker_;

View file

@ -277,6 +277,7 @@ rtc_library("video_stream_buffer_controller") {
"../api/video:video_rtp_headers",
"../modules/video_coding",
"../modules/video_coding:frame_helpers",
"../modules/video_coding:video_codec_interface",
"../modules/video_coding/timing:inter_frame_delay_variation_calculator",
"../modules/video_coding/timing:jitter_estimator",
"../modules/video_coding/timing:timing_module",

View file

@ -17,6 +17,7 @@
#include "api/task_queue/task_queue_base.h"
#include "api/video/encoded_frame.h"
#include "api/video/frame_buffer.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h"
#include "modules/video_coding/timing/jitter_estimator.h"
#include "modules/video_coding/timing/timing.h"