mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-14 14:20:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Moved FrameKey to api/video/encoded_frame.h and renamed it to VideoLayerFrameId.

Browse source 
Since we want the VideoStreamDecoder to callback with the last
continuous frame we need to move the FrameKey into the public API.

Bug: webrtc:8909
Change-Id: I39634145d848b8163778e31a1e0d04d91f9bbeb8
Reviewed-on: https://webrtc-review.googlesource.com/60864
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22495}
This commit is contained in:
philipel 2018-03-19 15:34:53 +01:00 committed by Commit Bot
parent 9c1ee368e0
commit 0fa82a60e9
9 changed files with 121 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
api/video/encoded_frame.h
View file

@ -16,7 +16,39 @@
namespace webrtc { namespace webrtc {
namespace video_coding { namespace video_coding {
// NOTE: This class is still under development and may change without notice.
struct VideoLayerFrameId {
// TODO(philipel): The default ctor is currently used internaly, but have a
// look if we can remove it.
VideoLayerFrameId() : picture_id(-1), spatial_layer(0) {}
VideoLayerFrameId(int64_t picture_id, uint8_t spatial_layer)
: picture_id(picture_id), spatial_layer(spatial_layer) {}
bool operator==(const VideoLayerFrameId& rhs) const {
return picture_id == rhs.picture_id && spatial_layer == rhs.spatial_layer;
}
bool operator!=(const VideoLayerFrameId& rhs) const {
return !(*this == rhs);
}
bool operator<(const VideoLayerFrameId& rhs) const {
if (picture_id == rhs.picture_id)
return spatial_layer < rhs.spatial_layer;
return picture_id < rhs.picture_id;
}
bool operator<=(const VideoLayerFrameId& rhs) const { return !(rhs < *this); }
bool operator>(const VideoLayerFrameId& rhs) const { return rhs < *this; }
bool operator>=(const VideoLayerFrameId& rhs) const { return rhs <= *this; }
int64_t picture_id;
uint8_t spatial_layer;
};
// TODO(philipel): Remove webrtc::VCMEncodedFrame inheritance. // TODO(philipel): Remove webrtc::VCMEncodedFrame inheritance.
// TODO(philipel): Move transport specific info out of EncodedFrame.
// NOTE: This class is still under development and may change without notice.
class EncodedFrame : public webrtc::VCMEncodedFrame { class EncodedFrame : public webrtc::VCMEncodedFrame {
public: public:
static const uint8_t kMaxFrameReferences = 5; static const uint8_t kMaxFrameReferences = 5;
@ -44,11 +76,11 @@ class EncodedFrame : public webrtc::VCMEncodedFrame {
bool is_keyframe() const { return num_references == 0; } bool is_keyframe() const { return num_references == 0; }
// The tuple (|picture_id|, |spatial_layer|) uniquely identifies a frame VideoLayerFrameId id;
// object. For codec types that don't necessarily have picture ids they // TODO(philipel): Remove the two references below when downstream projects
// have to be constructed from the header data relevant to that codec. // have been updated.
int64_t picture_id = 0; int64_t& picture_id = id.picture_id;
uint8_t spatial_layer = 0; uint8_t& spatial_layer = id.spatial_layer;
uint32_t timestamp = 0; uint32_t timestamp = 0;
// TODO(philipel): Add simple modify/access functions to prevent adding too // TODO(philipel): Add simple modify/access functions to prevent adding too

60
modules/video_coding/frame_buffer2.cc
View file

@ -172,8 +172,9 @@ FrameBuffer::ReturnReason FrameBuffer::NextFrame(
// Sanity check for RTP timestamp monotonicity. // Sanity check for RTP timestamp monotonicity.
if (last_decoded_frame_it_ != frames_.end()) { if (last_decoded_frame_it_ != frames_.end()) {
const FrameKey& last_decoded_frame_key = last_decoded_frame_it_->first; const VideoLayerFrameId& last_decoded_frame_key =
const FrameKey& frame_key = next_frame_it_->first; last_decoded_frame_it_->first;
const VideoLayerFrameId& frame_key = next_frame_it_->first;
const bool frame_is_higher_spatial_layer_of_last_decoded_frame = const bool frame_is_higher_spatial_layer_of_last_decoded_frame =
last_decoded_frame_timestamp_ == frame->timestamp && last_decoded_frame_timestamp_ == frame->timestamp &&
@ -186,8 +187,8 @@ FrameBuffer::ReturnReason FrameBuffer::NextFrame(
// these conditions. // these conditions.
RTC_LOG(LS_WARNING) RTC_LOG(LS_WARNING)
<< "Frame with (timestamp:picture_id:spatial_id) (" << "Frame with (timestamp:picture_id:spatial_id) ("
<< frame->timestamp << ":" << frame->picture_id << ":" << frame->timestamp << ":" << frame->id.picture_id << ":"
<< static_cast<int>(frame->spatial_layer) << ")" << static_cast<int>(frame->id.spatial_layer) << ")"
<< " sent to decoder after frame with" << " sent to decoder after frame with"
<< " (timestamp:picture_id:spatial_id) (" << " (timestamp:picture_id:spatial_id) ("
<< last_decoded_frame_timestamp_ << ":" << last_decoded_frame_timestamp_ << ":"
@ -263,11 +264,11 @@ void FrameBuffer::UpdateRtt(int64_t rtt_ms) {
} }
bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const {
if (frame.picture_id < 0) if (frame.id.picture_id < 0)
return false; return false;
for (size_t i = 0; i < frame.num_references; ++i) { for (size_t i = 0; i < frame.num_references; ++i) {
if (frame.references[i] < 0 || frame.references[i] >= frame.picture_id) if (frame.references[i] < 0 || frame.references[i] >= frame.id.picture_id)
return false; return false;
for (size_t j = i + 1; j < frame.num_references; ++j) { for (size_t j = i + 1; j < frame.num_references; ++j) {
@ -276,7 +277,7 @@ bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const {
} }
} }
if (frame.inter_layer_predicted && frame.spatial_layer == 0) if (frame.inter_layer_predicted && frame.id.spatial_layer == 0)
return false; return false;
return true; return true;
@ -301,7 +302,7 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
if (stats_callback_) if (stats_callback_)
stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(), stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
frame->contentType()); frame->contentType());
FrameKey key(frame->picture_id, frame->spatial_layer); const VideoLayerFrameId& id = frame->id;
rtc::CritScope lock(&crit_); rtc::CritScope lock(&crit_);
@ -312,8 +313,8 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
if (!ValidReferences(*frame)) { if (!ValidReferences(*frame)) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":" << id.picture_id << ":"
<< static_cast<int>(key.spatial_layer) << static_cast<int>(id.spatial_layer)
<< ") has invalid frame references, dropping frame."; << ") has invalid frame references, dropping frame.";
return last_continuous_picture_id; return last_continuous_picture_id;
} }
@ -321,15 +322,15 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
if (num_frames_buffered_ >= kMaxFramesBuffered) { if (num_frames_buffered_ >= kMaxFramesBuffered) {
if (frame->is_keyframe()) { if (frame->is_keyframe()) {
RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) (" RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) ("
<< key.picture_id << ":" << id.picture_id << ":"
<< static_cast<int>(key.spatial_layer) << static_cast<int>(id.spatial_layer)
<< ") but buffer is full, clearing" << ") but buffer is full, clearing"
<< " buffer and inserting the frame."; << " buffer and inserting the frame.";
ClearFramesAndHistory(); ClearFramesAndHistory();
} else { } else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":" << id.picture_id << ":"
<< static_cast<int>(key.spatial_layer) << static_cast<int>(id.spatial_layer)
<< ") could not be inserted due to the frame " << ") could not be inserted due to the frame "
<< "buffer being full, dropping frame."; << "buffer being full, dropping frame.";
return last_continuous_picture_id; return last_continuous_picture_id;
@ -337,7 +338,7 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
} }
if (last_decoded_frame_it_ != frames_.end() && if (last_decoded_frame_it_ != frames_.end() &&
key <= last_decoded_frame_it_->first) { id <= last_decoded_frame_it_->first) {
if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) && if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) &&
frame->is_keyframe()) { frame->is_keyframe()) {
// If this frame has a newer timestamp but an earlier picture id then we // If this frame has a newer timestamp but an earlier picture id then we
@ -351,8 +352,8 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
last_continuous_picture_id = -1; last_continuous_picture_id = -1;
} else { } else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":" << id.picture_id << ":"
<< static_cast<int>(key.spatial_layer) << static_cast<int>(id.spatial_layer)
<< ") inserted after frame (" << ") inserted after frame ("
<< last_decoded_frame_it_->first.picture_id << ":" << last_decoded_frame_it_->first.picture_id << ":"
<< static_cast<int>( << static_cast<int>(
@ -365,21 +366,20 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
// Test if inserting this frame would cause the order of the frames to become // Test if inserting this frame would cause the order of the frames to become
// ambiguous (covering more than half the interval of 2^16). This can happen // ambiguous (covering more than half the interval of 2^16). This can happen
// when the picture id make large jumps mid stream. // when the picture id make large jumps mid stream.
if (!frames_.empty() && if (!frames_.empty() && id < frames_.begin()->first &&
key < frames_.begin()->first && frames_.rbegin()->first < id) {
frames_.rbegin()->first < key) {
RTC_LOG(LS_WARNING) RTC_LOG(LS_WARNING)
<< "A jump in picture id was detected, clearing buffer."; << "A jump in picture id was detected, clearing buffer.";
ClearFramesAndHistory(); ClearFramesAndHistory();
last_continuous_picture_id = -1; last_continuous_picture_id = -1;
} }
auto info = frames_.insert(std::make_pair(key, FrameInfo())).first; auto info = frames_.emplace(id, FrameInfo()).first;
if (info->second.frame) { if (info->second.frame) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":" << id.picture_id << ":"
<< static_cast<int>(key.spatial_layer) << static_cast<int>(id.spatial_layer)
<< ") already inserted, dropping frame."; << ") already inserted, dropping frame.";
return last_continuous_picture_id; return last_continuous_picture_id;
} }
@ -482,7 +482,7 @@ void FrameBuffer::AdvanceLastDecodedFrame(FrameMap::iterator decoded) {
bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
FrameMap::iterator info) { FrameMap::iterator info) {
TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame"); TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
FrameKey key(frame.picture_id, frame.spatial_layer); const VideoLayerFrameId& id = frame.id;
info->second.num_missing_continuous = frame.num_references; info->second.num_missing_continuous = frame.num_references;
info->second.num_missing_decodable = frame.num_references; info->second.num_missing_decodable = frame.num_references;
@ -491,7 +491,7 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
// Check how many dependencies that have already been fulfilled. // Check how many dependencies that have already been fulfilled.
for (size_t i = 0; i < frame.num_references; ++i) { for (size_t i = 0; i < frame.num_references; ++i) {
FrameKey ref_key(frame.references[i], frame.spatial_layer); VideoLayerFrameId ref_key(frame.references[i], frame.id.spatial_layer);
auto ref_info = frames_.find(ref_key); auto ref_info = frames_.find(ref_key);
// Does |frame| depend on a frame earlier than the last decoded frame? // Does |frame| depend on a frame earlier than the last decoded frame?
@ -501,8 +501,8 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
int64_t now_ms = clock_->TimeInMilliseconds(); int64_t now_ms = clock_->TimeInMilliseconds();
if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) { if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
RTC_LOG(LS_WARNING) RTC_LOG(LS_WARNING)
<< "Frame with (picture_id:spatial_id) (" << key.picture_id << ":" << "Frame with (picture_id:spatial_id) (" << id.picture_id << ":"
<< static_cast<int>(key.spatial_layer) << static_cast<int>(id.spatial_layer)
<< ") depends on a non-decoded frame more previous than" << ") depends on a non-decoded frame more previous than"
<< " the last decoded frame, dropping frame."; << " the last decoded frame, dropping frame.";
last_log_non_decoded_ms_ = now_ms; last_log_non_decoded_ms_ = now_ms;
@ -522,7 +522,7 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
// Add backwards reference so |frame| can be updated when new // Add backwards reference so |frame| can be updated when new
// frames are inserted or decoded. // frames are inserted or decoded.
ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] = ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
key; id;
RTC_DCHECK_LT(ref_info->second.num_dependent_frames, RTC_DCHECK_LT(ref_info->second.num_dependent_frames,
(FrameInfo::kMaxNumDependentFrames - 1)); (FrameInfo::kMaxNumDependentFrames - 1));
// TODO(philipel): Look into why this could happen and handle // TODO(philipel): Look into why this could happen and handle
@ -541,7 +541,7 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
++info->second.num_missing_continuous; ++info->second.num_missing_continuous;
++info->second.num_missing_decodable; ++info->second.num_missing_decodable;
FrameKey ref_key(frame.picture_id, frame.spatial_layer - 1); VideoLayerFrameId ref_key(frame.id.picture_id, frame.id.spatial_layer - 1);
// Gets or create the FrameInfo for the referenced frame. // Gets or create the FrameInfo for the referenced frame.
auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first; auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
if (ref_info->second.continuous) if (ref_info->second.continuous)
@ -551,7 +551,7 @@ bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
--info->second.num_missing_decodable; --info->second.num_missing_decodable;
} else { } else {
ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] = ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
key; id;
++ref_info->second.num_dependent_frames; ++ref_info->second.num_dependent_frames;
} }
RTC_DCHECK_LE(ref_info->second.num_missing_continuous, RTC_DCHECK_LE(ref_info->second.num_missing_continuous,

21
modules/video_coding/frame_buffer2.h
View file

@ -78,23 +78,6 @@ class FrameBuffer {
void UpdateRtt(int64_t rtt_ms); void UpdateRtt(int64_t rtt_ms);
private: private:
struct FrameKey {
FrameKey() : picture_id(-1), spatial_layer(0) {}
FrameKey(int64_t picture_id, uint8_t spatial_layer)
: picture_id(picture_id), spatial_layer(spatial_layer) {}
bool operator<(const FrameKey& rhs) const {
if (picture_id == rhs.picture_id)
return spatial_layer < rhs.spatial_layer;
return picture_id < rhs.picture_id;
}
bool operator<=(const FrameKey& rhs) const { return !(rhs < *this); }
int64_t picture_id;
uint8_t spatial_layer;
};
struct FrameInfo { struct FrameInfo {
// The maximum number of frames that can depend on this frame. // The maximum number of frames that can depend on this frame.
static constexpr size_t kMaxNumDependentFrames = 8; static constexpr size_t kMaxNumDependentFrames = 8;
@ -103,7 +86,7 @@ class FrameBuffer {
// on this frame. // on this frame.
// TODO(philipel): Add simple modify/access functions to prevent adding too // TODO(philipel): Add simple modify/access functions to prevent adding too
// many |dependent_frames|. // many |dependent_frames|.
FrameKey dependent_frames[kMaxNumDependentFrames]; VideoLayerFrameId dependent_frames[kMaxNumDependentFrames];
size_t num_dependent_frames = 0; size_t num_dependent_frames = 0;
// A frame is continiuous if it has all its referenced/indirectly // A frame is continiuous if it has all its referenced/indirectly
@ -124,7 +107,7 @@ class FrameBuffer {
std::unique_ptr<EncodedFrame> frame; std::unique_ptr<EncodedFrame> frame;
}; };
using FrameMap = std::map<FrameKey, FrameInfo>; using FrameMap = std::map<VideoLayerFrameId, FrameInfo>;
// Check that the references of |frame| are valid. // Check that the references of |frame| are valid.
bool ValidReferences(const EncodedFrame& frame) const; bool ValidReferences(const EncodedFrame& frame) const;

13
modules/video_coding/frame_buffer2_unittest.cc
View file

@ -160,8 +160,8 @@ class TestFrameBuffer2 : public ::testing::Test {
{rtc::checked_cast<uint16_t>(refs)...}}; {rtc::checked_cast<uint16_t>(refs)...}};
std::unique_ptr<FrameObjectFake> frame(new FrameObjectFake()); std::unique_ptr<FrameObjectFake> frame(new FrameObjectFake());
frame->picture_id = picture_id; frame->id.picture_id = picture_id;
frame->spatial_layer = spatial_layer; frame->id.spatial_layer = spatial_layer;
frame->timestamp = ts_ms * 90; frame->timestamp = ts_ms * 90;
frame->num_references = references.size(); frame->num_references = references.size();
frame->inter_layer_predicted = inter_layer_predicted; frame->inter_layer_predicted = inter_layer_predicted;
@ -193,8 +193,8 @@ class TestFrameBuffer2 : public ::testing::Test {
rtc::CritScope lock(&crit_); rtc::CritScope lock(&crit_);
ASSERT_LT(index, frames_.size()); ASSERT_LT(index, frames_.size());
ASSERT_TRUE(frames_[index]); ASSERT_TRUE(frames_[index]);
ASSERT_EQ(picture_id, frames_[index]->picture_id); ASSERT_EQ(picture_id, frames_[index]->id.picture_id);
ASSERT_EQ(spatial_layer, frames_[index]->spatial_layer); ASSERT_EQ(spatial_layer, frames_[index]->id.spatial_layer);
} }
void CheckNoFrame(size_t index) { void CheckNoFrame(size_t index) {
@ -269,6 +269,7 @@ TEST_F(TestFrameBuffer2, OneSuperFrame) {
TEST_F(TestFrameBuffer2, SetPlayoutDelay) { TEST_F(TestFrameBuffer2, SetPlayoutDelay) {
const PlayoutDelay kPlayoutDelayMs = {123, 321}; const PlayoutDelay kPlayoutDelayMs = {123, 321};
std::unique_ptr<FrameObjectFake> test_frame(new FrameObjectFake()); std::unique_ptr<FrameObjectFake> test_frame(new FrameObjectFake());
test_frame->id.picture_id = 0;
test_frame->SetPlayoutDelay(kPlayoutDelayMs); test_frame->SetPlayoutDelay(kPlayoutDelayMs);
buffer_.InsertFrame(std::move(test_frame)); buffer_.InsertFrame(std::move(test_frame));
EXPECT_EQ(kPlayoutDelayMs.min_ms, timing_.min_playout_delay()); EXPECT_EQ(kPlayoutDelayMs.min_ms, timing_.min_playout_delay());
@ -500,8 +501,8 @@ TEST_F(TestFrameBuffer2, StatsCallback) {
{ {
std::unique_ptr<FrameObjectFake> frame(new FrameObjectFake()); std::unique_ptr<FrameObjectFake> frame(new FrameObjectFake());
frame->SetSize(kFrameSize); frame->SetSize(kFrameSize);
frame->picture_id = pid; frame->id.picture_id = pid;
frame->spatial_layer = 0; frame->id.spatial_layer = 0;
frame->timestamp = ts; frame->timestamp = ts;
frame->num_references = 0; frame->num_references = 0;
frame->inter_layer_predicted = false; frame->inter_layer_predicted = false;

4
modules/video_coding/packet_buffer.cc
View file

@ -430,8 +430,8 @@ bool PacketBuffer::GetBitstream(const RtpFrameObject& frame,
RTC_DCHECK_EQ(data_buffer_[index].seqNum, sequence_buffer_[index].seq_num); RTC_DCHECK_EQ(data_buffer_[index].seqNum, sequence_buffer_[index].seq_num);
size_t length = data_buffer_[index].sizeBytes; size_t length = data_buffer_[index].sizeBytes;
if (destination + length > destination_end) { if (destination + length > destination_end) {
RTC_LOG(LS_WARNING) << "Frame (" << frame.picture_id << ":" RTC_LOG(LS_WARNING) << "Frame (" << frame.id.picture_id << ":"
<< static_cast<int>(frame.spatial_layer) << ")" << static_cast<int>(frame.id.spatial_layer) << ")"
<< " bitstream buffer is not large enough."; << " bitstream buffer is not large enough.";
return false; return false;
} }

76
modules/video_coding/rtp_frame_reference_finder.cc
View file

@ -178,9 +178,9 @@ RtpFrameReferenceFinder::ManageFrameGeneric(RtpFrameObject* frame,
if (last_unwrap_ == -1) if (last_unwrap_ == -1)
last_unwrap_ = picture_id; last_unwrap_ = picture_id;
frame->picture_id = unwrapper_.Unwrap(picture_id); frame->id.picture_id = unwrapper_.Unwrap(picture_id);
frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1; frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1;
frame->references[0] = frame->picture_id - 1; frame->references[0] = frame->id.picture_id - 1;
return kHandOff; return kHandOff;
} }
@ -229,17 +229,17 @@ RtpFrameReferenceFinder::ManageFrameGeneric(RtpFrameObject* frame,
// Since keyframes can cause reordering we can't simply assign the // Since keyframes can cause reordering we can't simply assign the
// picture id according to some incrementing counter. // picture id according to some incrementing counter.
frame->picture_id = frame->last_seq_num(); frame->id.picture_id = frame->last_seq_num();
frame->num_references = frame->frame_type() == kVideoFrameDelta; frame->num_references = frame->frame_type() == kVideoFrameDelta;
frame->references[0] = generic_unwrapper_.Unwrap(last_picture_id_gop); frame->references[0] = generic_unwrapper_.Unwrap(last_picture_id_gop);
if (AheadOf<uint16_t>(frame->picture_id, last_picture_id_gop)) { if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) {
seq_num_it->second.first = frame->picture_id; seq_num_it->second.first = frame->id.picture_id;
seq_num_it->second.second = frame->picture_id; seq_num_it->second.second = frame->id.picture_id;
} }
last_picture_id_ = frame->picture_id; last_picture_id_ = frame->id.picture_id;
UpdateLastPictureIdWithPadding(frame->picture_id); UpdateLastPictureIdWithPadding(frame->id.picture_id);
frame->picture_id = generic_unwrapper_.Unwrap(frame->picture_id); frame->id.picture_id = generic_unwrapper_.Unwrap(frame->id.picture_id);
return kHandOff; return kHandOff;
} }
@ -260,21 +260,21 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8(
return ManageFrameGeneric(std::move(frame), codec_header.pictureId); return ManageFrameGeneric(std::move(frame), codec_header.pictureId);
} }
frame->picture_id = codec_header.pictureId % kPicIdLength; frame->id.picture_id = codec_header.pictureId % kPicIdLength;
if (last_unwrap_ == -1) if (last_unwrap_ == -1)
last_unwrap_ = codec_header.pictureId; last_unwrap_ = codec_header.pictureId;
if (last_picture_id_ == -1) if (last_picture_id_ == -1)
last_picture_id_ = frame->picture_id; last_picture_id_ = frame->id.picture_id;
// Find if there has been a gap in fully received frames and save the picture // Find if there has been a gap in fully received frames and save the picture
// id of those frames in |not_yet_received_frames_|. // id of those frames in |not_yet_received_frames_|.
if (AheadOf<uint16_t, kPicIdLength>(frame->picture_id, last_picture_id_)) { if (AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, last_picture_id_)) {
do { do {
last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1); last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
not_yet_received_frames_.insert(last_picture_id_); not_yet_received_frames_.insert(last_picture_id_);
} while (last_picture_id_ != frame->picture_id); } while (last_picture_id_ != frame->id.picture_id);
} }
// Clean up info for base layers that are too old. // Clean up info for base layers that are too old.
@ -284,7 +284,7 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8(
// Clean up info about not yet received frames that are too old. // Clean up info about not yet received frames that are too old.
uint16_t old_picture_id = uint16_t old_picture_id =
Subtract<kPicIdLength>(frame->picture_id, kMaxNotYetReceivedFrames); Subtract<kPicIdLength>(frame->id.picture_id, kMaxNotYetReceivedFrames);
auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id); auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id);
not_yet_received_frames_.erase(not_yet_received_frames_.begin(), not_yet_received_frames_.erase(not_yet_received_frames_.begin(),
clean_frames_to); clean_frames_to);
@ -339,7 +339,7 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8(
// a layer sync frame has been received after this frame for the same // a layer sync frame has been received after this frame for the same
// base layer frame, drop this frame. // base layer frame, drop this frame.
if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer], if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer],
frame->picture_id)) { frame->id.picture_id)) {
return kDrop; return kDrop;
} }
@ -348,14 +348,14 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8(
auto not_received_frame_it = auto not_received_frame_it =
not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
if (not_received_frame_it != not_yet_received_frames_.end() && if (not_received_frame_it != not_yet_received_frames_.end() &&
AheadOf<uint16_t, kPicIdLength>(frame->picture_id, AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id,
*not_received_frame_it)) { *not_received_frame_it)) {
return kStash; return kStash;
} }
if (!(AheadOf<uint16_t, kPicIdLength>(frame->picture_id, if (!(AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id,
layer_info_it->second[layer]))) { layer_info_it->second[layer]))) {
RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->picture_id RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id
<< " and packet range [" << frame->first_seq_num() << " and packet range [" << frame->first_seq_num()
<< ", " << frame->last_seq_num() << ", " << frame->last_seq_num()
<< "] already received, " << "] already received, "
@ -382,17 +382,17 @@ void RtpFrameReferenceFinder::UpdateLayerInfoVp8(
while (layer_info_it != layer_info_.end()) { while (layer_info_it != layer_info_.end()) {
if (layer_info_it->second[temporal_index] != -1 && if (layer_info_it->second[temporal_index] != -1 &&
AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_index], AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_index],
frame->picture_id)) { frame->id.picture_id)) {
// The frame was not newer, then no subsequent layer info have to be // The frame was not newer, then no subsequent layer info have to be
// update. // update.
break; break;
} }
layer_info_it->second[codec_header.temporalIdx] = frame->picture_id; layer_info_it->second[codec_header.temporalIdx] = frame->id.picture_id;
++tl0_pic_idx; ++tl0_pic_idx;
layer_info_it = layer_info_.find(tl0_pic_idx); layer_info_it = layer_info_.find(tl0_pic_idx);
} }
not_yet_received_frames_.erase(frame->picture_id); not_yet_received_frames_.erase(frame->id.picture_id);
UnwrapPictureIds(frame); UnwrapPictureIds(frame);
} }
@ -413,21 +413,21 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
return ManageFrameGeneric(std::move(frame), codec_header.picture_id); return ManageFrameGeneric(std::move(frame), codec_header.picture_id);
} }
frame->spatial_layer = codec_header.spatial_idx; frame->id.spatial_layer = codec_header.spatial_idx;
frame->inter_layer_predicted = codec_header.inter_layer_predicted; frame->inter_layer_predicted = codec_header.inter_layer_predicted;
frame->picture_id = codec_header.picture_id % kPicIdLength; frame->id.picture_id = codec_header.picture_id % kPicIdLength;
if (last_unwrap_ == -1) if (last_unwrap_ == -1)
last_unwrap_ = codec_header.picture_id; last_unwrap_ = codec_header.picture_id;
if (last_picture_id_ == -1) if (last_picture_id_ == -1)
last_picture_id_ = frame->picture_id; last_picture_id_ = frame->id.picture_id;
if (codec_header.flexible_mode) { if (codec_header.flexible_mode) {
frame->num_references = codec_header.num_ref_pics; frame->num_references = codec_header.num_ref_pics;
for (size_t i = 0; i < frame->num_references; ++i) { for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] = frame->references[i] = Subtract<kPicIdLength>(frame->id.picture_id,
Subtract<kPicIdLength>(frame->picture_id, codec_header.pid_diff[i]); codec_header.pid_diff[i]);
} }
UnwrapPictureIds(frame); UnwrapPictureIds(frame);
@ -443,10 +443,10 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
} else { } else {
current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1); current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1);
scalability_structures_[current_ss_idx_] = codec_header.gof; scalability_structures_[current_ss_idx_] = codec_header.gof;
scalability_structures_[current_ss_idx_].pid_start = frame->picture_id; scalability_structures_[current_ss_idx_].pid_start = frame->id.picture_id;
GofInfo info(&scalability_structures_[current_ss_idx_], GofInfo info(&scalability_structures_[current_ss_idx_],
frame->picture_id); frame->id.picture_id);
gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, info)); gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, info));
} }
} }
@ -463,7 +463,7 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
frame->num_references = 0; frame->num_references = 0;
GofInfo info = gof_info_.find(codec_header.tl0_pic_idx)->second; GofInfo info = gof_info_.find(codec_header.tl0_pic_idx)->second;
FrameReceivedVp9(frame->picture_id, &info); FrameReceivedVp9(frame->id.picture_id, &info);
UnwrapPictureIds(frame); UnwrapPictureIds(frame);
return kHandOff; return kHandOff;
} }
@ -478,16 +478,16 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
return kStash; return kStash;
GofInfo* info = &gof_info_it->second; GofInfo* info = &gof_info_it->second;
FrameReceivedVp9(frame->picture_id, info); FrameReceivedVp9(frame->id.picture_id, info);
// Make sure we don't miss any frame that could potentially have the // Make sure we don't miss any frame that could potentially have the
// up switch flag set. // up switch flag set.
if (MissingRequiredFrameVp9(frame->picture_id, *info)) if (MissingRequiredFrameVp9(frame->id.picture_id, *info))
return kStash; return kStash;
if (codec_header.temporal_up_switch) { if (codec_header.temporal_up_switch) {
auto pid_tidx = auto pid_tidx =
std::make_pair(frame->picture_id, codec_header.temporal_idx); std::make_pair(frame->id.picture_id, codec_header.temporal_idx);
up_switch_.insert(pid_tidx); up_switch_.insert(pid_tidx);
} }
@ -495,28 +495,28 @@ RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
// then gof info has already been inserted earlier, so we only want to // then gof info has already been inserted earlier, so we only want to
// insert if we haven't done so already. // insert if we haven't done so already.
if (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) { if (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) {
GofInfo new_info(info->gof, frame->picture_id); GofInfo new_info(info->gof, frame->id.picture_id);
gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, new_info)); gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, new_info));
} }
// Clean out old info about up switch frames. // Clean out old info about up switch frames.
uint16_t old_picture_id = Subtract<kPicIdLength>(frame->picture_id, 50); uint16_t old_picture_id = Subtract<kPicIdLength>(frame->id.picture_id, 50);
auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id); auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id);
up_switch_.erase(up_switch_.begin(), up_switch_erase_to); up_switch_.erase(up_switch_.begin(), up_switch_erase_to);
size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
frame->picture_id); frame->id.picture_id);
size_t gof_idx = diff % info->gof->num_frames_in_gof; size_t gof_idx = diff % info->gof->num_frames_in_gof;
// Populate references according to the scalability structure. // Populate references according to the scalability structure.
frame->num_references = info->gof->num_ref_pics[gof_idx]; frame->num_references = info->gof->num_ref_pics[gof_idx];
for (size_t i = 0; i < frame->num_references; ++i) { for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] = Subtract<kPicIdLength>( frame->references[i] = Subtract<kPicIdLength>(
frame->picture_id, info->gof->pid_diff[gof_idx][i]); frame->id.picture_id, info->gof->pid_diff[gof_idx][i]);
// If this is a reference to a frame earlier than the last up switch point, // If this is a reference to a frame earlier than the last up switch point,
// then ignore this reference. // then ignore this reference.
if (UpSwitchInIntervalVp9(frame->picture_id, codec_header.temporal_idx, if (UpSwitchInIntervalVp9(frame->id.picture_id, codec_header.temporal_idx,
frame->references[i])) { frame->references[i])) {
--frame->num_references; --frame->num_references;
} }
@ -615,7 +615,7 @@ bool RtpFrameReferenceFinder::UpSwitchInIntervalVp9(uint16_t picture_id,
void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) { void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) {
for (size_t i = 0; i < frame->num_references; ++i) for (size_t i = 0; i < frame->num_references; ++i)
frame->references[i] = unwrapper_.Unwrap(frame->references[i]); frame->references[i] = unwrapper_.Unwrap(frame->references[i]);
frame->picture_id = unwrapper_.Unwrap(frame->picture_id); frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id);
} }
} // namespace video_coding } // namespace video_coding

4
modules/video_coding/rtp_frame_reference_finder_unittest.cc
View file

@ -65,8 +65,8 @@ class TestRtpFrameReferenceFinder : public ::testing::Test,
uint16_t Rand() { return rand_.Rand<uint16_t>(); } uint16_t Rand() { return rand_.Rand<uint16_t>(); }
void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override { void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) override {
int64_t pid = frame->picture_id; int64_t pid = frame->id.picture_id;
uint16_t sidx = frame->spatial_layer; uint16_t sidx = frame->id.spatial_layer;
auto frame_it = frames_from_callback_.find(std::make_pair(pid, sidx)); auto frame_it = frames_from_callback_.find(std::make_pair(pid, sidx));
if (frame_it != frames_from_callback_.end()) { if (frame_it != frames_from_callback_.end()) {
ADD_FAILURE() << "Already received frame with (pid:sidx): (" << pid << ":" ADD_FAILURE() << "Already received frame with (pid:sidx): (" << pid << ":"

3
video/rtp_video_stream_receiver.cc
View file

@ -368,7 +368,8 @@ void RtpVideoStreamReceiver::OnCompleteFrame(
rtc::CritScope lock(&last_seq_num_cs_); rtc::CritScope lock(&last_seq_num_cs_);
video_coding::RtpFrameObject* rtp_frame = video_coding::RtpFrameObject* rtp_frame =
static_cast<video_coding::RtpFrameObject*>(frame.get()); static_cast<video_coding::RtpFrameObject*>(frame.get());
last_seq_num_for_pic_id_[rtp_frame->picture_id] = rtp_frame->last_seq_num(); last_seq_num_for_pic_id_[rtp_frame->id.picture_id] =
rtp_frame->last_seq_num();
} }
complete_frame_callback_->OnCompleteFrame(std::move(frame)); complete_frame_callback_->OnCompleteFrame(std::move(frame));
} }

2
video/video_receive_stream.cc
View file

@ -434,7 +434,7 @@ bool VideoReceiveStream::Decode() {
decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) { decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) {
keyframe_required_ = false; keyframe_required_ = false;
frame_decoded_ = true; frame_decoded_ = true;
rtp_video_stream_receiver_.FrameDecoded(frame->picture_id); rtp_video_stream_receiver_.FrameDecoded(frame->id.picture_id);
if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME) if (decode_result == WEBRTC_VIDEO_CODEC_OK_REQUEST_KEYFRAME)
RequestKeyFrame(); RequestKeyFrame();