mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-14 14:20:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Decode Target Information for VP8 libvpx encoder.

Browse source 
In this CL:
 - Created static helper function GenericFrameInfo::DecodeTargetInfo to
   convert DTI symbols to a list of GenericFrameInfo::OperatingPointIndication.
 - Added per frame DTI information for the different stream structures.

Bug: webrtc:10342
Change-Id: I62ff2e9fc9b380fe1d0447ff071e86b6b35ab249
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/129923
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27350}
This commit is contained in:
philipel 2019-03-28 16:09:52 +01:00 committed by Commit Bot
parent 1f928d3316
commit b19f27a5f9
6 changed files with 168 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
common_video/generic_frame_descriptor/generic_frame_info.cc
View file

@ -15,6 +15,24 @@
namespace webrtc { namespace webrtc {
absl::InlinedVector<GenericFrameInfo::DecodeTargetIndication, 10>
GenericFrameInfo::DecodeTargetInfo(absl::string_view indication_symbols) {
absl::InlinedVector<DecodeTargetIndication, 10> decode_targets;
for (char symbol : indication_symbols) {
DecodeTargetIndication indication;
switch (symbol) {
case '-': indication = DecodeTargetIndication::kNotPresent; break;
case 'D': indication = DecodeTargetIndication::kDiscardable; break;
case 'R': indication = DecodeTargetIndication::kRequired; break;
case 'S': indication = DecodeTargetIndication::kSwitch; break;
default: RTC_NOTREACHED();
}
decode_targets.push_back(indication);
}
return decode_targets;
}
GenericFrameInfo::GenericFrameInfo() = default; GenericFrameInfo::GenericFrameInfo() = default;
GenericFrameInfo::GenericFrameInfo(const GenericFrameInfo&) = default; GenericFrameInfo::GenericFrameInfo(const GenericFrameInfo&) = default;
GenericFrameInfo::~GenericFrameInfo() = default; GenericFrameInfo::~GenericFrameInfo() = default;
@ -38,19 +56,7 @@ GenericFrameInfo::Builder& GenericFrameInfo::Builder::S(int spatial_id) {
GenericFrameInfo::Builder& GenericFrameInfo::Builder::Dtis( GenericFrameInfo::Builder& GenericFrameInfo::Builder::Dtis(
absl::string_view indication_symbols) { absl::string_view indication_symbols) {
for (const auto& symbol : indication_symbols) { info_.decode_target_indications = DecodeTargetInfo(indication_symbols);
DecodeTargetIndication indication;
switch (symbol) {
case '-': indication = DecodeTargetIndication::kNotPresent; break;
case 'D': indication = DecodeTargetIndication::kDiscardable; break;
case 'R': indication = DecodeTargetIndication::kRequired; break;
case 'S': indication = DecodeTargetIndication::kSwitch; break;
default: RTC_NOTREACHED();
}
info_.decode_target_indications.push_back(indication);
}
return *this; return *this;
} }

11
common_video/generic_frame_descriptor/generic_frame_info.h
View file

@ -22,12 +22,15 @@ namespace webrtc {
struct GenericFrameInfo { struct GenericFrameInfo {
enum class DecodeTargetIndication { enum class DecodeTargetIndication {
kNotPresent, // GenericFrameInfo::Builder symbol '-' kNotPresent, // DecodeTargetInfo symbol '-'
kDiscardable, // GenericFrameInfo::Builder symbol 'D' kDiscardable, // DecodeTargetInfo symbol 'D'
kSwitch, // GenericFrameInfo::Builder symbol 'S' kSwitch, // DecodeTargetInfo symbol 'S'
kRequired // GenericFrameInfo::Builder symbol 'R' kRequired // DecodeTargetInfo symbol 'R'
}; };
static absl::InlinedVector<DecodeTargetIndication, 10> DecodeTargetInfo(
absl::string_view indication_symbols);
class Builder; class Builder;
GenericFrameInfo(); GenericFrameInfo();

147
modules/video_coding/codecs/vp8/default_temporal_layers.cc
View file

@ -31,10 +31,10 @@ DefaultTemporalLayers::PendingFrame::PendingFrame() = default;
DefaultTemporalLayers::PendingFrame::PendingFrame( DefaultTemporalLayers::PendingFrame::PendingFrame(
bool expired, bool expired,
uint8_t updated_buffers_mask, uint8_t updated_buffers_mask,
const Vp8FrameConfig& frame_config) const DependencyInfo& dependency_info)
: expired(expired), : expired(expired),
updated_buffer_mask(updated_buffers_mask), updated_buffer_mask(updated_buffers_mask),
frame_config(frame_config) {} dependency_info(dependency_info) {}
namespace { namespace {
using Buffer = Vp8FrameConfig::Buffer; using Buffer = Vp8FrameConfig::Buffer;
@ -99,27 +99,10 @@ uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) {
} }
return flags; return flags;
} }
// Find the set of buffers that are never updated by the given pattern.
std::set<Vp8BufferReference> FindKfBuffers(
const std::vector<Vp8FrameConfig>& frame_configs) {
std::set<Vp8BufferReference> kf_buffers(kAllBuffers.begin(),
kAllBuffers.end());
for (Vp8FrameConfig config : frame_configs) {
// Get bit-masked set of update buffers for this frame config.
uint8_t updated_buffers = GetUpdatedBuffers(config);
for (Vp8BufferReference buffer : kAllBuffers) {
if (static_cast<uint8_t>(buffer) & updated_buffers) {
kf_buffers.erase(buffer);
}
}
}
return kf_buffers;
}
} // namespace } // namespace
std::vector<Vp8FrameConfig> DefaultTemporalLayers::GetTemporalPattern( std::vector<DefaultTemporalLayers::DependencyInfo>
size_t num_layers) { DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) {
// For indexing in the patterns described below (which temporal layers they // For indexing in the patterns described below (which temporal layers they
// belong to), see the diagram above. // belong to), see the diagram above.
// Layer sync is done similarly for all patterns (except single stream) and // Layer sync is done similarly for all patterns (except single stream) and
@ -133,10 +116,11 @@ std::vector<Vp8FrameConfig> DefaultTemporalLayers::GetTemporalPattern(
// so that if scene changes occur (user walks between rooms or rotates webcam) // so that if scene changes occur (user walks between rooms or rotates webcam)
// the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant // the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant
// keyframe. // keyframe.
switch (num_layers) { switch (num_layers) {
case 1: case 1:
// Always reference and update the same buffer. // Always reference and update the same buffer.
return {Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone)}; return {{"S", {kReferenceAndUpdate, kNone, kNone}}};
case 2: case 2:
// All layers can reference but not update the 'alt' buffer, this means // All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe. // that the 'alt' buffer reference is effectively the last keyframe.
@ -147,23 +131,23 @@ std::vector<Vp8FrameConfig> DefaultTemporalLayers::GetTemporalPattern(
// 1---1 1---1 ... // 1---1 1---1 ...
// / / / / // / / / /
// 0---0---0---0 ... // 0---0---0---0 ...
return {Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kUpdate, kNone), {"-S", {kReference, kUpdate, kNone}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), {"SR", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kReference, kNone, kFreezeEntropy)}; {"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
} else { } else {
// "Default" 8-frame pattern: // "Default" 8-frame pattern:
// 1---1---1---1 1---1---1---1 ... // 1---1---1---1 1---1---1---1 ...
// / / / / / / / / // / / / / / / / /
// 0---0---0---0---0---0---0---0 ... // 0---0---0---0---0---0---0---0 ...
return {Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kUpdate, kNone), {"-S", {kReference, kUpdate, kNone}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), {"SR", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), {"-R", {kReference, kReferenceAndUpdate, kNone}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), {"SR", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), {"-R", {kReference, kReferenceAndUpdate, kNone}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), {"SR", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kReference, kNone, kFreezeEntropy)}; {"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
} }
case 3: case 3:
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
@ -183,62 +167,59 @@ std::vector<Vp8FrameConfig> DefaultTemporalLayers::GetTemporalPattern(
// TL1 references 'last' and references and updates 'golden'. // TL1 references 'last' and references and updates 'golden'.
// TL2 references both 'last' & 'golden' and references and updates // TL2 references both 'last' & 'golden' and references and updates
// 'arf'. // 'arf'.
return { return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), {"--S", {kReference, kNone, kUpdate}},
Vp8FrameConfig(kReference, kNone, kUpdate), {"-DR", {kReference, kUpdate, kNone}},
Vp8FrameConfig(kReference, kUpdate, kNone), {"--D", {kReference, kReference, kReference, kFreezeEntropy}}};
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)};
} else { } else {
// All layers can reference but not update the 'alt' buffer, this means // All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe. // that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer. // TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'. // TL1 also references 'last' and references and updates 'golden'.
// TL2 references both 'last' and 'golden' but updates no buffer. // TL2 references both 'last' and 'golden' but updates no buffer.
return { return {{"SSS", {kReferenceAndUpdate, kNone, kReference}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), {"--D", {kReference, kNone, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kNone, kReference, kFreezeEntropy), {"-SS", {kReference, kUpdate, kReference}},
Vp8FrameConfig(kReference, kUpdate, kReference), {"--D", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"SRR", {kReferenceAndUpdate, kNone, kReference}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), {"--D", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"-DS", {kReference, kReferenceAndUpdate, kReference}},
Vp8FrameConfig(kReference, kReferenceAndUpdate, kReference), {"--D", {kReference, kReference, kReference, kFreezeEntropy}}};
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)};
} }
case 4: case 4:
// TL0 references and updates only the 'last' buffer. // TL0 references and updates only the 'last' buffer.
// TL1 references 'last' and updates and references 'golden'. // TL1 references 'last' and updates and references 'golden'.
// TL2 references 'last' and 'golden', and references and updates 'arf'. // TL2 references 'last' and 'golden', and references and updates 'arf'.
// TL3 references all buffers but update none of them. // TL3 references all buffers but update none of them.
return { // TODO(philipel): Set decode target information for this structure.
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), return {{"----", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kNone, kNone, kFreezeEntropy), {"----", {kReference, kNone, kNone, kFreezeEntropy}},
Vp8FrameConfig(kReference, kNone, kUpdate), {"----", {kReference, kNone, kUpdate}},
Vp8FrameConfig(kReference, kNone, kReference, kFreezeEntropy), {"----", {kReference, kNone, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kUpdate, kNone), {"----", {kReference, kUpdate, kNone}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"----", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), {"----", {kReference, kReference, kReferenceAndUpdate}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"----", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), {"----", {kReferenceAndUpdate, kNone, kNone}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"----", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), {"----", {kReference, kReference, kReferenceAndUpdate}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"----", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), {"----", {kReference, kReferenceAndUpdate, kNone}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), {"----", {kReference, kReference, kReference, kFreezeEntropy}},
Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), {"----", {kReference, kReference, kReferenceAndUpdate}},
Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; {"----", {kReference, kReference, kReference, kFreezeEntropy}}};
default: default:
RTC_NOTREACHED(); RTC_NOTREACHED();
break; break;
} }
RTC_NOTREACHED(); RTC_NOTREACHED();
return {Vp8FrameConfig(kNone, kNone, kNone)}; return {{"", {kNone, kNone, kNone}}};
} }
DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
: num_layers_(std::max(1, number_of_temporal_layers)), : num_layers_(std::max(1, number_of_temporal_layers)),
temporal_ids_(GetTemporalIds(num_layers_)), temporal_ids_(GetTemporalIds(num_layers_)),
temporal_pattern_(GetTemporalPattern(num_layers_)), temporal_pattern_(GetDependencyInfo(num_layers_)),
kf_buffers_(FindKfBuffers(temporal_pattern_)),
pattern_idx_(kUninitializedPatternIndex) { pattern_idx_(kUninitializedPatternIndex) {
RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
RTC_CHECK_GE(number_of_temporal_layers, 0); RTC_CHECK_GE(number_of_temporal_layers, 0);
@ -257,6 +238,16 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
for (Vp8BufferReference buffer : kAllBuffers) { for (Vp8BufferReference buffer : kAllBuffers) {
frames_since_buffer_refresh_[buffer] = 0; frames_since_buffer_refresh_[buffer] = 0;
} }
kf_buffers_ = {kAllBuffers.begin(), kAllBuffers.end()};
for (DependencyInfo info : temporal_pattern_) {
uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config);
for (Vp8BufferReference buffer : kAllBuffers) {
if (static_cast<uint8_t>(buffer) & updated_buffers)
kf_buffers_.erase(buffer);
}
}
} }
DefaultTemporalLayers::~DefaultTemporalLayers() = default; DefaultTemporalLayers::~DefaultTemporalLayers() = default;
@ -347,7 +338,8 @@ Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(size_t stream_index,
RTC_DCHECK_GT(temporal_pattern_.size(), 0); RTC_DCHECK_GT(temporal_pattern_.size(), 0);
pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size();
Vp8FrameConfig tl_config = temporal_pattern_[pattern_idx_]; DependencyInfo dependency_info = temporal_pattern_[pattern_idx_];
Vp8FrameConfig& tl_config = dependency_info.frame_config;
tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx = tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx =
temporal_ids_[pattern_idx_ % temporal_ids_.size()]; temporal_ids_[pattern_idx_ % temporal_ids_.size()];
@ -386,7 +378,7 @@ Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(size_t stream_index,
// Add frame to set of pending frames, awaiting completion. // Add frame to set of pending frames, awaiting completion.
pending_frames_[timestamp] = pending_frames_[timestamp] =
PendingFrame{false, GetUpdatedBuffers(tl_config), tl_config}; PendingFrame{false, GetUpdatedBuffers(tl_config), dependency_info};
#if RTC_DCHECK_IS_ON #if RTC_DCHECK_IS_ON
// Checker does not yet support encoder frame dropping, so validate flags // Checker does not yet support encoder frame dropping, so validate flags
@ -472,10 +464,11 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
} }
PendingFrame& frame = pending_frame->second; PendingFrame& frame = pending_frame->second;
const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config;
#if RTC_DCHECK_IS_ON #if RTC_DCHECK_IS_ON
if (is_keyframe) { if (is_keyframe) {
// Signal key-frame so checker resets state. // Signal key-frame so checker resets state.
RTC_DCHECK(checker_->CheckTemporalConfig(true, frame.frame_config)); RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config));
} }
#endif #endif
@ -503,8 +496,8 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
} }
} else { } else {
// Delta frame, update codec specifics with temporal id and sync flag. // Delta frame, update codec specifics with temporal id and sync flag.
vp8_info.temporalIdx = frame.frame_config.packetizer_temporal_idx; vp8_info.temporalIdx = frame_config.packetizer_temporal_idx;
vp8_info.layerSync = frame.frame_config.layer_sync; vp8_info.layerSync = frame_config.layer_sync;
} }
} }
@ -513,13 +506,13 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
for (int i = 0; i < static_cast<int>(Buffer::kCount); ++i) { for (int i = 0; i < static_cast<int>(Buffer::kCount); ++i) {
if (!is_keyframe && frame.frame_config.References(static_cast<Buffer>(i))) { if (!is_keyframe && frame_config.References(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.referencedBuffersCount, RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
arraysize(CodecSpecificInfoVP8::referencedBuffers)); arraysize(CodecSpecificInfoVP8::referencedBuffers));
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
} }
if (is_keyframe || frame.frame_config.Updates(static_cast<Buffer>(i))) { if (is_keyframe || frame_config.Updates(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.updatedBuffersCount, RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
arraysize(CodecSpecificInfoVP8::updatedBuffers)); arraysize(CodecSpecificInfoVP8::updatedBuffers));
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
@ -532,6 +525,10 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
info->template_structure = GetTemplateStructure(num_layers_); info->template_structure = GetTemplateStructure(num_layers_);
} }
GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
generic_frame_info.decode_target_indications =
frame.dependency_info.decode_target_indications;
if (!frame.expired) { if (!frame.expired) {
for (Vp8BufferReference buffer : kAllBuffers) { for (Vp8BufferReference buffer : kAllBuffers) {
if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) { if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) {

24
modules/video_coding/codecs/vp8/default_temporal_layers.h
View file

@ -18,6 +18,7 @@
#include <map> #include <map>
#include <memory> #include <memory>
#include <set> #include <set>
#include <utility>
#include <vector> #include <vector>
#include "absl/types/optional.h" #include "absl/types/optional.h"
@ -61,7 +62,20 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
void OnRttUpdate(int64_t rtt_ms) override; void OnRttUpdate(int64_t rtt_ms) override;
private: private:
static std::vector<Vp8FrameConfig> GetTemporalPattern(size_t num_layers); struct DependencyInfo {
DependencyInfo() = default;
DependencyInfo(absl::string_view indication_symbols,
Vp8FrameConfig frame_config)
: decode_target_indications(
GenericFrameInfo::DecodeTargetInfo(indication_symbols)),
frame_config(frame_config) {}
absl::InlinedVector<GenericFrameInfo::DecodeTargetIndication, 10>
decode_target_indications;
Vp8FrameConfig frame_config;
};
static std::vector<DependencyInfo> GetDependencyInfo(size_t num_layers);
bool IsSyncFrame(const Vp8FrameConfig& config) const; bool IsSyncFrame(const Vp8FrameConfig& config) const;
void ValidateReferences(Vp8FrameConfig::BufferFlags* flags, void ValidateReferences(Vp8FrameConfig::BufferFlags* flags,
Vp8FrameConfig::Vp8BufferReference ref) const; Vp8FrameConfig::Vp8BufferReference ref) const;
@ -69,9 +83,9 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
const size_t num_layers_; const size_t num_layers_;
const std::vector<unsigned int> temporal_ids_; const std::vector<unsigned int> temporal_ids_;
const std::vector<Vp8FrameConfig> temporal_pattern_; const std::vector<DependencyInfo> temporal_pattern_;
// Set of buffers that are never updated except by keyframes. // Set of buffers that are never updated except by keyframes.
const std::set<Vp8FrameConfig::Vp8BufferReference> kf_buffers_; std::set<Vp8FrameConfig::Vp8BufferReference> kf_buffers_;
TemplateStructure GetTemplateStructure(int num_layers) const; TemplateStructure GetTemplateStructure(int num_layers) const;
uint8_t pattern_idx_; uint8_t pattern_idx_;
@ -82,7 +96,7 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
PendingFrame(); PendingFrame();
PendingFrame(bool expired, PendingFrame(bool expired,
uint8_t updated_buffers_mask, uint8_t updated_buffers_mask,
const Vp8FrameConfig& frame_config); const DependencyInfo& dependency_info);
// Flag indicating if this frame has expired, ie it belongs to a previous // Flag indicating if this frame has expired, ie it belongs to a previous
// iteration of the temporal pattern. // iteration of the temporal pattern.
bool expired = false; bool expired = false;
@ -90,7 +104,7 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
// updates. // updates.
uint8_t updated_buffer_mask = 0; uint8_t updated_buffer_mask = 0;
// The frame config return by UpdateLayerConfig() for this frame. // The frame config return by UpdateLayerConfig() for this frame.
Vp8FrameConfig frame_config; DependencyInfo dependency_info;
}; };
// Map from rtp timestamp to pending frame status. Reset on pattern loop. // Map from rtp timestamp to pending frame status. Reset on pattern loop.
std::map<uint32_t, PendingFrame> pending_frames_; std::map<uint32_t, PendingFrame> pending_frames_;

60
modules/video_coding/codecs/vp8/screenshare_layers.cc
View file

@ -90,16 +90,16 @@ Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(size_t stream_index,
auto it = pending_frame_configs_.find(timestamp); auto it = pending_frame_configs_.find(timestamp);
if (it != pending_frame_configs_.end()) { if (it != pending_frame_configs_.end()) {
// Drop and re-encode, reuse the previous config. // Drop and re-encode, reuse the previous config.
return it->second; return it->second.frame_config;
} }
if (number_of_temporal_layers_ <= 1) { if (number_of_temporal_layers_ <= 1) {
// No flags needed for 1 layer screenshare. // No flags needed for 1 layer screenshare.
// TODO(pbos): Consider updating only last, and not all buffers. // TODO(pbos): Consider updating only last, and not all buffers.
Vp8FrameConfig tl_config(kReferenceAndUpdate, kReferenceAndUpdate, DependencyInfo dependency_info{
kReferenceAndUpdate); "S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}};
pending_frame_configs_[timestamp] = tl_config; pending_frame_configs_[timestamp] = dependency_info;
return tl_config; return dependency_info.frame_config;
} }
const int64_t now_ms = rtc::TimeMillis(); const int64_t now_ms = rtc::TimeMillis();
@ -199,35 +199,35 @@ Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(size_t stream_index,
RTC_NOTREACHED(); RTC_NOTREACHED();
} }
Vp8FrameConfig tl_config; DependencyInfo dependency_info;
// TODO(pbos): Consider referencing but not updating the 'alt' buffer for all // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
// layers. // layers.
switch (layer_state) { switch (layer_state) {
case TemporalLayerState::kDrop: case TemporalLayerState::kDrop:
tl_config = Vp8FrameConfig(kNone, kNone, kNone); dependency_info = {"", {kNone, kNone, kNone}};
break; break;
case TemporalLayerState::kTl0: case TemporalLayerState::kTl0:
// TL0 only references and updates 'last'. // TL0 only references and updates 'last'.
tl_config = Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone); dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}};
tl_config.packetizer_temporal_idx = 0; dependency_info.frame_config.packetizer_temporal_idx = 0;
break; break;
case TemporalLayerState::kTl1: case TemporalLayerState::kTl1:
// TL1 references both 'last' and 'golden' but only updates 'golden'. // TL1 references both 'last' and 'golden' but only updates 'golden'.
tl_config = Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone); dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}};
tl_config.packetizer_temporal_idx = 1; dependency_info.frame_config.packetizer_temporal_idx = 1;
break; break;
case TemporalLayerState::kTl1Sync: case TemporalLayerState::kTl1Sync:
// Predict from only TL0 to allow participants to switch to the high // Predict from only TL0 to allow participants to switch to the high
// bitrate stream. Updates 'golden' so that TL1 can continue to refer to // bitrate stream. Updates 'golden' so that TL1 can continue to refer to
// and update 'golden' from this point on. // and update 'golden' from this point on.
tl_config = Vp8FrameConfig(kReference, kUpdate, kNone); dependency_info = {"-S", {kReference, kUpdate, kNone}};
tl_config.packetizer_temporal_idx = 1; dependency_info.frame_config.packetizer_temporal_idx = 1;
dependency_info.frame_config.layer_sync = true;
break; break;
} }
tl_config.layer_sync = layer_state == TemporalLayerState::kTl1Sync; pending_frame_configs_[timestamp] = dependency_info;
pending_frame_configs_[timestamp] = tl_config; return dependency_info.frame_config;
return tl_config;
} }
void ScreenshareLayers::OnRatesUpdated( void ScreenshareLayers::OnRatesUpdated(
@ -286,28 +286,38 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index,
return; return;
} }
absl::optional<Vp8FrameConfig> frame_config; absl::optional<DependencyInfo> dependency_info;
auto it = pending_frame_configs_.find(rtp_timestamp); auto it = pending_frame_configs_.find(rtp_timestamp);
if (it != pending_frame_configs_.end()) { if (it != pending_frame_configs_.end()) {
frame_config = it->second; dependency_info = it->second;
pending_frame_configs_.erase(it); pending_frame_configs_.erase(it);
if (checker_) { if (checker_) {
RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe, *frame_config)); RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe,
dependency_info->frame_config));
} }
} }
CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
if (number_of_temporal_layers_ == 1) { if (number_of_temporal_layers_ == 1) {
vp8_info.temporalIdx = kNoTemporalIdx; vp8_info.temporalIdx = kNoTemporalIdx;
vp8_info.layerSync = false; vp8_info.layerSync = false;
generic_frame_info.decode_target_indications =
GenericFrameInfo::DecodeTargetInfo("S");
} else { } else {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp); int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
if (frame_config) { if (dependency_info) {
vp8_info.temporalIdx = frame_config->packetizer_temporal_idx; vp8_info.temporalIdx =
vp8_info.layerSync = frame_config->layer_sync; dependency_info->frame_config.packetizer_temporal_idx;
vp8_info.layerSync = dependency_info->frame_config.layer_sync;
generic_frame_info.decode_target_indications =
dependency_info->decode_target_indications;
} else { } else {
RTC_DCHECK(is_keyframe); RTC_DCHECK(is_keyframe);
generic_frame_info.decode_target_indications =
GenericFrameInfo::DecodeTargetInfo("SS");
} }
if (is_keyframe) { if (is_keyframe) {
@ -328,13 +338,15 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index,
// Note that |frame_config| is not derefernced if |is_keyframe|, // Note that |frame_config| is not derefernced if |is_keyframe|,
// meaning it's never dereferenced if the optional may be unset. // meaning it's never dereferenced if the optional may be unset.
for (int i = 0; i < static_cast<int>(Buffer::kCount); ++i) { for (int i = 0; i < static_cast<int>(Buffer::kCount); ++i) {
if (!is_keyframe && frame_config->References(static_cast<Buffer>(i))) { if (!is_keyframe &&
dependency_info->frame_config.References(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.referencedBuffersCount, RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
arraysize(CodecSpecificInfoVP8::referencedBuffers)); arraysize(CodecSpecificInfoVP8::referencedBuffers));
vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
} }
if (is_keyframe || frame_config->Updates(static_cast<Buffer>(i))) { if (is_keyframe ||
dependency_info->frame_config.Updates(static_cast<Buffer>(i))) {
RTC_DCHECK_LT(vp8_info.updatedBuffersCount, RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
arraysize(CodecSpecificInfoVP8::updatedBuffers)); arraysize(CodecSpecificInfoVP8::updatedBuffers));
vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;

16
modules/video_coding/codecs/vp8/screenshare_layers.h
View file

@ -11,6 +11,7 @@
#include <map> #include <map>
#include <memory> #include <memory>
#include <utility>
#include <vector> #include <vector>
#include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_frame_config.h"
@ -67,6 +68,19 @@ class ScreenshareLayers final : public Vp8FrameBufferController {
private: private:
enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };
struct DependencyInfo {
DependencyInfo() = default;
DependencyInfo(absl::string_view indication_symbols,
Vp8FrameConfig frame_config)
: decode_target_indications(
GenericFrameInfo::DecodeTargetInfo(indication_symbols)),
frame_config(frame_config) {}
absl::InlinedVector<GenericFrameInfo::DecodeTargetIndication, 10>
decode_target_indications;
Vp8FrameConfig frame_config;
};
bool TimeToSync(int64_t timestamp) const; bool TimeToSync(int64_t timestamp) const;
uint32_t GetCodecTargetBitrateKbps() const; uint32_t GetCodecTargetBitrateKbps() const;
@ -81,7 +95,7 @@ class ScreenshareLayers final : public Vp8FrameBufferController {
int max_qp_; int max_qp_;
uint32_t max_debt_bytes_; uint32_t max_debt_bytes_;
std::map<uint32_t, Vp8FrameConfig> pending_frame_configs_; std::map<uint32_t, DependencyInfo> pending_frame_configs_;
// Configured max framerate. // Configured max framerate.
absl::optional<uint32_t> target_framerate_; absl::optional<uint32_t> target_framerate_;