mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-14 14:20:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc
Niels Möller 150dcb0a9a Delete logic to set picture id and tl0 pic idx in encoders. 
It would be nice to also delete the fields from CodecSpecificInfo,
but those fields are used on the receive side.

Bug: webrtc:8830
Change-Id: I1a3f13ea2c024cbd73b33fd9dd58e531d3576a55
Reviewed-on: https://webrtc-review.googlesource.com/64780
Commit-Queue: Niels Moller <nisse@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22625}
2018-03-27 13:11:16 +00:00

538 lines
23 KiB
C++
Raw Blame History

/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
#include <stdlib.h>
#include <string.h>
#include <algorithm>
#include <memory>
#include <set>
#include <vector>
#include "modules/include/module_common_types.h"
#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
TemporalLayers::FrameConfig::FrameConfig()
: FrameConfig(kNone, kNone, kNone, false) {}
TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
TemporalLayers::BufferFlags golden,
TemporalLayers::BufferFlags arf)
: FrameConfig(last, golden, arf, false) {}
TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
TemporalLayers::BufferFlags golden,
TemporalLayers::BufferFlags arf,
FreezeEntropy)
: FrameConfig(last, golden, arf, true) {}
TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
TemporalLayers::BufferFlags golden,
TemporalLayers::BufferFlags arf,
bool freeze_entropy)
: drop_frame(last == TemporalLayers::kNone &&
golden == TemporalLayers::kNone &&
arf == TemporalLayers::kNone),
last_buffer_flags(last),
golden_buffer_flags(golden),
arf_buffer_flags(arf),
encoder_layer_id(0),
packetizer_temporal_idx(kNoTemporalIdx),
layer_sync(false),
freeze_entropy(freeze_entropy) {}
namespace {
std::vector<unsigned int> GetTemporalIds(size_t num_layers) {
switch (num_layers) {
case 1:
// Temporal layer structure (single layer):
// 0 0 0 0 ...
return {0};
case 2:
// Temporal layer structure:
// 1 1 ...
// 0 0 ...
return {0, 1};
case 3:
// Temporal layer structure:
// 2 2 2 2 ...
// 1 1 ...
// 0 0 ...
return {0, 2, 1, 2};
case 4:
// Temporal layer structure:
// 3 3 3 3 3 3 3 3 ...
// 2 2 2 2 ...
// 1 1 ...
// 0 0 ...
return {0, 3, 2, 3, 1, 3, 2, 3};
default:
RTC_NOTREACHED();
break;
}
RTC_NOTREACHED();
return {0};
}
std::vector<bool> GetTemporalLayerSync(size_t num_layers) {
switch (num_layers) {
case 1:
return {false};
case 2:
return {false, true, false, false, false, false, false, false};
case 3:
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
return {false, true, true, false};
} else {
return {false, true, true, false, false, false, false, false};
}
case 4:
return {false, true, true, false, true, false, false, false,
false, false, false, false, false, false, false, false};
default:
break;
}
RTC_NOTREACHED() << num_layers;
return {};
}
std::vector<TemporalLayers::FrameConfig> GetTemporalPattern(size_t num_layers) {
// For indexing in the patterns described below (which temporal layers they
// belong to), see the diagram above.
// Layer sync is done similarly for all patterns (except single stream) and
// happens every 8 frames:
// TL1 layer syncs by periodically by only referencing TL0 ('last'), but still
// updating 'golden', so it can be used as a reference by future TL1 frames.
// TL2 layer syncs just before TL1 by only depending on TL0 (and not depending
// on TL1's buffer before TL1 has layer synced).
// TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for
// the base layer in 1-3TL instead of 'last' periodically on long intervals,
// so that if scene changes occur (user walks between rooms or rotates webcam)
// the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant
// keyframe.
switch (num_layers) {
case 1:
// All frames reference all buffers and the 'last' buffer is updated.
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference,
TemporalLayers::kReference)};
case 2:
// All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'.
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy)};
case 3:
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
// This field trial is intended to check if it is worth using a shorter
// temporal pattern, trading some coding efficiency for less risk of
// dropped frames.
// The coding efficiency will decrease somewhat since the higher layer
// state is more volatile, but it will be offset slightly by updating
// the altref buffer with TL2 frames, instead of just referencing lower
// layers.
// If a frame is dropped in a higher layer, the jitter
// buffer on the receive side won't be able to decode any higher layer
// frame until the next sync frame. So we expect a noticeable decrease
// in frame drops on links with high packet loss.
// TL0 references and updates the 'last' buffer.
// TL1 references 'last' and references and updates 'golden'.
// TL2 references both 'last' & 'golden' and references and updates
// 'arf'.
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kNone,
TemporalLayers::kUpdate),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kUpdate,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kFreezeEntropy)};
} else {
// All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'.
// TL2 references both 'last' and 'golden' but updates no buffer.
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kNone,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kFreezeEntropy)};
}
case 4:
// TL0 references and updates only the 'last' buffer.
// TL1 references 'last' and updates and references 'golden'.
// TL2 references 'last' and 'golden', and references and updates 'arf'.
// TL3 references all buffers but update none of them.
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kNone,
TemporalLayers::kNone, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kNone,
TemporalLayers::kUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kNone,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kUpdate,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, TemporalLayers::kFreezeEntropy)};
default:
RTC_NOTREACHED();
break;
}
RTC_NOTREACHED();
return {TemporalLayers::FrameConfig(
TemporalLayers::kNone, TemporalLayers::kNone, TemporalLayers::kNone)};
}
} // namespace
DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
: num_layers_(std::max(1, number_of_temporal_layers)),
temporal_ids_(GetTemporalIds(num_layers_)),
temporal_layer_sync_(GetTemporalLayerSync(num_layers_)),
temporal_pattern_(GetTemporalPattern(num_layers_)),
pattern_idx_(255),
last_base_layer_sync_(false) {
RTC_DCHECK_EQ(temporal_pattern_.size(), temporal_layer_sync_.size());
RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
RTC_CHECK_GE(number_of_temporal_layers, 0);
RTC_CHECK_LE(number_of_temporal_layers, 4);
// pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if
// temporal_ids_ are ever longer. If this is no longer correct it needs to
// wrap at max(temporal_ids_.size(), temporal_pattern_.size()).
RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size());
}
void DefaultTemporalLayers::OnRatesUpdated(
const std::vector<uint32_t>& bitrates_bps,
int framerate_fps) {
RTC_DCHECK_GT(bitrates_bps.size(), 0);
RTC_DCHECK_LE(bitrates_bps.size(), num_layers_);
// |bitrates_bps| uses individual rate per layer, but Vp8EncoderConfig wants
// the accumulated rate, so sum them up.
new_bitrates_bps_ = bitrates_bps;
new_bitrates_bps_->resize(num_layers_);
for (size_t i = 1; i < num_layers_; ++i) {
(*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1];
}
}
bool DefaultTemporalLayers::UpdateConfiguration(Vp8EncoderConfig* cfg) {
if (!new_bitrates_bps_) {
return false;
}
for (size_t i = 0; i < num_layers_; ++i) {
cfg->ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000;
// ..., 4, 2, 1
cfg->ts_rate_decimator[i] = 1 << (num_layers_ - i - 1);
}
cfg->ts_number_layers = num_layers_;
cfg->ts_periodicity = temporal_ids_.size();
memcpy(cfg->ts_layer_id, &temporal_ids_[0],
sizeof(unsigned int) * temporal_ids_.size());
new_bitrates_bps_.reset();
return true;
}
TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig(
uint32_t timestamp) {
RTC_DCHECK_GT(num_layers_, 0);
RTC_DCHECK_LT(0, temporal_pattern_.size());
pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size();
TemporalLayers::FrameConfig tl_config = temporal_pattern_[pattern_idx_];
tl_config.layer_sync =
temporal_layer_sync_[pattern_idx_ % temporal_layer_sync_.size()];
tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx =
temporal_ids_[pattern_idx_ % temporal_ids_.size()];
return tl_config;
}
void DefaultTemporalLayers::PopulateCodecSpecific(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
RTC_DCHECK_GT(num_layers_, 0);
if (num_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
} else {
vp8_info->temporalIdx = tl_config.packetizer_temporal_idx;
vp8_info->layerSync = tl_config.layer_sync;
if (frame_is_keyframe) {
vp8_info->temporalIdx = 0;
vp8_info->layerSync = true;
}
if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
// Regardless of pattern the frame after a base layer sync will always
// be a layer sync.
vp8_info->layerSync = true;
}
last_base_layer_sync_ = frame_is_keyframe;
}
}
// Returns list of temporal dependencies for each frame in the temporal pattern.
// Values are lists of indecies in the pattern.
std::vector<std::set<uint8_t>> GetTemporalDependencies(
int num_temporal_layers) {
switch (num_temporal_layers) {
case 1:
return {{0}};
case 2:
return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}};
case 3:
if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
return {{0}, {0}, {0}, {0, 1, 2}};
} else {
return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}};
}
case 4:
return {{8}, {0}, {0}, {0, 2},
{0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6},
{0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10},
{4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}};
default:
RTC_NOTREACHED();
return {};
}
}
DefaultTemporalLayersChecker::DefaultTemporalLayersChecker(
int num_temporal_layers)
: TemporalLayersChecker(num_temporal_layers),
num_layers_(std::max(1, num_temporal_layers)),
temporal_ids_(GetTemporalIds(num_layers_)),
temporal_dependencies_(GetTemporalDependencies(num_layers_)),
pattern_idx_(255) {
int i = 0;
while (temporal_ids_.size() < temporal_dependencies_.size()) {
temporal_ids_.push_back(temporal_ids_[i++]);
}
}
bool DefaultTemporalLayersChecker::CheckTemporalConfig(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& frame_config) {
if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe,
frame_config)) {
return false;
}
if (frame_config.drop_frame) {
return true;
}
++pattern_idx_;
if (pattern_idx_ == temporal_ids_.size()) {
// All non key-frame buffers should be updated each pattern cycle.
if (!last_.is_keyframe && !last_.is_updated_this_cycle) {
RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle.";
return false;
}
if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) {
RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle.";
return false;
}
if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) {
RTC_LOG(LS_ERROR)
<< "Golden buffer was not updated during pattern cycle.";
return false;
}
last_.is_updated_this_cycle = false;
arf_.is_updated_this_cycle = false;
golden_.is_updated_this_cycle = false;
pattern_idx_ = 0;
}
uint8_t expected_tl_idx = temporal_ids_[pattern_idx_];
if (frame_config.packetizer_temporal_idx != expected_tl_idx) {
RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: "
<< static_cast<int>(expected_tl_idx) << " Actual: "
<< static_cast<int>(frame_config.packetizer_temporal_idx);
return false;
}
bool need_sync = temporal_ids_[pattern_idx_] > 0 &&
temporal_ids_[pattern_idx_] != kNoTemporalIdx;
std::vector<int> dependencies;
if (frame_config.last_buffer_flags &
TemporalLayers::BufferFlags::kReference) {
uint8_t referenced_layer = temporal_ids_[last_.pattern_idx];
if (referenced_layer > 0) {
need_sync = false;
}
if (!last_.is_keyframe) {
dependencies.push_back(last_.pattern_idx);
}
}
if (frame_config.arf_buffer_flags & TemporalLayers::BufferFlags::kReference) {
uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx];
if (referenced_layer > 0) {
need_sync = false;
}
if (!arf_.is_keyframe) {
dependencies.push_back(arf_.pattern_idx);
}
}
if (frame_config.golden_buffer_flags &
TemporalLayers::BufferFlags::kReference) {
uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx];
if (referenced_layer > 0) {
need_sync = false;
}
if (!golden_.is_keyframe) {
dependencies.push_back(golden_.pattern_idx);
}
}
if (need_sync != frame_config.layer_sync) {
RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
<< need_sync << " Actual: " << frame_config.layer_sync;
return false;
}
if (!frame_is_keyframe) {
size_t i;
for (i = 0; i < dependencies.size(); ++i) {
if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) ==
temporal_dependencies_[pattern_idx_].end()) {
RTC_LOG(LS_ERROR)
<< "Illegal temporal dependency out of defined pattern "
"from position "
<< static_cast<int>(pattern_idx_) << " to position "
<< static_cast<int>(dependencies[i]);
return false;
}
}
}
if (frame_config.last_buffer_flags & TemporalLayers::BufferFlags::kUpdate) {
last_.is_updated_this_cycle = true;
last_.pattern_idx = pattern_idx_;
last_.is_keyframe = false;
}
if (frame_config.arf_buffer_flags & TemporalLayers::BufferFlags::kUpdate) {
arf_.is_updated_this_cycle = true;
arf_.pattern_idx = pattern_idx_;
arf_.is_keyframe = false;
}
if (frame_config.golden_buffer_flags & TemporalLayers::BufferFlags::kUpdate) {
golden_.is_updated_this_cycle = true;
golden_.pattern_idx = pattern_idx_;
golden_.is_keyframe = false;
}
if (frame_is_keyframe) {
last_.is_keyframe = true;
arf_.is_keyframe = true;
golden_.is_keyframe = true;
}
return true;
}
} // namespace webrtc
Reference in a new issue View git blame Copy permalink