mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-12 21:30:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/video/video_stream_encoder.cc
Danil Chapovalov 44ab20021d In EncoderStreamFactory pass field trials as required parameter 
Instead of passing it as optional parameter during construction, pass field trials as required parameters on use.
Test that create the EncoderStreamFactory might not have an easy access to the actual field trials, but prod code has appropriate field trials when uses the factory.

This way EncoderStreamFactory doesn't need to depend on global field trial string through FieldTrialBaseConfig class.

Bug: webrtc:10335
Change-Id: I8f7030e41579ff2c5dd362c491a4e1624b23e690
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/347700
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42098}
2024-04-17 12:53:30 +00:00

2625 lines
107 KiB
C++
Raw Blame History

/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_stream_encoder.h"
#include <algorithm>
#include <array>
#include <limits>
#include <memory>
#include <numeric>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/cleanup/cleanup.h"
#include "absl/types/optional.h"
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/render_resolution.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_layers_allocation.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_processor.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_adapter.h"
#include "media/base/media_channel.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include "modules/video_coding/utility/vp8_constants.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "rtc_base/experiments/encoder_info_settings.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/thread_annotations.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/metrics.h"
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include "video/alignment_adjuster.h"
#include "video/config/encoder_stream_factory.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_dumping_encoder.h"
namespace webrtc {
namespace {
// Time interval for logging frame counts.
const int64_t kFrameLogIntervalMs = 60000;
// Time to keep a single cached pending frame in paused state.
const int64_t kPendingFrameTimeoutMs = 1000;
constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
// TODO(bugs.webrtc.org/13572): Remove this kill switch after deploying the
// feature.
constexpr char kSwitchEncoderOnInitializationFailuresFieldTrial[] =
"WebRTC-SwitchEncoderOnInitializationFailures";
const size_t kDefaultPayloadSize = 1440;
const int64_t kParameterUpdateIntervalMs = 1000;
// Animation is capped to 720p.
constexpr int kMaxAnimationPixels = 1280 * 720;
constexpr int kDefaultMinScreenSharebps = 1200000;
int GetNumSpatialLayers(const VideoCodec& codec) {
if (codec.codecType == kVideoCodecVP9) {
return codec.VP9().numberOfSpatialLayers;
} else if (codec.codecType == kVideoCodecAV1 &&
codec.GetScalabilityMode().has_value()) {
return ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
} else {
return 0;
}
}
absl::optional<EncodedImageCallback::DropReason> MaybeConvertDropReason(
VideoStreamEncoderObserver::DropReason reason) {
switch (reason) {
case VideoStreamEncoderObserver::DropReason::kMediaOptimization:
return EncodedImageCallback::DropReason::kDroppedByMediaOptimizations;
case VideoStreamEncoderObserver::DropReason::kEncoder:
return EncodedImageCallback::DropReason::kDroppedByEncoder;
default:
return absl::nullopt;
}
}
bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
const VideoCodec& new_send_codec,
bool was_encode_called_since_last_initialization) {
// Does not check max/minBitrate or maxFramerate.
if (new_send_codec.codecType != prev_send_codec.codecType ||
new_send_codec.width != prev_send_codec.width ||
new_send_codec.height != prev_send_codec.height ||
new_send_codec.qpMax != prev_send_codec.qpMax ||
new_send_codec.numberOfSimulcastStreams !=
prev_send_codec.numberOfSimulcastStreams ||
new_send_codec.mode != prev_send_codec.mode ||
new_send_codec.GetFrameDropEnabled() !=
prev_send_codec.GetFrameDropEnabled()) {
return true;
}
if (!was_encode_called_since_last_initialization &&
(new_send_codec.startBitrate != prev_send_codec.startBitrate)) {
// If start bitrate has changed reconfigure encoder only if encoding had not
// yet started.
return true;
}
switch (new_send_codec.codecType) {
case kVideoCodecVP8:
if (new_send_codec.VP8() != prev_send_codec.VP8()) {
return true;
}
break;
case kVideoCodecVP9:
if (new_send_codec.VP9() != prev_send_codec.VP9()) {
return true;
}
break;
case kVideoCodecH264:
if (new_send_codec.H264() != prev_send_codec.H264()) {
return true;
}
break;
case kVideoCodecH265:
// TODO(bugs.webrtc.org/13485): Implement new send codec H265
[[fallthrough]];
default:
break;
}
for (unsigned char i = 0; i < new_send_codec.numberOfSimulcastStreams; ++i) {
if (!new_send_codec.simulcastStream[i].active) {
// No need to reset when stream is inactive.
continue;
}
if (!prev_send_codec.simulcastStream[i].active ||
new_send_codec.simulcastStream[i].width !=
prev_send_codec.simulcastStream[i].width ||
new_send_codec.simulcastStream[i].height !=
prev_send_codec.simulcastStream[i].height ||
new_send_codec.simulcastStream[i].numberOfTemporalLayers !=
prev_send_codec.simulcastStream[i].numberOfTemporalLayers ||
new_send_codec.simulcastStream[i].qpMax !=
prev_send_codec.simulcastStream[i].qpMax) {
return true;
}
}
if (new_send_codec.codecType == kVideoCodecVP9) {
size_t num_spatial_layers = new_send_codec.VP9().numberOfSpatialLayers;
for (unsigned char i = 0; i < num_spatial_layers; ++i) {
if (!new_send_codec.spatialLayers[i].active) {
// No need to reset when layer is inactive.
continue;
}
if (new_send_codec.spatialLayers[i].width !=
prev_send_codec.spatialLayers[i].width ||
new_send_codec.spatialLayers[i].height !=
prev_send_codec.spatialLayers[i].height ||
new_send_codec.spatialLayers[i].numberOfTemporalLayers !=
prev_send_codec.spatialLayers[i].numberOfTemporalLayers ||
new_send_codec.spatialLayers[i].qpMax !=
prev_send_codec.spatialLayers[i].qpMax ||
!prev_send_codec.spatialLayers[i].active) {
return true;
}
}
}
if (new_send_codec.GetScalabilityMode() !=
prev_send_codec.GetScalabilityMode()) {
return true;
}
return false;
}
// Limit allocation across TLs in bitrate allocation according to number of TLs
// in EncoderInfo.
VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
const VideoBitrateAllocation& allocation,
const VideoEncoder::EncoderInfo& encoder_info) {
if (allocation.get_sum_bps() == 0) {
return allocation;
}
VideoBitrateAllocation new_allocation;
for (int si = 0; si < kMaxSpatialLayers; ++si) {
if (encoder_info.fps_allocation[si].size() == 1 &&
allocation.IsSpatialLayerUsed(si)) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at ti:0).
new_allocation.SetBitrate(si, 0, allocation.GetSpatialLayerSum(si));
} else {
for (int ti = 0; ti < kMaxTemporalStreams; ++ti) {
if (allocation.HasBitrate(si, ti))
new_allocation.SetBitrate(si, ti, allocation.GetBitrate(si, ti));
}
}
}
new_allocation.set_bw_limited(allocation.is_bw_limited());
return new_allocation;
}
// Converts a VideoBitrateAllocation that contains allocated bitrate per layer,
// and an EncoderInfo that contains information about the actual encoder
// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast
// etc.
VideoLayersAllocation CreateVideoLayersAllocation(
const VideoCodec& encoder_config,
const VideoEncoder::RateControlParameters& current_rate,
const VideoEncoder::EncoderInfo& encoder_info) {
const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate;
VideoLayersAllocation layers_allocation;
if (target_bitrate.get_sum_bps() == 0) {
return layers_allocation;
}
if (encoder_config.numberOfSimulcastStreams > 1) {
layers_allocation.resolution_and_frame_rate_is_valid = true;
for (int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) {
if (!target_bitrate.IsSpatialLayerUsed(si) ||
target_bitrate.GetSpatialLayerSum(si) == 0) {
continue;
}
layers_allocation.active_spatial_layers.emplace_back();
VideoLayersAllocation::SpatialLayer& spatial_layer =
layers_allocation.active_spatial_layers.back();
spatial_layer.width = encoder_config.simulcastStream[si].width;
spatial_layer.height = encoder_config.simulcastStream[si].height;
spatial_layer.rtp_stream_index = si;
spatial_layer.spatial_id = 0;
auto frame_rate_fraction =
VideoEncoder::EncoderInfo::kMaxFramerateFraction;
if (encoder_info.fps_allocation[si].size() == 1) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at tl:0).
spatial_layer.target_bitrate_per_temporal_layer.push_back(
DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si)));
frame_rate_fraction = encoder_info.fps_allocation[si][0];
} else { // Temporal layers are supported.
uint32_t temporal_layer_bitrate_bps = 0;
for (size_t ti = 0;
ti < encoder_config.simulcastStream[si].numberOfTemporalLayers;
++ti) {
if (!target_bitrate.HasBitrate(si, ti)) {
break;
}
if (ti < encoder_info.fps_allocation[si].size()) {
// Use frame rate of the top used temporal layer.
frame_rate_fraction = encoder_info.fps_allocation[si][ti];
}
temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti);
spatial_layer.target_bitrate_per_temporal_layer.push_back(
DataRate::BitsPerSec(temporal_layer_bitrate_bps));
}
}
// Encoder may drop frames internally if `maxFramerate` is set.
spatial_layer.frame_rate_fps = std::min<uint8_t>(
encoder_config.simulcastStream[si].maxFramerate,
rtc::saturated_cast<uint8_t>(
(current_rate.framerate_fps * frame_rate_fraction) /
VideoEncoder::EncoderInfo::kMaxFramerateFraction));
}
} else if (encoder_config.numberOfSimulcastStreams == 1) {
// TODO(bugs.webrtc.org/12000): Implement support for AV1 with
// scalability.
const bool higher_spatial_depend_on_lower =
encoder_config.codecType == kVideoCodecVP9 &&
encoder_config.VP9().interLayerPred == InterLayerPredMode::kOn;
layers_allocation.resolution_and_frame_rate_is_valid = true;
std::vector<DataRate> aggregated_spatial_bitrate(
webrtc::kMaxTemporalStreams, DataRate::Zero());
for (int si = 0; si < webrtc::kMaxSpatialLayers; ++si) {
layers_allocation.resolution_and_frame_rate_is_valid = true;
if (!target_bitrate.IsSpatialLayerUsed(si) ||
target_bitrate.GetSpatialLayerSum(si) == 0) {
break;
}
layers_allocation.active_spatial_layers.emplace_back();
VideoLayersAllocation::SpatialLayer& spatial_layer =
layers_allocation.active_spatial_layers.back();
spatial_layer.width = encoder_config.spatialLayers[si].width;
spatial_layer.height = encoder_config.spatialLayers[si].height;
spatial_layer.rtp_stream_index = 0;
spatial_layer.spatial_id = si;
auto frame_rate_fraction =
VideoEncoder::EncoderInfo::kMaxFramerateFraction;
if (encoder_info.fps_allocation[si].size() == 1) {
// One TL is signalled to be used by the encoder. Do not distribute
// bitrate allocation across TLs (use sum at tl:0).
DataRate aggregated_temporal_bitrate =
DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si));
aggregated_spatial_bitrate[0] += aggregated_temporal_bitrate;
if (higher_spatial_depend_on_lower) {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_spatial_bitrate[0]);
} else {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_temporal_bitrate);
}
frame_rate_fraction = encoder_info.fps_allocation[si][0];
} else { // Temporal layers are supported.
DataRate aggregated_temporal_bitrate = DataRate::Zero();
for (size_t ti = 0;
ti < encoder_config.spatialLayers[si].numberOfTemporalLayers;
++ti) {
if (!target_bitrate.HasBitrate(si, ti)) {
break;
}
if (ti < encoder_info.fps_allocation[si].size()) {
// Use frame rate of the top used temporal layer.
frame_rate_fraction = encoder_info.fps_allocation[si][ti];
}
aggregated_temporal_bitrate +=
DataRate::BitsPerSec(target_bitrate.GetBitrate(si, ti));
if (higher_spatial_depend_on_lower) {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_temporal_bitrate + aggregated_spatial_bitrate[ti]);
aggregated_spatial_bitrate[ti] += aggregated_temporal_bitrate;
} else {
spatial_layer.target_bitrate_per_temporal_layer.push_back(
aggregated_temporal_bitrate);
}
}
}
// Encoder may drop frames internally if `maxFramerate` is set.
spatial_layer.frame_rate_fps = std::min<uint8_t>(
encoder_config.spatialLayers[si].maxFramerate,
rtc::saturated_cast<uint8_t>(
(current_rate.framerate_fps * frame_rate_fraction) /
VideoEncoder::EncoderInfo::kMaxFramerateFraction));
}
}
return layers_allocation;
}
VideoEncoder::EncoderInfo GetEncoderInfoWithBitrateLimitUpdate(
const VideoEncoder::EncoderInfo& info,
const VideoEncoderConfig& encoder_config,
bool default_limits_allowed) {
if (!default_limits_allowed || !info.resolution_bitrate_limits.empty() ||
encoder_config.simulcast_layers.size() <= 1) {
return info;
}
// Bitrate limits are not configured and more than one layer is used, use
// the default limits (bitrate limits are not used for simulcast).
VideoEncoder::EncoderInfo new_info = info;
new_info.resolution_bitrate_limits =
EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
encoder_config.codec_type);
return new_info;
}
int NumActiveStreams(const std::vector<VideoStream>& streams) {
int num_active = 0;
for (const auto& stream : streams) {
if (stream.active)
++num_active;
}
return num_active;
}
void ApplySpatialLayerBitrateLimits(
const VideoEncoder::EncoderInfo& encoder_info,
const VideoEncoderConfig& encoder_config,
VideoCodec* codec) {
if (!(GetNumSpatialLayers(*codec) > 0)) {
// ApplySpatialLayerBitrateLimits() supports VP9 and AV1 (the latter with
// scalability mode set) only.
return;
}
if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
encoder_config, *codec) ||
encoder_config.simulcast_layers.size() <= 1) {
// Resolution bitrate limits usage is restricted to singlecast.
return;
}
// Get bitrate limits for active stream.
absl::optional<uint32_t> pixels =
VideoStreamAdapter::GetSingleActiveLayerPixels(*codec);
if (!pixels.has_value()) {
return;
}
absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
encoder_info.GetEncoderBitrateLimitsForResolution(*pixels);
if (!bitrate_limits.has_value()) {
return;
}
// Index for the active stream.
absl::optional<size_t> index;
for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
if (encoder_config.simulcast_layers[i].active)
index = i;
}
if (!index.has_value()) {
return;
}
int min_bitrate_bps;
if (encoder_config.simulcast_layers[*index].min_bitrate_bps <= 0) {
min_bitrate_bps = bitrate_limits->min_bitrate_bps;
} else {
min_bitrate_bps =
std::max(bitrate_limits->min_bitrate_bps,
encoder_config.simulcast_layers[*index].min_bitrate_bps);
}
int max_bitrate_bps;
if (encoder_config.simulcast_layers[*index].max_bitrate_bps <= 0) {
max_bitrate_bps = bitrate_limits->max_bitrate_bps;
} else {
max_bitrate_bps =
std::min(bitrate_limits->max_bitrate_bps,
encoder_config.simulcast_layers[*index].max_bitrate_bps);
}
if (min_bitrate_bps >= max_bitrate_bps) {
RTC_LOG(LS_WARNING) << "Bitrate limits not used, min_bitrate_bps "
<< min_bitrate_bps << " >= max_bitrate_bps "
<< max_bitrate_bps;
return;
}
for (int i = 0; i < GetNumSpatialLayers(*codec); ++i) {
if (codec->spatialLayers[i].active) {
codec->spatialLayers[i].minBitrate = min_bitrate_bps / 1000;
codec->spatialLayers[i].maxBitrate = max_bitrate_bps / 1000;
codec->spatialLayers[i].targetBitrate =
std::min(codec->spatialLayers[i].targetBitrate,
codec->spatialLayers[i].maxBitrate);
break;
}
}
}
void ApplyEncoderBitrateLimitsIfSingleActiveStream(
const VideoEncoder::EncoderInfo& encoder_info,
const std::vector<VideoStream>& encoder_config_layers,
std::vector<VideoStream>* streams) {
// Apply limits if simulcast with one active stream (expect lowest).
bool single_active_stream =
streams->size() > 1 && NumActiveStreams(*streams) == 1 &&
!streams->front().active && NumActiveStreams(encoder_config_layers) == 1;
if (!single_active_stream) {
return;
}
// Index for the active stream.
size_t index = 0;
for (size_t i = 0; i < encoder_config_layers.size(); ++i) {
if (encoder_config_layers[i].active)
index = i;
}
if (streams->size() < (index + 1) || !(*streams)[index].active) {
return;
}
// Get bitrate limits for active stream.
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
encoder_info.GetEncoderBitrateLimitsForResolution(
(*streams)[index].width * (*streams)[index].height);
if (!encoder_bitrate_limits) {
return;
}
// If bitrate limits are set by RtpEncodingParameters, use intersection.
int min_bitrate_bps;
if (encoder_config_layers[index].min_bitrate_bps <= 0) {
min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
} else {
min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
(*streams)[index].min_bitrate_bps);
}
int max_bitrate_bps;
if (encoder_config_layers[index].max_bitrate_bps <= 0) {
max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
} else {
max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
(*streams)[index].max_bitrate_bps);
}
if (min_bitrate_bps >= max_bitrate_bps) {
RTC_LOG(LS_WARNING) << "Encoder bitrate limits"
<< " (min=" << encoder_bitrate_limits->min_bitrate_bps
<< ", max=" << encoder_bitrate_limits->max_bitrate_bps
<< ") do not intersect with stream limits"
<< " (min=" << (*streams)[index].min_bitrate_bps
<< ", max=" << (*streams)[index].max_bitrate_bps
<< "). Encoder bitrate limits not used.";
return;
}
(*streams)[index].min_bitrate_bps = min_bitrate_bps;
(*streams)[index].max_bitrate_bps = max_bitrate_bps;
(*streams)[index].target_bitrate_bps =
std::min((*streams)[index].target_bitrate_bps,
encoder_bitrate_limits->max_bitrate_bps);
}
absl::optional<int> ParseVp9LowTierCoreCountThreshold(
const FieldTrialsView& trials) {
FieldTrialFlag disable_low_tier("Disabled");
FieldTrialParameter<int> max_core_count("max_core_count", 2);
ParseFieldTrial({&disable_low_tier, &max_core_count},
trials.Lookup("WebRTC-VP9-LowTierOptimizations"));
if (disable_low_tier.Get()) {
return absl::nullopt;
}
return max_core_count.Get();
}
absl::optional<int> ParseEncoderThreadLimit(const FieldTrialsView& trials) {
FieldTrialOptional<int> encoder_thread_limit("encoder_thread_limit");
ParseFieldTrial({&encoder_thread_limit},
trials.Lookup("WebRTC-VideoEncoderSettings"));
return encoder_thread_limit.GetOptional();
}
absl::optional<VideoSourceRestrictions> MergeRestrictions(
const std::vector<absl::optional<VideoSourceRestrictions>>& list) {
absl::optional<VideoSourceRestrictions> return_value;
for (const auto& res : list) {
if (!res) {
continue;
}
if (!return_value) {
return_value = *res;
continue;
}
return_value->UpdateMin(*res);
}
return return_value;
}
} // namespace
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
: rate_control(),
encoder_target(DataRate::Zero()),
stable_encoder_target(DataRate::Zero()) {}
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings(
const VideoBitrateAllocation& bitrate,
double framerate_fps,
DataRate bandwidth_allocation,
DataRate encoder_target,
DataRate stable_encoder_target)
: rate_control(bitrate, framerate_fps, bandwidth_allocation),
encoder_target(encoder_target),
stable_encoder_target(stable_encoder_target) {}
bool VideoStreamEncoder::EncoderRateSettings::operator==(
const EncoderRateSettings& rhs) const {
return rate_control == rhs.rate_control &&
encoder_target == rhs.encoder_target &&
stable_encoder_target == rhs.stable_encoder_target;
}
bool VideoStreamEncoder::EncoderRateSettings::operator!=(
const EncoderRateSettings& rhs) const {
return !(*this == rhs);
}
class VideoStreamEncoder::DegradationPreferenceManager
: public DegradationPreferenceProvider {
public:
explicit DegradationPreferenceManager(
VideoStreamAdapter* video_stream_adapter)
: degradation_preference_(DegradationPreference::DISABLED),
is_screenshare_(false),
effective_degradation_preference_(DegradationPreference::DISABLED),
video_stream_adapter_(video_stream_adapter) {
RTC_DCHECK(video_stream_adapter_);
sequence_checker_.Detach();
}
~DegradationPreferenceManager() override = default;
DegradationPreference degradation_preference() const override {
RTC_DCHECK_RUN_ON(&sequence_checker_);
return effective_degradation_preference_;
}
void SetDegradationPreference(DegradationPreference degradation_preference) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
degradation_preference_ = degradation_preference;
MaybeUpdateEffectiveDegradationPreference();
}
void SetIsScreenshare(bool is_screenshare) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
is_screenshare_ = is_screenshare;
MaybeUpdateEffectiveDegradationPreference();
}
private:
void MaybeUpdateEffectiveDegradationPreference()
RTC_RUN_ON(&sequence_checker_) {
DegradationPreference effective_degradation_preference =
(is_screenshare_ &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
if (effective_degradation_preference != effective_degradation_preference_) {
effective_degradation_preference_ = effective_degradation_preference;
video_stream_adapter_->SetDegradationPreference(
effective_degradation_preference);
}
}
RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
DegradationPreference degradation_preference_
RTC_GUARDED_BY(&sequence_checker_);
bool is_screenshare_ RTC_GUARDED_BY(&sequence_checker_);
DegradationPreference effective_degradation_preference_
RTC_GUARDED_BY(&sequence_checker_);
VideoStreamAdapter* video_stream_adapter_ RTC_GUARDED_BY(&sequence_checker_);
};
VideoStreamEncoder::VideoStreamEncoder(
const Environment& env,
uint32_t number_of_cores,
VideoStreamEncoderObserver* encoder_stats_observer,
const VideoStreamEncoderSettings& settings,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter,
std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
encoder_queue,
BitrateAllocationCallbackType allocation_cb_type,
webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector)
: env_(env),
worker_queue_(TaskQueueBase::Current()),
number_of_cores_(number_of_cores),
settings_(settings),
allocation_cb_type_(allocation_cb_type),
rate_control_settings_(
RateControlSettings::ParseFromKeyValueConfig(&env_.field_trials())),
encoder_selector_from_constructor_(encoder_selector),
encoder_selector_from_factory_(
encoder_selector_from_constructor_
? nullptr
: settings.encoder_factory->GetEncoderSelector()),
encoder_selector_(encoder_selector_from_constructor_
? encoder_selector_from_constructor_
: encoder_selector_from_factory_.get()),
encoder_stats_observer_(encoder_stats_observer),
frame_cadence_adapter_(std::move(frame_cadence_adapter)),
delta_ntp_internal_ms_(env_.clock().CurrentNtpInMilliseconds() -
env_.clock().TimeInMilliseconds()),
last_frame_log_ms_(env_.clock().TimeInMilliseconds()),
next_frame_types_(1, VideoFrameType::kVideoFrameDelta),
automatic_animation_detection_experiment_(
ParseAutomatincAnimationDetectionFieldTrial()),
input_state_provider_(encoder_stats_observer),
video_stream_adapter_(
std::make_unique<VideoStreamAdapter>(&input_state_provider_,
encoder_stats_observer,
env_.field_trials())),
degradation_preference_manager_(
std::make_unique<DegradationPreferenceManager>(
video_stream_adapter_.get())),
stream_resource_manager_(&input_state_provider_,
encoder_stats_observer,
&env_.clock(),
settings_.experiment_cpu_load_estimator,
std::move(overuse_detector),
degradation_preference_manager_.get(),
env_.field_trials()),
video_source_sink_controller_(/*sink=*/frame_cadence_adapter_.get(),
/*source=*/nullptr),
default_limits_allowed_(!env_.field_trials().IsEnabled(
"WebRTC-DefaultBitrateLimitsKillSwitch")),
qp_parsing_allowed_(
!env_.field_trials().IsEnabled("WebRTC-QpParsingKillSwitch")),
switch_encoder_on_init_failures_(!env_.field_trials().IsDisabled(
kSwitchEncoderOnInitializationFailuresFieldTrial)),
vp9_low_tier_core_threshold_(
ParseVp9LowTierCoreCountThreshold(env_.field_trials())),
experimental_encoder_thread_limit_(
ParseEncoderThreadLimit(env_.field_trials())),
encoder_queue_(std::move(encoder_queue)) {
TRACE_EVENT0("webrtc", "VideoStreamEncoder::VideoStreamEncoder");
RTC_DCHECK_RUN_ON(worker_queue_);
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK_GE(number_of_cores, 1);
frame_cadence_adapter_->Initialize(&cadence_callback_);
stream_resource_manager_.Initialize(encoder_queue_.get());
encoder_queue_->PostTask([this] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
resource_adaptation_processor_ =
std::make_unique<ResourceAdaptationProcessor>(
video_stream_adapter_.get());
stream_resource_manager_.SetAdaptationProcessor(
resource_adaptation_processor_.get(), video_stream_adapter_.get());
resource_adaptation_processor_->AddResourceLimitationsListener(
&stream_resource_manager_);
video_stream_adapter_->AddRestrictionsListener(&stream_resource_manager_);
video_stream_adapter_->AddRestrictionsListener(this);
stream_resource_manager_.MaybeInitializePixelLimitResource();
// Add the stream resource manager's resources to the processor.
adaptation_constraints_ = stream_resource_manager_.AdaptationConstraints();
for (auto* constraint : adaptation_constraints_) {
video_stream_adapter_->AddAdaptationConstraint(constraint);
}
});
}
VideoStreamEncoder::~VideoStreamEncoder() {
RTC_DCHECK_RUN_ON(worker_queue_);
RTC_DCHECK(!video_source_sink_controller_.HasSource())
<< "Must call ::Stop() before destruction.";
// The queue must be destroyed before its pointer is invalidated to avoid race
// between destructor and running task that check if function is called on the
// encoder_queue_.
// std::unique_ptr destructor does the same two operations in reverse order as
// it doesn't expect member would be used after its destruction has started.
encoder_queue_.get_deleter()(encoder_queue_.get());
encoder_queue_.release();
}
void VideoStreamEncoder::Stop() {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetSource(nullptr);
rtc::Event shutdown_event;
absl::Cleanup shutdown = [&shutdown_event] { shutdown_event.Set(); };
encoder_queue_->PostTask([this, shutdown = std::move(shutdown)] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
if (resource_adaptation_processor_) {
stream_resource_manager_.StopManagedResources();
for (auto* constraint : adaptation_constraints_) {
video_stream_adapter_->RemoveAdaptationConstraint(constraint);
}
for (auto& resource : additional_resources_) {
stream_resource_manager_.RemoveResource(resource);
}
additional_resources_.clear();
video_stream_adapter_->RemoveRestrictionsListener(this);
video_stream_adapter_->RemoveRestrictionsListener(
&stream_resource_manager_);
resource_adaptation_processor_->RemoveResourceLimitationsListener(
&stream_resource_manager_);
stream_resource_manager_.SetAdaptationProcessor(nullptr, nullptr);
resource_adaptation_processor_.reset();
}
rate_allocator_ = nullptr;
ReleaseEncoder();
encoder_ = nullptr;
frame_cadence_adapter_ = nullptr;
});
shutdown_event.Wait(rtc::Event::kForever);
}
void VideoStreamEncoder::SetFecControllerOverride(
FecControllerOverride* fec_controller_override) {
encoder_queue_->PostTask([this, fec_controller_override] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_DCHECK(!fec_controller_override_);
fec_controller_override_ = fec_controller_override;
if (encoder_) {
encoder_->SetFecControllerOverride(fec_controller_override_);
}
});
}
void VideoStreamEncoder::AddAdaptationResource(
rtc::scoped_refptr<Resource> resource) {
RTC_DCHECK_RUN_ON(worker_queue_);
TRACE_EVENT0("webrtc", "VideoStreamEncoder::AddAdaptationResource");
// Map any externally added resources as kCpu for the sake of stats reporting.
// TODO(hbos): Make the manager map any unknown resources to kCpu and get rid
// of this MapResourceToReason() call.
TRACE_EVENT_ASYNC_BEGIN0(
"webrtc", "VideoStreamEncoder::AddAdaptationResource(latency)", this);
encoder_queue_->PostTask([this, resource = std::move(resource)] {
TRACE_EVENT_ASYNC_END0(
"webrtc", "VideoStreamEncoder::AddAdaptationResource(latency)", this);
RTC_DCHECK_RUN_ON(encoder_queue_.get());
additional_resources_.push_back(resource);
stream_resource_manager_.AddResource(resource, VideoAdaptationReason::kCpu);
});
}
std::vector<rtc::scoped_refptr<Resource>>
VideoStreamEncoder::GetAdaptationResources() {
RTC_DCHECK_RUN_ON(worker_queue_);
// In practice, this method is only called by tests to verify operations that
// run on the encoder queue. So rather than force PostTask() operations to
// be accompanied by an event and a `Wait()`, we'll use PostTask + Wait()
// here.
rtc::Event event;
std::vector<rtc::scoped_refptr<Resource>> resources;
encoder_queue_->PostTask([&] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
resources = resource_adaptation_processor_->GetResources();
event.Set();
});
event.Wait(rtc::Event::kForever);
return resources;
}
void VideoStreamEncoder::SetSource(
rtc::VideoSourceInterface<VideoFrame>* source,
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetSource(source);
input_state_provider_.OnHasInputChanged(source);
// This may trigger reconfiguring the QualityScaler on the encoder queue.
encoder_queue_->PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
degradation_preference_manager_->SetDegradationPreference(
degradation_preference);
stream_resource_manager_.SetDegradationPreferences(degradation_preference);
if (encoder_) {
stream_resource_manager_.ConfigureQualityScaler(
encoder_->GetEncoderInfo());
stream_resource_manager_.ConfigureBandwidthQualityScaler(
encoder_->GetEncoderInfo());
}
});
}
void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetRotationApplied(rotation_applied);
video_source_sink_controller_.PushSourceSinkSettings();
encoder_queue_->PostTask([this, sink] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
sink_ = sink;
});
}
void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
encoder_queue_->PostTask([this, start_bitrate_bps] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_LOG(LS_INFO) << "SetStartBitrate " << start_bitrate_bps;
encoder_target_bitrate_bps_ =
start_bitrate_bps != 0 ? absl::optional<uint32_t>(start_bitrate_bps)
: absl::nullopt;
stream_resource_manager_.SetStartBitrate(
DataRate::BitsPerSec(start_bitrate_bps));
});
}
void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
size_t max_data_payload_length) {
ConfigureEncoder(std::move(config), max_data_payload_length, nullptr);
}
void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
size_t max_data_payload_length,
SetParametersCallback callback) {
RTC_DCHECK_RUN_ON(worker_queue_);
// Inform source about max configured framerate,
// requested_resolution and which layers are active.
int max_framerate = -1;
// Is any layer active.
bool active = false;
// The max requested_resolution.
absl::optional<rtc::VideoSinkWants::FrameSize> requested_resolution;
for (const auto& stream : config.simulcast_layers) {
active |= stream.active;
if (stream.active) {
max_framerate = std::max(stream.max_framerate, max_framerate);
}
// Note: we propagate the highest requested_resolution regardless
// if layer is active or not.
if (stream.requested_resolution) {
if (!requested_resolution) {
requested_resolution.emplace(stream.requested_resolution->width,
stream.requested_resolution->height);
} else {
requested_resolution.emplace(
std::max(stream.requested_resolution->width,
requested_resolution->width),
std::max(stream.requested_resolution->height,
requested_resolution->height));
}
}
}
if (requested_resolution !=
video_source_sink_controller_.requested_resolution() ||
active != video_source_sink_controller_.active() ||
max_framerate !=
video_source_sink_controller_.frame_rate_upper_limit().value_or(-1)) {
video_source_sink_controller_.SetRequestedResolution(requested_resolution);
if (max_framerate >= 0) {
video_source_sink_controller_.SetFrameRateUpperLimit(max_framerate);
} else {
video_source_sink_controller_.SetFrameRateUpperLimit(absl::nullopt);
}
video_source_sink_controller_.SetActive(active);
video_source_sink_controller_.PushSourceSinkSettings();
}
encoder_queue_->PostTask([this, config = std::move(config),
max_data_payload_length,
callback = std::move(callback)]() mutable {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_DCHECK(sink_);
RTC_LOG(LS_INFO) << "ConfigureEncoder requested.";
// Set up the frame cadence adapter according to if we're going to do
// screencast. The final number of spatial layers is based on info
// in `send_codec_`, which is computed based on incoming frame
// dimensions which can only be determined later.
//
// Note: zero-hertz mode isn't enabled by this alone. Constraints also
// have to be set up with min_fps = 0 and max_fps > 0.
if (config.content_type == VideoEncoderConfig::ContentType::kScreen) {
frame_cadence_adapter_->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{});
} else {
frame_cadence_adapter_->SetZeroHertzModeEnabled(absl::nullopt);
}
pending_encoder_creation_ =
(!encoder_ || encoder_config_.video_format != config.video_format ||
max_data_payload_length_ != max_data_payload_length);
encoder_config_ = std::move(config);
max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ = true;
// Reconfigure the encoder now if the frame resolution is known.
// Otherwise, the reconfiguration is deferred until the next frame to
// minimize the number of reconfigurations. The codec configuration
// depends on incoming video frame size.
if (last_frame_info_) {
if (callback) {
encoder_configuration_callbacks_.push_back(std::move(callback));
}
ReconfigureEncoder();
} else {
webrtc::InvokeSetParametersCallback(callback, webrtc::RTCError::OK());
}
});
}
// We should reduce the number of 'full' ReconfigureEncoder(). If only need
// subset of it at runtime, consider handle it in
// VideoStreamEncoder::EncodeVideoFrame() when encoder_info_ != info.
void VideoStreamEncoder::ReconfigureEncoder() {
// Running on the encoder queue.
RTC_DCHECK(pending_encoder_reconfiguration_);
RTC_LOG(LS_INFO) << "[VSE] " << __func__
<< " [encoder_config=" << encoder_config_.ToString() << "]";
bool encoder_reset_required = false;
if (pending_encoder_creation_) {
// Destroy existing encoder instance before creating a new one. Otherwise
// attempt to create another instance will fail if encoder factory
// supports only single instance of encoder of given type.
encoder_.reset();
encoder_ = MaybeCreateFrameDumpingEncoderWrapper(
settings_.encoder_factory->Create(env_, encoder_config_.video_format),
env_.field_trials());
if (!encoder_) {
RTC_LOG(LS_ERROR) << "CreateVideoEncoder failed, failing encoder format: "
<< encoder_config_.video_format.ToString();
RequestEncoderSwitch();
return;
}
if (encoder_selector_) {
encoder_selector_->OnCurrentEncoder(encoder_config_.video_format);
}
encoder_->SetFecControllerOverride(fec_controller_override_);
encoder_reset_required = true;
}
// TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
// Possibly adjusts scale_resolution_down_by in `encoder_config_` to limit the
// alignment value.
AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
encoder_->GetEncoderInfo(), &encoder_config_, absl::nullopt);
std::vector<VideoStream> streams;
if (encoder_config_.video_stream_factory) {
// Note: only tests set their own EncoderStreamFactory...
streams = encoder_config_.video_stream_factory->CreateEncoderStreams(
env_.field_trials(), last_frame_info_->width, last_frame_info_->height,
encoder_config_);
} else {
auto factory = rtc::make_ref_counted<cricket::EncoderStreamFactory>(
encoder_config_.video_format.name, encoder_config_.max_qp,
encoder_config_.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen,
encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo(),
MergeRestrictions({latest_restrictions_, animate_restrictions_}));
streams = factory->CreateEncoderStreams(
env_.field_trials(), last_frame_info_->width, last_frame_info_->height,
encoder_config_);
}
// TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
// Get alignment when actual number of layers are known.
int alignment = AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
encoder_->GetEncoderInfo(), &encoder_config_, streams.size());
// Check that the higher layers do not try to set number of temporal layers
// to less than 1.
// TODO(brandtr): Get rid of the wrapping optional as it serves no purpose
// at this layer.
#if RTC_DCHECK_IS_ON
for (const auto& stream : streams) {
RTC_DCHECK_GE(stream.num_temporal_layers.value_or(1), 1);
}
#endif
// TODO(ilnik): If configured resolution is significantly less than provided,
// e.g. because there are not enough SSRCs for all simulcast streams,
// signal new resolutions via SinkWants to video source.
// Stream dimensions may be not equal to given because of a simulcast
// restrictions.
auto highest_stream = absl::c_max_element(
streams, [](const webrtc::VideoStream& a, const webrtc::VideoStream& b) {
return std::tie(a.width, a.height) < std::tie(b.width, b.height);
});
int highest_stream_width = static_cast<int>(highest_stream->width);
int highest_stream_height = static_cast<int>(highest_stream->height);
// Dimension may be reduced to be, e.g. divisible by 4.
RTC_CHECK_GE(last_frame_info_->width, highest_stream_width);
RTC_CHECK_GE(last_frame_info_->height, highest_stream_height);
crop_width_ = last_frame_info_->width - highest_stream_width;
crop_height_ = last_frame_info_->height - highest_stream_height;
if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(true)) {
// when qp is not trusted, we priorities to using the
// |resolution_bitrate_limits| provided by the decoder.
const std::vector<VideoEncoder::ResolutionBitrateLimits>& bitrate_limits =
encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()
? EncoderInfoSettings::
GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()
: encoder_->GetEncoderInfo().resolution_bitrate_limits;
// For BandwidthQualityScaler, its implement based on a certain pixel_count
// correspond a certain bps interval. In fact, WebRTC default max_bps is
// 2500Kbps when width * height > 960 * 540. For example, we assume:
// 1.the camera support 1080p.
// 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps].
// 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps].
// We will never be stable at 720p due to actual encoding bps of 720p and
// 1080p are both 2500Kbps. So it is necessary to do a linear interpolation
// to get a certain bitrate for certain pixel_count. It also doesn't work
// for 960*540 and 640*520, we will nerver be stable at 640*520 due to their
// |target_bitrate_bps| are both 2000Kbps.
absl::optional<VideoEncoder::ResolutionBitrateLimits>
qp_untrusted_bitrate_limit = EncoderInfoSettings::
GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
last_frame_info_->width * last_frame_info_->height,
bitrate_limits);
if (qp_untrusted_bitrate_limit) {
// bandwidth_quality_scaler is only used for singlecast.
if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
VideoStream& stream = streams.back();
stream.max_bitrate_bps =
std::min(stream.max_bitrate_bps,
qp_untrusted_bitrate_limit->max_bitrate_bps);
stream.min_bitrate_bps =
std::min(stream.max_bitrate_bps,
qp_untrusted_bitrate_limit->min_bitrate_bps);
// If it is screen share mode, the minimum value of max_bitrate should
// be greater than/equal to 1200kbps.
if (encoder_config_.content_type ==
VideoEncoderConfig::ContentType::kScreen) {
stream.max_bitrate_bps =
std::max(stream.max_bitrate_bps, kDefaultMinScreenSharebps);
}
stream.target_bitrate_bps = stream.max_bitrate_bps;
}
}
} else {
absl::optional<VideoEncoder::ResolutionBitrateLimits>
encoder_bitrate_limits =
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
last_frame_info_->width * last_frame_info_->height);
if (encoder_bitrate_limits) {
if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
// Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
// or/and can be provided by encoder. In presence of both set of
// limits, the final set is derived as their intersection.
int min_bitrate_bps;
if (encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
} else {
min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
streams.back().min_bitrate_bps);
}
int max_bitrate_bps;
// The API max bitrate comes from both `encoder_config_.max_bitrate_bps`
// and `encoder_config_.simulcast_layers[0].max_bitrate_bps`.
absl::optional<int> api_max_bitrate_bps;
if (encoder_config_.simulcast_layers[0].max_bitrate_bps > 0) {
api_max_bitrate_bps =
encoder_config_.simulcast_layers[0].max_bitrate_bps;
}
if (encoder_config_.max_bitrate_bps > 0) {
api_max_bitrate_bps = api_max_bitrate_bps.has_value()
? std::min(encoder_config_.max_bitrate_bps,
*api_max_bitrate_bps)
: encoder_config_.max_bitrate_bps;
}
if (!api_max_bitrate_bps.has_value()) {
max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
} else {
max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
streams.back().max_bitrate_bps);
}
if (min_bitrate_bps < max_bitrate_bps) {
streams.back().min_bitrate_bps = min_bitrate_bps;
streams.back().max_bitrate_bps = max_bitrate_bps;
streams.back().target_bitrate_bps =
std::min(streams.back().target_bitrate_bps,
encoder_bitrate_limits->max_bitrate_bps);
} else {
RTC_LOG(LS_WARNING)
<< "Bitrate limits provided by encoder"
<< " (min=" << encoder_bitrate_limits->min_bitrate_bps
<< ", max=" << encoder_bitrate_limits->max_bitrate_bps
<< ") do not intersect with limits set by app"
<< " (min=" << streams.back().min_bitrate_bps
<< ", max=" << api_max_bitrate_bps.value_or(-1)
<< "). The app bitrate limits will be used.";
}
}
}
}
ApplyEncoderBitrateLimitsIfSingleActiveStream(
GetEncoderInfoWithBitrateLimitUpdate(
encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_),
encoder_config_.simulcast_layers, &streams);
VideoCodec codec = VideoCodecInitializer::SetupCodec(
env_.field_trials(), encoder_config_, streams);
if (encoder_config_.codec_type == kVideoCodecVP9 ||
encoder_config_.codec_type == kVideoCodecAV1) {
// Spatial layers configuration might impose some parity restrictions,
// thus some cropping might be needed.
RTC_CHECK_GE(last_frame_info_->width, codec.width);
RTC_CHECK_GE(last_frame_info_->height, codec.height);
crop_width_ = last_frame_info_->width - codec.width;
crop_height_ = last_frame_info_->height - codec.height;
ApplySpatialLayerBitrateLimits(
GetEncoderInfoWithBitrateLimitUpdate(encoder_->GetEncoderInfo(),
encoder_config_,
default_limits_allowed_),
encoder_config_, &codec);
}
char log_stream_buf[4 * 1024];
rtc::SimpleStringBuilder log_stream(log_stream_buf);
log_stream << "ReconfigureEncoder: simulcast streams: ";
for (size_t i = 0; i < codec.numberOfSimulcastStreams; ++i) {
absl::optional<ScalabilityMode> scalability_mode =
codec.simulcastStream[i].GetScalabilityMode();
if (scalability_mode) {
log_stream << "{" << i << ": " << codec.simulcastStream[i].width << "x"
<< codec.simulcastStream[i].height << " "
<< ScalabilityModeToString(*scalability_mode)
<< ", min_kbps: " << codec.simulcastStream[i].minBitrate
<< ", target_kbps: " << codec.simulcastStream[i].targetBitrate
<< ", max_kbps: " << codec.simulcastStream[i].maxBitrate
<< ", max_fps: " << codec.simulcastStream[i].maxFramerate
<< ", max_qp: " << codec.simulcastStream[i].qpMax
<< ", num_tl: "
<< codec.simulcastStream[i].numberOfTemporalLayers
<< ", active: "
<< (codec.simulcastStream[i].active ? "true" : "false") << "}";
}
}
if (encoder_config_.codec_type == kVideoCodecVP9 ||
encoder_config_.codec_type == kVideoCodecAV1) {
log_stream << ", spatial layers: ";
for (int i = 0; i < GetNumSpatialLayers(codec); ++i) {
log_stream << "{" << i << ": " << codec.spatialLayers[i].width << "x"
<< codec.spatialLayers[i].height
<< ", min_kbps: " << codec.spatialLayers[i].minBitrate
<< ", target_kbps: " << codec.spatialLayers[i].targetBitrate
<< ", max_kbps: " << codec.spatialLayers[i].maxBitrate
<< ", max_fps: " << codec.spatialLayers[i].maxFramerate
<< ", max_qp: " << codec.spatialLayers[i].qpMax << ", num_tl: "
<< codec.spatialLayers[i].numberOfTemporalLayers
<< ", active: "
<< (codec.spatialLayers[i].active ? "true" : "false") << "}";
}
}
RTC_LOG(LS_INFO) << "[VSE] " << log_stream.str();
codec.startBitrate = std::max(encoder_target_bitrate_bps_.value_or(0) / 1000,
codec.minBitrate);
codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
codec.expect_encode_from_texture = last_frame_info_->is_texture;
// Make sure the start bit rate is sane...
RTC_DCHECK_LE(codec.startBitrate, 1000000);
max_framerate_ = codec.maxFramerate;
// The resolutions that we're actually encoding with.
std::vector<rtc::VideoSinkWants::FrameSize> encoder_resolutions;
// TODO(hbos): For the case of SVC, also make use of `codec.spatialLayers`.
// For now, SVC layers are handled by the VP9 encoder.
for (const auto& simulcastStream : codec.simulcastStream) {
if (!simulcastStream.active)
continue;
encoder_resolutions.emplace_back(simulcastStream.width,
simulcastStream.height);
}
worker_queue_->PostTask(SafeTask(
task_safety_.flag(),
[this, alignment,
encoder_resolutions = std::move(encoder_resolutions)]() {
RTC_DCHECK_RUN_ON(worker_queue_);
if (alignment != video_source_sink_controller_.resolution_alignment() ||
encoder_resolutions !=
video_source_sink_controller_.resolutions()) {
video_source_sink_controller_.SetResolutionAlignment(alignment);
video_source_sink_controller_.SetResolutions(
std::move(encoder_resolutions));
video_source_sink_controller_.PushSourceSinkSettings();
}
}));
rate_allocator_ =
settings_.bitrate_allocator_factory->CreateVideoBitrateAllocator(codec);
rate_allocator_->SetLegacyConferenceMode(
encoder_config_.legacy_conference_mode);
// Reset (release existing encoder) if one exists and anything except
// start bitrate or max framerate has changed.
if (!encoder_reset_required) {
encoder_reset_required = RequiresEncoderReset(
send_codec_, codec, was_encode_called_since_last_initialization_);
}
if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
number_of_cores_ <= vp9_low_tier_core_threshold_.value_or(0)) {
codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityLow);
}
send_codec_ = codec;
// Keep the same encoder, as long as the video_format is unchanged.
// Encoder creation block is split in two since EncoderInfo needed to start
// CPU adaptation with the correct settings should be polled after
// encoder_->InitEncode().
if (encoder_reset_required) {
ReleaseEncoder();
const size_t max_data_payload_length = max_data_payload_length_ > 0
? max_data_payload_length_
: kDefaultPayloadSize;
VideoEncoder::Settings settings = VideoEncoder::Settings(
settings_.capabilities, number_of_cores_, max_data_payload_length);
settings.encoder_thread_limit = experimental_encoder_thread_limit_;
int error = encoder_->InitEncode(&send_codec_, settings);
if (error != 0) {
RTC_LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
"codec type: "
<< CodecTypeToPayloadString(send_codec_.codecType)
<< " (" << send_codec_.codecType
<< "). Error: " << error;
ReleaseEncoder();
} else {
encoder_initialized_ = true;
encoder_->RegisterEncodeCompleteCallback(this);
frame_encode_metadata_writer_.OnEncoderInit(send_codec_);
next_frame_types_.clear();
next_frame_types_.resize(
std::max(static_cast<int>(codec.numberOfSimulcastStreams), 1),
VideoFrameType::kVideoFrameKey);
}
frame_encode_metadata_writer_.Reset();
last_encode_info_ms_ = absl::nullopt;
was_encode_called_since_last_initialization_ = false;
}
// Inform dependents of updated encoder settings.
OnEncoderSettingsChanged();
if (encoder_initialized_) {
RTC_LOG(LS_VERBOSE) << " max bitrate " << codec.maxBitrate
<< " start bitrate " << codec.startBitrate
<< " max frame rate " << codec.maxFramerate
<< " max payload size " << max_data_payload_length_;
} else {
RTC_LOG(LS_ERROR) << "[VSE] Failed to configure encoder.";
rate_allocator_ = nullptr;
}
if (pending_encoder_creation_) {
stream_resource_manager_.ConfigureEncodeUsageResource();
pending_encoder_creation_ = false;
}
int num_layers;
if (codec.codecType == kVideoCodecVP8) {
num_layers = codec.VP8()->numberOfTemporalLayers;
} else if (codec.codecType == kVideoCodecVP9) {
num_layers = codec.VP9()->numberOfTemporalLayers;
} else if (codec.codecType == kVideoCodecAV1 &&
codec.GetScalabilityMode().has_value()) {
num_layers =
ScalabilityModeToNumTemporalLayers(*(codec.GetScalabilityMode()));
} else if (codec.codecType == kVideoCodecH264) {
num_layers = codec.H264()->numberOfTemporalLayers;
} else if (codec.codecType == kVideoCodecGeneric &&
codec.numberOfSimulcastStreams > 0) {
// This is mainly for unit testing, disabling frame dropping.
// TODO(sprang): Add a better way to disable frame dropping.
num_layers = codec.simulcastStream[0].numberOfTemporalLayers;
} else {
// TODO(bugs.webrtc.org/13485): Implement H265 temporal layer
num_layers = 1;
}
frame_dropper_.Reset();
frame_dropper_.SetRates(codec.startBitrate, max_framerate_);
// Force-disable frame dropper if either:
// * We have screensharing with layers.
// * "WebRTC-FrameDropper" field trial is "Disabled".
force_disable_frame_dropper_ =
env_.field_trials().IsDisabled(kFrameDropperFieldTrial) ||
(num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing);
const VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
if (rate_control_settings_.UseEncoderBitrateAdjuster()) {
bitrate_adjuster_ =
std::make_unique<EncoderBitrateAdjuster>(codec, env_.field_trials());
bitrate_adjuster_->OnEncoderInfo(info);
}
if (rate_allocator_ && last_encoder_rate_settings_) {
// We have a new rate allocator instance and already configured target
// bitrate. Update the rate allocation and notify observers.
// We must invalidate the last_encoder_rate_settings_ to ensure
// the changes get propagated to all listeners.
EncoderRateSettings rate_settings = *last_encoder_rate_settings_;
last_encoder_rate_settings_.reset();
rate_settings.rate_control.framerate_fps = GetInputFramerateFps();
SetEncoderRates(UpdateBitrateAllocation(rate_settings));
}
encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams);
pending_encoder_reconfiguration_ = false;
bool is_svc = false;
bool single_stream_or_non_first_inactive = true;
for (size_t i = 1; i < encoder_config_.number_of_streams; ++i) {
if (encoder_config_.simulcast_layers[i].active) {
single_stream_or_non_first_inactive = false;
break;
}
}
// Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9
// and AV1 and leave only one stream containing all necessary information.
if ((encoder_config_.codec_type == kVideoCodecVP9 ||
encoder_config_.codec_type == kVideoCodecAV1) &&
single_stream_or_non_first_inactive) {
// Lower max bitrate to the level codec actually can produce.
streams[0].max_bitrate_bps =
std::min(streams[0].max_bitrate_bps,
SvcRateAllocator::GetMaxBitrate(codec).bps<int>());
streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000;
// target_bitrate_bps specifies the maximum padding bitrate.
streams[0].target_bitrate_bps =
SvcRateAllocator::GetPaddingBitrate(codec).bps<int>();
streams[0].width = streams.back().width;
streams[0].height = streams.back().height;
is_svc = GetNumSpatialLayers(codec) > 1;
streams.resize(1);
}
sink_->OnEncoderConfigurationChanged(
std::move(streams), is_svc, encoder_config_.content_type,
encoder_config_.min_transmit_bitrate_bps);
stream_resource_manager_.ConfigureQualityScaler(info);
stream_resource_manager_.ConfigureBandwidthQualityScaler(info);
webrtc::RTCError encoder_configuration_result = webrtc::RTCError::OK();
if (!encoder_initialized_) {
RTC_LOG(LS_WARNING) << "Failed to initialize "
<< CodecTypeToPayloadString(codec.codecType)
<< " encoder."
<< "switch_encoder_on_init_failures: "
<< switch_encoder_on_init_failures_;
if (switch_encoder_on_init_failures_) {
RequestEncoderSwitch();
} else {
encoder_configuration_result =
webrtc::RTCError(RTCErrorType::UNSUPPORTED_OPERATION);
}
}
if (!encoder_configuration_callbacks_.empty()) {
for (auto& callback : encoder_configuration_callbacks_) {
webrtc::InvokeSetParametersCallback(callback,
encoder_configuration_result);
}
encoder_configuration_callbacks_.clear();
}
}
void VideoStreamEncoder::RequestEncoderSwitch() {
bool is_encoder_switching_supported =
settings_.encoder_switch_request_callback != nullptr;
bool is_encoder_selector_available = encoder_selector_ != nullptr;
RTC_LOG(LS_INFO) << "RequestEncoderSwitch."
<< " is_encoder_selector_available: "
<< is_encoder_selector_available
<< " is_encoder_switching_supported: "
<< is_encoder_switching_supported;
if (!is_encoder_switching_supported) {
return;
}
// If encoder selector is available, switch to the encoder it prefers.
// Otherwise try switching to VP8 (default WebRTC codec).
absl::optional<SdpVideoFormat> preferred_fallback_encoder;
if (is_encoder_selector_available) {
preferred_fallback_encoder = encoder_selector_->OnEncoderBroken();
}
if (!preferred_fallback_encoder) {
preferred_fallback_encoder =
SdpVideoFormat(CodecTypeToPayloadString(kVideoCodecVP8));
}
settings_.encoder_switch_request_callback->RequestEncoderSwitch(
*preferred_fallback_encoder, /*allow_default_fallback=*/true);
}
void VideoStreamEncoder::OnEncoderSettingsChanged() {
EncoderSettings encoder_settings(
GetEncoderInfoWithBitrateLimitUpdate(
encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_),
encoder_config_.Copy(), send_codec_);
stream_resource_manager_.SetEncoderSettings(encoder_settings);
input_state_provider_.OnEncoderSettingsChanged(encoder_settings);
bool is_screenshare = encoder_settings.encoder_config().content_type ==
VideoEncoderConfig::ContentType::kScreen;
degradation_preference_manager_->SetIsScreenshare(is_screenshare);
if (is_screenshare) {
frame_cadence_adapter_->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{
send_codec_.numberOfSimulcastStreams});
}
}
void VideoStreamEncoder::OnFrame(Timestamp post_time,
bool queue_overload,
const VideoFrame& video_frame) {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
VideoFrame incoming_frame = video_frame;
// In some cases, e.g., when the frame from decoder is fed to encoder,
// the timestamp may be set to the future. As the encoding pipeline assumes
// capture time to be less than present time, we should reset the capture
// timestamps here. Otherwise there may be issues with RTP send stream.
if (incoming_frame.timestamp_us() > post_time.us())
incoming_frame.set_timestamp_us(post_time.us());
// Capture time may come from clock with an offset and drift from clock_.
int64_t capture_ntp_time_ms;
if (video_frame.ntp_time_ms() > 0) {
capture_ntp_time_ms = video_frame.ntp_time_ms();
} else if (video_frame.render_time_ms() != 0) {
capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
} else {
capture_ntp_time_ms = post_time.ms() + delta_ntp_internal_ms_;
}
incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);
// Convert NTP time, in ms, to RTP timestamp.
const int kMsToRtpTimestamp = 90;
incoming_frame.set_rtp_timestamp(
kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms()));
// Identifier should remain the same for newly produced incoming frame and the
// received |video_frame|.
incoming_frame.set_capture_time_identifier(
video_frame.capture_time_identifier());
if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
// We don't allow the same capture time for two frames, drop this one.
RTC_LOG(LS_WARNING) << "Same/old NTP timestamp ("
<< incoming_frame.ntp_time_ms()
<< " <= " << last_captured_timestamp_
<< ") for incoming frame. Dropping.";
ProcessDroppedFrame(incoming_frame,
VideoStreamEncoderObserver::DropReason::kBadTimestamp);
return;
}
bool log_stats = false;
if (post_time.ms() - last_frame_log_ms_ > kFrameLogIntervalMs) {
last_frame_log_ms_ = post_time.ms();
log_stats = true;
}
last_captured_timestamp_ = incoming_frame.ntp_time_ms();
encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(),
incoming_frame.height());
++captured_frame_count_;
CheckForAnimatedContent(incoming_frame, post_time.us());
bool cwnd_frame_drop =
cwnd_frame_drop_interval_ &&
(cwnd_frame_counter_++ % cwnd_frame_drop_interval_.value() == 0);
if (!queue_overload && !cwnd_frame_drop) {
MaybeEncodeVideoFrame(incoming_frame, post_time.us());
} else {
if (cwnd_frame_drop) {
// Frame drop by congestion window pushback. Do not encode this
// frame.
++dropped_frame_cwnd_pushback_count_;
} else {
// There is a newer frame in flight. Do not encode this frame.
RTC_LOG(LS_VERBOSE)
<< "Incoming frame dropped due to that the encoder is blocked.";
++dropped_frame_encoder_block_count_;
}
ProcessDroppedFrame(
incoming_frame,
cwnd_frame_drop
? VideoStreamEncoderObserver::DropReason::kCongestionWindow
: VideoStreamEncoderObserver::DropReason::kEncoderQueue);
}
if (log_stats) {
RTC_LOG(LS_INFO) << "Number of frames: captured " << captured_frame_count_
<< ", dropped (due to congestion window pushback) "
<< dropped_frame_cwnd_pushback_count_
<< ", dropped (due to encoder blocked) "
<< dropped_frame_encoder_block_count_ << ", interval_ms "
<< kFrameLogIntervalMs;
captured_frame_count_ = 0;
dropped_frame_cwnd_pushback_count_ = 0;
dropped_frame_encoder_block_count_ = 0;
}
}
void VideoStreamEncoder::OnDiscardedFrame() {
encoder_stats_observer_->OnFrameDropped(
VideoStreamEncoderObserver::DropReason::kSource);
}
bool VideoStreamEncoder::EncoderPaused() const {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
// Pause video if paused by caller or as long as the network is down or the
// pacer queue has grown too large in buffered mode.
// If the pacer queue has grown too large or the network is down,
// `last_encoder_rate_settings_->encoder_target` will be 0.
return !last_encoder_rate_settings_ ||
last_encoder_rate_settings_->encoder_target == DataRate::Zero();
}
void VideoStreamEncoder::TraceFrameDropStart() {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
// Start trace event only on the first frame after encoder is paused.
if (!encoder_paused_and_dropped_frame_) {
TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this);
}
encoder_paused_and_dropped_frame_ = true;
}
void VideoStreamEncoder::TraceFrameDropEnd() {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
// End trace event on first frame after encoder resumes, if frame was dropped.
if (encoder_paused_and_dropped_frame_) {
TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
}
encoder_paused_and_dropped_frame_ = false;
}
VideoStreamEncoder::EncoderRateSettings
VideoStreamEncoder::UpdateBitrateAllocation(
const EncoderRateSettings& rate_settings) {
VideoBitrateAllocation new_allocation;
// Only call allocators if bitrate > 0 (ie, not suspended), otherwise they
// might cap the bitrate to the min bitrate configured.
if (rate_allocator_ && rate_settings.encoder_target > DataRate::Zero()) {
new_allocation = rate_allocator_->Allocate(VideoBitrateAllocationParameters(
rate_settings.encoder_target, rate_settings.stable_encoder_target,
rate_settings.rate_control.framerate_fps));
}
EncoderRateSettings new_rate_settings = rate_settings;
new_rate_settings.rate_control.target_bitrate = new_allocation;
new_rate_settings.rate_control.bitrate = new_allocation;
// VideoBitrateAllocator subclasses may allocate a bitrate higher than the
// target in order to sustain the min bitrate of the video codec. In this
// case, make sure the bandwidth allocation is at least equal the allocation
// as that is part of the document contract for that field.
new_rate_settings.rate_control.bandwidth_allocation =
std::max(new_rate_settings.rate_control.bandwidth_allocation,
DataRate::BitsPerSec(
new_rate_settings.rate_control.bitrate.get_sum_bps()));
if (bitrate_adjuster_) {
VideoBitrateAllocation adjusted_allocation =
bitrate_adjuster_->AdjustRateAllocation(new_rate_settings.rate_control);
RTC_LOG(LS_VERBOSE) << "Adjusting allocation, fps = "
<< rate_settings.rate_control.framerate_fps << ", from "
<< new_allocation.ToString() << ", to "
<< adjusted_allocation.ToString();
new_rate_settings.rate_control.bitrate = adjusted_allocation;
}
return new_rate_settings;
}
uint32_t VideoStreamEncoder::GetInputFramerateFps() {
const uint32_t default_fps = max_framerate_ != -1 ? max_framerate_ : 30;
// This method may be called after we cleared out the frame_cadence_adapter_
// reference in Stop(). In such a situation it's probably not important with a
// decent estimate.
absl::optional<uint32_t> input_fps =
frame_cadence_adapter_ ? frame_cadence_adapter_->GetInputFrameRateFps()
: absl::nullopt;
if (!input_fps || *input_fps == 0) {
return default_fps;
}
return *input_fps;
}
void VideoStreamEncoder::SetEncoderRates(
const EncoderRateSettings& rate_settings) {
RTC_DCHECK_GT(rate_settings.rate_control.framerate_fps, 0.0);
bool rate_control_changed =
(!last_encoder_rate_settings_.has_value() ||
last_encoder_rate_settings_->rate_control != rate_settings.rate_control);
// For layer allocation signal we care only about the target bitrate (not the
// adjusted one) and the target fps.
bool layer_allocation_changed =
!last_encoder_rate_settings_.has_value() ||
last_encoder_rate_settings_->rate_control.target_bitrate !=
rate_settings.rate_control.target_bitrate ||
last_encoder_rate_settings_->rate_control.framerate_fps !=
rate_settings.rate_control.framerate_fps;
if (last_encoder_rate_settings_ != rate_settings) {
last_encoder_rate_settings_ = rate_settings;
}
if (!encoder_)
return;
// Make the cadence adapter know if streams were disabled.
for (int spatial_index = 0;
spatial_index != send_codec_.numberOfSimulcastStreams; ++spatial_index) {
frame_cadence_adapter_->UpdateLayerStatus(
spatial_index,
/*enabled=*/rate_settings.rate_control.target_bitrate
.GetSpatialLayerSum(spatial_index) > 0);
}
// `bitrate_allocation` is 0 it means that the network is down or the send
// pacer is full. We currently don't pass this on to the encoder since it is
// unclear how current encoder implementations behave when given a zero target
// bitrate.
// TODO(perkj): Make sure all known encoder implementations handle zero
// target bitrate and remove this check.
if (rate_settings.rate_control.bitrate.get_sum_bps() == 0)
return;
if (rate_control_changed) {
encoder_->SetRates(rate_settings.rate_control);
encoder_stats_observer_->OnBitrateAllocationUpdated(
send_codec_, rate_settings.rate_control.bitrate);
frame_encode_metadata_writer_.OnSetRates(
rate_settings.rate_control.bitrate,
static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5));
stream_resource_manager_.SetEncoderRates(rate_settings.rate_control);
if (layer_allocation_changed &&
allocation_cb_type_ ==
BitrateAllocationCallbackType::kVideoLayersAllocation) {
sink_->OnVideoLayersAllocationUpdated(CreateVideoLayersAllocation(
send_codec_, rate_settings.rate_control, encoder_->GetEncoderInfo()));
}
}
if ((allocation_cb_type_ ==
BitrateAllocationCallbackType::kVideoBitrateAllocation) ||
(encoder_config_.content_type ==
VideoEncoderConfig::ContentType::kScreen &&
allocation_cb_type_ == BitrateAllocationCallbackType::
kVideoBitrateAllocationWhenScreenSharing)) {
sink_->OnBitrateAllocationUpdated(
// Update allocation according to info from encoder. An encoder may
// choose to not use all layers due to for example HW.
UpdateAllocationFromEncoderInfo(
rate_settings.rate_control.target_bitrate,
encoder_->GetEncoderInfo()));
}
}
void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
int64_t time_when_posted_us) {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
input_state_provider_.OnFrameSizeObserved(video_frame.size());
if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
video_frame.height() != last_frame_info_->height ||
video_frame.is_texture() != last_frame_info_->is_texture) {
if ((!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
video_frame.height() != last_frame_info_->height) &&
settings_.encoder_switch_request_callback && encoder_selector_) {
if (auto encoder = encoder_selector_->OnResolutionChange(
{video_frame.width(), video_frame.height()})) {
settings_.encoder_switch_request_callback->RequestEncoderSwitch(
*encoder, /*allow_default_fallback=*/false);
}
}
pending_encoder_reconfiguration_ = true;
last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(),
video_frame.is_texture());
RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions="
<< last_frame_info_->width << "x"
<< last_frame_info_->height
<< ", texture=" << last_frame_info_->is_texture << ".";
// Force full frame update, since resolution has changed.
accumulated_update_rect_ =
VideoFrame::UpdateRect{0, 0, video_frame.width(), video_frame.height()};
}
// We have to create the encoder before the frame drop logic,
// because the latter depends on encoder_->GetScalingSettings.
// According to the testcase
// InitialFrameDropOffWhenEncoderDisabledScaling, the return value
// from GetScalingSettings should enable or disable the frame drop.
uint32_t framerate_fps = GetInputFramerateFps();
int64_t now_ms = env_.clock().TimeInMilliseconds();
if (pending_encoder_reconfiguration_) {
ReconfigureEncoder();
last_parameters_update_ms_.emplace(now_ms);
} else if (!last_parameters_update_ms_ ||
now_ms - *last_parameters_update_ms_ >=
kParameterUpdateIntervalMs) {
if (last_encoder_rate_settings_) {
// Clone rate settings before update, so that SetEncoderRates() will
// actually detect the change between the input and
// `last_encoder_rate_setings_`, triggering the call to SetRate() on the
// encoder.
EncoderRateSettings new_rate_settings = *last_encoder_rate_settings_;
new_rate_settings.rate_control.framerate_fps =
static_cast<double>(framerate_fps);
SetEncoderRates(UpdateBitrateAllocation(new_rate_settings));
}
last_parameters_update_ms_.emplace(now_ms);
}
// Because pending frame will be dropped in any case, we need to
// remember its updated region.
if (pending_frame_) {
ProcessDroppedFrame(*pending_frame_,
VideoStreamEncoderObserver::DropReason::kEncoderQueue);
}
if (DropDueToSize(video_frame.size())) {
RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
stream_resource_manager_.OnFrameDroppedDueToSize();
// Storing references to a native buffer risks blocking frame capture.
if (video_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kNative) {
pending_frame_ = video_frame;
pending_frame_post_time_us_ = time_when_posted_us;
} else {
// Ensure that any previously stored frame is dropped.
pending_frame_.reset();
ProcessDroppedFrame(
video_frame, VideoStreamEncoderObserver::DropReason::kEncoderQueue);
}
return;
}
stream_resource_manager_.OnMaybeEncodeFrame();
if (EncoderPaused()) {
// Storing references to a native buffer risks blocking frame capture.
if (video_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kNative) {
if (pending_frame_)
TraceFrameDropStart();
pending_frame_ = video_frame;
pending_frame_post_time_us_ = time_when_posted_us;
} else {
// Ensure that any previously stored frame is dropped.
pending_frame_.reset();
TraceFrameDropStart();
ProcessDroppedFrame(
video_frame, VideoStreamEncoderObserver::DropReason::kEncoderQueue);
}
return;
}
pending_frame_.reset();
frame_dropper_.Leak(framerate_fps);
// Frame dropping is enabled iff frame dropping is not force-disabled, and
// rate controller is not trusted.
const bool frame_dropping_enabled =
!force_disable_frame_dropper_ &&
!encoder_info_.has_trusted_rate_controller;
frame_dropper_.Enable(frame_dropping_enabled);
if (frame_dropping_enabled && frame_dropper_.DropFrame()) {
RTC_LOG(LS_VERBOSE)
<< "Drop Frame: "
"target bitrate "
<< (last_encoder_rate_settings_
? last_encoder_rate_settings_->encoder_target.bps()
: 0)
<< ", input frame rate " << framerate_fps;
ProcessDroppedFrame(
video_frame,
VideoStreamEncoderObserver::DropReason::kMediaOptimization);
return;
}
EncodeVideoFrame(video_frame, time_when_posted_us);
}
void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
int64_t time_when_posted_us) {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_LOG(LS_VERBOSE) << __func__ << " posted " << time_when_posted_us
<< " ntp time " << video_frame.ntp_time_ms();
// If the encoder fail we can't continue to encode frames. When this happens
// the WebrtcVideoSender is notified and the whole VideoSendStream is
// recreated.
if (encoder_failed_ || !encoder_initialized_)
return;
// It's possible that EncodeVideoFrame can be called after we've completed
// a Stop() operation. Check if the encoder_ is set before continuing.
// See: bugs.webrtc.org/12857
if (!encoder_)
return;
TraceFrameDropEnd();
// Encoder metadata needs to be updated before encode complete callback.
const VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
if (info.implementation_name != encoder_info_.implementation_name ||
info.is_hardware_accelerated != encoder_info_.is_hardware_accelerated) {
encoder_stats_observer_->OnEncoderImplementationChanged({
.name = info.implementation_name,
.is_hardware_accelerated = info.is_hardware_accelerated,
});
if (bitrate_adjuster_) {
// Encoder implementation changed, reset overshoot detector states.
bitrate_adjuster_->Reset();
}
}
if (encoder_info_ != info) {
OnEncoderSettingsChanged();
stream_resource_manager_.ConfigureEncodeUsageResource();
// Re-configure scalers when encoder info changed. Consider two cases:
// 1. When the status of the scaler changes from enabled to disabled, if we
// don't do this CL, scaler will adapt up/down to trigger an unnecessary
// full ReconfigureEncoder() when the scaler should be banned.
// 2. When the status of the scaler changes from disabled to enabled, if we
// don't do this CL, scaler will not work until some code trigger
// ReconfigureEncoder(). In extreme cases, the scaler doesn't even work for
// a long time when we expect that the scaler should work.
stream_resource_manager_.ConfigureQualityScaler(info);
stream_resource_manager_.ConfigureBandwidthQualityScaler(info);
RTC_LOG(LS_INFO) << "[VSE] Encoder info changed to " << info.ToString();
}
if (bitrate_adjuster_) {
for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) {
bitrate_adjuster_->OnEncoderInfo(info);
break;
}
}
}
encoder_info_ = info;
last_encode_info_ms_ = env_.clock().TimeInMilliseconds();
VideoFrame out_frame(video_frame);
// Crop or scale the frame if needed. Dimension may be reduced to fit encoder
// requirements, e.g. some encoders may require them to be divisible by 4.
if ((crop_width_ > 0 || crop_height_ > 0) &&
(out_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kNative ||
!info.supports_native_handle)) {
int cropped_width = video_frame.width() - crop_width_;
int cropped_height = video_frame.height() - crop_height_;
rtc::scoped_refptr<VideoFrameBuffer> cropped_buffer;
// TODO(ilnik): Remove scaling if cropping is too big, as it should never
// happen after SinkWants signaled correctly from ReconfigureEncoder.
VideoFrame::UpdateRect update_rect = video_frame.update_rect();
if (crop_width_ < 4 && crop_height_ < 4) {
// The difference is small, crop without scaling.
int offset_x = (crop_width_ + 1) / 2;
int offset_y = (crop_height_ + 1) / 2;
// Make sure offset is even so that u/v plane becomes aligned if u/v plane
// is subsampled.
offset_x -= offset_x % 2;
offset_y -= offset_y % 2;
cropped_buffer = video_frame.video_frame_buffer()->CropAndScale(
offset_x, offset_y, cropped_width, cropped_height, cropped_width,
cropped_height);
update_rect.offset_x -= offset_x;
update_rect.offset_y -= offset_y;
update_rect.Intersect(
VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height});
} else {
// The difference is large, scale it.
cropped_buffer = video_frame.video_frame_buffer()->Scale(cropped_width,
cropped_height);
if (!update_rect.IsEmpty()) {
// Since we can't reason about pixels after scaling, we invalidate whole
// picture, if anything changed.
update_rect =
VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height};
}
}
if (!cropped_buffer) {
RTC_LOG(LS_ERROR) << "Cropping and scaling frame failed, dropping frame.";
return;
}
out_frame.set_video_frame_buffer(cropped_buffer);
out_frame.set_update_rect(update_rect);
out_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
out_frame.set_capture_time_identifier(
video_frame.capture_time_identifier());
// Since accumulated_update_rect_ is constructed before cropping,
// we can't trust it. If any changes were pending, we invalidate whole
// frame here.
if (!accumulated_update_rect_.IsEmpty()) {
accumulated_update_rect_ =
VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()};
accumulated_update_rect_is_valid_ = false;
}
}
if (!accumulated_update_rect_is_valid_) {
out_frame.clear_update_rect();
} else if (!accumulated_update_rect_.IsEmpty() &&
out_frame.has_update_rect()) {
accumulated_update_rect_.Union(out_frame.update_rect());
accumulated_update_rect_.Intersect(
VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()});
out_frame.set_update_rect(accumulated_update_rect_);
accumulated_update_rect_.MakeEmptyUpdate();
}
accumulated_update_rect_is_valid_ = true;
TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
"Encode");
stream_resource_manager_.OnEncodeStarted(out_frame, time_when_posted_us);
// The encoder should get the size that it expects.
RTC_DCHECK(send_codec_.width <= out_frame.width() &&
send_codec_.height <= out_frame.height())
<< "Encoder configured to " << send_codec_.width << "x"
<< send_codec_.height << " received a too small frame "
<< out_frame.width() << "x" << out_frame.height();
TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
out_frame.rtp_timestamp());
frame_encode_metadata_writer_.OnEncodeStarted(out_frame);
const int32_t encode_status = encoder_->Encode(out_frame, &next_frame_types_);
was_encode_called_since_last_initialization_ = true;
if (encode_status < 0) {
RTC_LOG(LS_ERROR) << "Encoder failed, failing encoder format: "
<< encoder_config_.video_format.ToString();
RequestEncoderSwitch();
return;
}
for (auto& it : next_frame_types_) {
it = VideoFrameType::kVideoFrameDelta;
}
}
void VideoStreamEncoder::RequestRefreshFrame() {
worker_queue_->PostTask(SafeTask(task_safety_.flag(), [this] {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.RequestRefreshFrame();
}));
}
void VideoStreamEncoder::SendKeyFrame(
const std::vector<VideoFrameType>& layers) {
if (!encoder_queue_->IsCurrent()) {
encoder_queue_->PostTask([this, layers] { SendKeyFrame(layers); });
return;
}
RTC_DCHECK_RUN_ON(encoder_queue_.get());
TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
RTC_DCHECK(!next_frame_types_.empty());
if (frame_cadence_adapter_)
frame_cadence_adapter_->ProcessKeyFrameRequest();
if (!encoder_) {
RTC_DLOG(LS_INFO) << __func__ << " no encoder.";
return; // Shutting down, or not configured yet.
}
if (!layers.empty()) {
RTC_DCHECK_EQ(layers.size(), next_frame_types_.size());
for (size_t i = 0; i < layers.size() && i < next_frame_types_.size(); i++) {
next_frame_types_[i] = layers[i];
}
} else {
std::fill(next_frame_types_.begin(), next_frame_types_.end(),
VideoFrameType::kVideoFrameKey);
}
}
void VideoStreamEncoder::OnLossNotification(
const VideoEncoder::LossNotification& loss_notification) {
if (!encoder_queue_->IsCurrent()) {
encoder_queue_->PostTask(
[this, loss_notification] { OnLossNotification(loss_notification); });
return;
}
RTC_DCHECK_RUN_ON(encoder_queue_.get());
if (encoder_) {
encoder_->OnLossNotification(loss_notification);
}
}
EncodedImage VideoStreamEncoder::AugmentEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) {
EncodedImage image_copy(encoded_image);
// We could either have simulcast layers or spatial layers.
// TODO(https://crbug.com/webrtc/14891): If we want to support a mix of
// simulcast and SVC we'll also need to consider the case where we have both
// simulcast and spatial indices.
int stream_idx = encoded_image.SpatialIndex().value_or(
encoded_image.SimulcastIndex().value_or(0));
frame_encode_metadata_writer_.FillTimingInfo(stream_idx, &image_copy);
frame_encode_metadata_writer_.UpdateBitstream(codec_specific_info,
&image_copy);
VideoCodecType codec_type = codec_specific_info
? codec_specific_info->codecType
: VideoCodecType::kVideoCodecGeneric;
if (image_copy.qp_ < 0 && qp_parsing_allowed_) {
// Parse encoded frame QP if that was not provided by encoder.
image_copy.qp_ =
qp_parser_
.Parse(codec_type, stream_idx, image_copy.data(), image_copy.size())
.value_or(-1);
}
TRACE_EVENT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage",
"stream_idx", stream_idx, "qp", image_copy.qp_);
RTC_LOG(LS_VERBOSE) << __func__ << " ntp time " << encoded_image.NtpTimeMs()
<< " stream_idx " << stream_idx << " qp "
<< image_copy.qp_;
image_copy.SetAtTargetQuality(codec_type == kVideoCodecVP8 &&
image_copy.qp_ <= kVp8SteadyStateQpThreshold);
return image_copy;
}
EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) {
TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
"timestamp", encoded_image.RtpTimestamp());
const size_t simulcast_index = encoded_image.SimulcastIndex().value_or(0);
const VideoCodecType codec_type = codec_specific_info
? codec_specific_info->codecType
: VideoCodecType::kVideoCodecGeneric;
EncodedImage image_copy =
AugmentEncodedImage(encoded_image, codec_specific_info);
// Post a task because `send_codec_` requires `encoder_queue_` lock and we
// need to update on quality convergence.
unsigned int image_width = image_copy._encodedWidth;
unsigned int image_height = image_copy._encodedHeight;
encoder_queue_->PostTask([this, codec_type, image_width, image_height,
simulcast_index,
at_target_quality =
image_copy.IsAtTargetQuality()] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
// Let the frame cadence adapter know about quality convergence.
if (frame_cadence_adapter_)
frame_cadence_adapter_->UpdateLayerQualityConvergence(simulcast_index,
at_target_quality);
// Currently, the internal quality scaler is used for VP9 instead of the
// webrtc qp scaler (in the no-svc case or if only a single spatial layer is
// encoded). It has to be explicitly detected and reported to adaptation
// metrics.
if (codec_type == VideoCodecType::kVideoCodecVP9 &&
send_codec_.VP9()->automaticResizeOn) {
unsigned int expected_width = send_codec_.width;
unsigned int expected_height = send_codec_.height;
int num_active_layers = 0;
for (int i = 0; i < send_codec_.VP9()->numberOfSpatialLayers; ++i) {
if (send_codec_.spatialLayers[i].active) {
++num_active_layers;
expected_width = send_codec_.spatialLayers[i].width;
expected_height = send_codec_.spatialLayers[i].height;
}
}
RTC_DCHECK_LE(num_active_layers, 1)
<< "VP9 quality scaling is enabled for "
"SVC with several active layers.";
encoder_stats_observer_->OnEncoderInternalScalerUpdate(
image_width < expected_width || image_height < expected_height);
}
});
// Encoded is called on whatever thread the real encoder implementation run
// on. In the case of hardware encoders, there might be several encoders
// running in parallel on different threads.
encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info);
EncodedImageCallback::Result result =
sink_->OnEncodedImage(image_copy, codec_specific_info);
// We are only interested in propagating the meta-data about the image, not
// encoded data itself, to the post encode function. Since we cannot be sure
// the pointer will still be valid when run on the task queue, set it to null.
DataSize frame_size = DataSize::Bytes(image_copy.size());
image_copy.ClearEncodedData();
int temporal_index = 0;
if (codec_specific_info) {
if (codec_specific_info->codecType == kVideoCodecVP9) {
temporal_index = codec_specific_info->codecSpecific.VP9.temporal_idx;
} else if (codec_specific_info->codecType == kVideoCodecVP8) {
temporal_index = codec_specific_info->codecSpecific.VP8.temporalIdx;
}
}
if (temporal_index == kNoTemporalIdx) {
temporal_index = 0;
}
RunPostEncode(image_copy, env_.clock().CurrentTime().us(), temporal_index,
frame_size);
if (result.error == Result::OK) {
// In case of an internal encoder running on a separate thread, the
// decision to drop a frame might be a frame late and signaled via
// atomic flag. This is because we can't easily wait for the worker thread
// without risking deadlocks, eg during shutdown when the worker thread
// might be waiting for the internal encoder threads to stop.
if (pending_frame_drops_.load() > 0) {
int pending_drops = pending_frame_drops_.fetch_sub(1);
RTC_DCHECK_GT(pending_drops, 0);
result.drop_next_frame = true;
}
}
return result;
}
void VideoStreamEncoder::OnDroppedFrame(DropReason reason) {
sink_->OnDroppedFrame(reason);
encoder_queue_->PostTask([this, reason] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
stream_resource_manager_.OnFrameDropped(reason);
});
}
DataRate VideoStreamEncoder::UpdateTargetBitrate(DataRate target_bitrate,
double cwnd_reduce_ratio) {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
DataRate updated_target_bitrate = target_bitrate;
// Drop frames when congestion window pushback ratio is larger than 1
// percent and target bitrate is larger than codec min bitrate.
// When target_bitrate is 0 means codec is paused, skip frame dropping.
if (cwnd_reduce_ratio > 0.01 && target_bitrate.bps() > 0 &&
target_bitrate.bps() > send_codec_.minBitrate * 1000) {
int reduce_bitrate_bps = std::min(
static_cast<int>(target_bitrate.bps() * cwnd_reduce_ratio),
static_cast<int>(target_bitrate.bps() - send_codec_.minBitrate * 1000));
if (reduce_bitrate_bps > 0) {
// At maximum the congestion window can drop 1/2 frames.
cwnd_frame_drop_interval_ = std::max(
2, static_cast<int>(target_bitrate.bps() / reduce_bitrate_bps));
// Reduce target bitrate accordingly.
updated_target_bitrate =
target_bitrate - (target_bitrate / cwnd_frame_drop_interval_.value());
return updated_target_bitrate;
}
}
cwnd_frame_drop_interval_.reset();
return updated_target_bitrate;
}
void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
DataRate stable_target_bitrate,
DataRate link_allocation,
uint8_t fraction_lost,
int64_t round_trip_time_ms,
double cwnd_reduce_ratio) {
RTC_DCHECK_GE(link_allocation, target_bitrate);
if (!encoder_queue_->IsCurrent()) {
encoder_queue_->PostTask([this, target_bitrate, stable_target_bitrate,
link_allocation, fraction_lost,
round_trip_time_ms, cwnd_reduce_ratio] {
DataRate updated_target_bitrate =
UpdateTargetBitrate(target_bitrate, cwnd_reduce_ratio);
OnBitrateUpdated(updated_target_bitrate, stable_target_bitrate,
link_allocation, fraction_lost, round_trip_time_ms,
cwnd_reduce_ratio);
});
return;
}
RTC_DCHECK_RUN_ON(encoder_queue_.get());
const bool video_is_suspended = target_bitrate == DataRate::Zero();
const bool video_suspension_changed = video_is_suspended != EncoderPaused();
if (!video_is_suspended && settings_.encoder_switch_request_callback &&
encoder_selector_) {
if (auto encoder = encoder_selector_->OnAvailableBitrate(link_allocation)) {
settings_.encoder_switch_request_callback->RequestEncoderSwitch(
*encoder, /*allow_default_fallback=*/false);
}
}
RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active.";
RTC_LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << target_bitrate.bps()
<< " stable bitrate = " << stable_target_bitrate.bps()
<< " link allocation bitrate = " << link_allocation.bps()
<< " packet loss " << static_cast<int>(fraction_lost)
<< " rtt " << round_trip_time_ms;
if (encoder_) {
encoder_->OnPacketLossRateUpdate(static_cast<float>(fraction_lost) / 256.f);
encoder_->OnRttUpdate(round_trip_time_ms);
}
uint32_t framerate_fps = GetInputFramerateFps();
frame_dropper_.SetRates((target_bitrate.bps() + 500) / 1000, framerate_fps);
EncoderRateSettings new_rate_settings{
VideoBitrateAllocation(), static_cast<double>(framerate_fps),
link_allocation, target_bitrate, stable_target_bitrate};
SetEncoderRates(UpdateBitrateAllocation(new_rate_settings));
if (target_bitrate.bps() != 0)
encoder_target_bitrate_bps_ = target_bitrate.bps();
stream_resource_manager_.SetTargetBitrate(target_bitrate);
if (video_suspension_changed) {
RTC_LOG(LS_INFO) << "Video suspend state changed to: "
<< (video_is_suspended ? "suspended" : "not suspended");
encoder_stats_observer_->OnSuspendChange(video_is_suspended);
if (!video_is_suspended && pending_frame_ &&
!DropDueToSize(pending_frame_->size())) {
// A pending stored frame can be processed.
int64_t pending_time_us =
env_.clock().CurrentTime().us() - pending_frame_post_time_us_;
if (pending_time_us < kPendingFrameTimeoutMs * 1000)
EncodeVideoFrame(*pending_frame_, pending_frame_post_time_us_);
pending_frame_.reset();
} else if (!video_is_suspended && !pending_frame_ &&
encoder_paused_and_dropped_frame_) {
// A frame was enqueued during pause-state, but since it was a native
// frame we could not store it in `pending_frame_` so request a
// refresh-frame instead.
RequestRefreshFrame();
}
}
}
bool VideoStreamEncoder::DropDueToSize(uint32_t source_pixel_count) const {
if (!encoder_ || !stream_resource_manager_.DropInitialFrames() ||
!encoder_target_bitrate_bps_ ||
!stream_resource_manager_.SingleActiveStreamPixels()) {
return false;
}
int pixel_count = std::min(
source_pixel_count, *stream_resource_manager_.SingleActiveStreamPixels());
uint32_t bitrate_bps =
stream_resource_manager_.UseBandwidthAllocationBps().value_or(
encoder_target_bitrate_bps_.value());
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
GetEncoderInfoWithBitrateLimitUpdate(
encoder_->GetEncoderInfo(), encoder_config_, default_limits_allowed_)
.GetEncoderBitrateLimitsForResolution(pixel_count);
if (encoder_bitrate_limits.has_value()) {
// Use bitrate limits provided by encoder.
return bitrate_bps <
static_cast<uint32_t>(encoder_bitrate_limits->min_start_bitrate_bps);
}
if (bitrate_bps < 300000 /* qvga */) {
return pixel_count > 320 * 240;
} else if (bitrate_bps < 500000 /* vga */) {
return pixel_count > 640 * 480;
}
return false;
}
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason,
const VideoSourceRestrictions& unfiltered_restrictions) {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_LOG(LS_INFO) << "Updating sink restrictions from "
<< (reason ? reason->Name() : std::string("<null>"))
<< " to " << restrictions.ToString();
if (frame_cadence_adapter_) {
frame_cadence_adapter_->UpdateVideoSourceRestrictions(
restrictions.max_frame_rate());
}
// TODO(webrtc:14451) Split video_source_sink_controller_
// so that ownership on restrictions/wants is kept on &encoder_queue_
latest_restrictions_ = restrictions;
worker_queue_->PostTask(SafeTask(
task_safety_.flag(), [this, restrictions = std::move(restrictions)]() {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetRestrictions(std::move(restrictions));
video_source_sink_controller_.PushSourceSinkSettings();
}));
}
void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image,
int64_t time_sent_us,
int temporal_index,
DataSize frame_size) {
if (!encoder_queue_->IsCurrent()) {
encoder_queue_->PostTask([this, encoded_image, time_sent_us, temporal_index,
frame_size] {
RunPostEncode(encoded_image, time_sent_us, temporal_index, frame_size);
});
return;
}
RTC_DCHECK_RUN_ON(encoder_queue_.get());
absl::optional<int> encode_duration_us;
if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) {
encode_duration_us =
TimeDelta::Millis(encoded_image.timing_.encode_finish_ms -
encoded_image.timing_.encode_start_ms)
.us();
}
// Run post encode tasks, such as overuse detection and frame rate/drop
// stats for internal encoders.
const bool keyframe =
encoded_image._frameType == VideoFrameType::kVideoFrameKey;
if (!frame_size.IsZero()) {
frame_dropper_.Fill(frame_size.bytes(), !keyframe);
}
stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us,
encode_duration_us, frame_size);
if (bitrate_adjuster_) {
// We could either have simulcast layers or spatial layers.
// TODO(https://crbug.com/webrtc/14891): If we want to support a mix of
// simulcast and SVC we'll also need to consider the case where we have both
// simulcast and spatial indices.
int stream_index = encoded_image.SpatialIndex().value_or(
encoded_image.SimulcastIndex().value_or(0));
bitrate_adjuster_->OnEncodedFrame(frame_size, stream_index, temporal_index);
}
}
void VideoStreamEncoder::ReleaseEncoder() {
if (!encoder_ || !encoder_initialized_) {
return;
}
encoder_->Release();
encoder_initialized_ = false;
TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release");
}
VideoStreamEncoder::AutomaticAnimationDetectionExperiment
VideoStreamEncoder::ParseAutomatincAnimationDetectionFieldTrial() const {
AutomaticAnimationDetectionExperiment result;
result.Parser()->Parse(env_.field_trials().Lookup(
"WebRTC-AutomaticAnimationDetectionScreenshare"));
if (!result.enabled) {
RTC_LOG(LS_INFO) << "Automatic animation detection experiment is disabled.";
return result;
}
RTC_LOG(LS_INFO) << "Automatic animation detection experiment settings:"
" min_duration_ms="
<< result.min_duration_ms
<< " min_area_ration=" << result.min_area_ratio
<< " min_fps=" << result.min_fps;
return result;
}
void VideoStreamEncoder::CheckForAnimatedContent(
const VideoFrame& frame,
int64_t time_when_posted_in_us) {
if (!automatic_animation_detection_experiment_.enabled ||
encoder_config_.content_type !=
VideoEncoderConfig::ContentType::kScreen ||
stream_resource_manager_.degradation_preference() !=
DegradationPreference::BALANCED) {
return;
}
if (expect_resize_state_ == ExpectResizeState::kResize && last_frame_info_ &&
last_frame_info_->width != frame.width() &&
last_frame_info_->height != frame.height()) {
// On applying resolution cap there will be one frame with no/different
// update, which should be skipped.
// It can be delayed by several frames.
expect_resize_state_ = ExpectResizeState::kFirstFrameAfterResize;
return;
}
if (expect_resize_state_ == ExpectResizeState::kFirstFrameAfterResize) {
// The first frame after resize should have new, scaled update_rect.
if (frame.has_update_rect()) {
last_update_rect_ = frame.update_rect();
} else {
last_update_rect_ = absl::nullopt;
}
expect_resize_state_ = ExpectResizeState::kNoResize;
}
bool should_cap_resolution = false;
if (!frame.has_update_rect()) {
last_update_rect_ = absl::nullopt;
animation_start_time_ = Timestamp::PlusInfinity();
} else if ((!last_update_rect_ ||
frame.update_rect() != *last_update_rect_)) {
last_update_rect_ = frame.update_rect();
animation_start_time_ = Timestamp::Micros(time_when_posted_in_us);
} else {
TimeDelta animation_duration =
Timestamp::Micros(time_when_posted_in_us) - animation_start_time_;
float area_ratio = static_cast<float>(last_update_rect_->width *
last_update_rect_->height) /
(frame.width() * frame.height());
if (animation_duration.ms() >=
automatic_animation_detection_experiment_.min_duration_ms &&
area_ratio >=
automatic_animation_detection_experiment_.min_area_ratio &&
encoder_stats_observer_->GetInputFrameRate() >=
automatic_animation_detection_experiment_.min_fps) {
should_cap_resolution = true;
}
}
if (cap_resolution_due_to_video_content_ != should_cap_resolution) {
expect_resize_state_ = should_cap_resolution ? ExpectResizeState::kResize
: ExpectResizeState::kNoResize;
cap_resolution_due_to_video_content_ = should_cap_resolution;
if (should_cap_resolution) {
RTC_LOG(LS_INFO) << "Applying resolution cap due to animation detection.";
} else {
RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent "
"animation detection.";
}
// TODO(webrtc:14451) Split video_source_sink_controller_
// so that ownership on restrictions/wants is kept on &encoder_queue_
if (should_cap_resolution) {
animate_restrictions_ =
VideoSourceRestrictions(kMaxAnimationPixels,
/* target_pixels_per_frame= */ absl::nullopt,
/* max_frame_rate= */ absl::nullopt);
} else {
animate_restrictions_.reset();
}
worker_queue_->PostTask(
SafeTask(task_safety_.flag(), [this, should_cap_resolution]() {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetPixelsPerFrameUpperLimit(
should_cap_resolution
? absl::optional<size_t>(kMaxAnimationPixels)
: absl::nullopt);
video_source_sink_controller_.PushSourceSinkSettings();
}));
}
}
void VideoStreamEncoder::InjectAdaptationResource(
rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason) {
encoder_queue_->PostTask([this, resource = std::move(resource), reason] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
additional_resources_.push_back(resource);
stream_resource_manager_.AddResource(resource, reason);
});
}
void VideoStreamEncoder::InjectAdaptationConstraint(
AdaptationConstraint* adaptation_constraint) {
rtc::Event event;
encoder_queue_->PostTask([this, adaptation_constraint, &event] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
if (!resource_adaptation_processor_) {
// The VideoStreamEncoder was stopped and the processor destroyed before
// this task had a chance to execute. No action needed.
return;
}
adaptation_constraints_.push_back(adaptation_constraint);
video_stream_adapter_->AddAdaptationConstraint(adaptation_constraint);
event.Set();
});
event.Wait(rtc::Event::kForever);
}
void VideoStreamEncoder::AddRestrictionsListenerForTesting(
VideoSourceRestrictionsListener* restrictions_listener) {
rtc::Event event;
encoder_queue_->PostTask([this, restrictions_listener, &event] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_DCHECK(resource_adaptation_processor_);
video_stream_adapter_->AddRestrictionsListener(restrictions_listener);
event.Set();
});
event.Wait(rtc::Event::kForever);
}
void VideoStreamEncoder::RemoveRestrictionsListenerForTesting(
VideoSourceRestrictionsListener* restrictions_listener) {
rtc::Event event;
encoder_queue_->PostTask([this, restrictions_listener, &event] {
RTC_DCHECK_RUN_ON(encoder_queue_.get());
RTC_DCHECK(resource_adaptation_processor_);
video_stream_adapter_->RemoveRestrictionsListener(restrictions_listener);
event.Set();
});
event.Wait(rtc::Event::kForever);
}
// RTC_RUN_ON(&encoder_queue_)
void VideoStreamEncoder::ProcessDroppedFrame(
const VideoFrame& frame,
VideoStreamEncoderObserver::DropReason reason) {
accumulated_update_rect_.Union(frame.update_rect());
accumulated_update_rect_is_valid_ &= frame.has_update_rect();
if (auto converted_reason = MaybeConvertDropReason(reason)) {
OnDroppedFrame(*converted_reason);
}
encoder_stats_observer_->OnFrameDropped(reason);
}
} // namespace webrtc
Reference in a new issue View git blame Copy permalink