mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-18 08:07:56 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/media/engine/webrtc_video_engine.cc
Amit Hilbuch e7a5f7bfae Modifying MediaChannel to accept CopyOnWriteBuffer by value. 
MediaChannel accepted the RtpPacket buffers through non-const pointer.
This is both unclear and introduces questions regarding if the buffer is
actually copied or not.
This change modifies the method to accept by value to reduce ambiguity.
Usage of the non-const data() method which could potentially copy the
buffer contents is also reduced in favor of cdata() which never copies.

Bug: None
Change-Id: I3b2daef0d31cb6aacceb46c86da3a40ce836242b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/127340
Reviewed-by: Steve Anton <steveanton@webrtc.org>
Reviewed-by: Seth Hampson <shampson@webrtc.org>
Commit-Queue: Amit Hilbuch <amithi@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27090}
2019-03-12 23:49:57 +00:00

2917 lines
107 KiB
C++
Raw Blame History

/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "media/engine/webrtc_video_engine.h"
#include <stdio.h>
#include <algorithm>
#include <set>
#include <string>
#include <utility>
#include "absl/strings/match.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "call/call.h"
#include "media/engine/constants.h"
#include "media/engine/simulcast.h"
#include "media/engine/webrtc_media_engine.h"
#include "media/engine/webrtc_voice_engine.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/field_trial.h"
namespace cricket {
namespace {
const int kMinLayerSize = 16;
// If this field trial is enabled, we will enable sending FlexFEC and disable
// sending ULPFEC whenever the former has been negotiated in the SDPs.
bool IsFlexfecFieldTrialEnabled() {
return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03");
}
// If this field trial is enabled, the "flexfec-03" codec will be advertised
// as being supported. This means that "flexfec-03" will appear in the default
// SDP offer, and we therefore need to be ready to receive FlexFEC packets from
// the remote. It also means that FlexFEC SSRCs will be generated by
// MediaSession and added as "a=ssrc:" and "a=ssrc-group:" lines in the local
// SDP.
bool IsFlexfecAdvertisedFieldTrialEnabled() {
return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03-Advertised");
}
void AddDefaultFeedbackParams(VideoCodec* codec) {
// Don't add any feedback params for RED and ULPFEC.
if (codec->name == kRedCodecName || codec->name == kUlpfecCodecName)
return;
codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamRemb, kParamValueEmpty));
codec->AddFeedbackParam(
FeedbackParam(kRtcpFbParamTransportCc, kParamValueEmpty));
// Don't add any more feedback params for FLEXFEC.
if (codec->name == kFlexfecCodecName)
return;
codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamCcm, kRtcpFbCcmParamFir));
codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kParamValueEmpty));
codec->AddFeedbackParam(FeedbackParam(kRtcpFbParamNack, kRtcpFbNackParamPli));
}
// This function will assign dynamic payload types (in the range [96, 127]) to
// the input codecs, and also add ULPFEC, RED, FlexFEC, and associated RTX
// codecs for recognized codecs (VP8, VP9, H264, and RED). It will also add
// default feedback params to the codecs.
std::vector<VideoCodec> AssignPayloadTypesAndDefaultCodecs(
std::vector<webrtc::SdpVideoFormat> input_formats) {
if (input_formats.empty())
return std::vector<VideoCodec>();
static const int kFirstDynamicPayloadType = 96;
static const int kLastDynamicPayloadType = 127;
int payload_type = kFirstDynamicPayloadType;
input_formats.push_back(webrtc::SdpVideoFormat(kRedCodecName));
input_formats.push_back(webrtc::SdpVideoFormat(kUlpfecCodecName));
if (IsFlexfecAdvertisedFieldTrialEnabled()) {
webrtc::SdpVideoFormat flexfec_format(kFlexfecCodecName);
// This value is currently arbitrarily set to 10 seconds. (The unit
// is microseconds.) This parameter MUST be present in the SDP, but
// we never use the actual value anywhere in our code however.
// TODO(brandtr): Consider honouring this value in the sender and receiver.
flexfec_format.parameters = {{kFlexfecFmtpRepairWindow, "10000000"}};
input_formats.push_back(flexfec_format);
}
std::vector<VideoCodec> output_codecs;
for (const webrtc::SdpVideoFormat& format : input_formats) {
VideoCodec codec(format);
codec.id = payload_type;
AddDefaultFeedbackParams(&codec);
output_codecs.push_back(codec);
// Increment payload type.
++payload_type;
if (payload_type > kLastDynamicPayloadType) {
RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest.";
break;
}
// Add associated RTX codec for non-FEC codecs.
if (!absl::EqualsIgnoreCase(codec.name, kUlpfecCodecName) &&
!absl::EqualsIgnoreCase(codec.name, kFlexfecCodecName)) {
output_codecs.push_back(
VideoCodec::CreateRtxCodec(payload_type, codec.id));
// Increment payload type.
++payload_type;
if (payload_type > kLastDynamicPayloadType) {
RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest.";
break;
}
}
}
return output_codecs;
}
std::vector<VideoCodec> AssignPayloadTypesAndDefaultCodecs(
const webrtc::VideoEncoderFactory* encoder_factory) {
return encoder_factory ? AssignPayloadTypesAndDefaultCodecs(
encoder_factory->GetSupportedFormats())
: std::vector<VideoCodec>();
}
int GetMaxFramerate(const webrtc::VideoEncoderConfig& encoder_config,
size_t num_layers) {
int max_fps = -1;
for (size_t i = 0; i < num_layers; ++i) {
int fps = (encoder_config.simulcast_layers[i].max_framerate > 0)
? encoder_config.simulcast_layers[i].max_framerate
: kDefaultVideoMaxFramerate;
max_fps = std::max(fps, max_fps);
}
return max_fps;
}
bool IsTemporalLayersSupported(const std::string& codec_name) {
return absl::EqualsIgnoreCase(codec_name, kVp8CodecName) ||
absl::EqualsIgnoreCase(codec_name, kVp9CodecName);
}
static std::string CodecVectorToString(const std::vector<VideoCodec>& codecs) {
rtc::StringBuilder out;
out << "{";
for (size_t i = 0; i < codecs.size(); ++i) {
out << codecs[i].ToString();
if (i != codecs.size() - 1) {
out << ", ";
}
}
out << "}";
return out.Release();
}
static bool ValidateCodecFormats(const std::vector<VideoCodec>& codecs) {
bool has_video = false;
for (size_t i = 0; i < codecs.size(); ++i) {
if (!codecs[i].ValidateCodecFormat()) {
return false;
}
if (codecs[i].GetCodecType() == VideoCodec::CODEC_VIDEO) {
has_video = true;
}
}
if (!has_video) {
RTC_LOG(LS_ERROR) << "Setting codecs without a video codec is invalid: "
<< CodecVectorToString(codecs);
return false;
}
return true;
}
static bool ValidateStreamParams(const StreamParams& sp) {
if (sp.ssrcs.empty()) {
RTC_LOG(LS_ERROR) << "No SSRCs in stream parameters: " << sp.ToString();
return false;
}
std::vector<uint32_t> primary_ssrcs;
sp.GetPrimarySsrcs(&primary_ssrcs);
std::vector<uint32_t> rtx_ssrcs;
sp.GetFidSsrcs(primary_ssrcs, &rtx_ssrcs);
for (uint32_t rtx_ssrc : rtx_ssrcs) {
bool rtx_ssrc_present = false;
for (uint32_t sp_ssrc : sp.ssrcs) {
if (sp_ssrc == rtx_ssrc) {
rtx_ssrc_present = true;
break;
}
}
if (!rtx_ssrc_present) {
RTC_LOG(LS_ERROR) << "RTX SSRC '" << rtx_ssrc
<< "' missing from StreamParams ssrcs: "
<< sp.ToString();
return false;
}
}
if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) {
RTC_LOG(LS_ERROR)
<< "RTX SSRCs exist, but don't cover all SSRCs (unsupported): "
<< sp.ToString();
return false;
}
return true;
}
// Returns true if the given codec is disallowed from doing simulcast.
bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) {
return !webrtc::field_trial::IsDisabled("WebRTC-H264Simulcast")
? absl::EqualsIgnoreCase(codec_name, kVp9CodecName)
: absl::EqualsIgnoreCase(codec_name, kH264CodecName) ||
absl::EqualsIgnoreCase(codec_name, kVp9CodecName);
}
// The selected thresholds for QVGA and VGA corresponded to a QP around 10.
// The change in QP declined above the selected bitrates.
static int GetMaxDefaultVideoBitrateKbps(int width,
int height,
bool is_screenshare) {
int max_bitrate;
if (width * height <= 320 * 240) {
max_bitrate = 600;
} else if (width * height <= 640 * 480) {
max_bitrate = 1700;
} else if (width * height <= 960 * 540) {
max_bitrate = 2000;
} else {
max_bitrate = 2500;
}
if (is_screenshare)
max_bitrate = std::max(max_bitrate, 1200);
return max_bitrate;
}
bool GetVp9LayersFromFieldTrialGroup(size_t* num_spatial_layers,
size_t* num_temporal_layers) {
std::string group = webrtc::field_trial::FindFullName("WebRTC-SupportVP9SVC");
if (group.empty())
return false;
if (sscanf(group.c_str(), "EnabledByFlag_%zuSL%zuTL", num_spatial_layers,
num_temporal_layers) != 2) {
return false;
}
if (*num_spatial_layers > webrtc::kMaxSpatialLayers ||
*num_spatial_layers < 1)
return false;
const size_t kMaxTemporalLayers = 3;
if (*num_temporal_layers > kMaxTemporalLayers || *num_temporal_layers < 1)
return false;
return true;
}
absl::optional<size_t> GetVp9SpatialLayersFromFieldTrial() {
size_t num_sl;
size_t num_tl;
if (GetVp9LayersFromFieldTrialGroup(&num_sl, &num_tl)) {
return num_sl;
}
return absl::nullopt;
}
absl::optional<size_t> GetVp9TemporalLayersFromFieldTrial() {
size_t num_sl;
size_t num_tl;
if (GetVp9LayersFromFieldTrialGroup(&num_sl, &num_tl)) {
return num_tl;
}
return absl::nullopt;
}
const char kForcedFallbackFieldTrial[] =
"WebRTC-VP8-Forced-Fallback-Encoder-v2";
absl::optional<int> GetFallbackMinBpsFromFieldTrial() {
if (!webrtc::field_trial::IsEnabled(kForcedFallbackFieldTrial))
return absl::nullopt;
std::string group =
webrtc::field_trial::FindFullName(kForcedFallbackFieldTrial);
if (group.empty())
return absl::nullopt;
int min_pixels;
int max_pixels;
int min_bps;
if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels,
&min_bps) != 3) {
return absl::nullopt;
}
if (min_bps <= 0)
return absl::nullopt;
return min_bps;
}
int GetMinVideoBitrateBps() {
return GetFallbackMinBpsFromFieldTrial().value_or(kMinVideoBitrateBps);
}
} // namespace
// This constant is really an on/off, lower-level configurable NACK history
// duration hasn't been implemented.
static const int kNackHistoryMs = 1000;
static const int kDefaultRtcpReceiverReportSsrc = 1;
// Minimum time interval for logging stats.
static const int64_t kStatsLogIntervalMs = 10000;
rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
WebRtcVideoChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings(
const VideoCodec& codec) {
RTC_DCHECK_RUN_ON(&thread_checker_);
bool is_screencast = parameters_.options.is_screencast.value_or(false);
// No automatic resizing when using simulcast or screencast.
bool automatic_resize =
!is_screencast && parameters_.config.rtp.ssrcs.size() == 1;
bool frame_dropping = !is_screencast;
bool denoising;
bool codec_default_denoising = false;
if (is_screencast) {
denoising = false;
} else {
// Use codec default if video_noise_reduction is unset.
codec_default_denoising = !parameters_.options.video_noise_reduction;
denoising = parameters_.options.video_noise_reduction.value_or(false);
}
if (absl::EqualsIgnoreCase(codec.name, kH264CodecName)) {
webrtc::VideoCodecH264 h264_settings =
webrtc::VideoEncoder::GetDefaultH264Settings();
h264_settings.frameDroppingOn = frame_dropping;
return new rtc::RefCountedObject<
webrtc::VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
}
if (absl::EqualsIgnoreCase(codec.name, kVp8CodecName)) {
webrtc::VideoCodecVP8 vp8_settings =
webrtc::VideoEncoder::GetDefaultVp8Settings();
vp8_settings.automaticResizeOn = automatic_resize;
// VP8 denoising is enabled by default.
vp8_settings.denoisingOn = codec_default_denoising ? true : denoising;
vp8_settings.frameDroppingOn = frame_dropping;
return new rtc::RefCountedObject<
webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
}
if (absl::EqualsIgnoreCase(codec.name, kVp9CodecName)) {
webrtc::VideoCodecVP9 vp9_settings =
webrtc::VideoEncoder::GetDefaultVp9Settings();
const size_t default_num_spatial_layers =
parameters_.config.rtp.ssrcs.size();
const size_t num_spatial_layers =
GetVp9SpatialLayersFromFieldTrial().value_or(
default_num_spatial_layers);
const size_t default_num_temporal_layers =
num_spatial_layers > 1 ? kConferenceDefaultNumTemporalLayers : 1;
const size_t num_temporal_layers =
GetVp9TemporalLayersFromFieldTrial().value_or(
default_num_temporal_layers);
vp9_settings.numberOfSpatialLayers = std::min<unsigned char>(
num_spatial_layers, kConferenceMaxNumSpatialLayers);
vp9_settings.numberOfTemporalLayers = std::min<unsigned char>(
num_temporal_layers, kConferenceMaxNumTemporalLayers);
// VP9 denoising is disabled by default.
vp9_settings.denoisingOn = codec_default_denoising ? true : denoising;
vp9_settings.automaticResizeOn = automatic_resize;
// Ensure frame dropping is always enabled.
RTC_DCHECK(vp9_settings.frameDroppingOn);
if (!is_screencast) {
// Limit inter-layer prediction to key pictures.
vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOnKeyPic;
} else {
// 3 spatial layers vp9 screenshare needs flexible mode.
vp9_settings.flexibleMode = vp9_settings.numberOfSpatialLayers > 2;
}
return new rtc::RefCountedObject<
webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
}
return nullptr;
}
DefaultUnsignalledSsrcHandler::DefaultUnsignalledSsrcHandler()
: default_sink_(nullptr) {}
UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc(
WebRtcVideoChannel* channel,
uint32_t ssrc) {
absl::optional<uint32_t> default_recv_ssrc =
channel->GetDefaultReceiveStreamSsrc();
if (default_recv_ssrc) {
RTC_LOG(LS_INFO) << "Destroying old default receive stream for SSRC="
<< ssrc << ".";
channel->RemoveRecvStream(*default_recv_ssrc);
}
StreamParams sp = channel->unsignaled_stream_params();
sp.ssrcs.push_back(ssrc);
RTC_LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc
<< ".";
if (!channel->AddRecvStream(sp, /*default_stream=*/true)) {
RTC_LOG(LS_WARNING) << "Could not create default receive stream.";
}
// SSRC 0 returns default_recv_base_minimum_delay_ms.
const int unsignaled_ssrc = 0;
int default_recv_base_minimum_delay_ms =
channel->GetBaseMinimumPlayoutDelayMs(unsignaled_ssrc).value_or(0);
// Set base minimum delay if it was set before for the default receive stream.
channel->SetBaseMinimumPlayoutDelayMs(ssrc,
default_recv_base_minimum_delay_ms);
channel->SetSink(ssrc, default_sink_);
return kDeliverPacket;
}
rtc::VideoSinkInterface<webrtc::VideoFrame>*
DefaultUnsignalledSsrcHandler::GetDefaultSink() const {
return default_sink_;
}
void DefaultUnsignalledSsrcHandler::SetDefaultSink(
WebRtcVideoChannel* channel,
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
default_sink_ = sink;
absl::optional<uint32_t> default_recv_ssrc =
channel->GetDefaultReceiveStreamSsrc();
if (default_recv_ssrc) {
channel->SetSink(*default_recv_ssrc, default_sink_);
}
}
WebRtcVideoEngine::WebRtcVideoEngine(
std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
video_bitrate_allocator_factory)
: decoder_factory_(std::move(video_decoder_factory)),
encoder_factory_(std::move(video_encoder_factory)),
bitrate_allocator_factory_(std::move(video_bitrate_allocator_factory)) {
RTC_LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()";
}
WebRtcVideoEngine::~WebRtcVideoEngine() {
RTC_LOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine";
}
VideoMediaChannel* WebRtcVideoEngine::CreateMediaChannel(
webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const webrtc::CryptoOptions& crypto_options) {
RTC_LOG(LS_INFO) << "CreateMediaChannel. Options: " << options.ToString();
return new WebRtcVideoChannel(call, config, options, crypto_options,
encoder_factory_.get(), decoder_factory_.get(),
bitrate_allocator_factory_.get());
}
std::vector<VideoCodec> WebRtcVideoEngine::codecs() const {
return AssignPayloadTypesAndDefaultCodecs(encoder_factory_.get());
}
RtpCapabilities WebRtcVideoEngine::GetCapabilities() const {
RtpCapabilities capabilities;
int id = 1;
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kTimestampOffsetUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kVideoRotationUri, id++));
capabilities.header_extensions.push_back(webrtc::RtpExtension(
webrtc::RtpExtension::kTransportSequenceNumberUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kPlayoutDelayUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kVideoContentTypeUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kVideoTimingUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kFrameMarkingUri, id++));
capabilities.header_extensions.push_back(
webrtc::RtpExtension(webrtc::RtpExtension::kColorSpaceUri, id++));
if (webrtc::field_trial::IsEnabled("WebRTC-GenericDescriptorAdvertised")) {
capabilities.header_extensions.push_back(webrtc::RtpExtension(
webrtc::RtpExtension::kGenericFrameDescriptorUri00, id++));
capabilities.header_extensions.push_back(webrtc::RtpExtension(
webrtc::RtpExtension::kGenericFrameDescriptorUri01, id++));
}
return capabilities;
}
WebRtcVideoChannel::WebRtcVideoChannel(
webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const webrtc::CryptoOptions& crypto_options,
webrtc::VideoEncoderFactory* encoder_factory,
webrtc::VideoDecoderFactory* decoder_factory,
webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory)
: VideoMediaChannel(config),
call_(call),
unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_),
video_config_(config.video),
encoder_factory_(encoder_factory),
decoder_factory_(decoder_factory),
bitrate_allocator_factory_(bitrate_allocator_factory),
default_send_options_(options),
last_stats_log_ms_(-1),
discard_unknown_ssrc_packets_(webrtc::field_trial::IsEnabled(
"WebRTC-Video-DiscardPacketsWithUnknownSsrc")),
crypto_options_(crypto_options) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc;
sending_ = false;
recv_codecs_ =
MapCodecs(AssignPayloadTypesAndDefaultCodecs(encoder_factory_));
recv_flexfec_payload_type_ = recv_codecs_.front().flexfec_payload_type;
}
WebRtcVideoChannel::~WebRtcVideoChannel() {
for (auto& kv : send_streams_)
delete kv.second;
for (auto& kv : receive_streams_)
delete kv.second;
}
absl::optional<WebRtcVideoChannel::VideoCodecSettings>
WebRtcVideoChannel::SelectSendVideoCodec(
const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
const std::vector<VideoCodec> local_supported_codecs =
AssignPayloadTypesAndDefaultCodecs(encoder_factory_);
// Select the first remote codec that is supported locally.
for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
// For H264, we will limit the encode level to the remote offered level
// regardless if level asymmetry is allowed or not. This is strictly not
// following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
// since we should limit the encode level to the lower of local and remote
// level when level asymmetry is not allowed.
if (FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec))
return remote_mapped_codec;
}
// No remote codec was supported.
return absl::nullopt;
}
bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
std::vector<VideoCodecSettings> before,
std::vector<VideoCodecSettings> after) {
// The receive codec order doesn't matter, so we sort the codecs before
// comparing. This is necessary because currently the
// only way to change the send codec is to munge SDP, which causes
// the receive codec list to change order, which causes the streams
// to be recreates which causes a "blink" of black video. In order
// to support munging the SDP in this way without recreating receive
// streams, we ignore the order of the received codecs so that
// changing the order doesn't cause this "blink".
auto comparison = [](const VideoCodecSettings& codec1,
const VideoCodecSettings& codec2) {
return codec1.codec.id > codec2.codec.id;
};
absl::c_sort(before, comparison);
absl::c_sort(after, comparison);
// Changes in FlexFEC payload type are handled separately in
// WebRtcVideoChannel::GetChangedRecvParameters, so disregard FlexFEC in the
// comparison here.
return !absl::c_equal(before, after,
VideoCodecSettings::EqualsDisregardingFlexfec);
}
bool WebRtcVideoChannel::GetChangedSendParameters(
const VideoSendParameters& params,
ChangedSendParameters* changed_params) const {
if (!ValidateCodecFormats(params.codecs) ||
!ValidateRtpExtensions(params.extensions)) {
return false;
}
// Select one of the remote codecs that will be used as send codec.
absl::optional<VideoCodecSettings> selected_send_codec =
SelectSendVideoCodec(MapCodecs(params.codecs));
if (!selected_send_codec) {
RTC_LOG(LS_ERROR) << "No video codecs supported.";
return false;
}
// Never enable sending FlexFEC, unless we are in the experiment.
if (!IsFlexfecFieldTrialEnabled()) {
if (selected_send_codec->flexfec_payload_type != -1) {
RTC_LOG(LS_INFO)
<< "Remote supports flexfec-03, but we will not send since "
<< "WebRTC-FlexFEC-03 field trial is not enabled.";
}
selected_send_codec->flexfec_payload_type = -1;
}
if (!send_codec_ || *selected_send_codec != *send_codec_)
changed_params->codec = selected_send_codec;
// Handle RTP header extensions.
if (params.extmap_allow_mixed != ExtmapAllowMixed()) {
changed_params->extmap_allow_mixed = params.extmap_allow_mixed;
}
std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions(
params.extensions, webrtc::RtpExtension::IsSupportedForVideo, true);
if (!send_rtp_extensions_ || (*send_rtp_extensions_ != filtered_extensions)) {
changed_params->rtp_header_extensions =
absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions);
}
if (params.mid != send_params_.mid) {
changed_params->mid = params.mid;
}
// Handle max bitrate.
if (params.max_bandwidth_bps != send_params_.max_bandwidth_bps &&
params.max_bandwidth_bps >= -1) {
// 0 or -1 uncaps max bitrate.
// TODO(pbos): Reconsider how 0 should be treated. It is not mentioned as a
// special value and might very well be used for stopping sending.
changed_params->max_bandwidth_bps =
params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps;
}
// Handle conference mode.
if (params.conference_mode != send_params_.conference_mode) {
changed_params->conference_mode = params.conference_mode;
}
// Handle RTCP mode.
if (params.rtcp.reduced_size != send_params_.rtcp.reduced_size) {
changed_params->rtcp_mode = params.rtcp.reduced_size
? webrtc::RtcpMode::kReducedSize
: webrtc::RtcpMode::kCompound;
}
return true;
}
bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSendParameters");
RTC_LOG(LS_INFO) << "SetSendParameters: " << params.ToString();
ChangedSendParameters changed_params;
if (!GetChangedSendParameters(params, &changed_params)) {
return false;
}
if (changed_params.codec) {
const VideoCodecSettings& codec_settings = *changed_params.codec;
send_codec_ = codec_settings;
RTC_LOG(LS_INFO) << "Using codec: " << codec_settings.codec.ToString();
}
if (changed_params.extmap_allow_mixed) {
SetExtmapAllowMixed(*changed_params.extmap_allow_mixed);
}
if (changed_params.rtp_header_extensions) {
send_rtp_extensions_ = changed_params.rtp_header_extensions;
}
if (changed_params.codec || changed_params.max_bandwidth_bps) {
if (params.max_bandwidth_bps == -1) {
// Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is
// -1, which corresponds to no "b=AS" attribute in SDP. Note that the
// global max bitrate may be set below in GetBitrateConfigForCodec, from
// the codec max bitrate.
// TODO(pbos): This should be reconsidered (codec max bitrate should
// probably not affect global call max bitrate).
bitrate_config_.max_bitrate_bps = -1;
}
if (send_codec_) {
// TODO(holmer): Changing the codec parameters shouldn't necessarily mean
// that we change the min/max of bandwidth estimation. Reevaluate this.
bitrate_config_ = GetBitrateConfigForCodec(send_codec_->codec);
if (!changed_params.codec) {
// If the codec isn't changing, set the start bitrate to -1 which means
// "unchanged" so that BWE isn't affected.
bitrate_config_.start_bitrate_bps = -1;
}
}
if (params.max_bandwidth_bps >= 0) {
// Note that max_bandwidth_bps intentionally takes priority over the
// bitrate config for the codec. This allows FEC to be applied above the
// codec target bitrate.
// TODO(pbos): Figure out whether b=AS means max bitrate for this
// WebRtcVideoChannel (in which case we're good), or per sender (SSRC),
// in which case this should not set a BitrateConstraints but rather
// reconfigure all senders.
bitrate_config_.max_bitrate_bps =
params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps;
}
call_->GetTransportControllerSend()->SetSdpBitrateParameters(
bitrate_config_);
}
for (auto& kv : send_streams_) {
kv.second->SetSendParameters(changed_params);
}
if (changed_params.codec || changed_params.rtcp_mode) {
// Update receive feedback parameters from new codec or RTCP mode.
RTC_LOG(LS_INFO)
<< "SetFeedbackOptions on all the receive streams because the send "
"codec or RTCP mode has changed.";
for (auto& kv : receive_streams_) {
RTC_DCHECK(kv.second != nullptr);
kv.second->SetFeedbackParameters(
HasNack(send_codec_->codec), HasRemb(send_codec_->codec),
HasTransportCc(send_codec_->codec),
params.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize
: webrtc::RtcpMode::kCompound);
}
}
send_params_ = params;
return true;
}
webrtc::RtpParameters WebRtcVideoChannel::GetRtpSendParameters(
uint32_t ssrc) const {
RTC_DCHECK_RUN_ON(&thread_checker_);
auto it = send_streams_.find(ssrc);
if (it == send_streams_.end()) {
RTC_LOG(LS_WARNING) << "Attempting to get RTP send parameters for stream "
<< "with ssrc " << ssrc << " which doesn't exist.";
return webrtc::RtpParameters();
}
webrtc::RtpParameters rtp_params = it->second->GetRtpParameters();
// Need to add the common list of codecs to the send stream-specific
// RTP parameters.
for (const VideoCodec& codec : send_params_.codecs) {
rtp_params.codecs.push_back(codec.ToCodecParameters());
}
return rtp_params;
}
webrtc::RTCError WebRtcVideoChannel::SetRtpSendParameters(
uint32_t ssrc,
const webrtc::RtpParameters& parameters) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRtpSendParameters");
auto it = send_streams_.find(ssrc);
if (it == send_streams_.end()) {
RTC_LOG(LS_ERROR) << "Attempting to set RTP send parameters for stream "
<< "with ssrc " << ssrc << " which doesn't exist.";
return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR);
}
// TODO(deadbeef): Handle setting parameters with a list of codecs in a
// different order (which should change the send codec).
webrtc::RtpParameters current_parameters = GetRtpSendParameters(ssrc);
if (current_parameters.codecs != parameters.codecs) {
RTC_DLOG(LS_ERROR) << "Using SetParameters to change the set of codecs "
<< "is not currently supported.";
return webrtc::RTCError(webrtc::RTCErrorType::INTERNAL_ERROR);
}
if (!parameters.encodings.empty()) {
const auto& priority = parameters.encodings[0].network_priority;
rtc::DiffServCodePoint new_dscp = rtc::DSCP_DEFAULT;
if (priority == 0.5 * webrtc::kDefaultBitratePriority) {
new_dscp = rtc::DSCP_CS1;
} else if (priority == webrtc::kDefaultBitratePriority) {
new_dscp = rtc::DSCP_DEFAULT;
} else if (priority == 2.0 * webrtc::kDefaultBitratePriority) {
new_dscp = rtc::DSCP_AF42;
} else if (priority == 4.0 * webrtc::kDefaultBitratePriority) {
new_dscp = rtc::DSCP_AF41;
} else {
RTC_LOG(LS_WARNING) << "Received invalid send network priority: "
<< priority;
return webrtc::RTCError(webrtc::RTCErrorType::INVALID_RANGE);
}
SetPreferredDscp(new_dscp);
}
return it->second->SetRtpParameters(parameters);
}
webrtc::RtpParameters WebRtcVideoChannel::GetRtpReceiveParameters(
uint32_t ssrc) const {
RTC_DCHECK_RUN_ON(&thread_checker_);
webrtc::RtpParameters rtp_params;
// SSRC of 0 represents an unsignaled receive stream.
if (ssrc == 0) {
if (!default_unsignalled_ssrc_handler_.GetDefaultSink()) {
RTC_LOG(LS_WARNING)
<< "Attempting to get RTP parameters for the default, "
"unsignaled video receive stream, but not yet "
"configured to receive such a stream.";
return rtp_params;
}
rtp_params.encodings.emplace_back();
} else {
auto it = receive_streams_.find(ssrc);
if (it == receive_streams_.end()) {
RTC_LOG(LS_WARNING)
<< "Attempting to get RTP receive parameters for stream "
<< "with SSRC " << ssrc << " which doesn't exist.";
return webrtc::RtpParameters();
}
rtp_params = it->second->GetRtpParameters();
}
// Add codecs, which any stream is prepared to receive.
for (const VideoCodec& codec : recv_params_.codecs) {
rtp_params.codecs.push_back(codec.ToCodecParameters());
}
return rtp_params;
}
bool WebRtcVideoChannel::SetRtpReceiveParameters(
uint32_t ssrc,
const webrtc::RtpParameters& parameters) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRtpReceiveParameters");
// SSRC of 0 represents an unsignaled receive stream.
if (ssrc == 0) {
if (!default_unsignalled_ssrc_handler_.GetDefaultSink()) {
RTC_LOG(LS_WARNING)
<< "Attempting to set RTP parameters for the default, "
"unsignaled video receive stream, but not yet "
"configured to receive such a stream.";
return false;
}
} else {
auto it = receive_streams_.find(ssrc);
if (it == receive_streams_.end()) {
RTC_LOG(LS_WARNING)
<< "Attempting to set RTP receive parameters for stream "
<< "with SSRC " << ssrc << " which doesn't exist.";
return false;
}
}
webrtc::RtpParameters current_parameters = GetRtpReceiveParameters(ssrc);
if (current_parameters != parameters) {
RTC_DLOG(LS_ERROR) << "Changing the RTP receive parameters is currently "
<< "unsupported.";
return false;
}
return true;
}
bool WebRtcVideoChannel::GetChangedRecvParameters(
const VideoRecvParameters& params,
ChangedRecvParameters* changed_params) const {
if (!ValidateCodecFormats(params.codecs) ||
!ValidateRtpExtensions(params.extensions)) {
return false;
}
// Handle receive codecs.
const std::vector<VideoCodecSettings> mapped_codecs =
MapCodecs(params.codecs);
if (mapped_codecs.empty()) {
RTC_LOG(LS_ERROR) << "SetRecvParameters called without any video codecs.";
return false;
}
// Verify that every mapped codec is supported locally.
const std::vector<VideoCodec> local_supported_codecs =
AssignPayloadTypesAndDefaultCodecs(encoder_factory_);
for (const VideoCodecSettings& mapped_codec : mapped_codecs) {
if (!FindMatchingCodec(local_supported_codecs, mapped_codec.codec)) {
RTC_LOG(LS_ERROR)
<< "SetRecvParameters called with unsupported video codec: "
<< mapped_codec.codec.ToString();
return false;
}
}
if (NonFlexfecReceiveCodecsHaveChanged(recv_codecs_, mapped_codecs)) {
changed_params->codec_settings =
absl::optional<std::vector<VideoCodecSettings>>(mapped_codecs);
}
// Handle RTP header extensions.
std::vector<webrtc::RtpExtension> filtered_extensions = FilterRtpExtensions(
params.extensions, webrtc::RtpExtension::IsSupportedForVideo, false);
if (filtered_extensions != recv_rtp_extensions_) {
changed_params->rtp_header_extensions =
absl::optional<std::vector<webrtc::RtpExtension>>(filtered_extensions);
}
int flexfec_payload_type = mapped_codecs.front().flexfec_payload_type;
if (flexfec_payload_type != recv_flexfec_payload_type_) {
changed_params->flexfec_payload_type = flexfec_payload_type;
}
return true;
}
bool WebRtcVideoChannel::SetRecvParameters(const VideoRecvParameters& params) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetRecvParameters");
RTC_LOG(LS_INFO) << "SetRecvParameters: " << params.ToString();
ChangedRecvParameters changed_params;
if (!GetChangedRecvParameters(params, &changed_params)) {
return false;
}
if (changed_params.flexfec_payload_type) {
RTC_LOG(LS_INFO) << "Changing FlexFEC payload type (recv) from "
<< recv_flexfec_payload_type_ << " to "
<< *changed_params.flexfec_payload_type;
recv_flexfec_payload_type_ = *changed_params.flexfec_payload_type;
}
if (changed_params.rtp_header_extensions) {
recv_rtp_extensions_ = *changed_params.rtp_header_extensions;
}
if (changed_params.codec_settings) {
RTC_LOG(LS_INFO) << "Changing recv codecs from "
<< CodecSettingsVectorToString(recv_codecs_) << " to "
<< CodecSettingsVectorToString(
*changed_params.codec_settings);
recv_codecs_ = *changed_params.codec_settings;
}
for (auto& kv : receive_streams_) {
kv.second->SetRecvParameters(changed_params);
}
recv_params_ = params;
return true;
}
std::string WebRtcVideoChannel::CodecSettingsVectorToString(
const std::vector<VideoCodecSettings>& codecs) {
rtc::StringBuilder out;
out << "{";
for (size_t i = 0; i < codecs.size(); ++i) {
out << codecs[i].codec.ToString();
if (i != codecs.size() - 1) {
out << ", ";
}
}
out << "}";
return out.Release();
}
bool WebRtcVideoChannel::GetSendCodec(VideoCodec* codec) {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (!send_codec_) {
RTC_LOG(LS_VERBOSE) << "GetSendCodec: No send codec set.";
return false;
}
*codec = send_codec_->codec;
return true;
}
bool WebRtcVideoChannel::SetSend(bool send) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "WebRtcVideoChannel::SetSend");
RTC_LOG(LS_VERBOSE) << "SetSend: " << (send ? "true" : "false");
if (send && !send_codec_) {
RTC_DLOG(LS_ERROR) << "SetSend(true) called before setting codec.";
return false;
}
for (const auto& kv : send_streams_) {
kv.second->SetSend(send);
}
sending_ = send;
return true;
}
bool WebRtcVideoChannel::SetVideoSend(
uint32_t ssrc,
const VideoOptions* options,
rtc::VideoSourceInterface<webrtc::VideoFrame>* source) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "SetVideoSend");
RTC_DCHECK(ssrc != 0);
RTC_LOG(LS_INFO) << "SetVideoSend (ssrc= " << ssrc << ", options: "
<< (options ? options->ToString() : "nullptr")
<< ", source = " << (source ? "(source)" : "nullptr") << ")";
const auto& kv = send_streams_.find(ssrc);
if (kv == send_streams_.end()) {
// Allow unknown ssrc only if source is null.
RTC_CHECK(source == nullptr);
RTC_LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc;
return false;
}
return kv->second->SetVideoSend(options, source);
}
bool WebRtcVideoChannel::ValidateSendSsrcAvailability(
const StreamParams& sp) const {
for (uint32_t ssrc : sp.ssrcs) {
if (send_ssrcs_.find(ssrc) != send_ssrcs_.end()) {
RTC_LOG(LS_ERROR) << "Send stream with SSRC '" << ssrc
<< "' already exists.";
return false;
}
}
return true;
}
bool WebRtcVideoChannel::ValidateReceiveSsrcAvailability(
const StreamParams& sp) const {
for (uint32_t ssrc : sp.ssrcs) {
if (receive_ssrcs_.find(ssrc) != receive_ssrcs_.end()) {
RTC_LOG(LS_ERROR) << "Receive stream with SSRC '" << ssrc
<< "' already exists.";
return false;
}
}
return true;
}
bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_LOG(LS_INFO) << "AddSendStream: " << sp.ToString();
if (!ValidateStreamParams(sp))
return false;
if (!ValidateSendSsrcAvailability(sp))
return false;
for (uint32_t used_ssrc : sp.ssrcs)
send_ssrcs_.insert(used_ssrc);
webrtc::VideoSendStream::Config config(this, media_transport());
for (const RidDescription& rid : sp.rids()) {
config.rtp.rids.push_back(rid.rid);
}
config.suspend_below_min_bitrate = video_config_.suspend_below_min_bitrate;
config.periodic_alr_bandwidth_probing =
video_config_.periodic_alr_bandwidth_probing;
config.encoder_settings.experiment_cpu_load_estimator =
video_config_.experiment_cpu_load_estimator;
config.encoder_settings.encoder_factory = encoder_factory_;
config.encoder_settings.bitrate_allocator_factory =
bitrate_allocator_factory_;
config.crypto_options = crypto_options_;
config.rtp.extmap_allow_mixed = ExtmapAllowMixed();
config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms;
WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(
call_, sp, std::move(config), default_send_options_,
video_config_.enable_cpu_adaptation, bitrate_config_.max_bitrate_bps,
send_codec_, send_rtp_extensions_, send_params_);
uint32_t ssrc = sp.first_ssrc();
RTC_DCHECK(ssrc != 0);
send_streams_[ssrc] = stream;
if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) {
rtcp_receiver_report_ssrc_ = ssrc;
RTC_LOG(LS_INFO)
<< "SetLocalSsrc on all the receive streams because we added "
"a send stream.";
for (auto& kv : receive_streams_)
kv.second->SetLocalSsrc(ssrc);
}
if (sending_) {
stream->SetSend(true);
}
return true;
}
bool WebRtcVideoChannel::RemoveSendStream(uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_LOG(LS_INFO) << "RemoveSendStream: " << ssrc;
WebRtcVideoSendStream* removed_stream;
std::map<uint32_t, WebRtcVideoSendStream*>::iterator it =
send_streams_.find(ssrc);
if (it == send_streams_.end()) {
return false;
}
for (uint32_t old_ssrc : it->second->GetSsrcs())
send_ssrcs_.erase(old_ssrc);
removed_stream = it->second;
send_streams_.erase(it);
// Switch receiver report SSRCs, the one in use is no longer valid.
if (rtcp_receiver_report_ssrc_ == ssrc) {
rtcp_receiver_report_ssrc_ = send_streams_.empty()
? kDefaultRtcpReceiverReportSsrc
: send_streams_.begin()->first;
RTC_LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because the "
"previous local SSRC was removed.";
for (auto& kv : receive_streams_) {
kv.second->SetLocalSsrc(rtcp_receiver_report_ssrc_);
}
}
delete removed_stream;
return true;
}
void WebRtcVideoChannel::DeleteReceiveStream(
WebRtcVideoChannel::WebRtcVideoReceiveStream* stream) {
for (uint32_t old_ssrc : stream->GetSsrcs())
receive_ssrcs_.erase(old_ssrc);
delete stream;
}
bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp) {
return AddRecvStream(sp, false);
}
bool WebRtcVideoChannel::AddRecvStream(const StreamParams& sp,
bool default_stream) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_LOG(LS_INFO) << "AddRecvStream"
<< (default_stream ? " (default stream)" : "") << ": "
<< sp.ToString();
if (!sp.has_ssrcs()) {
// This is a StreamParam with unsignaled SSRCs. Store it, so it can be used
// later when we know the SSRC on the first packet arrival.
unsignaled_stream_params_ = sp;
return true;
}
if (!ValidateStreamParams(sp))
return false;
uint32_t ssrc = sp.first_ssrc();
RTC_DCHECK(ssrc != 0); // TODO(pbos): Is this ever valid?
// Remove running stream if this was a default stream.
const auto& prev_stream = receive_streams_.find(ssrc);
if (prev_stream != receive_streams_.end()) {
if (default_stream || !prev_stream->second->IsDefaultStream()) {
RTC_LOG(LS_ERROR) << "Receive stream for SSRC '" << ssrc
<< "' already exists.";
return false;
}
DeleteReceiveStream(prev_stream->second);
receive_streams_.erase(prev_stream);
}
if (!ValidateReceiveSsrcAvailability(sp))
return false;
for (uint32_t used_ssrc : sp.ssrcs)
receive_ssrcs_.insert(used_ssrc);
webrtc::VideoReceiveStream::Config config(this, media_transport());
webrtc::FlexfecReceiveStream::Config flexfec_config(this);
ConfigureReceiverRtp(&config, &flexfec_config, sp);
config.crypto_options = crypto_options_;
config.enable_prerenderer_smoothing =
video_config_.enable_prerenderer_smoothing;
if (!sp.stream_ids().empty()) {
config.sync_group = sp.stream_ids()[0];
}
receive_streams_[ssrc] = new WebRtcVideoReceiveStream(
call_, sp, std::move(config), decoder_factory_, default_stream,
recv_codecs_, flexfec_config);
return true;
}
void WebRtcVideoChannel::ConfigureReceiverRtp(
webrtc::VideoReceiveStream::Config* config,
webrtc::FlexfecReceiveStream::Config* flexfec_config,
const StreamParams& sp) const {
uint32_t ssrc = sp.first_ssrc();
config->rtp.remote_ssrc = ssrc;
config->rtp.local_ssrc = rtcp_receiver_report_ssrc_;
// TODO(pbos): This protection is against setting the same local ssrc as
// remote which is not permitted by the lower-level API. RTCP requires a
// corresponding sender SSRC. Figure out what to do when we don't have
// (receive-only) or know a good local SSRC.
if (config->rtp.remote_ssrc == config->rtp.local_ssrc) {
if (config->rtp.local_ssrc != kDefaultRtcpReceiverReportSsrc) {
config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc;
} else {
config->rtp.local_ssrc = kDefaultRtcpReceiverReportSsrc + 1;
}
}
// Whether or not the receive stream sends reduced size RTCP is determined
// by the send params.
// TODO(deadbeef): Once we change "send_params" to "sender_params" and
// "recv_params" to "receiver_params", we should get this out of
// receiver_params_.
config->rtp.rtcp_mode = send_params_.rtcp.reduced_size
? webrtc::RtcpMode::kReducedSize
: webrtc::RtcpMode::kCompound;
config->rtp.remb = send_codec_ ? HasRemb(send_codec_->codec) : false;
config->rtp.transport_cc =
send_codec_ ? HasTransportCc(send_codec_->codec) : false;
sp.GetFidSsrc(ssrc, &config->rtp.rtx_ssrc);
config->rtp.extensions = recv_rtp_extensions_;
// TODO(brandtr): Generalize when we add support for multistream protection.
flexfec_config->payload_type = recv_flexfec_payload_type_;
if (IsFlexfecAdvertisedFieldTrialEnabled() &&
sp.GetFecFrSsrc(ssrc, &flexfec_config->remote_ssrc)) {
flexfec_config->protected_media_ssrcs = {ssrc};
flexfec_config->local_ssrc = config->rtp.local_ssrc;
flexfec_config->rtcp_mode = config->rtp.rtcp_mode;
// TODO(brandtr): We should be spec-compliant and set |transport_cc| here
// based on the rtcp-fb for the FlexFEC codec, not the media codec.
flexfec_config->transport_cc = config->rtp.transport_cc;
flexfec_config->rtp_header_extensions = config->rtp.extensions;
}
}
bool WebRtcVideoChannel::RemoveRecvStream(uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_LOG(LS_INFO) << "RemoveRecvStream: " << ssrc;
if (ssrc == 0) {
// This indicates that we need to remove the unsignaled stream parameters
// that are cached.
unsignaled_stream_params_ = StreamParams();
return true;
}
std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator stream =
receive_streams_.find(ssrc);
if (stream == receive_streams_.end()) {
RTC_LOG(LS_ERROR) << "Stream not found for ssrc: " << ssrc;
return false;
}
DeleteReceiveStream(stream->second);
receive_streams_.erase(stream);
return true;
}
bool WebRtcVideoChannel::SetSink(
uint32_t ssrc,
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_LOG(LS_INFO) << "SetSink: ssrc:" << ssrc << " "
<< (sink ? "(ptr)" : "nullptr");
if (ssrc == 0) {
default_unsignalled_ssrc_handler_.SetDefaultSink(this, sink);
return true;
}
std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it =
receive_streams_.find(ssrc);
if (it == receive_streams_.end()) {
return false;
}
it->second->SetSink(sink);
return true;
}
bool WebRtcVideoChannel::GetStats(VideoMediaInfo* info) {
RTC_DCHECK_RUN_ON(&thread_checker_);
TRACE_EVENT0("webrtc", "WebRtcVideoChannel::GetStats");
// Log stats periodically.
bool log_stats = false;
int64_t now_ms = rtc::TimeMillis();
if (last_stats_log_ms_ == -1 ||
now_ms - last_stats_log_ms_ > kStatsLogIntervalMs) {
last_stats_log_ms_ = now_ms;
log_stats = true;
}
info->Clear();
FillSenderStats(info, log_stats);
FillReceiverStats(info, log_stats);
FillSendAndReceiveCodecStats(info);
// TODO(holmer): We should either have rtt available as a metric on
// VideoSend/ReceiveStreams, or we should remove rtt from VideoSenderInfo.
// TODO(nisse): Arrange to get correct RTT also when using MediaTransport.
webrtc::Call::Stats stats = call_->GetStats();
if (stats.rtt_ms != -1) {
for (size_t i = 0; i < info->senders.size(); ++i) {
info->senders[i].rtt_ms = stats.rtt_ms;
}
}
if (log_stats)
RTC_LOG(LS_INFO) << stats.ToString(now_ms);
return true;
}
void WebRtcVideoChannel::FillSenderStats(VideoMediaInfo* video_media_info,
bool log_stats) {
for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator it =
send_streams_.begin();
it != send_streams_.end(); ++it) {
video_media_info->senders.push_back(
it->second->GetVideoSenderInfo(log_stats));
}
}
void WebRtcVideoChannel::FillReceiverStats(VideoMediaInfo* video_media_info,
bool log_stats) {
for (std::map<uint32_t, WebRtcVideoReceiveStream*>::iterator it =
receive_streams_.begin();
it != receive_streams_.end(); ++it) {
video_media_info->receivers.push_back(
it->second->GetVideoReceiverInfo(log_stats));
}
}
void WebRtcVideoChannel::FillBitrateInfo(BandwidthEstimationInfo* bwe_info) {
RTC_DCHECK_RUN_ON(&thread_checker_);
for (std::map<uint32_t, WebRtcVideoSendStream*>::iterator stream =
send_streams_.begin();
stream != send_streams_.end(); ++stream) {
stream->second->FillBitrateInfo(bwe_info);
}
}
void WebRtcVideoChannel::FillSendAndReceiveCodecStats(
VideoMediaInfo* video_media_info) {
for (const VideoCodec& codec : send_params_.codecs) {
webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
video_media_info->send_codecs.insert(
std::make_pair(codec_params.payload_type, std::move(codec_params)));
}
for (const VideoCodec& codec : recv_params_.codecs) {
webrtc::RtpCodecParameters codec_params = codec.ToCodecParameters();
video_media_info->receive_codecs.insert(
std::make_pair(codec_params.payload_type, std::move(codec_params)));
}
}
void WebRtcVideoChannel::OnPacketReceived(rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
RTC_DCHECK_RUN_ON(&thread_checker_);
const webrtc::PacketReceiver::DeliveryStatus delivery_result =
call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
packet_time_us);
switch (delivery_result) {
case webrtc::PacketReceiver::DELIVERY_OK:
return;
case webrtc::PacketReceiver::DELIVERY_PACKET_ERROR:
return;
case webrtc::PacketReceiver::DELIVERY_UNKNOWN_SSRC:
break;
}
if (discard_unknown_ssrc_packets_) {
return;
}
uint32_t ssrc = 0;
if (!GetRtpSsrc(packet.cdata(), packet.size(), &ssrc)) {
return;
}
int payload_type = 0;
if (!GetRtpPayloadType(packet.cdata(), packet.size(), &payload_type)) {
return;
}
// See if this payload_type is registered as one that usually gets its own
// SSRC (RTX) or at least is safe to drop either way (FEC). If it is, and
// it wasn't handled above by DeliverPacket, that means we don't know what
// stream it associates with, and we shouldn't ever create an implicit channel
// for these.
for (auto& codec : recv_codecs_) {
if (payload_type == codec.rtx_payload_type ||
payload_type == codec.ulpfec.red_rtx_payload_type ||
payload_type == codec.ulpfec.ulpfec_payload_type) {
return;
}
}
if (payload_type == recv_flexfec_payload_type_) {
return;
}
switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) {
case UnsignalledSsrcHandler::kDropPacket:
return;
case UnsignalledSsrcHandler::kDeliverPacket:
break;
}
if (call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
packet_time_us) !=
webrtc::PacketReceiver::DELIVERY_OK) {
RTC_LOG(LS_WARNING) << "Failed to deliver RTP packet on re-delivery.";
return;
}
}
void WebRtcVideoChannel::OnRtcpReceived(rtc::CopyOnWriteBuffer packet,
int64_t packet_time_us) {
RTC_DCHECK_RUN_ON(&thread_checker_);
// TODO(pbos): Check webrtc::PacketReceiver::DELIVERY_OK once we deliver
// for both audio and video on the same path. Since BundleFilter doesn't
// filter RTCP anymore incoming RTCP packets could've been going to audio (so
// logging failures spam the log).
call_->Receiver()->DeliverPacket(webrtc::MediaType::VIDEO, packet,
packet_time_us);
}
void WebRtcVideoChannel::OnReadyToSend(bool ready) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_LOG(LS_VERBOSE) << "OnReadyToSend: " << (ready ? "Ready." : "Not ready.");
call_->SignalChannelNetworkState(
webrtc::MediaType::VIDEO,
ready ? webrtc::kNetworkUp : webrtc::kNetworkDown);
}
void WebRtcVideoChannel::OnNetworkRouteChanged(
const std::string& transport_name,
const rtc::NetworkRoute& network_route) {
RTC_DCHECK_RUN_ON(&thread_checker_);
call_->GetTransportControllerSend()->OnNetworkRouteChanged(transport_name,
network_route);
call_->GetTransportControllerSend()->OnTransportOverheadChanged(
network_route.packet_overhead);
}
void WebRtcVideoChannel::SetInterface(
NetworkInterface* iface,
webrtc::MediaTransportInterface* media_transport) {
RTC_DCHECK_RUN_ON(&thread_checker_);
MediaChannel::SetInterface(iface, media_transport);
// Set the RTP recv/send buffer to a bigger size.
MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_RCVBUF,
kVideoRtpRecvBufferSize);
// Speculative change to increase the outbound socket buffer size.
// In b/15152257, we are seeing a significant number of packets discarded
// due to lack of socket buffer space, although it's not yet clear what the
// ideal value should be.
MediaChannel::SetOption(NetworkInterface::ST_RTP, rtc::Socket::OPT_SNDBUF,
kVideoRtpSendBufferSize);
}
void WebRtcVideoChannel::SetFrameDecryptor(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
RTC_DCHECK_RUN_ON(&thread_checker_);
auto matching_stream = receive_streams_.find(ssrc);
if (matching_stream != receive_streams_.end()) {
matching_stream->second->SetFrameDecryptor(frame_decryptor);
}
}
void WebRtcVideoChannel::SetFrameEncryptor(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) {
RTC_DCHECK_RUN_ON(&thread_checker_);
auto matching_stream = send_streams_.find(ssrc);
if (matching_stream != send_streams_.end()) {
matching_stream->second->SetFrameEncryptor(frame_encryptor);
} else {
RTC_LOG(LS_ERROR) << "No stream found to attach frame encryptor";
}
}
bool WebRtcVideoChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc,
int delay_ms) {
RTC_DCHECK_RUN_ON(&thread_checker_);
absl::optional<uint32_t> default_ssrc = GetDefaultReceiveStreamSsrc();
// SSRC of 0 represents the default receive stream.
if (ssrc == 0) {
default_recv_base_minimum_delay_ms_ = delay_ms;
}
if (ssrc == 0 && !default_ssrc) {
return true;
}
if (ssrc == 0 && default_ssrc) {
ssrc = default_ssrc.value();
}
auto stream = receive_streams_.find(ssrc);
if (stream != receive_streams_.end()) {
stream->second->SetBaseMinimumPlayoutDelayMs(delay_ms);
return true;
} else {
RTC_LOG(LS_ERROR) << "No stream found to set base minimum playout delay";
return false;
}
}
absl::optional<int> WebRtcVideoChannel::GetBaseMinimumPlayoutDelayMs(
uint32_t ssrc) const {
RTC_DCHECK_RUN_ON(&thread_checker_);
// SSRC of 0 represents the default receive stream.
if (ssrc == 0) {
return default_recv_base_minimum_delay_ms_;
}
auto stream = receive_streams_.find(ssrc);
if (stream != receive_streams_.end()) {
return stream->second->GetBaseMinimumPlayoutDelayMs();
} else {
RTC_LOG(LS_ERROR) << "No stream found to get base minimum playout delay";
return absl::nullopt;
}
}
absl::optional<uint32_t> WebRtcVideoChannel::GetDefaultReceiveStreamSsrc() {
RTC_DCHECK_RUN_ON(&thread_checker_);
absl::optional<uint32_t> ssrc;
for (auto it = receive_streams_.begin(); it != receive_streams_.end(); ++it) {
if (it->second->IsDefaultStream()) {
ssrc.emplace(it->first);
break;
}
}
return ssrc;
}
std::vector<webrtc::RtpSource> WebRtcVideoChannel::GetSources(
uint32_t ssrc) const {
RTC_DCHECK_RUN_ON(&thread_checker_);
auto it = receive_streams_.find(ssrc);
if (it == receive_streams_.end()) {
// TODO(bugs.webrtc.org/9781): Investigate standard compliance
// with sources for streams that has been removed.
RTC_LOG(LS_ERROR) << "Attempting to get contributing sources for SSRC:"
<< ssrc << " which doesn't exist.";
return {};
}
return it->second->GetSources();
}
bool WebRtcVideoChannel::SendRtp(const uint8_t* data,
size_t len,
const webrtc::PacketOptions& options) {
rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen);
rtc::PacketOptions rtc_options;
rtc_options.packet_id = options.packet_id;
if (DscpEnabled()) {
rtc_options.dscp = PreferredDscp();
}
rtc_options.info_signaled_after_sent.included_in_feedback =
options.included_in_feedback;
rtc_options.info_signaled_after_sent.included_in_allocation =
options.included_in_allocation;
return MediaChannel::SendPacket(&packet, rtc_options);
}
bool WebRtcVideoChannel::SendRtcp(const uint8_t* data, size_t len) {
rtc::CopyOnWriteBuffer packet(data, len, kMaxRtpPacketLen);
rtc::PacketOptions rtc_options;
if (DscpEnabled()) {
rtc_options.dscp = PreferredDscp();
}
return MediaChannel::SendRtcp(&packet, rtc_options);
}
WebRtcVideoChannel::WebRtcVideoSendStream::VideoSendStreamParameters::
VideoSendStreamParameters(
webrtc::VideoSendStream::Config config,
const VideoOptions& options,
int max_bitrate_bps,
const absl::optional<VideoCodecSettings>& codec_settings)
: config(std::move(config)),
options(options),
max_bitrate_bps(max_bitrate_bps),
conference_mode(false),
codec_settings(codec_settings) {}
WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream(
webrtc::Call* call,
const StreamParams& sp,
webrtc::VideoSendStream::Config config,
const VideoOptions& options,
bool enable_cpu_overuse_detection,
int max_bitrate_bps,
const absl::optional<VideoCodecSettings>& codec_settings,
const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
// TODO(deadbeef): Don't duplicate information between send_params,
// rtp_extensions, options, etc.
const VideoSendParameters& send_params)
: worker_thread_(rtc::Thread::Current()),
ssrcs_(sp.ssrcs),
ssrc_groups_(sp.ssrc_groups),
call_(call),
enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
source_(nullptr),
stream_(nullptr),
encoder_sink_(nullptr),
parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
rtp_parameters_(CreateRtpParametersWithEncodings(sp)),
sending_(false) {
parameters_.config.rtp.max_packet_size = kVideoMtu;
parameters_.conference_mode = send_params.conference_mode;
sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
// ValidateStreamParams should prevent this from happening.
RTC_CHECK(!parameters_.config.rtp.ssrcs.empty());
rtp_parameters_.encodings[0].ssrc = parameters_.config.rtp.ssrcs[0];
// RTX.
sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
&parameters_.config.rtp.rtx.ssrcs);
// FlexFEC SSRCs.
// TODO(brandtr): This code needs to be generalized when we add support for
// multistream protection.
if (IsFlexfecFieldTrialEnabled()) {
uint32_t flexfec_ssrc;
bool flexfec_enabled = false;
for (uint32_t primary_ssrc : parameters_.config.rtp.ssrcs) {
if (sp.GetFecFrSsrc(primary_ssrc, &flexfec_ssrc)) {
if (flexfec_enabled) {
RTC_LOG(LS_INFO)
<< "Multiple FlexFEC streams in local SDP, but "
"our implementation only supports a single FlexFEC "
"stream. Will not enable FlexFEC for proposed "
"stream with SSRC: "
<< flexfec_ssrc << ".";
continue;
}
flexfec_enabled = true;
parameters_.config.rtp.flexfec.ssrc = flexfec_ssrc;
parameters_.config.rtp.flexfec.protected_media_ssrcs = {primary_ssrc};
}
}
}
parameters_.config.rtp.c_name = sp.cname;
parameters_.config.track_id = sp.id;
if (rtp_extensions) {
parameters_.config.rtp.extensions = *rtp_extensions;
rtp_parameters_.header_extensions = *rtp_extensions;
}
parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
? webrtc::RtcpMode::kReducedSize
: webrtc::RtcpMode::kCompound;
parameters_.config.rtp.mid = send_params.mid;
rtp_parameters_.rtcp.reduced_size = send_params.rtcp.reduced_size;
if (codec_settings) {
SetCodec(*codec_settings);
}
}
WebRtcVideoChannel::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
if (stream_ != NULL) {
call_->DestroyVideoSendStream(stream_);
}
}
bool WebRtcVideoChannel::WebRtcVideoSendStream::SetVideoSend(
const VideoOptions* options,
rtc::VideoSourceInterface<webrtc::VideoFrame>* source) {
TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend");
RTC_DCHECK_RUN_ON(&thread_checker_);
if (options) {
VideoOptions old_options = parameters_.options;
parameters_.options.SetAll(*options);
if (parameters_.options.is_screencast.value_or(false) !=
old_options.is_screencast.value_or(false) &&
parameters_.codec_settings) {
// If screen content settings change, we may need to recreate the codec
// instance so that the correct type is used.
SetCodec(*parameters_.codec_settings);
// Mark screenshare parameter as being updated, then test for any other
// changes that may require codec reconfiguration.
old_options.is_screencast = options->is_screencast;
}
if (parameters_.options != old_options) {
ReconfigureEncoder();
}
}
if (source_ && stream_) {
stream_->SetSource(nullptr, webrtc::DegradationPreference::DISABLED);
}
// Switch to the new source.
source_ = source;
if (source && stream_) {
stream_->SetSource(this, GetDegradationPreference());
}
return true;
}
webrtc::DegradationPreference
WebRtcVideoChannel::WebRtcVideoSendStream::GetDegradationPreference() const {
// Do not adapt resolution for screen content as this will likely
// result in blurry and unreadable text.
// |this| acts like a VideoSource to make sure SinkWants are handled on the
// correct thread.
webrtc::DegradationPreference degradation_preference;
if (rtp_parameters_.degradation_preference !=
webrtc::DegradationPreference::BALANCED) {
// If the degradationPreference is different from the default value, assume
// it is what we want, regardless of trials or other internal settings.
degradation_preference = rtp_parameters_.degradation_preference;
} else if (!enable_cpu_overuse_detection_) {
degradation_preference = webrtc::DegradationPreference::DISABLED;
} else if (parameters_.options.is_screencast.value_or(false)) {
degradation_preference = webrtc::DegradationPreference::MAINTAIN_RESOLUTION;
} else if (webrtc::field_trial::IsEnabled(
"WebRTC-Video-BalancedDegradation")) {
degradation_preference = webrtc::DegradationPreference::BALANCED;
} else {
// TODO(orphis): The default should be BALANCED as the standard mandates.
// Right now, there is no way to set it to BALANCED as it would change
// the behavior for any project expecting MAINTAIN_FRAMERATE by default.
degradation_preference = webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
}
return degradation_preference;
}
const std::vector<uint32_t>&
WebRtcVideoChannel::WebRtcVideoSendStream::GetSsrcs() const {
return ssrcs_;
}
void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec(
const VideoCodecSettings& codec_settings) {
RTC_DCHECK_RUN_ON(&thread_checker_);
parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec);
RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
parameters_.config.rtp.payload_name = codec_settings.codec.name;
parameters_.config.rtp.payload_type = codec_settings.codec.id;
parameters_.config.rtp.ulpfec = codec_settings.ulpfec;
parameters_.config.rtp.flexfec.payload_type =
codec_settings.flexfec_payload_type;
// Set RTX payload type if RTX is enabled.
if (!parameters_.config.rtp.rtx.ssrcs.empty()) {
if (codec_settings.rtx_payload_type == -1) {
RTC_LOG(LS_WARNING)
<< "RTX SSRCs configured but there's no configured RTX "
"payload type. Ignoring.";
parameters_.config.rtp.rtx.ssrcs.clear();
} else {
parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type;
}
}
parameters_.config.rtp.nack.rtp_history_ms =
HasNack(codec_settings.codec) ? kNackHistoryMs : 0;
parameters_.codec_settings = codec_settings;
// TODO(nisse): Avoid recreation, it should be enough to call
// ReconfigureEncoder.
RTC_LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetCodec.";
RecreateWebRtcStream();
}
void WebRtcVideoChannel::WebRtcVideoSendStream::SetSendParameters(
const ChangedSendParameters& params) {
RTC_DCHECK_RUN_ON(&thread_checker_);
// |recreate_stream| means construction-time parameters have changed and the
// sending stream needs to be reset with the new config.
bool recreate_stream = false;
if (params.rtcp_mode) {
parameters_.config.rtp.rtcp_mode = *params.rtcp_mode;
rtp_parameters_.rtcp.reduced_size =
parameters_.config.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize;
recreate_stream = true;
}
if (params.extmap_allow_mixed) {
parameters_.config.rtp.extmap_allow_mixed = *params.extmap_allow_mixed;
recreate_stream = true;
}
if (params.rtp_header_extensions) {
parameters_.config.rtp.extensions = *params.rtp_header_extensions;
rtp_parameters_.header_extensions = *params.rtp_header_extensions;
recreate_stream = true;
}
if (params.mid) {
parameters_.config.rtp.mid = *params.mid;
recreate_stream = true;
}
if (params.max_bandwidth_bps) {
parameters_.max_bitrate_bps = *params.max_bandwidth_bps;
ReconfigureEncoder();
}
if (params.conference_mode) {
parameters_.conference_mode = *params.conference_mode;
}
// Set codecs and options.
if (params.codec) {
SetCodec(*params.codec);
recreate_stream = false; // SetCodec has already recreated the stream.
} else if (params.conference_mode && parameters_.codec_settings) {
SetCodec(*parameters_.codec_settings);
recreate_stream = false; // SetCodec has already recreated the stream.
}
if (recreate_stream) {
RTC_LOG(LS_INFO)
<< "RecreateWebRtcStream (send) because of SetSendParameters";
RecreateWebRtcStream();
}
}
webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
const webrtc::RtpParameters& new_parameters) {
RTC_DCHECK_RUN_ON(&thread_checker_);
webrtc::RTCError error = CheckRtpParametersInvalidModificationAndValues(
rtp_parameters_, new_parameters);
if (!error.ok()) {
return error;
}
bool new_param = false;
for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
if ((new_parameters.encodings[i].min_bitrate_bps !=
rtp_parameters_.encodings[i].min_bitrate_bps) ||
(new_parameters.encodings[i].max_bitrate_bps !=
rtp_parameters_.encodings[i].max_bitrate_bps) ||
(new_parameters.encodings[i].max_framerate !=
rtp_parameters_.encodings[i].max_framerate) ||
(new_parameters.encodings[i].scale_resolution_down_by !=
rtp_parameters_.encodings[i].scale_resolution_down_by) ||
(new_parameters.encodings[i].num_temporal_layers !=
rtp_parameters_.encodings[i].num_temporal_layers)) {
new_param = true;
break;
}
}
bool new_degradation_preference = false;
if (new_parameters.degradation_preference !=
rtp_parameters_.degradation_preference) {
new_degradation_preference = true;
}
// TODO(bugs.webrtc.org/8807): The bitrate priority really doesn't require an
// entire encoder reconfiguration, it just needs to update the bitrate
// allocator.
bool reconfigure_encoder =
new_param || (new_parameters.encodings[0].bitrate_priority !=
rtp_parameters_.encodings[0].bitrate_priority);
// TODO(bugs.webrtc.org/8807): The active field as well should not require
// a full encoder reconfiguration, but it needs to update both the bitrate
// allocator and the video bitrate allocator.
bool new_send_state = false;
for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
if (new_parameters.encodings[i].active !=
rtp_parameters_.encodings[i].active) {
new_send_state = true;
}
}
rtp_parameters_ = new_parameters;
// Codecs are currently handled at the WebRtcVideoChannel level.
rtp_parameters_.codecs.clear();
if (reconfigure_encoder || new_send_state) {
ReconfigureEncoder();
}
if (new_send_state) {
UpdateSendState();
}
if (new_degradation_preference) {
stream_->SetSource(this, GetDegradationPreference());
}
return webrtc::RTCError::OK();
}
webrtc::RtpParameters
WebRtcVideoChannel::WebRtcVideoSendStream::GetRtpParameters() const {
RTC_DCHECK_RUN_ON(&thread_checker_);
return rtp_parameters_;
}
void WebRtcVideoChannel::WebRtcVideoSendStream::SetFrameEncryptor(
rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) {
RTC_DCHECK_RUN_ON(&thread_checker_);
parameters_.config.frame_encryptor = frame_encryptor;
if (stream_) {
RecreateWebRtcStream();
}
}
void WebRtcVideoChannel::WebRtcVideoSendStream::UpdateSendState() {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (sending_) {
RTC_DCHECK(stream_ != nullptr);
std::vector<bool> active_layers(rtp_parameters_.encodings.size());
for (size_t i = 0; i < active_layers.size(); ++i) {
active_layers[i] = rtp_parameters_.encodings[i].active;
}
// This updates what simulcast layers are sending, and possibly starts
// or stops the VideoSendStream.
stream_->UpdateActiveSimulcastLayers(active_layers);
} else {
if (stream_ != nullptr) {
stream_->Stop();
}
}
}
webrtc::VideoEncoderConfig
WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig(
const VideoCodec& codec) const {
RTC_DCHECK_RUN_ON(&thread_checker_);
webrtc::VideoEncoderConfig encoder_config;
encoder_config.codec_type = webrtc::PayloadStringToCodecType(codec.name);
encoder_config.video_format =
webrtc::SdpVideoFormat(codec.name, codec.params);
bool is_screencast = parameters_.options.is_screencast.value_or(false);
if (is_screencast) {
encoder_config.min_transmit_bitrate_bps =
1000 * parameters_.options.screencast_min_bitrate_kbps.value_or(0);
encoder_config.content_type =
webrtc::VideoEncoderConfig::ContentType::kScreen;
} else {
encoder_config.min_transmit_bitrate_bps = 0;
encoder_config.content_type =
webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
}
// By default, the stream count for the codec configuration should match the
// number of negotiated ssrcs. But if the codec is blacklisted for simulcast
// or a screencast (and not in simulcast screenshare experiment), only
// configure a single stream.
encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
if (IsCodecBlacklistedForSimulcast(codec.name) ||
(is_screencast && (!ScreenshareSimulcastFieldTrialEnabled() ||
!parameters_.conference_mode))) {
encoder_config.number_of_streams = 1;
}
// parameters_.max_bitrate comes from the max bitrate set at the SDP
// (m-section) level with the attribute "b=AS." Note that we override this
// value below if the RtpParameters max bitrate set with
// RtpSender::SetParameters has a lower value.
int stream_max_bitrate = parameters_.max_bitrate_bps;
// When simulcast is enabled (when there are multiple encodings),
// encodings[i].max_bitrate_bps will be enforced by
// encoder_config.simulcast_layers[i].max_bitrate_bps. Otherwise, it's
// enforced by stream_max_bitrate, taking the minimum of the two maximums
// (one coming from SDP, the other coming from RtpParameters).
if (rtp_parameters_.encodings[0].max_bitrate_bps &&
rtp_parameters_.encodings.size() == 1) {
stream_max_bitrate =
webrtc::MinPositive(*(rtp_parameters_.encodings[0].max_bitrate_bps),
parameters_.max_bitrate_bps);
}
// The codec max bitrate comes from the "x-google-max-bitrate" parameter
// attribute set in the SDP for a specific codec. As done in
// WebRtcVideoChannel::SetSendParameters, this value does not override the
// stream max_bitrate set above.
int codec_max_bitrate_kbps;
if (codec.GetParam(kCodecParamMaxBitrate, &codec_max_bitrate_kbps) &&
stream_max_bitrate == -1) {
stream_max_bitrate = codec_max_bitrate_kbps * 1000;
}
encoder_config.max_bitrate_bps = stream_max_bitrate;
// The encoder config's default bitrate priority is set to 1.0,
// unless it is set through the sender's encoding parameters.
// The bitrate priority, which is used in the bitrate allocation, is done
// on a per sender basis, so we use the first encoding's value.
encoder_config.bitrate_priority =
rtp_parameters_.encodings[0].bitrate_priority;
// Application-controlled state is held in the encoder_config's
// simulcast_layers. Currently this is used to control which simulcast layers
// are active and for configuring the min/max bitrate and max framerate.
// The encoder_config's simulcast_layers is also used for non-simulcast (when
// there is a single layer).
RTC_DCHECK_GE(rtp_parameters_.encodings.size(),
encoder_config.number_of_streams);
RTC_DCHECK_GT(encoder_config.number_of_streams, 0);
// Copy all provided constraints.
encoder_config.simulcast_layers.resize(rtp_parameters_.encodings.size());
for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
encoder_config.simulcast_layers[i].active =
rtp_parameters_.encodings[i].active;
if (rtp_parameters_.encodings[i].min_bitrate_bps) {
encoder_config.simulcast_layers[i].min_bitrate_bps =
*rtp_parameters_.encodings[i].min_bitrate_bps;
}
if (rtp_parameters_.encodings[i].max_bitrate_bps) {
encoder_config.simulcast_layers[i].max_bitrate_bps =
*rtp_parameters_.encodings[i].max_bitrate_bps;
}
if (rtp_parameters_.encodings[i].max_framerate) {
encoder_config.simulcast_layers[i].max_framerate =
*rtp_parameters_.encodings[i].max_framerate;
}
if (rtp_parameters_.encodings[i].scale_resolution_down_by) {
encoder_config.simulcast_layers[i].scale_resolution_down_by =
*rtp_parameters_.encodings[i].scale_resolution_down_by;
}
if (rtp_parameters_.encodings[i].num_temporal_layers) {
encoder_config.simulcast_layers[i].num_temporal_layers =
*rtp_parameters_.encodings[i].num_temporal_layers;
}
}
int max_qp = kDefaultQpMax;
codec.GetParam(kCodecParamMaxQuantization, &max_qp);
encoder_config.video_stream_factory =
new rtc::RefCountedObject<EncoderStreamFactory>(
codec.name, max_qp, is_screencast, parameters_.conference_mode);
return encoder_config;
}
void WebRtcVideoChannel::WebRtcVideoSendStream::ReconfigureEncoder() {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (!stream_) {
// The webrtc::VideoSendStream |stream_| has not yet been created but other
// parameters has changed.
return;
}
RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0);
RTC_CHECK(parameters_.codec_settings);
VideoCodecSettings codec_settings = *parameters_.codec_settings;
webrtc::VideoEncoderConfig encoder_config =
CreateVideoEncoderConfig(codec_settings.codec);
encoder_config.encoder_specific_settings =
ConfigureVideoEncoderSettings(codec_settings.codec);
stream_->ReconfigureVideoEncoder(encoder_config.Copy());
encoder_config.encoder_specific_settings = NULL;
parameters_.encoder_config = std::move(encoder_config);
}
void WebRtcVideoChannel::WebRtcVideoSendStream::SetSend(bool send) {
RTC_DCHECK_RUN_ON(&thread_checker_);
sending_ = send;
UpdateSendState();
}
void WebRtcVideoChannel::WebRtcVideoSendStream::RemoveSink(
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK(encoder_sink_ == sink);
encoder_sink_ = nullptr;
source_->RemoveSink(sink);
}
void WebRtcVideoChannel::WebRtcVideoSendStream::AddOrUpdateSink(
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
const rtc::VideoSinkWants& wants) {
if (worker_thread_ == rtc::Thread::Current()) {
// AddOrUpdateSink is called on |worker_thread_| if this is the first
// registration of |sink|.
RTC_DCHECK_RUN_ON(&thread_checker_);
encoder_sink_ = sink;
source_->AddOrUpdateSink(encoder_sink_, wants);
} else {
// Subsequent calls to AddOrUpdateSink will happen on the encoder task
// queue.
invoker_.AsyncInvoke<void>(
RTC_FROM_HERE, worker_thread_, [this, sink, wants] {
RTC_DCHECK_RUN_ON(&thread_checker_);
// |sink| may be invalidated after this task was posted since
// RemoveSink is called on the worker thread.
bool encoder_sink_valid = (sink == encoder_sink_);
if (source_ && encoder_sink_valid) {
source_->AddOrUpdateSink(encoder_sink_, wants);
}
});
}
}
VideoSenderInfo WebRtcVideoChannel::WebRtcVideoSendStream::GetVideoSenderInfo(
bool log_stats) {
VideoSenderInfo info;
RTC_DCHECK_RUN_ON(&thread_checker_);
for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
info.add_ssrc(ssrc);
if (parameters_.codec_settings) {
info.codec_name = parameters_.codec_settings->codec.name;
info.codec_payload_type = parameters_.codec_settings->codec.id;
}
if (stream_ == NULL)
return info;
webrtc::VideoSendStream::Stats stats = stream_->GetStats();
if (log_stats)
RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
info.adapt_changes = stats.number_of_cpu_adapt_changes;
info.adapt_reason =
stats.cpu_limited_resolution ? ADAPTREASON_CPU : ADAPTREASON_NONE;
info.has_entered_low_resolution = stats.has_entered_low_resolution;
// Get bandwidth limitation info from stream_->GetStats().
// Input resolution (output from video_adapter) can be further scaled down or
// higher video layer(s) can be dropped due to bitrate constraints.
// Note, adapt_changes only include changes from the video_adapter.
if (stats.bw_limited_resolution)
info.adapt_reason |= ADAPTREASON_BANDWIDTH;
info.encoder_implementation_name = stats.encoder_implementation_name;
info.ssrc_groups = ssrc_groups_;
info.framerate_input = stats.input_frame_rate;
info.framerate_sent = stats.encode_frame_rate;
info.avg_encode_ms = stats.avg_encode_time_ms;
info.encode_usage_percent = stats.encode_usage_percent;
info.frames_encoded = stats.frames_encoded;
info.qp_sum = stats.qp_sum;
info.nominal_bitrate = stats.media_bitrate_bps;
info.content_type = stats.content_type;
info.huge_frames_sent = stats.huge_frames_sent;
info.send_frame_width = 0;
info.send_frame_height = 0;
for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it =
stats.substreams.begin();
it != stats.substreams.end(); ++it) {
// TODO(pbos): Wire up additional stats, such as padding bytes.
webrtc::VideoSendStream::StreamStats stream_stats = it->second;
info.bytes_sent += stream_stats.rtp_stats.transmitted.payload_bytes +
stream_stats.rtp_stats.transmitted.header_bytes +
stream_stats.rtp_stats.transmitted.padding_bytes;
info.packets_sent += stream_stats.rtp_stats.transmitted.packets;
info.packets_lost += stream_stats.rtcp_stats.packets_lost;
if (stream_stats.width > info.send_frame_width)
info.send_frame_width = stream_stats.width;
if (stream_stats.height > info.send_frame_height)
info.send_frame_height = stream_stats.height;
info.firs_rcvd += stream_stats.rtcp_packet_type_counts.fir_packets;
info.nacks_rcvd += stream_stats.rtcp_packet_type_counts.nack_packets;
info.plis_rcvd += stream_stats.rtcp_packet_type_counts.pli_packets;
}
if (!stats.substreams.empty()) {
// TODO(pbos): Report fraction lost per SSRC.
webrtc::VideoSendStream::StreamStats first_stream_stats =
stats.substreams.begin()->second;
info.fraction_lost =
static_cast<float>(first_stream_stats.rtcp_stats.fraction_lost) /
(1 << 8);
}
return info;
}
void WebRtcVideoChannel::WebRtcVideoSendStream::FillBitrateInfo(
BandwidthEstimationInfo* bwe_info) {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (stream_ == NULL) {
return;
}
webrtc::VideoSendStream::Stats stats = stream_->GetStats();
for (std::map<uint32_t, webrtc::VideoSendStream::StreamStats>::iterator it =
stats.substreams.begin();
it != stats.substreams.end(); ++it) {
bwe_info->transmit_bitrate += it->second.total_bitrate_bps;
bwe_info->retransmit_bitrate += it->second.retransmit_bitrate_bps;
}
bwe_info->target_enc_bitrate += stats.target_media_bitrate_bps;
bwe_info->actual_enc_bitrate += stats.media_bitrate_bps;
}
void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (stream_ != NULL) {
call_->DestroyVideoSendStream(stream_);
}
RTC_CHECK(parameters_.codec_settings);
RTC_DCHECK_EQ((parameters_.encoder_config.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen),
parameters_.options.is_screencast.value_or(false))
<< "encoder content type inconsistent with screencast option";
parameters_.encoder_config.encoder_specific_settings =
ConfigureVideoEncoderSettings(parameters_.codec_settings->codec);
webrtc::VideoSendStream::Config config = parameters_.config.Copy();
if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) {
RTC_LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX "
"payload type the set codec. Ignoring RTX.";
config.rtp.rtx.ssrcs.clear();
}
stream_ = call_->CreateVideoSendStream(std::move(config),
parameters_.encoder_config.Copy());
parameters_.encoder_config.encoder_specific_settings = NULL;
if (source_) {
stream_->SetSource(this, GetDegradationPreference());
}
// Call stream_->Start() if necessary conditions are met.
UpdateSendState();
}
WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
webrtc::Call* call,
const StreamParams& sp,
webrtc::VideoReceiveStream::Config config,
webrtc::VideoDecoderFactory* decoder_factory,
bool default_stream,
const std::vector<VideoCodecSettings>& recv_codecs,
const webrtc::FlexfecReceiveStream::Config& flexfec_config)
: call_(call),
stream_params_(sp),
stream_(NULL),
default_stream_(default_stream),
config_(std::move(config)),
flexfec_config_(flexfec_config),
flexfec_stream_(nullptr),
decoder_factory_(decoder_factory),
sink_(NULL),
first_frame_timestamp_(-1),
estimated_remote_start_ntp_time_ms_(0) {
config_.renderer = this;
ConfigureCodecs(recv_codecs);
ConfigureFlexfecCodec(flexfec_config.payload_type);
MaybeRecreateWebRtcFlexfecStream();
RecreateWebRtcVideoStream();
}
WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
if (flexfec_stream_) {
MaybeDissociateFlexfecFromVideo();
call_->DestroyFlexfecReceiveStream(flexfec_stream_);
}
call_->DestroyVideoReceiveStream(stream_);
}
const std::vector<uint32_t>&
WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSsrcs() const {
return stream_params_.ssrcs;
}
std::vector<webrtc::RtpSource>
WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSources() {
RTC_DCHECK(stream_);
return stream_->GetSources();
}
webrtc::RtpParameters
WebRtcVideoChannel::WebRtcVideoReceiveStream::GetRtpParameters() const {
webrtc::RtpParameters rtp_parameters;
std::vector<uint32_t> primary_ssrcs;
stream_params_.GetPrimarySsrcs(&primary_ssrcs);
for (uint32_t ssrc : primary_ssrcs) {
rtp_parameters.encodings.emplace_back();
rtp_parameters.encodings.back().ssrc = ssrc;
}
rtp_parameters.header_extensions = config_.rtp.extensions;
rtp_parameters.rtcp.reduced_size =
config_.rtp.rtcp_mode == webrtc::RtcpMode::kReducedSize;
return rtp_parameters;
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs(
const std::vector<VideoCodecSettings>& recv_codecs) {
RTC_DCHECK(!recv_codecs.empty());
config_.decoders.clear();
config_.rtp.rtx_associated_payload_types.clear();
for (const auto& recv_codec : recv_codecs) {
webrtc::SdpVideoFormat video_format(recv_codec.codec.name,
recv_codec.codec.params);
webrtc::VideoReceiveStream::Decoder decoder;
decoder.decoder_factory = decoder_factory_;
decoder.video_format = video_format;
decoder.payload_type = recv_codec.codec.id;
decoder.video_format =
webrtc::SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params);
config_.decoders.push_back(decoder);
config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] =
recv_codec.codec.id;
}
const auto& codec = recv_codecs.front();
config_.rtp.ulpfec_payload_type = codec.ulpfec.ulpfec_payload_type;
config_.rtp.red_payload_type = codec.ulpfec.red_payload_type;
config_.rtp.nack.rtp_history_ms = HasNack(codec.codec) ? kNackHistoryMs : 0;
config_.rtp.rtcp_xr.receiver_reference_time_report = HasRrtr(codec.codec);
if (codec.ulpfec.red_rtx_payload_type != -1) {
config_.rtp
.rtx_associated_payload_types[codec.ulpfec.red_rtx_payload_type] =
codec.ulpfec.red_payload_type;
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureFlexfecCodec(
int flexfec_payload_type) {
flexfec_config_.payload_type = flexfec_payload_type;
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetLocalSsrc(
uint32_t local_ssrc) {
// TODO(pbos): Consider turning this sanity check into a RTC_DCHECK. You
// should not be able to create a sender with the same SSRC as a receiver, but
// right now this can't be done due to unittests depending on receiving what
// they are sending from the same MediaChannel.
if (local_ssrc == config_.rtp.local_ssrc) {
RTC_DLOG(LS_INFO) << "Ignoring call to SetLocalSsrc because parameters are "
"unchanged; local_ssrc="
<< local_ssrc;
return;
}
config_.rtp.local_ssrc = local_ssrc;
flexfec_config_.local_ssrc = local_ssrc;
RTC_LOG(LS_INFO)
<< "RecreateWebRtcStream (recv) because of SetLocalSsrc; local_ssrc="
<< local_ssrc;
MaybeRecreateWebRtcFlexfecStream();
RecreateWebRtcVideoStream();
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFeedbackParameters(
bool nack_enabled,
bool remb_enabled,
bool transport_cc_enabled,
webrtc::RtcpMode rtcp_mode) {
int nack_history_ms = nack_enabled ? kNackHistoryMs : 0;
if (config_.rtp.nack.rtp_history_ms == nack_history_ms &&
config_.rtp.remb == remb_enabled &&
config_.rtp.transport_cc == transport_cc_enabled &&
config_.rtp.rtcp_mode == rtcp_mode) {
RTC_LOG(LS_INFO)
<< "Ignoring call to SetFeedbackParameters because parameters are "
"unchanged; nack="
<< nack_enabled << ", remb=" << remb_enabled
<< ", transport_cc=" << transport_cc_enabled;
return;
}
config_.rtp.remb = remb_enabled;
config_.rtp.nack.rtp_history_ms = nack_history_ms;
config_.rtp.transport_cc = transport_cc_enabled;
config_.rtp.rtcp_mode = rtcp_mode;
// TODO(brandtr): We should be spec-compliant and set |transport_cc| here
// based on the rtcp-fb for the FlexFEC codec, not the media codec.
flexfec_config_.transport_cc = config_.rtp.transport_cc;
flexfec_config_.rtcp_mode = config_.rtp.rtcp_mode;
RTC_LOG(LS_INFO)
<< "RecreateWebRtcStream (recv) because of SetFeedbackParameters; nack="
<< nack_enabled << ", remb=" << remb_enabled
<< ", transport_cc=" << transport_cc_enabled;
MaybeRecreateWebRtcFlexfecStream();
RecreateWebRtcVideoStream();
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
const ChangedRecvParameters& params) {
bool video_needs_recreation = false;
bool flexfec_needs_recreation = false;
if (params.codec_settings) {
ConfigureCodecs(*params.codec_settings);
video_needs_recreation = true;
}
if (params.rtp_header_extensions) {
config_.rtp.extensions = *params.rtp_header_extensions;
flexfec_config_.rtp_header_extensions = *params.rtp_header_extensions;
video_needs_recreation = true;
flexfec_needs_recreation = true;
}
if (params.flexfec_payload_type) {
ConfigureFlexfecCodec(*params.flexfec_payload_type);
flexfec_needs_recreation = true;
}
if (flexfec_needs_recreation) {
RTC_LOG(LS_INFO) << "MaybeRecreateWebRtcFlexfecStream (recv) because of "
"SetRecvParameters";
MaybeRecreateWebRtcFlexfecStream();
}
if (video_needs_recreation) {
RTC_LOG(LS_INFO)
<< "RecreateWebRtcVideoStream (recv) because of SetRecvParameters";
RecreateWebRtcVideoStream();
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() {
absl::optional<int> base_minimum_playout_delay_ms;
if (stream_) {
base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs();
MaybeDissociateFlexfecFromVideo();
call_->DestroyVideoReceiveStream(stream_);
stream_ = nullptr;
}
webrtc::VideoReceiveStream::Config config = config_.Copy();
config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr);
config.stream_id = stream_params_.id;
stream_ = call_->CreateVideoReceiveStream(std::move(config));
if (base_minimum_playout_delay_ms) {
stream_->SetBaseMinimumPlayoutDelayMs(
base_minimum_playout_delay_ms.value());
}
MaybeAssociateFlexfecWithVideo();
stream_->Start();
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::
MaybeRecreateWebRtcFlexfecStream() {
if (flexfec_stream_) {
MaybeDissociateFlexfecFromVideo();
call_->DestroyFlexfecReceiveStream(flexfec_stream_);
flexfec_stream_ = nullptr;
}
if (flexfec_config_.IsCompleteAndEnabled()) {
flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_);
MaybeAssociateFlexfecWithVideo();
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::
MaybeAssociateFlexfecWithVideo() {
if (stream_ && flexfec_stream_) {
stream_->AddSecondarySink(flexfec_stream_);
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::
MaybeDissociateFlexfecFromVideo() {
if (stream_ && flexfec_stream_) {
stream_->RemoveSecondarySink(flexfec_stream_);
}
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame(
const webrtc::VideoFrame& frame) {
rtc::CritScope crit(&sink_lock_);
int64_t time_now_ms = rtc::TimeMillis();
if (first_frame_timestamp_ < 0)
first_frame_timestamp_ = time_now_ms;
int64_t elapsed_time_ms = time_now_ms - first_frame_timestamp_;
if (frame.ntp_time_ms() > 0)
estimated_remote_start_ntp_time_ms_ = frame.ntp_time_ms() - elapsed_time_ms;
if (sink_ == NULL) {
RTC_LOG(LS_WARNING) << "VideoReceiveStream not connected to a VideoSink.";
return;
}
sink_->OnFrame(frame);
}
bool WebRtcVideoChannel::WebRtcVideoReceiveStream::IsDefaultStream() const {
return default_stream_;
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFrameDecryptor(
rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) {
config_.frame_decryptor = frame_decryptor;
if (stream_) {
RecreateWebRtcVideoStream();
}
}
bool WebRtcVideoChannel::WebRtcVideoReceiveStream::SetBaseMinimumPlayoutDelayMs(
int delay_ms) {
return stream_ ? stream_->SetBaseMinimumPlayoutDelayMs(delay_ms) : false;
}
int WebRtcVideoChannel::WebRtcVideoReceiveStream::GetBaseMinimumPlayoutDelayMs()
const {
return stream_ ? stream_->GetBaseMinimumPlayoutDelayMs() : 0;
}
void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetSink(
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
rtc::CritScope crit(&sink_lock_);
sink_ = sink;
}
std::string
WebRtcVideoChannel::WebRtcVideoReceiveStream::GetCodecNameFromPayloadType(
int payload_type) {
for (const webrtc::VideoReceiveStream::Decoder& decoder : config_.decoders) {
if (decoder.payload_type == payload_type) {
return decoder.video_format.name;
}
}
return "";
}
VideoReceiverInfo
WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo(
bool log_stats) {
VideoReceiverInfo info;
info.ssrc_groups = stream_params_.ssrc_groups;
info.add_ssrc(config_.rtp.remote_ssrc);
webrtc::VideoReceiveStream::Stats stats = stream_->GetStats();
info.decoder_implementation_name = stats.decoder_implementation_name;
if (stats.current_payload_type != -1) {
info.codec_payload_type = stats.current_payload_type;
}
info.bytes_rcvd = stats.rtp_stats.transmitted.payload_bytes +
stats.rtp_stats.transmitted.header_bytes +
stats.rtp_stats.transmitted.padding_bytes;
info.packets_rcvd = stats.rtp_stats.transmitted.packets;
info.packets_lost = stats.rtcp_stats.packets_lost;
info.fraction_lost =
static_cast<float>(stats.rtcp_stats.fraction_lost) / (1 << 8);
info.framerate_rcvd = stats.network_frame_rate;
info.framerate_decoded = stats.decode_frame_rate;
info.framerate_output = stats.render_frame_rate;
info.frame_width = stats.width;
info.frame_height = stats.height;
{
rtc::CritScope frame_cs(&sink_lock_);
info.capture_start_ntp_time_ms = estimated_remote_start_ntp_time_ms_;
}
info.decode_ms = stats.decode_ms;
info.max_decode_ms = stats.max_decode_ms;
info.current_delay_ms = stats.current_delay_ms;
info.target_delay_ms = stats.target_delay_ms;
info.jitter_buffer_ms = stats.jitter_buffer_ms;
info.min_playout_delay_ms = stats.min_playout_delay_ms;
info.render_delay_ms = stats.render_delay_ms;
info.frames_received =
stats.frame_counts.key_frames + stats.frame_counts.delta_frames;
info.frames_decoded = stats.frames_decoded;
info.frames_rendered = stats.frames_rendered;
info.qp_sum = stats.qp_sum;
info.first_frame_received_to_decoded_ms =
stats.first_frame_received_to_decoded_ms;
info.interframe_delay_max_ms = stats.interframe_delay_max_ms;
info.freeze_count = stats.freeze_count;
info.pause_count = stats.pause_count;
info.total_freezes_duration_ms = stats.total_freezes_duration_ms;
info.total_pauses_duration_ms = stats.total_pauses_duration_ms;
info.total_frames_duration_ms = stats.total_frames_duration_ms;
info.sum_squared_frame_durations = stats.sum_squared_frame_durations;
info.content_type = stats.content_type;
info.codec_name = GetCodecNameFromPayloadType(stats.current_payload_type);
info.firs_sent = stats.rtcp_packet_type_counts.fir_packets;
info.plis_sent = stats.rtcp_packet_type_counts.pli_packets;
info.nacks_sent = stats.rtcp_packet_type_counts.nack_packets;
info.timing_frame_info = stats.timing_frame_info;
if (log_stats)
RTC_LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
return info;
}
WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings()
: flexfec_payload_type(-1), rtx_payload_type(-1) {}
bool WebRtcVideoChannel::VideoCodecSettings::operator==(
const WebRtcVideoChannel::VideoCodecSettings& other) const {
return codec == other.codec && ulpfec == other.ulpfec &&
flexfec_payload_type == other.flexfec_payload_type &&
rtx_payload_type == other.rtx_payload_type;
}
bool WebRtcVideoChannel::VideoCodecSettings::EqualsDisregardingFlexfec(
const WebRtcVideoChannel::VideoCodecSettings& a,
const WebRtcVideoChannel::VideoCodecSettings& b) {
return a.codec == b.codec && a.ulpfec == b.ulpfec &&
a.rtx_payload_type == b.rtx_payload_type;
}
bool WebRtcVideoChannel::VideoCodecSettings::operator!=(
const WebRtcVideoChannel::VideoCodecSettings& other) const {
return !(*this == other);
}
std::vector<WebRtcVideoChannel::VideoCodecSettings>
WebRtcVideoChannel::MapCodecs(const std::vector<VideoCodec>& codecs) {
RTC_DCHECK(!codecs.empty());
std::vector<VideoCodecSettings> video_codecs;
std::map<int, bool> payload_used;
std::map<int, VideoCodec::CodecType> payload_codec_type;
// |rtx_mapping| maps video payload type to rtx payload type.
std::map<int, int> rtx_mapping;
webrtc::UlpfecConfig ulpfec_config;
int flexfec_payload_type = -1;
for (size_t i = 0; i < codecs.size(); ++i) {
const VideoCodec& in_codec = codecs[i];
int payload_type = in_codec.id;
if (payload_used[payload_type]) {
RTC_LOG(LS_ERROR) << "Payload type already registered: "
<< in_codec.ToString();
return std::vector<VideoCodecSettings>();
}
payload_used[payload_type] = true;
payload_codec_type[payload_type] = in_codec.GetCodecType();
switch (in_codec.GetCodecType()) {
case VideoCodec::CODEC_RED: {
// RED payload type, should not have duplicates.
RTC_DCHECK_EQ(-1, ulpfec_config.red_payload_type);
ulpfec_config.red_payload_type = in_codec.id;
continue;
}
case VideoCodec::CODEC_ULPFEC: {
// ULPFEC payload type, should not have duplicates.
RTC_DCHECK_EQ(-1, ulpfec_config.ulpfec_payload_type);
ulpfec_config.ulpfec_payload_type = in_codec.id;
continue;
}
case VideoCodec::CODEC_FLEXFEC: {
// FlexFEC payload type, should not have duplicates.
RTC_DCHECK_EQ(-1, flexfec_payload_type);
flexfec_payload_type = in_codec.id;
continue;
}
case VideoCodec::CODEC_RTX: {
int associated_payload_type;
if (!in_codec.GetParam(kCodecParamAssociatedPayloadType,
&associated_payload_type) ||
!IsValidRtpPayloadType(associated_payload_type)) {
RTC_LOG(LS_ERROR)
<< "RTX codec with invalid or no associated payload type: "
<< in_codec.ToString();
return std::vector<VideoCodecSettings>();
}
rtx_mapping[associated_payload_type] = in_codec.id;
continue;
}
case VideoCodec::CODEC_VIDEO:
break;
}
video_codecs.push_back(VideoCodecSettings());
video_codecs.back().codec = in_codec;
}
// One of these codecs should have been a video codec. Only having FEC
// parameters into this code is a logic error.
RTC_DCHECK(!video_codecs.empty());
for (std::map<int, int>::const_iterator it = rtx_mapping.begin();
it != rtx_mapping.end(); ++it) {
if (!payload_used[it->first]) {
RTC_LOG(LS_ERROR) << "RTX mapped to payload not in codec list.";
return std::vector<VideoCodecSettings>();
}
if (payload_codec_type[it->first] != VideoCodec::CODEC_VIDEO &&
payload_codec_type[it->first] != VideoCodec::CODEC_RED) {
RTC_LOG(LS_ERROR)
<< "RTX not mapped to regular video codec or RED codec.";
return std::vector<VideoCodecSettings>();
}
if (it->first == ulpfec_config.red_payload_type) {
ulpfec_config.red_rtx_payload_type = it->second;
}
}
for (size_t i = 0; i < video_codecs.size(); ++i) {
video_codecs[i].ulpfec = ulpfec_config;
video_codecs[i].flexfec_payload_type = flexfec_payload_type;
if (rtx_mapping[video_codecs[i].codec.id] != 0 &&
rtx_mapping[video_codecs[i].codec.id] !=
ulpfec_config.red_payload_type) {
video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id];
}
}
return video_codecs;
}
// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of
// EncoderStreamFactory and instead set this value individually for each stream
// in the VideoEncoderConfig.simulcast_layers.
EncoderStreamFactory::EncoderStreamFactory(
std::string codec_name,
int max_qp,
bool is_screenshare,
bool screenshare_config_explicitly_enabled)
: codec_name_(codec_name),
max_qp_(max_qp),
is_screenshare_(is_screenshare),
screenshare_config_explicitly_enabled_(
screenshare_config_explicitly_enabled) {}
std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) {
bool screenshare_simulcast_enabled =
screenshare_config_explicitly_enabled_ &&
cricket::ScreenshareSimulcastFieldTrialEnabled();
if (is_screenshare_ && !screenshare_simulcast_enabled) {
RTC_DCHECK_EQ(1, encoder_config.number_of_streams);
}
RTC_DCHECK_GT(encoder_config.number_of_streams, 0);
RTC_DCHECK_GE(encoder_config.simulcast_layers.size(),
encoder_config.number_of_streams);
std::vector<webrtc::VideoStream> layers;
if (encoder_config.number_of_streams > 1 ||
((absl::EqualsIgnoreCase(codec_name_, kVp8CodecName) ||
absl::EqualsIgnoreCase(codec_name_, kH264CodecName)) &&
is_screenshare_ && screenshare_config_explicitly_enabled_)) {
const bool temporal_layers_supported =
absl::EqualsIgnoreCase(codec_name_, kVp8CodecName)
|| absl::EqualsIgnoreCase(codec_name_, kH264CodecName);
layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height,
0 /*not used*/, encoder_config.bitrate_priority,
max_qp_, 0 /*not_used*/, is_screenshare_,
temporal_layers_supported);
// The maximum |max_framerate| is currently used for video.
const int max_framerate = GetMaxFramerate(encoder_config, layers.size());
// Update the active simulcast layers and configured bitrates.
bool is_highest_layer_max_bitrate_configured = false;
const bool has_scale_resolution_down_by =
std::any_of(encoder_config.simulcast_layers.begin(),
encoder_config.simulcast_layers.end(),
[](const webrtc::VideoStream& layer) {
return layer.scale_resolution_down_by != -1.;
});
const int normalized_width =
NormalizeSimulcastSize(width, encoder_config.number_of_streams);
const int normalized_height =
NormalizeSimulcastSize(height, encoder_config.number_of_streams);
for (size_t i = 0; i < layers.size(); ++i) {
layers[i].active = encoder_config.simulcast_layers[i].active;
if (!is_screenshare_) {
// Update simulcast framerates with max configured max framerate.
layers[i].max_framerate = max_framerate;
}
// Update with configured num temporal layers if supported by codec.
if (encoder_config.simulcast_layers[i].num_temporal_layers &&
IsTemporalLayersSupported(codec_name_)) {
layers[i].num_temporal_layers =
*encoder_config.simulcast_layers[i].num_temporal_layers;
}
if (has_scale_resolution_down_by) {
const double scale_resolution_down_by = std::max(
encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0);
layers[i].width = std::max(
static_cast<int>(normalized_width / scale_resolution_down_by),
kMinLayerSize);
layers[i].height = std::max(
static_cast<int>(normalized_height / scale_resolution_down_by),
kMinLayerSize);
}
// Update simulcast bitrates with configured min and max bitrate.
if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) {
layers[i].min_bitrate_bps =
encoder_config.simulcast_layers[i].min_bitrate_bps;
}
if (encoder_config.simulcast_layers[i].max_bitrate_bps > 0) {
layers[i].max_bitrate_bps =
encoder_config.simulcast_layers[i].max_bitrate_bps;
}
if (encoder_config.simulcast_layers[i].target_bitrate_bps > 0) {
layers[i].target_bitrate_bps =
encoder_config.simulcast_layers[i].target_bitrate_bps;
}
if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0 &&
encoder_config.simulcast_layers[i].max_bitrate_bps > 0) {
// Min and max bitrate are configured.
// Set target to 3/4 of the max bitrate (or to max if below min).
if (encoder_config.simulcast_layers[i].target_bitrate_bps <= 0)
layers[i].target_bitrate_bps = layers[i].max_bitrate_bps * 3 / 4;
if (layers[i].target_bitrate_bps < layers[i].min_bitrate_bps)
layers[i].target_bitrate_bps = layers[i].max_bitrate_bps;
} else if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) {
// Only min bitrate is configured, make sure target/max are above min.
layers[i].target_bitrate_bps =
std::max(layers[i].target_bitrate_bps, layers[i].min_bitrate_bps);
layers[i].max_bitrate_bps =
std::max(layers[i].max_bitrate_bps, layers[i].min_bitrate_bps);
} else if (encoder_config.simulcast_layers[i].max_bitrate_bps > 0) {
// Only max bitrate is configured, make sure min/target are below max.
layers[i].min_bitrate_bps =
std::min(layers[i].min_bitrate_bps, layers[i].max_bitrate_bps);
layers[i].target_bitrate_bps =
std::min(layers[i].target_bitrate_bps, layers[i].max_bitrate_bps);
}
if (i == layers.size() - 1) {
is_highest_layer_max_bitrate_configured =
encoder_config.simulcast_layers[i].max_bitrate_bps > 0;
}
}
if (!is_screenshare_ && !is_highest_layer_max_bitrate_configured) {
// No application-configured maximum for the largest layer.
// If there is bitrate leftover, give it to the largest layer.
BoostMaxSimulcastLayer(encoder_config.max_bitrate_bps, &layers);
}
return layers;
}
// For unset max bitrates set default bitrate for non-simulcast.
int max_bitrate_bps =
(encoder_config.max_bitrate_bps > 0)
? encoder_config.max_bitrate_bps
: GetMaxDefaultVideoBitrateKbps(width, height, is_screenshare_) *
1000;
int min_bitrate_bps = GetMinVideoBitrateBps();
if (encoder_config.simulcast_layers[0].min_bitrate_bps > 0) {
// Use set min bitrate.
min_bitrate_bps = encoder_config.simulcast_layers[0].min_bitrate_bps;
// If only min bitrate is configured, make sure max is above min.
if (encoder_config.max_bitrate_bps <= 0)
max_bitrate_bps = std::max(min_bitrate_bps, max_bitrate_bps);
}
int max_framerate = (encoder_config.simulcast_layers[0].max_framerate > 0)
? encoder_config.simulcast_layers[0].max_framerate
: kDefaultVideoMaxFramerate;
webrtc::VideoStream layer;
layer.width = width;
layer.height = height;
layer.max_framerate = max_framerate;
if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) {
layer.width = std::max<size_t>(
layer.width /
encoder_config.simulcast_layers[0].scale_resolution_down_by,
kMinLayerSize);
layer.height = std::max<size_t>(
layer.height /
encoder_config.simulcast_layers[0].scale_resolution_down_by,
kMinLayerSize);
}
// In the case that the application sets a max bitrate that's lower than the
// min bitrate, we adjust it down (see bugs.webrtc.org/9141).
layer.min_bitrate_bps = std::min(min_bitrate_bps, max_bitrate_bps);
if (encoder_config.simulcast_layers[0].target_bitrate_bps <= 0) {
layer.target_bitrate_bps = max_bitrate_bps;
} else {
layer.target_bitrate_bps =
encoder_config.simulcast_layers[0].target_bitrate_bps;
}
layer.max_bitrate_bps = max_bitrate_bps;
layer.max_qp = max_qp_;
layer.bitrate_priority = encoder_config.bitrate_priority;
if (absl::EqualsIgnoreCase(codec_name_, kVp9CodecName)) {
RTC_DCHECK(encoder_config.encoder_specific_settings);
// Use VP9 SVC layering from codec settings which might be initialized
// though field trial in ConfigureVideoEncoderSettings.
webrtc::VideoCodecVP9 vp9_settings;
encoder_config.encoder_specific_settings->FillVideoCodecVp9(&vp9_settings);
layer.num_temporal_layers = vp9_settings.numberOfTemporalLayers;
}
if (IsTemporalLayersSupported(codec_name_)) {
// Use configured number of temporal layers if set.
if (encoder_config.simulcast_layers[0].num_temporal_layers) {
layer.num_temporal_layers =
*encoder_config.simulcast_layers[0].num_temporal_layers;
}
}
layers.push_back(layer);
return layers;
}
} // namespace cricket
Reference in a new issue View git blame Copy permalink