mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-12 21:30:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/media/engine/webrtc_video_engine.h
Per K 9c166e064f Remove VideoSendStream::StartPerRtpStream 
Instead, always use VideoSendStream::Start.

VideoSendStream::StartPerRtpStream was used for controlling if
individual rtp stream for a RtpEncodingParameter should be able to send RTP packets. It was not used for controlling the actual encoder layers.

With this change RtpEncodingParameter.active still controls actual encoder layers but it does not control if RTP packets can be sent or not.

The cleanup is done to simplify code and in the future allow sending
probe packet on a RtpTransceiver that allows sending, regardless of the
RtpEncodingParameter.active flag.

Bug: webrtc:14928
Change-Id: I896c055ed4de76db58d76f452147c29783f77ae1
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/335042
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41619}
2024-01-26 09:19:50 +00:00

905 lines
39 KiB
C++
Raw Blame History

/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
#include <stddef.h>
#include <cstdint>
#include <functional>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "absl/functional/any_invocable.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/call/transport.h"
#include "api/crypto/crypto_options.h"
#include "api/crypto/frame_decryptor_interface.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/field_trials_view.h"
#include "api/frame_transformer_interface.h"
#include "api/rtc_error.h"
#include "api/rtp_headers.h"
#include "api/rtp_parameters.h"
#include "api/rtp_sender_interface.h"
#include "api/scoped_refptr.h"
#include "api/sequence_checker.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
#include "api/transport/bitrate_settings.h"
#include "api/transport/field_trial_based_config.h"
#include "api/transport/rtp/rtp_source.h"
#include "api/video/recordable_encoded_frame.h"
#include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_frame.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "call/call.h"
#include "call/flexfec_receive_stream.h"
#include "call/rtp_config.h"
#include "call/video_receive_stream.h"
#include "call/video_send_stream.h"
#include "media/base/codec.h"
#include "media/base/media_channel.h"
#include "media/base/media_channel_impl.h"
#include "media/base/media_config.h"
#include "media/base/media_engine.h"
#include "media/base/stream_params.h"
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "rtc_base/network/sent_packet.h"
#include "rtc_base/network_route.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/thread_annotations.h"
#include "video/config/video_encoder_config.h"
namespace webrtc {
class VideoDecoderFactory;
class VideoEncoderFactory;
} // namespace webrtc
namespace cricket {
// Public for testing.
// Inputs StreamStats for all types of substreams (kMedia, kRtx, kFlexfec) and
// merges any non-kMedia substream stats object into its referenced kMedia-type
// substream. The resulting substreams are all kMedia. This means, for example,
// that packet and byte counters of RTX and FlexFEC streams are accounted for in
// the relevant RTP media stream's stats. This makes the resulting StreamStats
// objects ready to be turned into "outbound-rtp" stats objects for GetStats()
// which does not create separate stream stats objects for complementary
// streams.
std::map<uint32_t, webrtc::VideoSendStream::StreamStats>
MergeInfoAboutOutboundRtpSubstreamsForTesting(
const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& substreams);
// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
class WebRtcVideoEngine : public VideoEngineInterface {
public:
// These video codec factories represents all video codecs, i.e. both software
// and external hardware codecs.
WebRtcVideoEngine(
std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
const webrtc::FieldTrialsView& trials);
~WebRtcVideoEngine() override;
std::unique_ptr<VideoMediaSendChannelInterface> CreateSendChannel(
webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const webrtc::CryptoOptions& crypto_options,
webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory)
override;
std::unique_ptr<VideoMediaReceiveChannelInterface> CreateReceiveChannel(
webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const webrtc::CryptoOptions& crypto_options) override;
std::vector<VideoCodec> send_codecs() const override {
return send_codecs(true);
}
std::vector<VideoCodec> recv_codecs() const override {
return recv_codecs(true);
}
std::vector<VideoCodec> send_codecs(bool include_rtx) const override;
std::vector<VideoCodec> recv_codecs(bool include_rtx) const override;
std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions()
const override;
private:
const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_;
const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
bitrate_allocator_factory_;
const webrtc::FieldTrialsView& trials_;
};
struct VideoCodecSettings {
explicit VideoCodecSettings(const VideoCodec& codec);
// Checks if all members of |*this| are equal to the corresponding members
// of `other`.
bool operator==(const VideoCodecSettings& other) const;
bool operator!=(const VideoCodecSettings& other) const;
// Checks if all members of `a`, except `flexfec_payload_type`, are equal
// to the corresponding members of `b`.
static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
const VideoCodecSettings& b);
VideoCodec codec;
webrtc::UlpfecConfig ulpfec;
int flexfec_payload_type; // -1 if absent.
int rtx_payload_type; // -1 if absent.
absl::optional<int> rtx_time;
};
class WebRtcVideoSendChannel : public MediaChannelUtil,
public VideoMediaSendChannelInterface,
public webrtc::EncoderSwitchRequestCallback {
public:
WebRtcVideoSendChannel(
webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const webrtc::CryptoOptions& crypto_options,
webrtc::VideoEncoderFactory* encoder_factory,
webrtc::VideoDecoderFactory* decoder_factory,
webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory);
~WebRtcVideoSendChannel() override;
MediaType media_type() const override { return MEDIA_TYPE_VIDEO; }
// Type manipulations
VideoMediaSendChannelInterface* AsVideoSendChannel() override { return this; }
VoiceMediaSendChannelInterface* AsVoiceSendChannel() override {
RTC_CHECK_NOTREACHED();
return nullptr;
}
// Functions imported from MediaChannelUtil
bool HasNetworkInterface() const override {
return MediaChannelUtil::HasNetworkInterface();
}
void SetExtmapAllowMixed(bool extmap_allow_mixed) override {
MediaChannelUtil::SetExtmapAllowMixed(extmap_allow_mixed);
}
bool ExtmapAllowMixed() const override {
return MediaChannelUtil::ExtmapAllowMixed();
}
// Common functions between sender and receiver
void SetInterface(MediaChannelNetworkInterface* iface) override;
// VideoMediaSendChannelInterface implementation
bool SetSenderParameters(const VideoSenderParameters& params) override;
webrtc::RTCError SetRtpSendParameters(
uint32_t ssrc,
const webrtc::RtpParameters& parameters,
webrtc::SetParametersCallback callback) override;
webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
absl::optional<Codec> GetSendCodec() const override;
bool SetSend(bool send) override;
bool SetVideoSend(
uint32_t ssrc,
const VideoOptions* options,
rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
bool AddSendStream(const StreamParams& sp) override;
bool RemoveSendStream(uint32_t ssrc) override;
void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
bool GetStats(VideoMediaSendInfo* info) override;
void OnPacketSent(const rtc::SentPacket& sent_packet) override;
void OnReadyToSend(bool ready) override;
void OnNetworkRouteChanged(absl::string_view transport_name,
const rtc::NetworkRoute& network_route) override;
// Set a frame encryptor to a particular ssrc that will intercept all
// outgoing video frames and attempt to encrypt them and forward the result
// to the packetizer.
void SetFrameEncryptor(uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameEncryptorInterface>
frame_encryptor) override;
// note: The encoder_selector object must remain valid for the lifetime of the
// MediaChannel, unless replaced.
void SetEncoderSelector(uint32_t ssrc,
webrtc::VideoEncoderFactory::EncoderSelectorInterface*
encoder_selector) override;
void SetSendCodecChangedCallback(
absl::AnyInvocable<void()> callback) override {
send_codec_changed_callback_ = std::move(callback);
}
void SetSsrcListChangedCallback(
absl::AnyInvocable<void(const std::set<uint32_t>&)> callback) override {
ssrc_list_changed_callback_ = std::move(callback);
}
// Implemented for VideoMediaChannelTest.
bool sending() const {
RTC_DCHECK_RUN_ON(&thread_checker_);
return sending_;
}
// AdaptReason is used for expressing why a WebRtcVideoSendStream request
// a lower input frame size than the currently configured camera input frame
// size. There can be more than one reason OR:ed together.
enum AdaptReason {
ADAPTREASON_NONE = 0,
ADAPTREASON_CPU = 1,
ADAPTREASON_BANDWIDTH = 2,
};
// TODO(webrtc:14852): Update downstream projects to use
// cricket::kDefaultVideoMaxQpVpx/H26x and remove.
static constexpr int kDefaultQpMax = 56;
// Implements webrtc::EncoderSwitchRequestCallback.
void RequestEncoderFallback() override;
void RequestEncoderSwitch(const webrtc::SdpVideoFormat& format,
bool allow_default_fallback) override;
void GenerateSendKeyFrame(uint32_t ssrc,
const std::vector<std::string>& rids) override;
void SetEncoderToPacketizerFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
override;
// Information queries to support SetReceiverFeedbackParameters
webrtc::RtcpMode SendCodecRtcpMode() const override {
RTC_DCHECK_RUN_ON(&thread_checker_);
return send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize
: webrtc::RtcpMode::kCompound;
}
bool SendCodecHasLntf() const override {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (!send_codec()) {
return false;
}
return HasLntf(send_codec()->codec);
}
bool SendCodecHasNack() const override {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (!send_codec()) {
return false;
}
return HasNack(send_codec()->codec);
}
absl::optional<int> SendCodecRtxTime() const override {
RTC_DCHECK_RUN_ON(&thread_checker_);
if (!send_codec()) {
return absl::nullopt;
}
return send_codec()->rtx_time;
}
private:
struct ChangedSenderParameters {
// These optionals are unset if not changed.
absl::optional<VideoCodecSettings> send_codec;
absl::optional<std::vector<VideoCodecSettings>> negotiated_codecs;
absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
absl::optional<std::string> mid;
absl::optional<bool> extmap_allow_mixed;
absl::optional<int> max_bandwidth_bps;
absl::optional<bool> conference_mode;
absl::optional<webrtc::RtcpMode> rtcp_mode;
};
bool GetChangedSenderParameters(const VideoSenderParameters& params,
ChangedSenderParameters* changed_params) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
bool ApplyChangedParams(const ChangedSenderParameters& changed_params);
bool ValidateSendSsrcAvailability(const StreamParams& sp) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
// Populates `rtx_associated_payload_types`, `raw_payload_types` and
// `decoders` based on codec settings provided by `recv_codecs`.
// `recv_codecs` must be non-empty and all other parameters must be empty.
static void ExtractCodecInformation(
rtc::ArrayView<const VideoCodecSettings> recv_codecs,
std::map<int, int>& rtx_associated_payload_types,
std::set<int>& raw_payload_types,
std::vector<webrtc::VideoReceiveStreamInterface::Decoder>& decoders);
// Wrapper for the sender part.
class WebRtcVideoSendStream {
public:
WebRtcVideoSendStream(
webrtc::Call* call,
const StreamParams& sp,
webrtc::VideoSendStream::Config config,
const VideoOptions& options,
bool enable_cpu_overuse_detection,
int max_bitrate_bps,
const absl::optional<VideoCodecSettings>& codec_settings,
const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
const VideoSenderParameters& send_params);
~WebRtcVideoSendStream();
void SetSenderParameters(const ChangedSenderParameters& send_params);
webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters,
webrtc::SetParametersCallback callback);
webrtc::RtpParameters GetRtpParameters() const;
void SetFrameEncryptor(
rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor);
bool SetVideoSend(const VideoOptions* options,
rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
// note: The encoder_selector object must remain valid for the lifetime of
// the MediaChannel, unless replaced.
void SetEncoderSelector(
webrtc::VideoEncoderFactory::EncoderSelectorInterface*
encoder_selector);
void SetSend(bool send);
const std::vector<uint32_t>& GetSsrcs() const;
// Returns per ssrc VideoSenderInfos. Useful for simulcast scenario.
std::vector<VideoSenderInfo> GetPerLayerVideoSenderInfos(bool log_stats);
// Aggregates per ssrc VideoSenderInfos to single VideoSenderInfo for
// legacy reasons. Used in old GetStats API and track stats.
VideoSenderInfo GetAggregatedVideoSenderInfo(
const std::vector<VideoSenderInfo>& infos) const;
void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
void SetEncoderToPacketizerFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
frame_transformer);
void GenerateKeyFrame(const std::vector<std::string>& rids);
private:
// Parameters needed to reconstruct the underlying stream.
// webrtc::VideoSendStream doesn't support setting a lot of options on the
// fly, so when those need to be changed we tear down and reconstruct with
// similar parameters depending on which options changed etc.
struct VideoSendStreamParameters {
VideoSendStreamParameters(
webrtc::VideoSendStream::Config config,
const VideoOptions& options,
int max_bitrate_bps,
const absl::optional<VideoCodecSettings>& codec_settings);
webrtc::VideoSendStream::Config config;
VideoOptions options;
int max_bitrate_bps;
bool conference_mode;
absl::optional<VideoCodecSettings> codec_settings;
// Sent resolutions + bitrates etc. by the underlying VideoSendStream,
// typically changes when setting a new resolution or reconfiguring
// bitrates.
webrtc::VideoEncoderConfig encoder_config;
};
rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
ConfigureVideoEncoderSettings(const VideoCodec& codec);
void SetCodec(const VideoCodecSettings& codec);
void RecreateWebRtcStream();
webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
const VideoCodec& codec) const;
void ReconfigureEncoder(webrtc::SetParametersCallback callback);
// Calls Start or Stop according to whether or not `sending_` is true.
void UpdateSendState();
webrtc::DegradationPreference GetDegradationPreference() const
RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_;
webrtc::TaskQueueBase* const worker_thread_;
const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_);
const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_);
webrtc::Call* const call_;
const bool enable_cpu_overuse_detection_;
rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
RTC_GUARDED_BY(&thread_checker_);
webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_);
// Contains settings that are the same for all streams in the MediaChannel,
// such as codecs, header extensions, and the global bitrate limit for the
// entire channel.
VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_);
// Contains settings that are unique for each stream, such as max_bitrate.
// Does *not* contain codecs, however.
// TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
// TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
// one stream per MediaChannel.
webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_);
bool sending_ RTC_GUARDED_BY(&thread_checker_);
// TODO(asapersson): investigate why setting
// DegrationPreferences::MAINTAIN_RESOLUTION isn't sufficient to disable
// downscaling everywhere in the pipeline.
const bool disable_automatic_resize_;
};
void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
// Get all codecs that are compatible with the receiver.
std::vector<VideoCodecSettings> SelectSendVideoCodecs(
const std::vector<VideoCodecSettings>& remote_mapped_codecs) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void FillSenderStats(VideoMediaSendInfo* info, bool log_stats)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
VideoMediaInfo* info)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void FillSendCodecStats(VideoMediaSendInfo* video_media_info)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
// Accessor function for send_codec_. Introduced in order to ensure
// that a receive channel does not touch the send codec directly.
// Can go away once these are different classes.
// TODO(bugs.webrtc.org/13931): Remove this function
absl::optional<VideoCodecSettings>& send_codec() { return send_codec_; }
const absl::optional<VideoCodecSettings>& send_codec() const {
return send_codec_;
}
webrtc::TaskQueueBase* const worker_thread_;
webrtc::ScopedTaskSafety task_safety_;
RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker network_thread_checker_{
webrtc::SequenceChecker::kDetached};
RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_;
uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
bool sending_ RTC_GUARDED_BY(thread_checker_);
bool receiving_ RTC_GUARDED_BY(&thread_checker_);
webrtc::Call* const call_;
rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_
RTC_GUARDED_BY(thread_checker_);
// Delay for unsignaled streams, which may be set before the stream exists.
int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0;
const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_);
// Using primary-ssrc (first ssrc) as key.
std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
RTC_GUARDED_BY(thread_checker_);
// When the channel and demuxer get reconfigured, there is a window of time
// where we have to be prepared for packets arriving based on the old demuxer
// criteria because the streams live on the worker thread and the demuxer
// lives on the network thread. Because packets are posted from the network
// thread to the worker thread, they can still be in-flight when streams are
// reconfgured. This can happen when `demuxer_criteria_id_` and
// `demuxer_criteria_completed_id_` don't match. During this time, we do not
// want to create unsignalled receive streams and should instead drop the
// packets. E.g:
// * If RemoveRecvStream(old_ssrc) was recently called, there may be packets
// in-flight for that ssrc. This happens when a receiver becomes inactive.
// * If we go from one to many m= sections, the demuxer may change from
// forwarding all packets to only forwarding the configured ssrcs, so there
// is a risk of receiving ssrcs for other, recently added m= sections.
uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0;
uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0;
absl::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_
RTC_GUARDED_BY(thread_checker_);
std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_);
std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_);
absl::optional<VideoCodecSettings> send_codec_
RTC_GUARDED_BY(thread_checker_);
std::vector<VideoCodecSettings> negotiated_codecs_
RTC_GUARDED_BY(thread_checker_);
std::vector<webrtc::RtpExtension> send_rtp_extensions_
RTC_GUARDED_BY(thread_checker_);
webrtc::VideoEncoderFactory* const encoder_factory_
RTC_GUARDED_BY(thread_checker_);
webrtc::VideoDecoderFactory* const decoder_factory_
RTC_GUARDED_BY(thread_checker_);
webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_
RTC_GUARDED_BY(thread_checker_);
std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_);
webrtc::RtpHeaderExtensionMap recv_rtp_extension_map_
RTC_GUARDED_BY(thread_checker_);
std::vector<webrtc::RtpExtension> recv_rtp_extensions_
RTC_GUARDED_BY(thread_checker_);
// See reason for keeping track of the FlexFEC payload type separately in
// comment in WebRtcVideoChannel::ChangedReceiverParameters.
int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_);
webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_);
// TODO(deadbeef): Don't duplicate information between
// send_params/recv_params, rtp_extensions, options, etc.
VideoSenderParameters send_params_ RTC_GUARDED_BY(thread_checker_);
VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_);
VideoReceiverParameters recv_params_ RTC_GUARDED_BY(thread_checker_);
int64_t last_send_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
int64_t last_receive_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_);
// This is a stream param that comes from the remote description, but wasn't
// signaled with any a=ssrc lines. It holds information that was signaled
// before the unsignaled receive stream is created when the first packet is
// received.
StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_);
// Per peer connection crypto options that last for the lifetime of the peer
// connection.
const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_);
// Optional frame transformer set on unsignaled streams.
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_);
// RTP parameters that need to be set when creating a video receive stream.
// Only used in Receiver mode - in Both mode, it reads those things from the
// codec.
webrtc::VideoReceiveStreamInterface::Config::Rtp rtp_config_;
// Callback invoked whenever the send codec changes.
// TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed.
absl::AnyInvocable<void()> send_codec_changed_callback_;
// Callback invoked whenever the list of SSRCs changes.
absl::AnyInvocable<void(const std::set<uint32_t>&)>
ssrc_list_changed_callback_;
};
class WebRtcVideoReceiveChannel : public MediaChannelUtil,
public VideoMediaReceiveChannelInterface {
public:
WebRtcVideoReceiveChannel(webrtc::Call* call,
const MediaConfig& config,
const VideoOptions& options,
const webrtc::CryptoOptions& crypto_options,
webrtc::VideoDecoderFactory* decoder_factory);
~WebRtcVideoReceiveChannel() override;
public:
MediaType media_type() const override { return MEDIA_TYPE_VIDEO; }
VideoMediaReceiveChannelInterface* AsVideoReceiveChannel() override {
return this;
}
VoiceMediaReceiveChannelInterface* AsVoiceReceiveChannel() override {
RTC_CHECK_NOTREACHED();
return nullptr;
}
// Common functions between sender and receiver
void SetInterface(MediaChannelNetworkInterface* iface) override;
// VideoMediaReceiveChannelInterface implementation
bool SetReceiverParameters(const VideoReceiverParameters& params) override;
webrtc::RtpParameters GetRtpReceiverParameters(uint32_t ssrc) const override;
webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override;
void SetReceive(bool receive) override;
bool AddRecvStream(const StreamParams& sp) override;
bool AddDefaultRecvStreamForTesting(const StreamParams& sp) override {
// Invokes private AddRecvStream variant function
return AddRecvStream(sp, true);
}
bool RemoveRecvStream(uint32_t ssrc) override;
void ResetUnsignaledRecvStream() override;
absl::optional<uint32_t> GetUnsignaledSsrc() const override;
void OnDemuxerCriteriaUpdatePending() override;
void OnDemuxerCriteriaUpdateComplete() override;
bool SetSink(uint32_t ssrc,
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
void SetDefaultSink(
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
bool GetStats(VideoMediaReceiveInfo* info) override;
void OnPacketReceived(const webrtc::RtpPacketReceived& packet) override;
bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override;
absl::optional<int> GetBaseMinimumPlayoutDelayMs(
uint32_t ssrc) const override;
// Choose one of the available SSRCs (or default if none) as the current
// receiver report SSRC.
void ChooseReceiverReportSsrc(const std::set<uint32_t>& choices) override;
// E2E Encrypted Video Frame API
// Set a frame decryptor to a particular ssrc that will intercept all
// incoming video frames and attempt to decrypt them before forwarding the
// result.
void SetFrameDecryptor(uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameDecryptorInterface>
frame_decryptor) override;
void SetRecordableEncodedFrameCallback(
uint32_t ssrc,
std::function<void(const webrtc::RecordableEncodedFrame&)> callback)
override;
void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
void RequestRecvKeyFrame(uint32_t ssrc) override;
void SetDepacketizerToDecoderFrameTransformer(
uint32_t ssrc,
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
override;
std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override;
void SetReceiverFeedbackParameters(bool lntf_enabled,
bool nack_enabled,
webrtc::RtcpMode rtcp_mode,
absl::optional<int> rtx_time) override;
private:
class WebRtcVideoReceiveStream;
struct ChangedReceiverParameters {
// These optionals are unset if not changed.
absl::optional<std::vector<VideoCodecSettings>> codec_settings;
absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
// Keep track of the FlexFEC payload type separately from `codec_settings`.
// This allows us to recreate the FlexfecReceiveStream separately from the
// VideoReceiveStreamInterface when the FlexFEC payload type is changed.
absl::optional<int> flexfec_payload_type;
};
// Finds VideoReceiveStreamInterface corresponding to ssrc. Aware of
// unsignalled ssrc handling.
WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void ProcessReceivedPacket(webrtc::RtpPacketReceived packet)
RTC_RUN_ON(thread_checker_);
// Expected to be invoked once per packet that belongs to this channel that
// can not be demuxed.
// Returns true if a new default stream has been created.
bool MaybeCreateDefaultReceiveStream(
const webrtc::RtpPacketReceived& parsed_packet)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void ReCreateDefaultReceiveStream(uint32_t ssrc,
absl::optional<uint32_t> rtx_ssrc);
// Add a receive stream. Used for testing.
bool AddRecvStream(const StreamParams& sp, bool default_stream);
void ConfigureReceiverRtp(
webrtc::VideoReceiveStreamInterface::Config* config,
webrtc::FlexfecReceiveStream::Config* flexfec_config,
const StreamParams& sp) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
// Called when the local ssrc changes. Sets `rtcp_receiver_report_ssrc_` and
// updates the receive streams.
void SetReceiverReportSsrc(uint32_t ssrc) RTC_RUN_ON(&thread_checker_);
// Wrapper for the receiver part, contains configs etc. that are needed to
// reconstruct the underlying VideoReceiveStreamInterface.
class WebRtcVideoReceiveStream
: public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
WebRtcVideoReceiveStream(
webrtc::Call* call,
const StreamParams& sp,
webrtc::VideoReceiveStreamInterface::Config config,
bool default_stream,
const std::vector<VideoCodecSettings>& recv_codecs,
const webrtc::FlexfecReceiveStream::Config& flexfec_config);
~WebRtcVideoReceiveStream();
webrtc::VideoReceiveStreamInterface& stream();
// Return value may be nullptr.
webrtc::FlexfecReceiveStream* flexfec_stream();
const std::vector<uint32_t>& GetSsrcs() const;
std::vector<webrtc::RtpSource> GetSources();
// Does not return codecs, nor header extensions, they are filled by the
// owning WebRtcVideoChannel.
webrtc::RtpParameters GetRtpParameters() const;
// TODO(deadbeef): Move these feedback parameters into the recv parameters.
void SetFeedbackParameters(bool lntf_enabled,
bool nack_enabled,
webrtc::RtcpMode rtcp_mode,
absl::optional<int> rtx_time);
void SetReceiverParameters(const ChangedReceiverParameters& recv_params);
void OnFrame(const webrtc::VideoFrame& frame) override;
bool IsDefaultStream() const;
void SetFrameDecryptor(
rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor);
bool SetBaseMinimumPlayoutDelayMs(int delay_ms);
int GetBaseMinimumPlayoutDelayMs() const;
void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
void SetRecordableEncodedFrameCallback(
std::function<void(const webrtc::RecordableEncodedFrame&)> callback);
void ClearRecordableEncodedFrameCallback();
void GenerateKeyFrame();
void SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
frame_transformer);
void SetLocalSsrc(uint32_t local_ssrc);
void UpdateRtxSsrc(uint32_t ssrc);
void StartReceiveStream();
void StopReceiveStream();
private:
// Attempts to reconfigure an already existing `flexfec_stream_`, create
// one if the configuration is now complete or remove a flexfec stream
// when disabled.
void SetFlexFecPayload(int payload_type);
void RecreateReceiveStream();
void CreateReceiveStream();
// Applies a new receive codecs configration to `config_`. Returns true
// if the internal stream needs to be reconstructed, or false if no changes
// were applied.
bool ReconfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
webrtc::Call* const call_;
const StreamParams stream_params_;
// Both `stream_` and `flexfec_stream_` are managed by `this`. They are
// destroyed by calling call_->DestroyVideoReceiveStream and
// call_->DestroyFlexfecReceiveStream, respectively.
webrtc::VideoReceiveStreamInterface* stream_;
const bool default_stream_;
webrtc::VideoReceiveStreamInterface::Config config_;
webrtc::FlexfecReceiveStream::Config flexfec_config_;
webrtc::FlexfecReceiveStream* flexfec_stream_;
webrtc::Mutex sink_lock_;
rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
RTC_GUARDED_BY(sink_lock_);
int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_);
// Start NTP time is estimated as current remote NTP time (estimated from
// RTCP) minus the elapsed time, as soon as remote NTP time is available.
int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_;
bool receiving_ RTC_GUARDED_BY(&thread_checker_);
};
bool GetChangedReceiverParameters(const VideoReceiverParameters& params,
ChangedReceiverParameters* changed_params)
const RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
RTC_GUARDED_BY(thread_checker_);
void FillReceiverStats(VideoMediaReceiveInfo* info, bool log_stats)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
void FillReceiveCodecStats(VideoMediaReceiveInfo* video_media_info)
RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
StreamParams unsignaled_stream_params() {
RTC_DCHECK_RUN_ON(&thread_checker_);
return unsignaled_stream_params_;
}
// Variables.
webrtc::TaskQueueBase* const worker_thread_;
webrtc::ScopedTaskSafety task_safety_;
RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker network_thread_checker_{
webrtc::SequenceChecker::kDetached};
RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_;
uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
bool receiving_ RTC_GUARDED_BY(&thread_checker_);
webrtc::Call* const call_;
rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_
RTC_GUARDED_BY(thread_checker_);
// Delay for unsignaled streams, which may be set before the stream exists.
int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0;
const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_);
// When the channel and demuxer get reconfigured, there is a window of time
// where we have to be prepared for packets arriving based on the old demuxer
// criteria because the streams live on the worker thread and the demuxer
// lives on the network thread. Because packets are posted from the network
// thread to the worker thread, they can still be in-flight when streams are
// reconfgured. This can happen when `demuxer_criteria_id_` and
// `demuxer_criteria_completed_id_` don't match. During this time, we do not
// want to create unsignalled receive streams and should instead drop the
// packets. E.g:
// * If RemoveRecvStream(old_ssrc) was recently called, there may be packets
// in-flight for that ssrc. This happens when a receiver becomes inactive.
// * If we go from one to many m= sections, the demuxer may change from
// forwarding all packets to only forwarding the configured ssrcs, so there
// is a risk of receiving ssrcs for other, recently added m= sections.
uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0;
uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0;
absl::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_
RTC_GUARDED_BY(thread_checker_);
std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_);
std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_);
absl::optional<VideoCodecSettings> send_codec_
RTC_GUARDED_BY(thread_checker_);
std::vector<VideoCodecSettings> negotiated_codecs_
RTC_GUARDED_BY(thread_checker_);
std::vector<webrtc::RtpExtension> send_rtp_extensions_
RTC_GUARDED_BY(thread_checker_);
webrtc::VideoDecoderFactory* const decoder_factory_
RTC_GUARDED_BY(thread_checker_);
std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_);
webrtc::RtpHeaderExtensionMap recv_rtp_extension_map_
RTC_GUARDED_BY(thread_checker_);
std::vector<webrtc::RtpExtension> recv_rtp_extensions_
RTC_GUARDED_BY(thread_checker_);
// See reason for keeping track of the FlexFEC payload type separately in
// comment in WebRtcVideoChannel::ChangedReceiverParameters.
int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_);
webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_);
// TODO(deadbeef): Don't duplicate information between
// send_params/recv_params, rtp_extensions, options, etc.
VideoSenderParameters send_params_ RTC_GUARDED_BY(thread_checker_);
VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_);
VideoReceiverParameters recv_params_ RTC_GUARDED_BY(thread_checker_);
int64_t last_receive_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_);
// This is a stream param that comes from the remote description, but wasn't
// signaled with any a=ssrc lines. It holds information that was signaled
// before the unsignaled receive stream is created when the first packet is
// received.
StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_);
// Per peer connection crypto options that last for the lifetime of the peer
// connection.
const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_);
// Optional frame transformer set on unsignaled streams.
rtc::scoped_refptr<webrtc::FrameTransformerInterface>
unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_);
// RTP parameters that need to be set when creating a video receive stream.
// Only used in Receiver mode - in Both mode, it reads those things from the
// codec.
webrtc::VideoReceiveStreamInterface::Config::Rtp rtp_config_;
// Callback invoked whenever the send codec changes.
// TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed.
absl::AnyInvocable<void()> send_codec_changed_callback_;
// Callback invoked whenever the list of SSRCs changes.
absl::AnyInvocable<void(const std::set<uint32_t>&)>
ssrc_list_changed_callback_;
const int receive_buffer_size_;
};
// Keeping the old name "WebRtcVideoChannel" around because some external
// customers are using cricket::WebRtcVideoChannel::AdaptReason
// TODO(bugs.webrtc.org/15216): Move this enum to an interface class and
// delete this workaround.
class WebRtcVideoChannel : public WebRtcVideoSendChannel {
public:
// Make all the values of AdaptReason available as
// WebRtcVideoChannel::ADAPT_xxx.
using WebRtcVideoSendChannel::AdaptReason;
};
} // namespace cricket
#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
Reference in a new issue View git blame Copy permalink