/* * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "media/engine/videoencodersoftwarefallbackwrapper.h" #include #include "media/base/h264_profile_level_id.h" #include "media/engine/internalencoderfactory.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/timeutils.h" #include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { const char kVp8ForceFallbackEncoderFieldTrial[] = "WebRTC-VP8-Forced-Fallback-Encoder"; bool EnableForcedFallback(const cricket::VideoCodec& codec) { if (!webrtc::field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial)) return false; return (PayloadStringToCodecType(codec.name) == kVideoCodecVP8); } bool IsForcedFallbackPossible(const VideoCodec& codec_settings) { return codec_settings.codecType == kVideoCodecVP8 && codec_settings.numberOfSimulcastStreams <= 1 && codec_settings.VP8().numberOfTemporalLayers == 1; } void GetForcedFallbackParamsFromFieldTrialGroup(uint32_t* param_low_kbps, uint32_t* param_high_kbps, int64_t* param_min_low_ms) { RTC_DCHECK(param_low_kbps); RTC_DCHECK(param_high_kbps); RTC_DCHECK(param_min_low_ms); std::string group = webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial); if (group.empty()) return; int low_kbps; int high_kbps; int min_low_ms; int min_pixels; if (sscanf(group.c_str(), "Enabled-%d,%d,%d,%d", &low_kbps, &high_kbps, &min_low_ms, &min_pixels) != 4) { LOG(LS_WARNING) << "Invalid number of forced fallback parameters provided."; return; } if (min_low_ms <= 0 || min_pixels <= 0 || low_kbps <= 0 || high_kbps <= low_kbps) { LOG(LS_WARNING) << "Invalid forced fallback parameter value provided."; return; } *param_low_kbps = low_kbps; *param_high_kbps = high_kbps; *param_min_low_ms = min_low_ms; } } // namespace VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( const cricket::VideoCodec& codec, std::unique_ptr encoder) : number_of_cores_(0), max_payload_size_(0), rates_set_(false), framerate_(0), channel_parameters_set_(false), packet_loss_(0), rtt_(0), codec_(codec), encoder_(std::move(encoder)), callback_(nullptr), forced_fallback_possible_(EnableForcedFallback(codec)) { if (forced_fallback_possible_) { GetForcedFallbackParamsFromFieldTrialGroup(&forced_fallback_.low_kbps, &forced_fallback_.high_kbps, &forced_fallback_.min_low_ms); } } bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() { LOG(LS_WARNING) << "Encoder falling back to software encoding."; MaybeModifyCodecForFallback(); cricket::InternalEncoderFactory internal_factory; if (!FindMatchingCodec(internal_factory.supported_codecs(), codec_)) { LOG(LS_WARNING) << "Encoder requesting fallback to codec not supported in software."; return false; } fallback_encoder_.reset(internal_factory.CreateVideoEncoder(codec_)); if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_, max_payload_size_) != WEBRTC_VIDEO_CODEC_OK) { LOG(LS_ERROR) << "Failed to initialize software-encoder fallback."; fallback_encoder_->Release(); fallback_encoder_.reset(); return false; } // Replay callback, rates, and channel parameters. if (callback_) fallback_encoder_->RegisterEncodeCompleteCallback(callback_); if (rates_set_) fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate_); if (channel_parameters_set_) fallback_encoder_->SetChannelParameters(packet_loss_, rtt_); fallback_implementation_name_ = std::string(fallback_encoder_->ImplementationName()) + " (fallback from: " + encoder_->ImplementationName() + ")"; // Since we're switching to the fallback encoder, Release the real encoder. It // may be re-initialized via InitEncode later, and it will continue to get // Set calls for rates and channel parameters in the meantime. encoder_->Release(); return true; } int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( const VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { // Store settings, in case we need to dynamically switch to the fallback // encoder after a failed Encode call. codec_settings_ = *codec_settings; number_of_cores_ = number_of_cores; max_payload_size_ = max_payload_size; // Clear stored rate/channel parameters. rates_set_ = false; channel_parameters_set_ = false; ValidateSettingsForForcedFallback(); // Try to reinit forced software codec if it is in use. if (TryReInitForcedFallbackEncoder()) { return WEBRTC_VIDEO_CODEC_OK; } forced_fallback_.Reset(); int32_t ret = encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size); if (ret == WEBRTC_VIDEO_CODEC_OK || codec_.name.empty()) { if (fallback_encoder_) { LOG(LS_WARNING) << "InitEncode OK, no longer using the software fallback encoder."; fallback_encoder_->Release(); } fallback_encoder_.reset(); if (callback_) encoder_->RegisterEncodeCompleteCallback(callback_); return ret; } // Try to instantiate software codec. if (InitFallbackEncoder()) { return WEBRTC_VIDEO_CODEC_OK; } // Software encoder failed, use original return code. return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { callback_ = callback; int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback); if (fallback_encoder_) return fallback_encoder_->RegisterEncodeCompleteCallback(callback); return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::Release() { // If the fallback_encoder_ is non-null, it means it was created via // InitFallbackEncoder which has Release()d encoder_, so we should only ever // need to Release() whichever one is active. if (fallback_encoder_) return fallback_encoder_->Release(); return encoder_->Release(); } int32_t VideoEncoderSoftwareFallbackWrapper::Encode( const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { if (TryReleaseForcedFallbackEncoder()) { // Frame may have been converted from kNative to kI420 during fallback. if (encoder_->SupportsNativeHandle() && frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { LOG(LS_WARNING) << "Encoder supports native frames, dropping one frame " << "to avoid possible reconfig due to format change."; return WEBRTC_VIDEO_CODEC_ERROR; } } if (fallback_encoder_) return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); int32_t ret = encoder_->Encode(frame, codec_specific_info, frame_types); // If requested, try a software fallback. bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) || (ret == WEBRTC_VIDEO_CODEC_OK && RequestForcedFallback()); if (fallback_requested && InitFallbackEncoder()) { // Fallback was successful. if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) forced_fallback_.Reset(); // Not a forced fallback. if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative && !fallback_encoder_->SupportsNativeHandle()) { LOG(LS_WARNING) << "Fallback encoder doesn't support native frames, " << "dropping one frame."; return WEBRTC_VIDEO_CODEC_ERROR; } // Start using the fallback with this frame. return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); } return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters( uint32_t packet_loss, int64_t rtt) { channel_parameters_set_ = true; packet_loss_ = packet_loss; rtt_ = rtt; int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt); if (fallback_encoder_) return fallback_encoder_->SetChannelParameters(packet_loss, rtt); return ret; } int32_t VideoEncoderSoftwareFallbackWrapper::SetRateAllocation( const BitrateAllocation& bitrate_allocation, uint32_t framerate) { rates_set_ = true; bitrate_allocation_ = bitrate_allocation; framerate_ = framerate; int32_t ret = encoder_->SetRateAllocation(bitrate_allocation_, framerate); if (fallback_encoder_) return fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate); return ret; } bool VideoEncoderSoftwareFallbackWrapper::SupportsNativeHandle() const { if (fallback_encoder_) return fallback_encoder_->SupportsNativeHandle(); return encoder_->SupportsNativeHandle(); } VideoEncoder::ScalingSettings VideoEncoderSoftwareFallbackWrapper::GetScalingSettings() const { if (forced_fallback_possible_ && fallback_encoder_) return fallback_encoder_->GetScalingSettings(); return encoder_->GetScalingSettings(); } const char *VideoEncoderSoftwareFallbackWrapper::ImplementationName() const { if (fallback_encoder_) return fallback_encoder_->ImplementationName(); return encoder_->ImplementationName(); } bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const { return (forced_fallback_possible_ && fallback_encoder_ && forced_fallback_.start_ms); } bool VideoEncoderSoftwareFallbackWrapper::RequestForcedFallback() { if (!forced_fallback_possible_ || fallback_encoder_ || !rates_set_) return false; // No fallback encoder. return forced_fallback_.ShouldStart(bitrate_allocation_.get_sum_kbps(), codec_settings_); } bool VideoEncoderSoftwareFallbackWrapper::TryReleaseForcedFallbackEncoder() { if (!IsForcedFallbackActive()) return false; if (!forced_fallback_.ShouldStop(bitrate_allocation_.get_sum_kbps(), codec_settings_)) { return false; } // Release the forced fallback encoder. if (encoder_->InitEncode(&codec_settings_, number_of_cores_, max_payload_size_) == WEBRTC_VIDEO_CODEC_OK) { LOG(LS_INFO) << "Stop forced SW encoder fallback, max bitrate exceeded."; fallback_encoder_->Release(); fallback_encoder_.reset(); forced_fallback_.Reset(); return true; } return false; } bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() { if (!IsForcedFallbackActive()) return false; // Encoder reconfigured. if (!forced_fallback_.IsValid(codec_settings_)) { LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded."; return false; } // Settings valid, reinitialize the forced fallback encoder. if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_, max_payload_size_) != WEBRTC_VIDEO_CODEC_OK) { LOG(LS_ERROR) << "Failed to init forced SW encoder fallback."; return false; } return true; } void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() { if (!forced_fallback_possible_) return; if (!IsForcedFallbackPossible(codec_settings_)) { if (IsForcedFallbackActive()) { fallback_encoder_->Release(); fallback_encoder_.reset(); } LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings."; forced_fallback_possible_ = false; } } bool VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams::ShouldStart( uint32_t bitrate_kbps, const VideoCodec& codec) { if (bitrate_kbps > low_kbps || !IsValid(codec)) { start_ms.reset(); return false; } // Has bitrate been below |low_kbps| for long enough duration. int64_t now_ms = rtc::TimeMillis(); if (!start_ms) start_ms.emplace(now_ms); if ((now_ms - *start_ms) >= min_low_ms) { LOG(LS_INFO) << "Request forced SW encoder fallback."; // In case the request fails, update time to avoid too frequent requests. start_ms.emplace(now_ms); return true; } return false; } bool VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams::ShouldStop( uint32_t bitrate_kbps, const VideoCodec& codec) const { return bitrate_kbps >= high_kbps && (codec.width * codec.height >= kMinPixelsStop); } void VideoEncoderSoftwareFallbackWrapper::MaybeModifyCodecForFallback() { // We have a specific case for H264 ConstrainedBaseline because that is the // only supported profile in Sw fallback. if (!cricket::CodecNamesEq(codec_.name.c_str(), cricket::kH264CodecName)) return; codec_.SetParam(cricket::kH264FmtpProfileLevelId, cricket::kH264ProfileLevelConstrainedBaseline); } } // namespace webrtc