mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 13:50:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc
Byoungchan Lee 11376fb992 Reset H.264 SVC Controller on key frame 
Sometimes OpenH264 returns a key frame even though we have not
requested one. However, SVC controller does not know about this
and will not reset its state. Since we are comparing expected tid
from SVC controller with actual tid from OpenH264, and drop frames
if they do not match, that causes a missing frame.

This CL resets the SVC controller state on key frames, ensuring
that it accurately maintains its state and does not drop frames.
Also, changes the message of the error log to be more descriptive.
Now, it will print the expected tid and actual tid.

Bug: webrtc:14877
Change-Id: I6c9e7532b2478773f03e5707bf7a1ca56e4f7b99
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/324001
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#40972}
2023-10-19 09:51:14 +00:00

736 lines
29 KiB
C++
Raw Blame History

/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*
*/
// Everything declared/defined in this header is only required when WebRTC is
// build with H264 support, please do not move anything out of the
// #ifdef unless needed and tested.
#ifdef WEBRTC_USE_H264
#include "modules/video_coding/codecs/h264/h264_encoder_impl.h"
#include <algorithm>
#include <limits>
#include <string>
#include "absl/strings/match.h"
#include "absl/types/optional.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/scalability_mode.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/svc/create_scalability_structure.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "modules/video_coding/utility/simulcast_utility.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/scale.h"
#include "third_party/openh264/src/codec/api/wels/codec_api.h"
#include "third_party/openh264/src/codec/api/wels/codec_app_def.h"
#include "third_party/openh264/src/codec/api/wels/codec_def.h"
#include "third_party/openh264/src/codec/api/wels/codec_ver.h"
namespace webrtc {
namespace {
const bool kOpenH264EncoderDetailedLogging = false;
// QP scaling thresholds.
static const int kLowH264QpThreshold = 24;
static const int kHighH264QpThreshold = 37;
// Used by histograms. Values of entries should not be changed.
enum H264EncoderImplEvent {
kH264EncoderEventInit = 0,
kH264EncoderEventError = 1,
kH264EncoderEventMax = 16,
};
int NumberOfThreads(absl::optional<int> encoder_thread_limit,
int width,
int height,
int number_of_cores) {
// TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac,
// see crbug.com/583348. Until further investigated, only use one thread.
// While this limitation is gone, this changes the bitstream format (see
// bugs.webrtc.org/14368) so still guarded by field trial to allow for
// experimentation using th experimental
// WebRTC-VideoEncoderSettings/encoder_thread_limit trial.
if (encoder_thread_limit.has_value()) {
int limit = encoder_thread_limit.value();
RTC_DCHECK_GE(limit, 1);
if (width * height >= 1920 * 1080 && number_of_cores > 8) {
return std::min(limit, 8); // 8 threads for 1080p on high perf machines.
} else if (width * height > 1280 * 960 && number_of_cores >= 6) {
return std::min(limit, 3); // 3 threads for 1080p.
} else if (width * height > 640 * 480 && number_of_cores >= 3) {
return std::min(limit, 2); // 2 threads for qHD/HD.
} else {
return 1; // 1 thread for VGA or less.
}
}
// TODO(sprang): Also check sSliceArgument.uiSliceNum on GetEncoderParams(),
// before enabling multithreading here.
return 1;
}
VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) {
switch (type) {
case videoFrameTypeIDR:
return VideoFrameType::kVideoFrameKey;
case videoFrameTypeSkip:
case videoFrameTypeI:
case videoFrameTypeP:
case videoFrameTypeIPMixed:
return VideoFrameType::kVideoFrameDelta;
case videoFrameTypeInvalid:
break;
}
RTC_DCHECK_NOTREACHED() << "Unexpected/invalid frame type: " << type;
return VideoFrameType::kEmptyFrame;
}
absl::optional<ScalabilityMode> ScalabilityModeFromTemporalLayers(
int num_temporal_layers) {
switch (num_temporal_layers) {
case 0:
break;
case 1:
return ScalabilityMode::kL1T1;
case 2:
return ScalabilityMode::kL1T2;
case 3:
return ScalabilityMode::kL1T3;
default:
RTC_DCHECK_NOTREACHED();
}
return absl::nullopt;
}
} // namespace
// Helper method used by H264EncoderImpl::Encode.
// Copies the encoded bytes from `info` to `encoded_image`. The
// `encoded_image->_buffer` may be deleted and reallocated if a bigger buffer is
// required.
//
// After OpenH264 encoding, the encoded bytes are stored in `info` spread out
// over a number of layers and "NAL units". Each NAL unit is a fragment starting
// with the four-byte start code {0,0,0,1}. All of this data (including the
// start codes) is copied to the `encoded_image->_buffer`.
static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) {
// Calculate minimum buffer size required to hold encoded data.
size_t required_capacity = 0;
size_t fragments_count = 0;
for (int layer = 0; layer < info->iLayerNum; ++layer) {
const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) {
RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0);
// Ensure `required_capacity` will not overflow.
RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal],
std::numeric_limits<size_t>::max() - required_capacity);
required_capacity += layerInfo.pNalLengthInByte[nal];
}
}
auto buffer = EncodedImageBuffer::Create(required_capacity);
encoded_image->SetEncodedData(buffer);
// Iterate layers and NAL units, note each NAL unit as a fragment and copy
// the data to `encoded_image->_buffer`.
const uint8_t start_code[4] = {0, 0, 0, 1};
size_t frag = 0;
encoded_image->set_size(0);
for (int layer = 0; layer < info->iLayerNum; ++layer) {
const SLayerBSInfo& layerInfo = info->sLayerInfo[layer];
// Iterate NAL units making up this layer, noting fragments.
size_t layer_len = 0;
for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) {
// Because the sum of all layer lengths, `required_capacity`, fits in a
// `size_t`, we know that any indices in-between will not overflow.
RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]);
RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]);
layer_len += layerInfo.pNalLengthInByte[nal];
}
// Copy the entire layer's data (including start codes).
memcpy(buffer->data() + encoded_image->size(), layerInfo.pBsBuf, layer_len);
encoded_image->set_size(encoded_image->size() + layer_len);
}
}
H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
: packetization_mode_(H264PacketizationMode::SingleNalUnit),
max_payload_size_(0),
number_of_cores_(0),
encoded_image_callback_(nullptr),
has_reported_init_(false),
has_reported_error_(false) {
RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
std::string packetization_mode_string;
if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
&packetization_mode_string) &&
packetization_mode_string == "1") {
packetization_mode_ = H264PacketizationMode::NonInterleaved;
}
downscaled_buffers_.reserve(kMaxSimulcastStreams - 1);
encoded_images_.reserve(kMaxSimulcastStreams);
encoders_.reserve(kMaxSimulcastStreams);
configurations_.reserve(kMaxSimulcastStreams);
tl0sync_limit_.reserve(kMaxSimulcastStreams);
svc_controllers_.reserve(kMaxSimulcastStreams);
}
H264EncoderImpl::~H264EncoderImpl() {
Release();
}
int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
const VideoEncoder::Settings& settings) {
ReportInit();
if (!inst || inst->codecType != kVideoCodecH264) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->maxFramerate == 0) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->width < 1 || inst->height < 1) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
int32_t release_ret = Release();
if (release_ret != WEBRTC_VIDEO_CODEC_OK) {
ReportError();
return release_ret;
}
int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
bool doing_simulcast = (number_of_streams > 1);
if (doing_simulcast &&
!SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
}
downscaled_buffers_.resize(number_of_streams - 1);
encoded_images_.resize(number_of_streams);
encoders_.resize(number_of_streams);
pictures_.resize(number_of_streams);
svc_controllers_.resize(number_of_streams);
scalability_modes_.resize(number_of_streams);
configurations_.resize(number_of_streams);
tl0sync_limit_.resize(number_of_streams);
max_payload_size_ = settings.max_payload_size;
number_of_cores_ = settings.number_of_cores;
encoder_thread_limit_ = settings.encoder_thread_limit;
codec_ = *inst;
// Code expects simulcastStream resolutions to be correct, make sure they are
// filled even when there are no simulcast layers.
if (codec_.numberOfSimulcastStreams == 0) {
codec_.simulcastStream[0].width = codec_.width;
codec_.simulcastStream[0].height = codec_.height;
}
for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
++i, --idx) {
ISVCEncoder* openh264_encoder;
// Create encoder.
if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
// Failed to create encoder.
RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
RTC_DCHECK(!openh264_encoder);
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(openh264_encoder);
if (kOpenH264EncoderDetailedLogging) {
int trace_level = WELS_LOG_DETAIL;
openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
}
// else WELS_LOG_DEFAULT is used by default.
// Store h264 encoder.
encoders_[i] = openh264_encoder;
// Set internal settings from codec_settings
configurations_[i].simulcast_idx = idx;
configurations_[i].sending = false;
configurations_[i].width = codec_.simulcastStream[idx].width;
configurations_[i].height = codec_.simulcastStream[idx].height;
configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
configurations_[i].frame_dropping_on = codec_.GetFrameDropEnabled();
configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
configurations_[i].num_temporal_layers =
std::max(codec_.H264()->numberOfTemporalLayers,
codec_.simulcastStream[idx].numberOfTemporalLayers);
// Create downscaled image buffers.
if (i > 0) {
downscaled_buffers_[i - 1] = I420Buffer::Create(
configurations_[i].width, configurations_[i].height,
configurations_[i].width, configurations_[i].width / 2,
configurations_[i].width / 2);
}
// Codec_settings uses kbits/second; encoder uses bits/second.
configurations_[i].max_bps = codec_.maxBitrate * 1000;
configurations_[i].target_bps = codec_.startBitrate * 1000;
// Create encoder parameters based on the layer configuration.
SEncParamExt encoder_params = CreateEncoderParams(i);
// Initialize.
if (openh264_encoder->InitializeExt(&encoder_params) != 0) {
RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
// TODO(pbos): Base init params on these values before submitting.
int video_format = EVideoFormatType::videoFormatI420;
openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format);
// Initialize encoded image. Default buffer size: size of unencoded data.
const size_t new_capacity =
CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width,
codec_.simulcastStream[idx].height);
encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity));
encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
encoded_images_[i].set_size(0);
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
scalability_modes_[i] = ScalabilityModeFromTemporalLayers(
configurations_[i].num_temporal_layers);
if (scalability_modes_[i].has_value()) {
svc_controllers_[i] = CreateScalabilityStructure(*scalability_modes_[i]);
if (svc_controllers_[i] == nullptr) {
RTC_LOG(LS_ERROR) << "Failed to create scalability structure";
Release();
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
}
SimulcastRateAllocator init_allocator(codec_);
VideoBitrateAllocation allocation =
init_allocator.Allocate(VideoBitrateAllocationParameters(
DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate));
SetRates(RateControlParameters(allocation, codec_.maxFramerate));
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264EncoderImpl::Release() {
while (!encoders_.empty()) {
ISVCEncoder* openh264_encoder = encoders_.back();
if (openh264_encoder) {
RTC_CHECK_EQ(0, openh264_encoder->Uninitialize());
WelsDestroySVCEncoder(openh264_encoder);
}
encoders_.pop_back();
}
downscaled_buffers_.clear();
configurations_.clear();
encoded_images_.clear();
pictures_.clear();
tl0sync_limit_.clear();
svc_controllers_.clear();
scalability_modes_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
encoded_image_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
void H264EncoderImpl::SetRates(const RateControlParameters& parameters) {
if (encoders_.empty()) {
RTC_LOG(LS_WARNING) << "SetRates() while uninitialized.";
return;
}
if (parameters.framerate_fps < 1.0) {
RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps;
return;
}
if (parameters.bitrate.get_sum_bps() == 0) {
// Encoder paused, turn off all encoding.
for (size_t i = 0; i < configurations_.size(); ++i) {
configurations_[i].SetStreamState(false);
}
return;
}
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps);
size_t stream_idx = encoders_.size() - 1;
for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
// Update layer config.
configurations_[i].target_bps =
parameters.bitrate.GetSpatialLayerSum(stream_idx);
configurations_[i].max_frame_rate = parameters.framerate_fps;
if (configurations_[i].target_bps) {
configurations_[i].SetStreamState(true);
// Update h264 encoder.
SBitrateInfo target_bitrate;
memset(&target_bitrate, 0, sizeof(SBitrateInfo));
target_bitrate.iLayer = SPATIAL_LAYER_ALL,
target_bitrate.iBitrate = configurations_[i].target_bps;
encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate);
encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE,
&configurations_[i].max_frame_rate);
} else {
configurations_[i].SetStreamState(false);
}
}
}
int32_t H264EncoderImpl::Encode(
const VideoFrame& input_frame,
const std::vector<VideoFrameType>* frame_types) {
if (encoders_.empty()) {
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!encoded_image_callback_) {
RTC_LOG(LS_WARNING)
<< "InitEncode() has been called, but a callback function "
"has not been set with RegisterEncodeCompleteCallback()";
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
rtc::scoped_refptr<I420BufferInterface> frame_buffer =
input_frame.video_frame_buffer()->ToI420();
if (!frame_buffer) {
RTC_LOG(LS_ERROR) << "Failed to convert "
<< VideoFrameBufferTypeToString(
input_frame.video_frame_buffer()->type())
<< " image to I420. Can't encode frame.";
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 ||
frame_buffer->type() == VideoFrameBuffer::Type::kI420A);
bool is_keyframe_needed = false;
for (size_t i = 0; i < configurations_.size(); ++i) {
if (configurations_[i].key_frame_request && configurations_[i].sending) {
// This is legacy behavior, generating a keyframe on all layers
// when generating one for a layer that became active for the first time
// or after being disabled.
is_keyframe_needed = true;
break;
}
}
RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
// Encode image for each layer.
for (size_t i = 0; i < encoders_.size(); ++i) {
// EncodeFrame input.
pictures_[i] = {0};
pictures_[i].iPicWidth = configurations_[i].width;
pictures_[i].iPicHeight = configurations_[i].height;
pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
// Downscale images on second and ongoing layers.
if (i == 0) {
pictures_[i].iStride[0] = frame_buffer->StrideY();
pictures_[i].iStride[1] = frame_buffer->StrideU();
pictures_[i].iStride[2] = frame_buffer->StrideV();
pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
} else {
pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
pictures_[i].pData[0] =
const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY());
pictures_[i].pData[1] =
const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU());
pictures_[i].pData[2] =
const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV());
// Scale the image down a number of times by downsampling factor.
libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
configurations_[i - 1].width,
configurations_[i - 1].height, pictures_[i].pData[0],
pictures_[i].iStride[0], pictures_[i].pData[1],
pictures_[i].iStride[1], pictures_[i].pData[2],
pictures_[i].iStride[2], configurations_[i].width,
configurations_[i].height, libyuv::kFilterBox);
}
if (!configurations_[i].sending) {
continue;
}
if (frame_types != nullptr && i < frame_types->size()) {
// Skip frame?
if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
continue;
}
}
// Send a key frame either when this layer is configured to require one
// or we have explicitly been asked to.
const size_t simulcast_idx =
static_cast<size_t>(configurations_[i].simulcast_idx);
bool send_key_frame =
is_keyframe_needed ||
(frame_types && simulcast_idx < frame_types->size() &&
(*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey);
if (send_key_frame) {
// API doc says ForceIntraFrame(false) does nothing, but calling this
// function forces a key frame regardless of the `bIDR` argument's value.
// (If every frame is a key frame we get lag/delays.)
encoders_[i]->ForceIntraFrame(true);
configurations_[i].key_frame_request = false;
}
// EncodeFrame output.
SFrameBSInfo info;
memset(&info, 0, sizeof(SFrameBSInfo));
std::vector<ScalableVideoController::LayerFrameConfig> layer_frames;
if (svc_controllers_[i]) {
layer_frames = svc_controllers_[i]->NextFrameConfig(send_key_frame);
RTC_CHECK_EQ(layer_frames.size(), 1);
}
// Encode!
int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
if (enc_ret != 0) {
RTC_LOG(LS_ERROR)
<< "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
<< ".";
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
encoded_images_[i]._encodedWidth = configurations_[i].width;
encoded_images_[i]._encodedHeight = configurations_[i].height;
encoded_images_[i].SetRtpTimestamp(input_frame.timestamp());
encoded_images_[i].SetColorSpace(input_frame.color_space());
encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
encoded_images_[i].SetSimulcastIndex(configurations_[i].simulcast_idx);
// Split encoded image up into fragments. This also updates
// `encoded_image_`.
RtpFragmentize(&encoded_images_[i], &info);
// Encoder can skip frames to save bandwidth in which case
// `encoded_images_[i]._length` == 0.
if (encoded_images_[i].size() > 0) {
// Parse QP.
h264_bitstream_parser_.ParseBitstream(encoded_images_[i]);
encoded_images_[i].qp_ =
h264_bitstream_parser_.GetLastSliceQp().value_or(-1);
// Deliver encoded image.
CodecSpecificInfo codec_specific;
codec_specific.codecType = kVideoCodecH264;
codec_specific.codecSpecific.H264.packetization_mode =
packetization_mode_;
codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
codec_specific.codecSpecific.H264.idr_frame =
info.eFrameType == videoFrameTypeIDR;
codec_specific.codecSpecific.H264.base_layer_sync = false;
if (configurations_[i].num_temporal_layers > 1) {
const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
codec_specific.codecSpecific.H264.temporal_idx = tid;
codec_specific.codecSpecific.H264.base_layer_sync =
tid > 0 && tid < tl0sync_limit_[i];
if (svc_controllers_[i]) {
if (encoded_images_[i]._frameType == VideoFrameType::kVideoFrameKey) {
// Reset the ScalableVideoController on key frame
// to reset the expected dependency structure.
layer_frames =
svc_controllers_[i]->NextFrameConfig(/* restart= */ true);
RTC_CHECK_EQ(layer_frames.size(), 1);
RTC_DCHECK_EQ(layer_frames[0].TemporalId(), 0);
RTC_DCHECK_EQ(layer_frames[0].IsKeyframe(), true);
}
if (layer_frames[0].TemporalId() != tid) {
RTC_LOG(LS_WARNING)
<< "Encoder produced a frame with temporal id " << tid
<< ", expected " << layer_frames[0].TemporalId() << ".";
continue;
}
encoded_images_[i].SetTemporalIndex(tid);
}
if (codec_specific.codecSpecific.H264.base_layer_sync) {
tl0sync_limit_[i] = tid;
}
if (tid == 0) {
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
}
}
if (svc_controllers_[i]) {
codec_specific.generic_frame_info =
svc_controllers_[i]->OnEncodeDone(layer_frames[0]);
if (send_key_frame && codec_specific.generic_frame_info.has_value()) {
codec_specific.template_structure =
svc_controllers_[i]->DependencyStructure();
}
codec_specific.scalability_mode = scalability_modes_[i];
}
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
&codec_specific);
}
}
return WEBRTC_VIDEO_CODEC_OK;
}
// Initialization parameters.
// There are two ways to initialize. There is SEncParamBase (cleared with
// memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
// which is a superset of SEncParamBase (cleared with GetDefaultParams) used
// in InitializeExt.
SEncParamExt H264EncoderImpl::CreateEncoderParams(size_t i) const {
SEncParamExt encoder_params;
encoders_[i]->GetDefaultParams(&encoder_params);
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
} else if (codec_.mode == VideoCodecMode::kScreensharing) {
encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
} else {
RTC_DCHECK_NOTREACHED();
}
encoder_params.iPicWidth = configurations_[i].width;
encoder_params.iPicHeight = configurations_[i].height;
encoder_params.iTargetBitrate = configurations_[i].target_bps;
// Keep unspecified. WebRTC's max codec bitrate is not the same setting
// as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543
encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE;
// Rate Control mode
encoder_params.iRCMode = RC_BITRATE_MODE;
encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate;
// The following parameters are extension parameters (they're in SEncParamExt,
// not in SEncParamBase).
encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on;
// `uiIntraPeriod` - multiple of GOP size
// `keyFrameInterval` - number of frames
encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval;
// Reuse SPS id if possible. This helps to avoid reset of chromium HW decoder
// on each key-frame.
// Note that WebRTC resets encoder on resolution change which makes all
// EParameterSetStrategy modes except INCREASING_ID (default) essentially
// equivalent to CONSTANT_ID.
encoder_params.eSpsPpsIdStrategy = SPS_LISTING;
encoder_params.uiMaxNalSize = 0;
// Threading model: use auto.
// 0: auto (dynamic imp. internal encoder)
// 1: single thread (default value)
// >1: number of threads
encoder_params.iMultipleThreadIdc =
NumberOfThreads(encoder_thread_limit_, encoder_params.iPicWidth,
encoder_params.iPicHeight, number_of_cores_);
// The base spatial layer 0 is the only one we use.
encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth;
encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight;
encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate;
encoder_params.sSpatialLayers[0].iSpatialBitrate =
encoder_params.iTargetBitrate;
encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
encoder_params.iMaxBitrate;
encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers;
if (encoder_params.iTemporalLayerNum > 1) {
// iNumRefFrame specifies total number of reference buffers to allocate.
// For N temporal layers we need at least (N - 1) buffers to store last
// encoded frames of all reference temporal layers.
// Note that there is no API in OpenH264 encoder to specify exact set of
// references to be used to prediction of a given frame. Encoder can
// theoretically use all available reference buffers.
encoder_params.iNumRefFrame = encoder_params.iTemporalLayerNum - 1;
}
RTC_LOG(LS_INFO) << "OpenH264 version is " << OPENH264_MAJOR << "."
<< OPENH264_MINOR;
switch (packetization_mode_) {
case H264PacketizationMode::SingleNalUnit:
// Limit the size of the packets produced.
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
SM_SIZELIMITED_SLICE;
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint =
static_cast<unsigned int>(max_payload_size_);
RTC_LOG(LS_INFO) << "Encoder is configured with NALU constraint: "
<< max_payload_size_ << " bytes";
break;
case H264PacketizationMode::NonInterleaved:
// When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto
// design it with cpu core number.
// TODO(sprang): Set to 0 when we understand why the rate controller borks
// when uiSliceNum > 1.
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode =
SM_FIXEDSLCNUM_SLICE;
break;
}
return encoder_params;
}
void H264EncoderImpl::ReportInit() {
if (has_reported_init_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
kH264EncoderEventInit, kH264EncoderEventMax);
has_reported_init_ = true;
}
void H264EncoderImpl::ReportError() {
if (has_reported_error_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
kH264EncoderEventError, kH264EncoderEventMax);
has_reported_error_ = true;
}
VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const {
EncoderInfo info;
info.supports_native_handle = false;
info.implementation_name = "OpenH264";
info.scaling_settings =
VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
info.is_hardware_accelerated = false;
info.supports_simulcast = true;
info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420};
return info;
}
void H264EncoderImpl::LayerConfig::SetStreamState(bool send_stream) {
if (send_stream && !sending) {
// Need a key frame if we have not sent this stream before.
key_frame_request = true;
}
sending = send_stream;
}
} // namespace webrtc
#endif // WEBRTC_USE_H264
Reference in a new issue View git blame Copy permalink