/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. * */ #include "modules/video_coding/codecs/h264/h264_encoder_impl.h" #include #include #include "third_party/openh264/src/codec/api/svc/codec_api.h" #include "third_party/openh264/src/codec/api/svc/codec_app_def.h" #include "third_party/openh264/src/codec/api/svc/codec_def.h" #include "third_party/openh264/src/codec/api/svc/codec_ver.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/timeutils.h" #include "system_wrappers/include/metrics.h" namespace webrtc { namespace { const bool kOpenH264EncoderDetailedLogging = false; // Used by histograms. Values of entries should not be changed. enum H264EncoderImplEvent { kH264EncoderEventInit = 0, kH264EncoderEventError = 1, kH264EncoderEventMax = 16, }; int NumberOfThreads(int width, int height, int number_of_cores) { // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac, // see crbug.com/583348. Until further investigated, only use one thread. // if (width * height >= 1920 * 1080 && number_of_cores > 8) { // return 8; // 8 threads for 1080p on high perf machines. // } else if (width * height > 1280 * 960 && number_of_cores >= 6) { // return 3; // 3 threads for 1080p. // } else if (width * height > 640 * 480 && number_of_cores >= 3) { // return 2; // 2 threads for qHD/HD. // } else { // return 1; // 1 thread for VGA or less. // } // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(), // before enabling multithreading here. return 1; } FrameType ConvertToVideoFrameType(EVideoFrameType type) { switch (type) { case videoFrameTypeIDR: return kVideoFrameKey; case videoFrameTypeSkip: case videoFrameTypeI: case videoFrameTypeP: case videoFrameTypeIPMixed: return kVideoFrameDelta; case videoFrameTypeInvalid: break; } RTC_NOTREACHED() << "Unexpected/invalid frame type: " << type; return kEmptyFrame; } } // namespace // Helper method used by H264EncoderImpl::Encode. // Copies the encoded bytes from |info| to |encoded_image| and updates the // fragmentation information of |frag_header|. The |encoded_image->_buffer| may // be deleted and reallocated if a bigger buffer is required. // // After OpenH264 encoding, the encoded bytes are stored in |info| spread out // over a number of layers and "NAL units". Each NAL unit is a fragment starting // with the four-byte start code {0,0,0,1}. All of this data (including the // start codes) is copied to the |encoded_image->_buffer| and the |frag_header| // is updated to point to each fragment, with offsets and lengths set as to // exclude the start codes. static void RtpFragmentize(EncodedImage* encoded_image, std::unique_ptr* encoded_image_buffer, const VideoFrameBuffer& frame_buffer, SFrameBSInfo* info, RTPFragmentationHeader* frag_header) { // Calculate minimum buffer size required to hold encoded data. size_t required_size = 0; size_t fragments_count = 0; for (int layer = 0; layer < info->iLayerNum; ++layer) { const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0); // Ensure |required_size| will not overflow. RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal], std::numeric_limits::max() - required_size); required_size += layerInfo.pNalLengthInByte[nal]; } } if (encoded_image->_size < required_size) { // Increase buffer size. Allocate enough to hold an unencoded image, this // should be more than enough to hold any encoded data of future frames of // the same size (avoiding possible future reallocation due to variations in // required size). encoded_image->_size = CalcBufferSize( VideoType::kI420, frame_buffer.width(), frame_buffer.height()); if (encoded_image->_size < required_size) { // Encoded data > unencoded data. Allocate required bytes. RTC_LOG(LS_WARNING) << "Encoding produced more bytes than the original image " << "data! Original bytes: " << encoded_image->_size << ", encoded bytes: " << required_size << "."; encoded_image->_size = required_size; } encoded_image->_buffer = new uint8_t[encoded_image->_size]; encoded_image_buffer->reset(encoded_image->_buffer); } // Iterate layers and NAL units, note each NAL unit as a fragment and copy // the data to |encoded_image->_buffer|. const uint8_t start_code[4] = {0, 0, 0, 1}; frag_header->VerifyAndAllocateFragmentationHeader(fragments_count); size_t frag = 0; encoded_image->_length = 0; for (int layer = 0; layer < info->iLayerNum; ++layer) { const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; // Iterate NAL units making up this layer, noting fragments. size_t layer_len = 0; for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { // Because the sum of all layer lengths, |required_size|, fits in a // |size_t|, we know that any indices in-between will not overflow. RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+0], start_code[0]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+1], start_code[1]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+2], start_code[2]); RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+3], start_code[3]); frag_header->fragmentationOffset[frag] = encoded_image->_length + layer_len + sizeof(start_code); frag_header->fragmentationLength[frag] = layerInfo.pNalLengthInByte[nal] - sizeof(start_code); layer_len += layerInfo.pNalLengthInByte[nal]; } // Copy the entire layer's data (including start codes). memcpy(encoded_image->_buffer + encoded_image->_length, layerInfo.pBsBuf, layer_len); encoded_image->_length += layer_len; } } H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec) : openh264_encoder_(nullptr), width_(0), height_(0), max_frame_rate_(0.0f), target_bps_(0), max_bps_(0), mode_(kRealtimeVideo), frame_dropping_on_(false), key_frame_interval_(0), packetization_mode_(H264PacketizationMode::SingleNalUnit), max_payload_size_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), has_reported_error_(false) { RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName)); std::string packetization_mode_string; if (codec.GetParam(cricket::kH264FmtpPacketizationMode, &packetization_mode_string) && packetization_mode_string == "1") { packetization_mode_ = H264PacketizationMode::NonInterleaved; } } H264EncoderImpl::~H264EncoderImpl() { Release(); } int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { ReportInit(); if (!codec_settings || codec_settings->codecType != kVideoCodecH264) { ReportError(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (codec_settings->maxFramerate == 0) { ReportError(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (codec_settings->width < 1 || codec_settings->height < 1) { ReportError(); return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } int32_t release_ret = Release(); if (release_ret != WEBRTC_VIDEO_CODEC_OK) { ReportError(); return release_ret; } RTC_DCHECK(!openh264_encoder_); // Create encoder. if (WelsCreateSVCEncoder(&openh264_encoder_) != 0) { // Failed to create encoder. RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder"; RTC_DCHECK(!openh264_encoder_); ReportError(); return WEBRTC_VIDEO_CODEC_ERROR; } RTC_DCHECK(openh264_encoder_); if (kOpenH264EncoderDetailedLogging) { int trace_level = WELS_LOG_DETAIL; openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level); } // else WELS_LOG_DEFAULT is used by default. number_of_cores_ = number_of_cores; // Set internal settings from codec_settings width_ = codec_settings->width; height_ = codec_settings->height; max_frame_rate_ = static_cast(codec_settings->maxFramerate); mode_ = codec_settings->mode; frame_dropping_on_ = codec_settings->H264().frameDroppingOn; key_frame_interval_ = codec_settings->H264().keyFrameInterval; max_payload_size_ = max_payload_size; // Codec_settings uses kbits/second; encoder uses bits/second. max_bps_ = codec_settings->maxBitrate * 1000; if (codec_settings->targetBitrate == 0) target_bps_ = codec_settings->startBitrate * 1000; else target_bps_ = codec_settings->targetBitrate * 1000; SEncParamExt encoder_params = CreateEncoderParams(); // Initialize. if (openh264_encoder_->InitializeExt(&encoder_params) != 0) { RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder"; Release(); ReportError(); return WEBRTC_VIDEO_CODEC_ERROR; } // TODO(pbos): Base init params on these values before submitting. int video_format = EVideoFormatType::videoFormatI420; openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format); // Initialize encoded image. Default buffer size: size of unencoded data. encoded_image_._size = CalcBufferSize(VideoType::kI420, codec_settings->width, codec_settings->height); encoded_image_._buffer = new uint8_t[encoded_image_._size]; encoded_image_buffer_.reset(encoded_image_._buffer); encoded_image_._completeFrame = true; encoded_image_._encodedWidth = 0; encoded_image_._encodedHeight = 0; encoded_image_._length = 0; return WEBRTC_VIDEO_CODEC_OK; } int32_t H264EncoderImpl::Release() { if (openh264_encoder_) { RTC_CHECK_EQ(0, openh264_encoder_->Uninitialize()); WelsDestroySVCEncoder(openh264_encoder_); openh264_encoder_ = nullptr; } encoded_image_._buffer = nullptr; encoded_image_buffer_.reset(); return WEBRTC_VIDEO_CODEC_OK; } int32_t H264EncoderImpl::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { encoded_image_callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } int32_t H264EncoderImpl::SetRateAllocation( const BitrateAllocation& bitrate_allocation, uint32_t framerate) { if (bitrate_allocation.get_sum_bps() <= 0 || framerate <= 0) return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; target_bps_ = bitrate_allocation.get_sum_bps(); max_frame_rate_ = static_cast(framerate); SBitrateInfo target_bitrate; memset(&target_bitrate, 0, sizeof(SBitrateInfo)); target_bitrate.iLayer = SPATIAL_LAYER_ALL, target_bitrate.iBitrate = target_bps_; openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate); openh264_encoder_->SetOption(ENCODER_OPTION_FRAME_RATE, &max_frame_rate_); return WEBRTC_VIDEO_CODEC_OK; } int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { if (!IsInitialized()) { ReportError(); return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (!encoded_image_callback_) { RTC_LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " << "has not been set with RegisterEncodeCompleteCallback()"; ReportError(); return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } bool force_key_frame = false; if (frame_types != nullptr) { // We only support a single stream. RTC_DCHECK_EQ(frame_types->size(), 1); // Skip frame? if ((*frame_types)[0] == kEmptyFrame) { return WEBRTC_VIDEO_CODEC_OK; } // Force key frame? force_key_frame = (*frame_types)[0] == kVideoFrameKey; } if (force_key_frame) { // API doc says ForceIntraFrame(false) does nothing, but calling this // function forces a key frame regardless of the |bIDR| argument's value. // (If every frame is a key frame we get lag/delays.) openh264_encoder_->ForceIntraFrame(true); } rtc::scoped_refptr frame_buffer = input_frame.video_frame_buffer()->ToI420(); // EncodeFrame input. SSourcePicture picture; memset(&picture, 0, sizeof(SSourcePicture)); picture.iPicWidth = frame_buffer->width(); picture.iPicHeight = frame_buffer->height(); picture.iColorFormat = EVideoFormatType::videoFormatI420; picture.uiTimeStamp = input_frame.ntp_time_ms(); picture.iStride[0] = frame_buffer->StrideY(); picture.iStride[1] = frame_buffer->StrideU(); picture.iStride[2] = frame_buffer->StrideV(); picture.pData[0] = const_cast(frame_buffer->DataY()); picture.pData[1] = const_cast(frame_buffer->DataU()); picture.pData[2] = const_cast(frame_buffer->DataV()); // EncodeFrame output. SFrameBSInfo info; memset(&info, 0, sizeof(SFrameBSInfo)); // Encode! int enc_ret = openh264_encoder_->EncodeFrame(&picture, &info); if (enc_ret != 0) { RTC_LOG(LS_ERROR) << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret << "."; ReportError(); return WEBRTC_VIDEO_CODEC_ERROR; } encoded_image_._encodedWidth = frame_buffer->width(); encoded_image_._encodedHeight = frame_buffer->height(); encoded_image_._timeStamp = input_frame.timestamp(); encoded_image_.ntp_time_ms_ = input_frame.ntp_time_ms(); encoded_image_.capture_time_ms_ = input_frame.render_time_ms(); encoded_image_.rotation_ = input_frame.rotation(); encoded_image_.content_type_ = (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; encoded_image_.timing_.flags = TimingFrameFlags::kInvalid; encoded_image_._frameType = ConvertToVideoFrameType(info.eFrameType); // Split encoded image up into fragments. This also updates |encoded_image_|. RTPFragmentationHeader frag_header; RtpFragmentize(&encoded_image_, &encoded_image_buffer_, *frame_buffer, &info, &frag_header); // Encoder can skip frames to save bandwidth in which case // |encoded_image_._length| == 0. if (encoded_image_._length > 0) { // Parse QP. h264_bitstream_parser_.ParseBitstream(encoded_image_._buffer, encoded_image_._length); h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_); // Deliver encoded image. CodecSpecificInfo codec_specific; codec_specific.codecType = kVideoCodecH264; codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_; encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); } return WEBRTC_VIDEO_CODEC_OK; } const char* H264EncoderImpl::ImplementationName() const { return "OpenH264"; } bool H264EncoderImpl::IsInitialized() const { return openh264_encoder_ != nullptr; } // Initialization parameters. // There are two ways to initialize. There is SEncParamBase (cleared with // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt // which is a superset of SEncParamBase (cleared with GetDefaultParams) used // in InitializeExt. SEncParamExt H264EncoderImpl::CreateEncoderParams() const { RTC_DCHECK(openh264_encoder_); SEncParamExt encoder_params; openh264_encoder_->GetDefaultParams(&encoder_params); if (mode_ == kRealtimeVideo) { encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME; } else if (mode_ == kScreensharing) { encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME; } else { RTC_NOTREACHED(); } encoder_params.iPicWidth = width_; encoder_params.iPicHeight = height_; encoder_params.iTargetBitrate = target_bps_; encoder_params.iMaxBitrate = max_bps_; // Rate Control mode encoder_params.iRCMode = RC_BITRATE_MODE; encoder_params.fMaxFrameRate = max_frame_rate_; // The following parameters are extension parameters (they're in SEncParamExt, // not in SEncParamBase). encoder_params.bEnableFrameSkip = frame_dropping_on_; // |uiIntraPeriod| - multiple of GOP size // |keyFrameInterval| - number of frames encoder_params.uiIntraPeriod = key_frame_interval_; encoder_params.uiMaxNalSize = 0; // Threading model: use auto. // 0: auto (dynamic imp. internal encoder) // 1: single thread (default value) // >1: number of threads encoder_params.iMultipleThreadIdc = NumberOfThreads( encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_); // The base spatial layer 0 is the only one we use. encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth; encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight; encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate; encoder_params.sSpatialLayers[0].iSpatialBitrate = encoder_params.iTargetBitrate; encoder_params.sSpatialLayers[0].iMaxSpatialBitrate = encoder_params.iMaxBitrate; RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "." << OPENH264_MINOR; switch (packetization_mode_) { case H264PacketizationMode::SingleNalUnit: // Limit the size of the packets produced. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = SM_SIZELIMITED_SLICE; encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = static_cast(max_payload_size_); break; case H264PacketizationMode::NonInterleaved: // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto // design it with cpu core number. // TODO(sprang): Set to 0 when we understand why the rate controller borks // when uiSliceNum > 1. encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = SM_FIXEDSLCNUM_SLICE; break; } return encoder_params; } void H264EncoderImpl::ReportInit() { if (has_reported_init_) return; RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event", kH264EncoderEventInit, kH264EncoderEventMax); has_reported_init_ = true; } void H264EncoderImpl::ReportError() { if (has_reported_error_) return; RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event", kH264EncoderEventError, kH264EncoderEventMax); has_reported_error_ = true; } int32_t H264EncoderImpl::SetChannelParameters( uint32_t packet_loss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) { return WEBRTC_VIDEO_CODEC_OK; } VideoEncoder::ScalingSettings H264EncoderImpl::GetScalingSettings() const { return VideoEncoder::ScalingSettings(true); } } // namespace webrtc