mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-13 13:50:40 +01:00
Enable H.264 temporal scalability in simulcast.
Bug: webrtc:10651 Change-Id: I58372186930ce33e925f85edb0f308657dbfe273 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/142840 Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Stefan Holmer <stefan@webrtc.org> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Cr-Commit-Position: refs/heads/master@{#28381}
This commit is contained in:
Johnny Lee
Commit Bot
parent
0d65fb5451
commit
c53850675b
5 changed files with 34 additions and 17 deletions
|
|
@ -585,6 +585,9 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
|
|||
// Turn off denoising for all streams but the highest resolution.
|
||||
stream_codec->VP8()->denoisingOn = false;
|
||||
}
|
||||
} else if (inst.codecType == webrtc::kVideoCodecH264) {
|
||||
stream_codec->H264()->numberOfTemporalLayers =
|
||||
inst.simulcastStream[stream_index].numberOfTemporalLayers;
|
||||
}
|
||||
// TODO(ronghuawu): what to do with targetBitrate.
|
||||
|
||||
|
|
|
|||
|
|
@ -155,9 +155,7 @@ H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
|
|||
number_of_cores_(0),
|
||||
encoded_image_callback_(nullptr),
|
||||
has_reported_init_(false),
|
||||
has_reported_error_(false),
|
||||
num_temporal_layers_(1),
|
||||
tl0sync_limit_(0) {
|
||||
has_reported_error_(false) {
|
||||
RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
|
||||
std::string packetization_mode_string;
|
||||
if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
|
||||
|
|
@ -169,6 +167,7 @@ H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
|
|||
encoded_images_.reserve(kMaxSimulcastStreams);
|
||||
encoders_.reserve(kMaxSimulcastStreams);
|
||||
configurations_.reserve(kMaxSimulcastStreams);
|
||||
tl0sync_limit_.reserve(kMaxSimulcastStreams);
|
||||
}
|
||||
|
||||
H264EncoderImpl::~H264EncoderImpl() {
|
||||
|
|
@ -209,6 +208,7 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
|
|||
encoders_.resize(number_of_streams);
|
||||
pictures_.resize(number_of_streams);
|
||||
configurations_.resize(number_of_streams);
|
||||
tl0sync_limit_.resize(number_of_streams);
|
||||
|
||||
number_of_cores_ = settings.number_of_cores;
|
||||
max_payload_size_ = settings.max_payload_size;
|
||||
|
|
@ -221,8 +221,6 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
|
|||
codec_.simulcastStream[0].height = codec_.height;
|
||||
}
|
||||
|
||||
num_temporal_layers_ = codec_.H264()->numberOfTemporalLayers;
|
||||
|
||||
for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
|
||||
++i, --idx) {
|
||||
ISVCEncoder* openh264_encoder;
|
||||
|
|
@ -253,6 +251,8 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
|
|||
configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
|
||||
configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
|
||||
configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
|
||||
configurations_[i].num_temporal_layers =
|
||||
codec_.simulcastStream[idx].numberOfTemporalLayers;
|
||||
|
||||
// Create downscaled image buffers.
|
||||
if (i > 0) {
|
||||
|
|
@ -290,6 +290,8 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
|
|||
encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
|
||||
encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
|
||||
encoded_images_[i].set_size(0);
|
||||
|
||||
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
|
||||
}
|
||||
|
||||
SimulcastRateAllocator init_allocator(codec_);
|
||||
|
|
@ -312,6 +314,7 @@ int32_t H264EncoderImpl::Release() {
|
|||
configurations_.clear();
|
||||
encoded_images_.clear();
|
||||
pictures_.clear();
|
||||
tl0sync_limit_.clear();
|
||||
return WEBRTC_VIDEO_CODEC_OK;
|
||||
}
|
||||
|
||||
|
|
@ -510,16 +513,16 @@ int32_t H264EncoderImpl::Encode(
|
|||
codec_specific.codecSpecific.H264.idr_frame =
|
||||
info.eFrameType == videoFrameTypeIDR;
|
||||
codec_specific.codecSpecific.H264.base_layer_sync = false;
|
||||
if (num_temporal_layers_ > 1) {
|
||||
if (configurations_[i].num_temporal_layers > 1) {
|
||||
const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
|
||||
codec_specific.codecSpecific.H264.temporal_idx = tid;
|
||||
codec_specific.codecSpecific.H264.base_layer_sync =
|
||||
tid > 0 && tid < tl0sync_limit_;
|
||||
tid > 0 && tid < tl0sync_limit_[i];
|
||||
if (codec_specific.codecSpecific.H264.base_layer_sync) {
|
||||
tl0sync_limit_ = tid;
|
||||
tl0sync_limit_[i] = tid;
|
||||
}
|
||||
if (tid == 0) {
|
||||
tl0sync_limit_ = num_temporal_layers_;
|
||||
tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
|
||||
}
|
||||
}
|
||||
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
|
||||
|
|
@ -573,7 +576,7 @@ SEncParamExt H264EncoderImpl::CreateEncoderParams(size_t i) const {
|
|||
encoder_params.iTargetBitrate;
|
||||
encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
|
||||
encoder_params.iMaxBitrate;
|
||||
encoder_params.iTemporalLayerNum = num_temporal_layers_;
|
||||
encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers;
|
||||
if (encoder_params.iTemporalLayerNum > 1) {
|
||||
encoder_params.iNumRefFrame = 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ class H264EncoderImpl : public H264Encoder {
|
|||
uint32_t max_bps = 0;
|
||||
bool frame_dropping_on = false;
|
||||
int key_frame_interval = 0;
|
||||
int num_temporal_layers = 1;
|
||||
|
||||
void SetStreamState(bool send_stream);
|
||||
};
|
||||
|
|
@ -107,8 +108,7 @@ class H264EncoderImpl : public H264Encoder {
|
|||
bool has_reported_init_;
|
||||
bool has_reported_error_;
|
||||
|
||||
int num_temporal_layers_;
|
||||
uint8_t tl0sync_limit_;
|
||||
std::vector<uint8_t> tl0sync_limit_;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
|
|||
|
|
@ -95,5 +95,10 @@ TEST(TestH264Simulcast, TestStrideEncodeDecode) {
|
|||
fixture->TestStrideEncodeDecode();
|
||||
}
|
||||
|
||||
TEST(TestH264Simulcast, TestSpatioTemporalLayers333PatternEncoder) {
|
||||
auto fixture = CreateSpecificSimulcastTestFixture();
|
||||
fixture->TestSpatioTemporalLayers333PatternEncoder();
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace webrtc
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ class SimulcastTestFixtureImpl::TestEncodedImageCallback
|
|||
const CodecSpecificInfo* codec_specific_info,
|
||||
const RTPFragmentationHeader* fragmentation) override {
|
||||
bool is_vp8 = (codec_specific_info->codecType == kVideoCodecVP8);
|
||||
bool is_h264 = (codec_specific_info->codecType == kVideoCodecH264);
|
||||
// Only store the base layer.
|
||||
if (encoded_image.SpatialIndex().value_or(0) == 0) {
|
||||
if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) {
|
||||
|
|
@ -102,6 +103,11 @@ class SimulcastTestFixtureImpl::TestEncodedImageCallback
|
|||
codec_specific_info->codecSpecific.VP8.layerSync;
|
||||
temporal_layer_[encoded_image.SpatialIndex().value_or(0)] =
|
||||
codec_specific_info->codecSpecific.VP8.temporalIdx;
|
||||
} else if (is_h264) {
|
||||
layer_sync_[encoded_image.SpatialIndex().value_or(0)] =
|
||||
codec_specific_info->codecSpecific.H264.base_layer_sync;
|
||||
temporal_layer_[encoded_image.SpatialIndex().value_or(0)] =
|
||||
codec_specific_info->codecSpecific.H264.temporal_idx;
|
||||
}
|
||||
return Result(Result::OK, encoded_image.Timestamp());
|
||||
}
|
||||
|
|
@ -263,8 +269,8 @@ SimulcastTestFixtureImpl::SimulcastTestFixtureImpl(
|
|||
: codec_type_(PayloadStringToCodecType(video_format.name)) {
|
||||
encoder_ = encoder_factory->CreateVideoEncoder(video_format);
|
||||
decoder_ = decoder_factory->CreateVideoDecoder(video_format);
|
||||
SetUpCodec(codec_type_ == kVideoCodecVP8 ? kDefaultTemporalLayerProfile
|
||||
: kNoTemporalLayerProfile);
|
||||
SetUpCodec((codec_type_ == kVideoCodecVP8 || codec_type_ == kVideoCodecH264)
|
||||
? kDefaultTemporalLayerProfile : kNoTemporalLayerProfile);
|
||||
}
|
||||
|
||||
SimulcastTestFixtureImpl::~SimulcastTestFixtureImpl() {
|
||||
|
|
@ -677,7 +683,7 @@ void SimulcastTestFixtureImpl::TestSwitchingToOneSmallStream() {
|
|||
// 3-3-3 pattern: 3 temporal layers for all spatial streams, so same
|
||||
// temporal_layer id and layer_sync is expected for all streams.
|
||||
void SimulcastTestFixtureImpl::TestSpatioTemporalLayers333PatternEncoder() {
|
||||
EXPECT_EQ(codec_type_, kVideoCodecVP8);
|
||||
bool is_h264 = codec_type_ == kVideoCodecH264;
|
||||
TestEncodedImageCallback encoder_callback;
|
||||
encoder_->RegisterEncodeCompleteCallback(&encoder_callback);
|
||||
SetRates(kMaxBitrates[2], 30); // To get all three streams.
|
||||
|
|
@ -688,7 +694,7 @@ void SimulcastTestFixtureImpl::TestSpatioTemporalLayers333PatternEncoder() {
|
|||
// First frame: #0.
|
||||
EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL));
|
||||
SetExpectedValues3<int>(0, 0, 0, expected_temporal_idx);
|
||||
SetExpectedValues3<bool>(true, true, true, expected_layer_sync);
|
||||
SetExpectedValues3<bool>(!is_h264, !is_h264, !is_h264, expected_layer_sync);
|
||||
VerifyTemporalIdxAndSyncForAllSpatialLayers(
|
||||
&encoder_callback, expected_temporal_idx, expected_layer_sync, 3);
|
||||
|
||||
|
|
@ -728,7 +734,7 @@ void SimulcastTestFixtureImpl::TestSpatioTemporalLayers333PatternEncoder() {
|
|||
input_frame_->set_timestamp(input_frame_->timestamp() + 3000);
|
||||
EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL));
|
||||
SetExpectedValues3<int>(2, 2, 2, expected_temporal_idx);
|
||||
SetExpectedValues3<bool>(false, false, false, expected_layer_sync);
|
||||
SetExpectedValues3<bool>(is_h264, is_h264, is_h264, expected_layer_sync);
|
||||
VerifyTemporalIdxAndSyncForAllSpatialLayers(
|
||||
&encoder_callback, expected_temporal_idx, expected_layer_sync, 3);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue