diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 994fcf4c75..9e6a7657c5 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -16,6 +16,8 @@ import sys # Files and directories that are *skipped* by cpplint in the presubmit script. CPPLINT_BLACKLIST = [ 'tools-webrtc', + 'webrtc/api/video_codecs/video_decoder.h', + 'webrtc/api/video_codecs/video_encoder.h', 'webrtc/base', 'webrtc/examples/objc', 'webrtc/media', @@ -38,8 +40,6 @@ CPPLINT_BLACKLIST = [ 'webrtc/call.h', 'webrtc/common_types.h', 'webrtc/common_types.cc', - 'webrtc/video_decoder.h', - 'webrtc/video_encoder.h', 'webrtc/video_send_stream.h', ] diff --git a/webrtc/DEPS b/webrtc/DEPS index 36a5b738db..d4ebc55da8 100644 --- a/webrtc/DEPS +++ b/webrtc/DEPS @@ -14,8 +14,6 @@ include_rules = [ "+webrtc/config.h", "+webrtc/transport.h", "+webrtc/typedefs.h", - "+webrtc/video_decoder.h", - "+webrtc/video_encoder.h", "+webrtc/video_frame.h", "+webrtc/video_receive_stream.h", "+webrtc/video_send_stream.h", diff --git a/webrtc/api/video_codecs/BUILD.gn b/webrtc/api/video_codecs/BUILD.gn new file mode 100644 index 0000000000..befd065f05 --- /dev/null +++ b/webrtc/api/video_codecs/BUILD.gn @@ -0,0 +1,28 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") +if (is_android) { + import("//build/config/android/config.gni") + import("//build/config/android/rules.gni") +} + +rtc_source_set("video_codecs_api") { + sources = [ + "video_decoder.h", + "video_encoder.h", + ] + + deps = [ + # TODO(ilnik): Add dependency on webrtc/video_frame.h when it will have it's + # own build target. + + "../..:webrtc_common", + "../../base:rtc_base_approved", + ] +} diff --git a/webrtc/api/video_codecs/video_decoder.h b/webrtc/api/video_codecs/video_decoder.h new file mode 100644 index 0000000000..ba62822dde --- /dev/null +++ b/webrtc/api/video_codecs/video_decoder.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_API_VIDEO_CODECS_VIDEO_DECODER_H_ +#define WEBRTC_API_VIDEO_CODECS_VIDEO_DECODER_H_ + +#include +#include +#include + +#include "webrtc/common_types.h" +#include "webrtc/typedefs.h" +#include "webrtc/video_frame.h" + +namespace webrtc { + +class RTPFragmentationHeader; +// TODO(pbos): Expose these through a public (root) header or change these APIs. +struct CodecSpecificInfo; +class VideoCodec; + +class DecodedImageCallback { + public: + virtual ~DecodedImageCallback() {} + + virtual int32_t Decoded(VideoFrame& decodedImage) = 0; + // Provides an alternative interface that allows the decoder to specify the + // decode time excluding waiting time for any previous pending frame to + // return. This is necessary for breaking positive feedback in the delay + // estimation when the decoder has a single output buffer. + virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) { + // The default implementation ignores custom decode time value. + return Decoded(decodedImage); + } + // TODO(sakal): Remove other implementations when upstream projects have been + // updated. + virtual void Decoded(VideoFrame& decodedImage, + rtc::Optional decode_time_ms, + rtc::Optional qp) { + Decoded(decodedImage, + decode_time_ms ? static_cast(*decode_time_ms) : -1); + } + + virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) { + return -1; + } + + virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId) { return -1; } +}; + +class VideoDecoder { + public: + virtual ~VideoDecoder() {} + + virtual int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) = 0; + + virtual int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info = NULL, + int64_t render_time_ms = -1) = 0; + + virtual int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) = 0; + + virtual int32_t Release() = 0; + + // Returns true if the decoder prefer to decode frames late. + // That is, it can not decode infinite number of frames before the decoded + // frame is consumed. + virtual bool PrefersLateDecoding() const { return true; } + + virtual const char* ImplementationName() const { return "unknown"; } +}; + +} // namespace webrtc + +#endif // WEBRTC_API_VIDEO_CODECS_VIDEO_DECODER_H_ diff --git a/webrtc/api/video_codecs/video_encoder.h b/webrtc/api/video_codecs/video_encoder.h new file mode 100644 index 0000000000..969de43209 --- /dev/null +++ b/webrtc/api/video_codecs/video_encoder.h @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_API_VIDEO_CODECS_VIDEO_ENCODER_H_ +#define WEBRTC_API_VIDEO_CODECS_VIDEO_ENCODER_H_ + +#include +#include +#include + +#include "webrtc/base/checks.h" +#include "webrtc/common_types.h" +#include "webrtc/typedefs.h" +#include "webrtc/video_frame.h" +#include "webrtc/base/optional.h" + +namespace webrtc { + +class RTPFragmentationHeader; +// TODO(pbos): Expose these through a public (root) header or change these APIs. +struct CodecSpecificInfo; +class VideoCodec; + +class EncodedImageCallback { + public: + virtual ~EncodedImageCallback() {} + + struct Result { + enum Error { + OK, + + // Failed to send the packet. + ERROR_SEND_FAILED, + }; + + Result(Error error) : error(error) {} + Result(Error error, uint32_t frame_id) : error(error), frame_id(frame_id) {} + + Error error; + + // Frame ID assigned to the frame. The frame ID should be the same as the ID + // seen by the receiver for this frame. RTP timestamp of the frame is used + // as frame ID when RTP is used to send video. Must be used only when + // error=OK. + uint32_t frame_id = 0; + + // Tells the encoder that the next frame is should be dropped. + bool drop_next_frame = false; + }; + + // Callback function which is called when an image has been encoded. + virtual Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) = 0; + + virtual void OnDroppedFrame() {} +}; + +class VideoEncoder { + public: + enum EncoderType { + kH264, + kVp8, + kVp9, + kUnsupportedCodec, + }; + struct QpThresholds { + QpThresholds(int l, int h) : low(l), high(h) {} + QpThresholds() : low(-1), high(-1) {} + int low; + int high; + }; + struct ScalingSettings { + ScalingSettings(bool on, int low, int high) + : enabled(on), + thresholds(rtc::Optional(QpThresholds(low, high))) {} + explicit ScalingSettings(bool on) : enabled(on) {} + const bool enabled; + const rtc::Optional thresholds; + }; + static VideoEncoder* Create(EncoderType codec_type); + // Returns true if this type of encoder can be created using + // VideoEncoder::Create. + static bool IsSupportedSoftware(EncoderType codec_type); + static EncoderType CodecToEncoderType(VideoCodecType codec_type); + + static VideoCodecVP8 GetDefaultVp8Settings(); + static VideoCodecVP9 GetDefaultVp9Settings(); + static VideoCodecH264 GetDefaultH264Settings(); + + virtual ~VideoEncoder() {} + + // Initialize the encoder with the information from the codecSettings + // + // Input: + // - codec_settings : Codec settings + // - number_of_cores : Number of cores available for the encoder + // - max_payload_size : The maximum size each payload is allowed + // to have. Usually MTU - overhead. + // + // Return value : Set bit rate if OK + // <0 - Errors: + // WEBRTC_VIDEO_CODEC_ERR_PARAMETER + // WEBRTC_VIDEO_CODEC_ERR_SIZE + // WEBRTC_VIDEO_CODEC_LEVEL_EXCEEDED + // WEBRTC_VIDEO_CODEC_MEMORY + // WEBRTC_VIDEO_CODEC_ERROR + virtual int32_t InitEncode(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) = 0; + + // Register an encode complete callback object. + // + // Input: + // - callback : Callback object which handles encoded images. + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. + virtual int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) = 0; + + // Free encoder memory. + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. + virtual int32_t Release() = 0; + + // Encode an I420 image (as a part of a video stream). The encoded image + // will be returned to the user through the encode complete callback. + // + // Input: + // - frame : Image to be encoded + // - frame_types : Frame type to be generated by the encoder. + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK + // <0 - Errors: + // WEBRTC_VIDEO_CODEC_ERR_PARAMETER + // WEBRTC_VIDEO_CODEC_MEMORY + // WEBRTC_VIDEO_CODEC_ERROR + // WEBRTC_VIDEO_CODEC_TIMEOUT + virtual int32_t Encode(const VideoFrame& frame, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) = 0; + + // Inform the encoder of the new packet loss rate and the round-trip time of + // the network. + // + // Input: + // - packet_loss : Fraction lost + // (loss rate in percent = 100 * packetLoss / 255) + // - rtt : Round-trip time in milliseconds + // Return value : WEBRTC_VIDEO_CODEC_OK if OK + // <0 - Errors: WEBRTC_VIDEO_CODEC_ERROR + virtual int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) = 0; + + // Inform the encoder about the new target bit rate. + // + // Input: + // - bitrate : New target bit rate + // - framerate : The target frame rate + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. + virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) { + RTC_NOTREACHED() << "SetRate(uint32_t, uint32_t) is deprecated."; + return -1; + } + + // Default fallback: Just use the sum of bitrates as the single target rate. + // TODO(sprang): Remove this default implementation when we remove SetRates(). + virtual int32_t SetRateAllocation(const BitrateAllocation& allocation, + uint32_t framerate) { + return SetRates(allocation.get_sum_kbps(), framerate); + } + + // Any encoder implementation wishing to use the WebRTC provided + // quality scaler must implement this method. + virtual ScalingSettings GetScalingSettings() const { + return ScalingSettings(false); + } + + virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; } + virtual bool SupportsNativeHandle() const { return false; } + virtual const char* ImplementationName() const { return "unknown"; } +}; + +} // namespace webrtc +#endif // WEBRTC_API_VIDEO_CODECS_VIDEO_ENCODER_H_ diff --git a/webrtc/media/BUILD.gn b/webrtc/media/BUILD.gn index 6b18392ee1..25bf806b31 100644 --- a/webrtc/media/BUILD.gn +++ b/webrtc/media/BUILD.gn @@ -226,6 +226,7 @@ rtc_static_library("rtc_media") { "../api:video_frame_api", "../api/audio_codecs:audio_codecs_api", "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/video_codecs:video_codecs_api", "../base:rtc_base", "../base:rtc_base_approved", "../call", @@ -305,6 +306,7 @@ if (rtc_include_tests) { "..:webrtc_common", "../api:call_api", "../api:video_frame_api", + "../api/video_codecs:video_codecs_api", "../base:rtc_base", "../base:rtc_base_approved", "../base:rtc_base_tests_main", @@ -438,6 +440,7 @@ if (rtc_include_tests) { ":rtc_unittest_main", "../api:video_frame_api", "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/video_codecs:video_codecs_api", "../audio", "../base:rtc_base", "../base:rtc_base_approved", diff --git a/webrtc/media/engine/fakewebrtcvideoengine.h b/webrtc/media/engine/fakewebrtcvideoengine.h index 2dce2c503f..f95fa7bdd8 100644 --- a/webrtc/media/engine/fakewebrtcvideoengine.h +++ b/webrtc/media/engine/fakewebrtcvideoengine.h @@ -14,7 +14,10 @@ #include #include #include +#include +#include "webrtc/api/video_codecs/video_decoder.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/basictypes.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/gunit.h" @@ -24,8 +27,6 @@ #include "webrtc/media/engine/webrtcvideodecoderfactory.h" #include "webrtc/media/engine/webrtcvideoencoderfactory.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" -#include "webrtc/video_decoder.h" -#include "webrtc/video_encoder.h" namespace cricket { static const int kEventTimeoutMs = 10000; diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h index 80d91f4663..3984e1f1cc 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper.h @@ -14,7 +14,7 @@ #include #include -#include "webrtc/video_decoder.h" +#include "webrtc/api/video_codecs/video_decoder.h" namespace webrtc { diff --git a/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc b/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc index 16991d21b7..4ac84856f2 100644 --- a/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc +++ b/webrtc/media/engine/videodecodersoftwarefallbackwrapper_unittest.cc @@ -8,8 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "webrtc/video_decoder.h" - +#include "webrtc/api/video_codecs/video_decoder.h" #include "webrtc/base/checks.h" #include "webrtc/media/engine/videodecodersoftwarefallbackwrapper.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h index 055dd2957a..61c1a11c8d 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h @@ -15,8 +15,8 @@ #include #include +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/media/base/codec.h" -#include "webrtc/video_encoder.h" namespace webrtc { diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index 742b34c5f0..087fc4e011 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -17,6 +17,8 @@ #include #include "webrtc/api/video/i420_buffer.h" +#include "webrtc/api/video_codecs/video_decoder.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/copyonwritebuffer.h" #include "webrtc/base/logging.h" #include "webrtc/base/stringutils.h" @@ -35,8 +37,6 @@ #include "webrtc/media/engine/webrtcvoiceengine.h" #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" #include "webrtc/system_wrappers/include/field_trial.h" -#include "webrtc/video_decoder.h" -#include "webrtc/video_encoder.h" using DegradationPreference = webrtc::VideoSendStream::DegradationPreference; diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index 18688342ce..f422ffcb6d 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -13,6 +13,7 @@ #include #include +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/arraysize.h" #include "webrtc/base/gunit.h" #include "webrtc/base/stringutils.h" @@ -30,7 +31,6 @@ #include "webrtc/media/engine/webrtcvideoengine2.h" #include "webrtc/media/engine/webrtcvoiceengine.h" #include "webrtc/test/field_trial.h" -#include "webrtc/video_encoder.h" using webrtc::RtpExtension; @@ -3803,7 +3803,6 @@ TEST_F(WebRtcVideoChannel2Test, RedRtxPacketDoesntCreateUnsignalledStream) { // The first unsignalled SSRC received will create a default receive stream. // Any different unsignalled SSRC received will replace the default. TEST_F(WebRtcVideoChannel2Test, ReceiveDifferentUnsignaledSsrc) { - // Allow receiving VP8, VP9, H264 (if enabled). cricket::VideoRecvParameters parameters; parameters.codecs.push_back(GetEngineCodec("VP8")); diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index c6cbfca009..9b611bb72f 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -129,6 +129,7 @@ rtc_static_library("video_coding_utility") { deps = [ "../..:webrtc_common", + "../../api/video_codecs:video_codecs_api", "../../base:rtc_base_approved", "../../base:rtc_numerics", "../../base:rtc_task_queue", @@ -224,6 +225,7 @@ rtc_static_library("webrtc_vp8") { deps = [ ":video_coding_utility", "../..:webrtc_common", + "../../api/video_codecs:video_codecs_api", "../../base:rtc_base_approved", "../../common_video", "../../system_wrappers", @@ -322,6 +324,7 @@ if (rtc_include_tests) { ":video_coding_utility", ":webrtc_vp8", "../..:webrtc_common", + "../../api/video_codecs:video_codecs_api", "../../base:rtc_base_approved", "../../common_video:common_video", "../../system_wrappers:system_wrappers", @@ -528,6 +531,7 @@ if (rtc_include_tests) { ":webrtc_vp9", "../..:webrtc_common", "../../api:video_frame_api", + "../../api/video_codecs:video_codecs_api", "../../base:rtc_base", "../../base:rtc_base_approved", "../../base:rtc_task_queue", diff --git a/webrtc/modules/video_coding/codecs/test/video_codec_test.h b/webrtc/modules/video_coding/codecs/test/video_codec_test.h index 65d20ee898..961927353f 100644 --- a/webrtc/modules/video_coding/codecs/test/video_codec_test.h +++ b/webrtc/modules/video_coding/codecs/test/video_codec_test.h @@ -13,12 +13,12 @@ #include +#include "webrtc/api/video_codecs/video_decoder.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/event.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/test/gtest.h" -#include "webrtc/video_decoder.h" -#include "webrtc/video_encoder.h" namespace webrtc { diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h index 8e7c7927bb..e558290e6b 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h @@ -16,10 +16,10 @@ #include #include +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/constructormagic.h" #include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" -#include "webrtc/video_encoder.h" namespace webrtc { diff --git a/webrtc/modules/video_coding/include/video_codec_interface.h b/webrtc/modules/video_coding/include/video_codec_interface.h index 871cb4b42a..0bef7dacd0 100644 --- a/webrtc/modules/video_coding/include/video_codec_interface.h +++ b/webrtc/modules/video_coding/include/video_codec_interface.h @@ -14,12 +14,12 @@ #include #include "webrtc/api/video/video_frame.h" +#include "webrtc/api/video_codecs/video_decoder.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/common_types.h" #include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" #include "webrtc/typedefs.h" -#include "webrtc/video_decoder.h" -#include "webrtc/video_encoder.h" namespace webrtc { diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index bf81e768e4..a07b1eabf2 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -14,7 +14,7 @@ #include #include "webrtc/common_types.h" -#include "webrtc/video_encoder.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/optional.h" #include "webrtc/base/sequenced_task_checker.h" #include "webrtc/modules/video_coding/utility/moving_average.h" diff --git a/webrtc/modules/video_coding/video_codec_initializer_unittest.cc b/webrtc/modules/video_coding/video_codec_initializer_unittest.cc index d9d1d03e37..cd269f46e0 100644 --- a/webrtc/modules/video_coding/video_codec_initializer_unittest.cc +++ b/webrtc/modules/video_coding/video_codec_initializer_unittest.cc @@ -8,12 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_types.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_codec_initializer.h" #include "webrtc/test/gtest.h" -#include "webrtc/video_encoder.h" namespace webrtc { diff --git a/webrtc/pc/BUILD.gn b/webrtc/pc/BUILD.gn index 8aba758aa0..95c484e0c4 100644 --- a/webrtc/pc/BUILD.gn +++ b/webrtc/pc/BUILD.gn @@ -147,6 +147,7 @@ rtc_static_library("libjingle_peerconnection") { ":rtc_pc", "../api:call_api", "../api:rtc_stats_api", + "../api/video_codecs:video_codecs_api", "../call", "../media", "../stats", diff --git a/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc b/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc index 093966ee43..1274b8e47f 100644 --- a/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc +++ b/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc @@ -13,15 +13,15 @@ #include "webrtc/sdk/android/src/jni/androidmediaencoder_jni.h" #include -#include #include +#include +#include +#include #include "third_party/libyuv/include/libyuv/convert.h" #include "third_party/libyuv/include/libyuv/convert_from.h" #include "third_party/libyuv/include/libyuv/video_common.h" -#include "webrtc/sdk/android/src/jni/androidmediacodeccommon.h" -#include "webrtc/sdk/android/src/jni/classreferenceholder.h" -#include "webrtc/sdk/android/src/jni/native_handle_impl.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" @@ -38,9 +38,11 @@ #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/utility/quality_scaler.h" #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" +#include "webrtc/sdk/android/src/jni/androidmediacodeccommon.h" +#include "webrtc/sdk/android/src/jni/classreferenceholder.h" +#include "webrtc/sdk/android/src/jni/native_handle_impl.h" #include "webrtc/system_wrappers/include/field_trial.h" #include "webrtc/system_wrappers/include/logcat_trace_context.h" -#include "webrtc/video_encoder.h" using rtc::Bind; using rtc::Thread; @@ -123,7 +125,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder { private: class EncodeTask : public rtc::QueuedTask { public: - EncodeTask(rtc::WeakPtr encoder); + explicit EncodeTask(rtc::WeakPtr encoder); bool Run() override; private: @@ -234,7 +236,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder { int64_t stat_start_time_ms_; // Start time for statistics. int current_frames_; // Number of frames in the current statistics interval. int current_bytes_; // Encoded bytes in the current statistics interval. - int current_acc_qp_; // Accumulated QP in the current statistics interval. + int current_acc_qp_; // Accumulated QP in the current statistics interval. int current_encoding_time_ms_; // Overall encoding time in the current second int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame. int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame. @@ -259,10 +261,10 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder { const webrtc::VideoRotation rotation; }; std::list input_frame_infos_; - int32_t output_timestamp_; // Last output frame timestamp from - // |input_frame_infos_|. - int64_t output_render_time_ms_; // Last output frame render time from - // |input_frame_infos_|. + int32_t output_timestamp_; // Last output frame timestamp from + // |input_frame_infos_|. + int64_t output_render_time_ms_; // Last output frame render time from + // |input_frame_infos_|. webrtc::VideoRotation output_rotation_; // Last output frame rotation from // |input_frame_infos_|. // Frame size in bytes fed to MediaCodec. @@ -276,8 +278,8 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder { webrtc::H264BitstreamParser h264_bitstream_parser_; // VP9 variables to populate codec specific structure. - webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for - // non-flexible VP9 mode. + webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for + // non-flexible VP9 mode. uint8_t tl0_pic_idx_; size_t gof_idx_; @@ -519,8 +521,9 @@ int32_t MediaCodecVideoEncoder::InitEncodeInternal(int width, ScopedLocalRefFrame local_ref_frame(jni); const VideoCodecType codec_type = GetCodecType(); - ALOGD << "InitEncodeInternal Type: " << (int)codec_type << ", " << width - << " x " << height << ". Bitrate: " << kbps << " kbps. Fps: " << fps; + ALOGD << "InitEncodeInternal Type: " << static_cast(codec_type) << ", " + << width << " x " << height << ". Bitrate: " << kbps + << " kbps. Fps: " << fps; if (kbps == 0) { kbps = last_set_bitrate_kbps_; } @@ -550,9 +553,10 @@ int32_t MediaCodecVideoEncoder::InitEncodeInternal(int width, input_frame_infos_.clear(); drop_next_input_frame_ = false; use_surface_ = use_surface; - picture_id_ = static_cast(rand()) & 0x7FFF; + // TODO(ilnik): Use rand_r() instead to avoid LINT warnings below. + picture_id_ = static_cast(rand()) & 0x7FFF; // NOLINT gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); - tl0_pic_idx_ = static_cast(rand()); + tl0_pic_idx_ = static_cast(rand()); // NOLINT gof_idx_ = 0; last_frame_received_ms_ = -1; frames_received_since_last_key_ = kMinKeyFrameInterval; @@ -676,7 +680,7 @@ int32_t MediaCodecVideoEncoder::Encode( } if (frames_encoded_ < kMaxEncodedLogFrames) { ALOGD << "Encoder frame in # " << (frames_received_ - 1) - << ". TS: " << (int)(current_timestamp_us_ / 1000) + << ". TS: " << static_cast(current_timestamp_us_ / 1000) << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_ << ". Kbps: " << last_set_bitrate_kbps_; } @@ -696,7 +700,7 @@ int32_t MediaCodecVideoEncoder::Encode( if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) { ALOGD << "Already " << input_frame_infos_.size() << " frames in the queue, dropping" - << ". TS: " << (int)(current_timestamp_us_ / 1000) + << ". TS: " << static_cast(current_timestamp_us_ / 1000) << ". Fps: " << last_set_fps_ << ". Consecutive drops: " << consecutive_full_queue_frame_drops_; current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; @@ -1139,14 +1143,13 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { frame_encoding_time_ms = rtc::TimeMillis() - encoding_start_time_ms; } if (frames_encoded_ < kMaxEncodedLogFrames) { - int current_latency = - (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_); - ALOGD << "Encoder frame out # " << frames_encoded_ << - ". Key: " << key_frame << - ". Size: " << payload_size << - ". TS: " << (int)last_output_timestamp_ms_ << - ". Latency: " << current_latency << - ". EncTime: " << frame_encoding_time_ms; + int current_latency = static_cast(last_input_timestamp_ms_ - + last_output_timestamp_ms_); + ALOGD << "Encoder frame out # " << frames_encoded_ + << ". Key: " << key_frame << ". Size: " << payload_size + << ". TS: " << static_cast(last_output_timestamp_ms_) + << ". Latency: " << current_latency + << ". EncTime: " << frame_encoding_time_ms; } // Calculate and print encoding statistics - every 3 seconds. diff --git a/webrtc/test/BUILD.gn b/webrtc/test/BUILD.gn index a159567c36..4050eaff85 100644 --- a/webrtc/test/BUILD.gn +++ b/webrtc/test/BUILD.gn @@ -403,6 +403,7 @@ rtc_source_set("test_common") { ":test_support", ":video_test_common", "..:webrtc_common", + "../api/video_codecs:video_codecs_api", "../audio", "../base:rtc_base_approved", "../call", diff --git a/webrtc/test/configurable_frame_size_encoder.h b/webrtc/test/configurable_frame_size_encoder.h index a3820baf7e..886c546685 100644 --- a/webrtc/test/configurable_frame_size_encoder.h +++ b/webrtc/test/configurable_frame_size_encoder.h @@ -14,7 +14,7 @@ #include #include -#include "webrtc/video_encoder.h" +#include "webrtc/api/video_codecs/video_encoder.h" namespace webrtc { namespace test { diff --git a/webrtc/test/fake_encoder.h b/webrtc/test/fake_encoder.h index c07fbb36e1..e3878ecd4a 100644 --- a/webrtc/test/fake_encoder.h +++ b/webrtc/test/fake_encoder.h @@ -14,12 +14,12 @@ #include #include +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/sequenced_task_checker.h" #include "webrtc/base/task_queue.h" #include "webrtc/common_types.h" #include "webrtc/system_wrappers/include/clock.h" -#include "webrtc/video_encoder.h" namespace webrtc { namespace test { diff --git a/webrtc/video/BUILD.gn b/webrtc/video/BUILD.gn index be12b07d38..c9d9a334e1 100644 --- a/webrtc/video/BUILD.gn +++ b/webrtc/video/BUILD.gn @@ -58,6 +58,7 @@ rtc_static_library("video") { deps = [ "..:webrtc_common", "../api:transport_api", + "../api/video_codecs:video_codecs_api", "../base:rtc_base_approved", "../base:rtc_numerics", "../base:rtc_task_queue", @@ -168,6 +169,7 @@ if (rtc_include_tests) { "replay.cc", ] deps = [ + "../api/video_codecs:video_codecs_api", "../system_wrappers:metrics_default", "../test:field_trial", "../test:run_test", diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index e320826ba5..ff0185c198 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -15,6 +15,7 @@ #include #include +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/checks.h" #include "webrtc/base/event.h" #include "webrtc/base/file.h" @@ -54,7 +55,6 @@ #include "webrtc/test/testsupport/fileutils.h" #include "webrtc/test/testsupport/perf_test.h" #include "webrtc/video/transport_adapter.h" -#include "webrtc/video_encoder.h" #if defined(MEMORY_SANITIZER) // Flaky under MemorySanitizer, see diff --git a/webrtc/video/payload_router.h b/webrtc/video/payload_router.h index f2d138dc2c..bcf1d9b977 100644 --- a/webrtc/video/payload_router.h +++ b/webrtc/video/payload_router.h @@ -13,12 +13,12 @@ #include +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/common_types.h" #include "webrtc/config.h" -#include "webrtc/video_encoder.h" #include "webrtc/system_wrappers/include/atomic32.h" namespace webrtc { diff --git a/webrtc/video/replay.cc b/webrtc/video/replay.cc index 6347dc9977..3bfdf92cd3 100644 --- a/webrtc/video/replay.cc +++ b/webrtc/video/replay.cc @@ -15,6 +15,7 @@ #include #include "gflags/gflags.h" +#include "webrtc/api/video_codecs/video_decoder.h" #include "webrtc/base/checks.h" #include "webrtc/call/call.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" @@ -32,7 +33,6 @@ #include "webrtc/test/video_capturer.h" #include "webrtc/test/video_renderer.h" #include "webrtc/typedefs.h" -#include "webrtc/video_decoder.h" namespace webrtc { namespace flags { diff --git a/webrtc/video/video_receive_stream_unittest.cc b/webrtc/video/video_receive_stream_unittest.cc index 9824890be8..29f2497b23 100644 --- a/webrtc/video/video_receive_stream_unittest.cc +++ b/webrtc/video/video_receive_stream_unittest.cc @@ -13,6 +13,7 @@ #include "webrtc/test/gtest.h" #include "webrtc/test/gmock.h" +#include "webrtc/api/video_codecs/video_decoder.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/event.h" #include "webrtc/media/base/fakevideorenderer.h" @@ -24,7 +25,6 @@ #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/system_wrappers/include/sleep.h" #include "webrtc/test/field_trial.h" -#include "webrtc/video_decoder.h" using testing::_; using testing::Invoke; diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 473a3c8284..213ff6d0a2 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -17,6 +17,7 @@ #include #include "webrtc/api/video/video_rotation.h" +#include "webrtc/api/video_codecs/video_encoder.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/event.h" #include "webrtc/base/sequenced_task_checker.h" @@ -31,7 +32,6 @@ #include "webrtc/system_wrappers/include/atomic32.h" #include "webrtc/typedefs.h" #include "webrtc/video/overuse_frame_detector.h" -#include "webrtc/video_encoder.h" #include "webrtc/video_send_stream.h" namespace webrtc { diff --git a/webrtc/video_decoder.h b/webrtc/video_decoder.h index 70c09129f7..65f9de9ca7 100644 --- a/webrtc/video_decoder.h +++ b/webrtc/video_decoder.h @@ -1,5 +1,6 @@ /* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * DEPRECATED: use api/video_codecs/video_decoder.h instead. + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -11,76 +12,6 @@ #ifndef WEBRTC_VIDEO_DECODER_H_ #define WEBRTC_VIDEO_DECODER_H_ -#include -#include -#include - -#include "webrtc/common_types.h" -#include "webrtc/typedefs.h" -#include "webrtc/video_frame.h" - -namespace webrtc { - -class RTPFragmentationHeader; -// TODO(pbos): Expose these through a public (root) header or change these APIs. -struct CodecSpecificInfo; -class VideoCodec; - -class DecodedImageCallback { - public: - virtual ~DecodedImageCallback() {} - - virtual int32_t Decoded(VideoFrame& decodedImage) = 0; - // Provides an alternative interface that allows the decoder to specify the - // decode time excluding waiting time for any previous pending frame to - // return. This is necessary for breaking positive feedback in the delay - // estimation when the decoder has a single output buffer. - virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) { - // The default implementation ignores custom decode time value. - return Decoded(decodedImage); - } - // TODO(sakal): Remove other implementations when upstream projects have been - // updated. - virtual void Decoded(VideoFrame& decodedImage, - rtc::Optional decode_time_ms, - rtc::Optional qp) { - Decoded(decodedImage, - decode_time_ms ? static_cast(*decode_time_ms) : -1); - } - - virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) { - return -1; - } - - virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId) { return -1; } -}; - -class VideoDecoder { - public: - virtual ~VideoDecoder() {} - - virtual int32_t InitDecode(const VideoCodec* codec_settings, - int32_t number_of_cores) = 0; - - virtual int32_t Decode(const EncodedImage& input_image, - bool missing_frames, - const RTPFragmentationHeader* fragmentation, - const CodecSpecificInfo* codec_specific_info = NULL, - int64_t render_time_ms = -1) = 0; - - virtual int32_t RegisterDecodeCompleteCallback( - DecodedImageCallback* callback) = 0; - - virtual int32_t Release() = 0; - - // Returns true if the decoder prefer to decode frames late. - // That is, it can not decode infinite number of frames before the decoded - // frame is consumed. - virtual bool PrefersLateDecoding() const { return true; } - - virtual const char* ImplementationName() const { return "unknown"; } -}; - -} // namespace webrtc +#include "webrtc/api/video_codecs/video_decoder.h" #endif // WEBRTC_VIDEO_DECODER_H_ diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index 0a21fa345a..e64c41bb2f 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -1,5 +1,6 @@ /* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * DEPRECATED: use api/video_codecs/video_encoder.h instead. + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -11,182 +12,6 @@ #ifndef WEBRTC_VIDEO_ENCODER_H_ #define WEBRTC_VIDEO_ENCODER_H_ -#include -#include -#include +#include "webrtc/api/video_codecs/video_encoder.h" -#include "webrtc/base/checks.h" -#include "webrtc/common_types.h" -#include "webrtc/typedefs.h" -#include "webrtc/video_frame.h" -#include "webrtc/base/optional.h" - -namespace webrtc { - -class RTPFragmentationHeader; -// TODO(pbos): Expose these through a public (root) header or change these APIs. -struct CodecSpecificInfo; -class VideoCodec; - -class EncodedImageCallback { - public: - virtual ~EncodedImageCallback() {} - - struct Result { - enum Error { - OK, - - // Failed to send the packet. - ERROR_SEND_FAILED, - }; - - Result(Error error) : error(error) {} - Result(Error error, uint32_t frame_id) : error(error), frame_id(frame_id) {} - - Error error; - - // Frame ID assigned to the frame. The frame ID should be the same as the ID - // seen by the receiver for this frame. RTP timestamp of the frame is used - // as frame ID when RTP is used to send video. Must be used only when - // error=OK. - uint32_t frame_id = 0; - - // Tells the encoder that the next frame is should be dropped. - bool drop_next_frame = false; - }; - - // Callback function which is called when an image has been encoded. - virtual Result OnEncodedImage( - const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info, - const RTPFragmentationHeader* fragmentation) = 0; - - virtual void OnDroppedFrame() {} -}; - -class VideoEncoder { - public: - enum EncoderType { - kH264, - kVp8, - kVp9, - kUnsupportedCodec, - }; - struct QpThresholds { - QpThresholds(int l, int h) : low(l), high(h) {} - QpThresholds() : low(-1), high(-1) {} - int low; - int high; - }; - struct ScalingSettings { - ScalingSettings(bool on, int low, int high) - : enabled(on), - thresholds(rtc::Optional(QpThresholds(low, high))) {} - explicit ScalingSettings(bool on) : enabled(on) {} - const bool enabled; - const rtc::Optional thresholds; - }; - static VideoEncoder* Create(EncoderType codec_type); - // Returns true if this type of encoder can be created using - // VideoEncoder::Create. - static bool IsSupportedSoftware(EncoderType codec_type); - static EncoderType CodecToEncoderType(VideoCodecType codec_type); - - static VideoCodecVP8 GetDefaultVp8Settings(); - static VideoCodecVP9 GetDefaultVp9Settings(); - static VideoCodecH264 GetDefaultH264Settings(); - - virtual ~VideoEncoder() {} - - // Initialize the encoder with the information from the codecSettings - // - // Input: - // - codec_settings : Codec settings - // - number_of_cores : Number of cores available for the encoder - // - max_payload_size : The maximum size each payload is allowed - // to have. Usually MTU - overhead. - // - // Return value : Set bit rate if OK - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERR_PARAMETER - // WEBRTC_VIDEO_CODEC_ERR_SIZE - // WEBRTC_VIDEO_CODEC_LEVEL_EXCEEDED - // WEBRTC_VIDEO_CODEC_MEMORY - // WEBRTC_VIDEO_CODEC_ERROR - virtual int32_t InitEncode(const VideoCodec* codec_settings, - int32_t number_of_cores, - size_t max_payload_size) = 0; - - // Register an encode complete callback object. - // - // Input: - // - callback : Callback object which handles encoded images. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t RegisterEncodeCompleteCallback( - EncodedImageCallback* callback) = 0; - - // Free encoder memory. - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t Release() = 0; - - // Encode an I420 image (as a part of a video stream). The encoded image - // will be returned to the user through the encode complete callback. - // - // Input: - // - frame : Image to be encoded - // - frame_types : Frame type to be generated by the encoder. - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK - // <0 - Errors: - // WEBRTC_VIDEO_CODEC_ERR_PARAMETER - // WEBRTC_VIDEO_CODEC_MEMORY - // WEBRTC_VIDEO_CODEC_ERROR - // WEBRTC_VIDEO_CODEC_TIMEOUT - virtual int32_t Encode(const VideoFrame& frame, - const CodecSpecificInfo* codec_specific_info, - const std::vector* frame_types) = 0; - - // Inform the encoder of the new packet loss rate and the round-trip time of - // the network. - // - // Input: - // - packet_loss : Fraction lost - // (loss rate in percent = 100 * packetLoss / 255) - // - rtt : Round-trip time in milliseconds - // Return value : WEBRTC_VIDEO_CODEC_OK if OK - // <0 - Errors: WEBRTC_VIDEO_CODEC_ERROR - virtual int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) = 0; - - // Inform the encoder about the new target bit rate. - // - // Input: - // - bitrate : New target bit rate - // - framerate : The target frame rate - // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) { - RTC_NOTREACHED() << "SetRate(uint32_t, uint32_t) is deprecated."; - return -1; - } - - // Default fallback: Just use the sum of bitrates as the single target rate. - // TODO(sprang): Remove this default implementation when we remove SetRates(). - virtual int32_t SetRateAllocation(const BitrateAllocation& allocation, - uint32_t framerate) { - return SetRates(allocation.get_sum_kbps(), framerate); - } - - // Any encoder implementation wishing to use the WebRTC provided - // quality scaler must implement this method. - virtual ScalingSettings GetScalingSettings() const { - return ScalingSettings(false); - } - - virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; } - virtual bool SupportsNativeHandle() const { return false; } - virtual const char* ImplementationName() const { return "unknown"; } -}; - -} // namespace webrtc #endif // WEBRTC_VIDEO_ENCODER_H_