mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 13:50:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/sdk/android/src/jni/androidmediadecoder.cc
Anders Carlsson 1e06d885dc Migrate legacy Android video codec factories. 
Prepare for building without built-in software codecs. When passing
null, inject the new type of factories but wrap them in the built-in
software codecs outside the videoengine.

Bug: webrtc:7925
Change-Id: I7408e6e46e6b9efdf346852954bf51a97e023b5c
Reviewed-on: https://webrtc-review.googlesource.com/83729
Commit-Queue: Anders Carlsson <andersc@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23897}
2018-07-10 08:01:05 +00:00

862 lines
32 KiB
C++
Raw Blame History

/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <algorithm>
#include <deque>
#include <memory>
#include <vector>
// NOTICE: androidmediadecoder_jni.h must be included before
// androidmediacodeccommon.h to avoid build errors.
#include "sdk/android/src/jni/androidmediadecoder_jni.h"
#include "api/video_codecs/sdp_video_format.h"
#include "common_video/h264/h264_bitstream_parser.h"
#include "common_video/include/i420_buffer_pool.h"
#include "media/base/mediaconstants.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/utility/vp8_header_parser.h"
#include "rtc_base/bind.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/ptr_util.h"
#include "rtc_base/scoped_ref_ptr.h"
#include "rtc_base/thread.h"
#include "rtc_base/timeutils.h"
#include "sdk/android/generated_video_jni/jni/MediaCodecVideoDecoder_jni.h"
#include "sdk/android/native_api/jni/java_types.h"
#include "sdk/android/src/jni/androidmediacodeccommon.h"
#include "sdk/android/src/jni/videoframe.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
#include "third_party/libyuv/include/libyuv/video_common.h"
using rtc::Bind;
using rtc::Thread;
using rtc::ThreadManager;
namespace webrtc {
namespace jni {
// Logging macros.
#define TAG_DECODER "MediaCodecVideoDecoder"
#ifdef TRACK_BUFFER_TIMING
#define ALOGV(...) \
__android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
#else
#define ALOGV(...)
#endif
#define ALOGD RTC_LOG_TAG(rtc::LS_INFO, TAG_DECODER)
#define ALOGW RTC_LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
#define ALOGE RTC_LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
enum { kMaxWarningLogFrames = 2 };
class MediaCodecVideoDecoder : public VideoDecoder, public rtc::MessageHandler {
public:
explicit MediaCodecVideoDecoder(JNIEnv* jni,
VideoCodecType codecType,
jobject render_egl_context);
~MediaCodecVideoDecoder() override;
int32_t InitDecode(const VideoCodec* codecSettings,
int32_t numberOfCores) override;
int32_t Decode(const EncodedImage& inputImage,
bool missingFrames,
const CodecSpecificInfo* codecSpecificInfo = NULL,
int64_t renderTimeMs = -1) override;
int32_t RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) override;
int32_t Release() override;
bool PrefersLateDecoding() const override { return true; }
// rtc::MessageHandler implementation.
void OnMessage(rtc::Message* msg) override;
const char* ImplementationName() const override;
private:
// CHECK-fail if not running on |codec_thread_|.
void CheckOnCodecThread();
int32_t InitDecodeOnCodecThread();
int32_t ResetDecodeOnCodecThread();
int32_t ReleaseOnCodecThread();
int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
// Deliver any outputs pending in the MediaCodec to our |callback_| and return
// true on success.
bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
int32_t ProcessHWErrorOnCodecThread();
void EnableFrameLogOnWarning();
void ResetVariables();
// Type of video codec.
VideoCodecType codecType_;
// Render EGL context - owned by factory, should not be allocated/destroyed
// by VideoDecoder.
jobject render_egl_context_;
bool key_frame_required_;
bool inited_;
bool sw_fallback_required_;
bool use_surface_;
VideoCodec codec_;
I420BufferPool decoded_frame_pool_;
DecodedImageCallback* callback_;
int frames_received_; // Number of frames received by decoder.
int frames_decoded_; // Number of frames decoded by decoder.
// Number of decoded frames for which log information is displayed.
int frames_decoded_logged_;
int64_t start_time_ms_; // Start time for statistics.
int current_frames_; // Number of frames in the current statistics interval.
int current_bytes_; // Encoded bytes in the current statistics interval.
int current_decoding_time_ms_; // Overall decoding time in the current second
int current_delay_time_ms_; // Overall delay time in the current second.
int32_t max_pending_frames_; // Maximum number of pending input frames.
H264BitstreamParser h264_bitstream_parser_;
std::deque<absl::optional<uint8_t>> pending_frame_qps_;
// State that is constant for the lifetime of this object once the ctor
// returns.
std::unique_ptr<Thread>
codec_thread_; // Thread on which to operate MediaCodec.
ScopedJavaGlobalRef<jobject> j_media_codec_video_decoder_;
// Global references; must be deleted in Release().
std::vector<ScopedJavaGlobalRef<jobject>> input_buffers_;
};
MediaCodecVideoDecoder::MediaCodecVideoDecoder(JNIEnv* jni,
VideoCodecType codecType,
jobject render_egl_context)
: codecType_(codecType),
render_egl_context_(render_egl_context),
key_frame_required_(true),
inited_(false),
sw_fallback_required_(false),
codec_thread_(Thread::Create()),
j_media_codec_video_decoder_(
jni,
Java_MediaCodecVideoDecoder_Constructor(jni)) {
codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
use_surface_ = (render_egl_context_ != NULL);
ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_;
memset(&codec_, 0, sizeof(codec_));
AllowBlockingCalls();
}
MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
// Call Release() to ensure no more callbacks to us after we are deleted.
Release();
}
int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
int32_t numberOfCores) {
ALOGD << "InitDecode.";
if (inst == NULL) {
ALOGE << "NULL VideoCodec instance";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Factory should guard against other codecs being used with us.
RTC_CHECK(inst->codecType == codecType_)
<< "Unsupported codec " << inst->codecType << " for " << codecType_;
if (sw_fallback_required_) {
ALOGE << "InitDecode() - fallback to SW decoder";
return WEBRTC_VIDEO_CODEC_OK;
}
// Save VideoCodec instance for later.
if (&codec_ != inst) {
codec_ = *inst;
}
// If maxFramerate is not set then assume 30 fps.
codec_.maxFramerate = (codec_.maxFramerate >= 1) ? codec_.maxFramerate : 30;
// Call Java init.
return codec_thread_->Invoke<int32_t>(
RTC_FROM_HERE,
Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this));
}
void MediaCodecVideoDecoder::ResetVariables() {
CheckOnCodecThread();
key_frame_required_ = true;
frames_received_ = 0;
frames_decoded_ = 0;
frames_decoded_logged_ = kMaxDecodedLogFrames;
start_time_ms_ = rtc::TimeMillis();
current_frames_ = 0;
current_bytes_ = 0;
current_decoding_time_ms_ = 0;
current_delay_time_ms_ = 0;
pending_frame_qps_.clear();
}
int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
CheckOnCodecThread();
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "InitDecodeOnCodecThread Type: " << static_cast<int>(codecType_)
<< ". " << codec_.width << " x " << codec_.height
<< ". Fps: " << static_cast<int>(codec_.maxFramerate);
// Release previous codec first if it was allocated before.
int ret_val = ReleaseOnCodecThread();
if (ret_val < 0) {
ALOGE << "Release failure: " << ret_val << " - fallback to SW codec";
sw_fallback_required_ = true;
return WEBRTC_VIDEO_CODEC_ERROR;
}
ResetVariables();
ScopedJavaLocalRef<jobject> j_video_codec_enum =
Java_VideoCodecType_fromNativeIndex(jni, codecType_);
jobject j_egl_context = use_surface_ ? render_egl_context_ : nullptr;
bool success = Java_MediaCodecVideoDecoder_initDecode(
jni, j_media_codec_video_decoder_, j_video_codec_enum, codec_.width,
codec_.height, JavaParamRef<jobject>(j_egl_context));
if (CheckException(jni) || !success) {
ALOGE << "Codec initialization error - fallback to SW codec.";
sw_fallback_required_ = true;
return WEBRTC_VIDEO_CODEC_ERROR;
}
inited_ = true;
switch (codecType_) {
case kVideoCodecVP8:
max_pending_frames_ = kMaxPendingFramesVp8;
break;
case kVideoCodecVP9:
max_pending_frames_ = kMaxPendingFramesVp9;
break;
case kVideoCodecH264:
max_pending_frames_ = kMaxPendingFramesH264;
break;
default:
max_pending_frames_ = 0;
}
ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
ScopedJavaLocalRef<jobjectArray> input_buffers =
Java_MediaCodecVideoDecoder_getInputBuffers(jni,
j_media_codec_video_decoder_);
input_buffers_ = JavaToNativeVector<ScopedJavaGlobalRef<jobject>>(
jni, input_buffers, [](JNIEnv* env, const JavaRef<jobject>& o) {
return ScopedJavaGlobalRef<jobject>(env, o);
});
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
CheckOnCodecThread();
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "ResetDecodeOnCodecThread Type: " << static_cast<int>(codecType_)
<< ". " << codec_.width << " x " << codec_.height;
ALOGD << " Frames received: " << frames_received_
<< ". Frames decoded: " << frames_decoded_;
inited_ = false;
rtc::MessageQueueManager::Clear(this);
ResetVariables();
Java_MediaCodecVideoDecoder_reset(jni, j_media_codec_video_decoder_,
codec_.width, codec_.height);
if (CheckException(jni)) {
ALOGE << "Soft reset error - fallback to SW codec.";
sw_fallback_required_ = true;
return WEBRTC_VIDEO_CODEC_ERROR;
}
inited_ = true;
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoDecoder::Release() {
ALOGD << "DecoderRelease request";
return codec_thread_->Invoke<int32_t>(
RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
}
int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
if (!inited_) {
return WEBRTC_VIDEO_CODEC_OK;
}
CheckOnCodecThread();
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ALOGD << "DecoderReleaseOnCodecThread: Frames received: " << frames_received_
<< ". Frames decoded: " << frames_decoded_;
ScopedLocalRefFrame local_ref_frame(jni);
input_buffers_.clear();
Java_MediaCodecVideoDecoder_release(jni, j_media_codec_video_decoder_);
inited_ = false;
rtc::MessageQueueManager::Clear(this);
if (CheckException(jni)) {
ALOGE << "Decoder release exception";
return WEBRTC_VIDEO_CODEC_ERROR;
}
ALOGD << "DecoderReleaseOnCodecThread done";
return WEBRTC_VIDEO_CODEC_OK;
}
void MediaCodecVideoDecoder::CheckOnCodecThread() {
RTC_CHECK(codec_thread_.get() == ThreadManager::Instance()->CurrentThread())
<< "Running on wrong thread!";
}
void MediaCodecVideoDecoder::EnableFrameLogOnWarning() {
// Log next 2 output frames.
frames_decoded_logged_ =
std::max(frames_decoded_logged_, frames_decoded_ + kMaxWarningLogFrames);
}
int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() {
CheckOnCodecThread();
int ret_val = ReleaseOnCodecThread();
if (ret_val < 0) {
ALOGE << "ProcessHWError: Release failure";
}
if (codecType_ == kVideoCodecH264) {
// For now there is no SW H.264 which can be used as fallback codec.
// So try to restart hw codec for now.
ret_val = InitDecodeOnCodecThread();
ALOGE << "Reset H.264 codec done. Status: " << ret_val;
if (ret_val == WEBRTC_VIDEO_CODEC_OK) {
// H.264 codec was succesfully reset - return regular error code.
return WEBRTC_VIDEO_CODEC_ERROR;
} else {
// Fail to restart H.264 codec - return error code which should stop the
// call.
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
} else {
sw_fallback_required_ = true;
ALOGE << "Return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE";
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
}
int32_t MediaCodecVideoDecoder::Decode(
const EncodedImage& inputImage,
bool missingFrames,
const CodecSpecificInfo* codecSpecificInfo,
int64_t renderTimeMs) {
if (sw_fallback_required_) {
ALOGE << "Decode() - fallback to SW codec";
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
if (callback_ == NULL) {
ALOGE << "Decode() - callback_ is NULL";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (inputImage._buffer == NULL && inputImage._length > 0) {
ALOGE << "Decode() - inputImage is incorrect";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (!inited_) {
ALOGE << "Decode() - decoder is not initialized";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
// Check if encoded frame dimension has changed.
if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
(inputImage._encodedWidth != codec_.width ||
inputImage._encodedHeight != codec_.height)) {
ALOGW << "Input resolution changed from " << codec_.width << " x "
<< codec_.height << " to " << inputImage._encodedWidth << " x "
<< inputImage._encodedHeight;
codec_.width = inputImage._encodedWidth;
codec_.height = inputImage._encodedHeight;
int32_t ret;
if (use_surface_ &&
(codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecH264)) {
// Soft codec reset - only for surface decoding.
ret = codec_thread_->Invoke<int32_t>(
RTC_FROM_HERE,
Bind(&MediaCodecVideoDecoder::ResetDecodeOnCodecThread, this));
} else {
// Hard codec reset.
ret = InitDecode(&codec_, 1);
}
if (ret < 0) {
ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec";
sw_fallback_required_ = true;
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
}
// Always start with a complete key frame.
if (key_frame_required_) {
if (inputImage._frameType != kVideoFrameKey) {
ALOGE << "Decode() - key frame is required";
return WEBRTC_VIDEO_CODEC_ERROR;
}
if (!inputImage._completeFrame) {
ALOGE << "Decode() - complete frame is required";
return WEBRTC_VIDEO_CODEC_ERROR;
}
key_frame_required_ = false;
}
if (inputImage._length == 0) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
return codec_thread_->Invoke<int32_t>(
RTC_FROM_HERE,
Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage));
}
int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
const EncodedImage& inputImage) {
CheckOnCodecThread();
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
// Try to drain the decoder and wait until output is not too
// much behind the input.
if (codecType_ == kVideoCodecH264 &&
frames_received_ > frames_decoded_ + max_pending_frames_) {
// Print warning for H.264 only - for VP8/VP9 one frame delay is ok.
ALOGW << "Decoder is too far behind. Try to drain. Received: "
<< frames_received_ << ". Decoded: " << frames_decoded_;
EnableFrameLogOnWarning();
}
const int64_t drain_start = rtc::TimeMillis();
while ((frames_received_ > frames_decoded_ + max_pending_frames_) &&
(rtc::TimeMillis() - drain_start) < kMediaCodecTimeoutMs) {
if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
ALOGE << "DeliverPendingOutputs error. Frames received: "
<< frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
}
}
if (frames_received_ > frames_decoded_ + max_pending_frames_) {
ALOGE << "Output buffer dequeue timeout. Frames received: "
<< frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
}
// Get input buffer.
int j_input_buffer_index = Java_MediaCodecVideoDecoder_dequeueInputBuffer(
jni, j_media_codec_video_decoder_);
if (CheckException(jni) || j_input_buffer_index < 0) {
ALOGE << "dequeueInputBuffer error: " << j_input_buffer_index
<< ". Retry DeliverPendingOutputs.";
EnableFrameLogOnWarning();
// Try to drain the decoder.
if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
ALOGE << "DeliverPendingOutputs error. Frames received: "
<< frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
}
// Try dequeue input buffer one last time.
j_input_buffer_index = Java_MediaCodecVideoDecoder_dequeueInputBuffer(
jni, j_media_codec_video_decoder_);
if (CheckException(jni) || j_input_buffer_index < 0) {
ALOGE << "dequeueInputBuffer critical error: " << j_input_buffer_index;
return ProcessHWErrorOnCodecThread();
}
}
// Copy encoded data to Java ByteBuffer.
jobject j_input_buffer = input_buffers_[j_input_buffer_index].obj();
uint8_t* buffer =
reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
RTC_CHECK(buffer) << "Indirect buffer??";
size_t buffer_capacity =
rtc::dchecked_cast<size_t>(jni->GetDirectBufferCapacity(j_input_buffer));
if (CheckException(jni) || buffer_capacity < inputImage._length) {
ALOGE << "Input frame size " << inputImage._length
<< " is bigger than buffer size " << buffer_capacity;
return ProcessHWErrorOnCodecThread();
}
jlong presentation_timestamp_us = static_cast<jlong>(
static_cast<int64_t>(frames_received_) * 1000000 / codec_.maxFramerate);
memcpy(buffer, inputImage._buffer, inputImage._length);
if (frames_decoded_ < frames_decoded_logged_) {
ALOGD << "Decoder frame in # " << frames_received_
<< ". Type: " << inputImage._frameType << ". Buffer # "
<< j_input_buffer_index
<< ". TS: " << presentation_timestamp_us / 1000
<< ". Size: " << inputImage._length;
}
// Save input image timestamps for later output.
frames_received_++;
current_bytes_ += inputImage._length;
absl::optional<uint8_t> qp;
if (codecType_ == kVideoCodecVP8) {
int qp_int;
if (vp8::GetQp(inputImage._buffer, inputImage._length, &qp_int)) {
qp = qp_int;
}
} else if (codecType_ == kVideoCodecH264) {
h264_bitstream_parser_.ParseBitstream(inputImage._buffer,
inputImage._length);
int qp_int;
if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) {
qp = qp_int;
}
}
pending_frame_qps_.push_back(qp);
// Feed input to decoder.
bool success = Java_MediaCodecVideoDecoder_queueInputBuffer(
jni, j_media_codec_video_decoder_, j_input_buffer_index,
static_cast<int>(inputImage._length), presentation_timestamp_us,
static_cast<int64_t>(inputImage._timeStamp), inputImage.ntp_time_ms_);
if (CheckException(jni) || !success) {
ALOGE << "queueInputBuffer error";
return ProcessHWErrorOnCodecThread();
}
// Try to drain the decoder
if (!DeliverPendingOutputs(jni, 0)) {
ALOGE << "DeliverPendingOutputs error";
return ProcessHWErrorOnCodecThread();
}
return WEBRTC_VIDEO_CODEC_OK;
}
bool MediaCodecVideoDecoder::DeliverPendingOutputs(JNIEnv* jni,
int dequeue_timeout_ms) {
CheckOnCodecThread();
if (frames_received_ <= frames_decoded_) {
// No need to query for output buffers - decoder is drained.
return true;
}
// Get decoder output.
ScopedJavaLocalRef<jobject> j_decoder_output_buffer =
(use_surface_ ? &Java_MediaCodecVideoDecoder_dequeueTextureBuffer
: &Java_MediaCodecVideoDecoder_dequeueOutputBuffer)(
jni, j_media_codec_video_decoder_, dequeue_timeout_ms);
if (CheckException(jni)) {
ALOGE << "dequeueOutputBuffer() error";
return false;
}
if (IsNull(jni, j_decoder_output_buffer)) {
// No decoded frame ready.
return true;
}
// Get decoded video frame properties.
int color_format = Java_MediaCodecVideoDecoder_getColorFormat(
jni, j_media_codec_video_decoder_);
int width =
Java_MediaCodecVideoDecoder_getWidth(jni, j_media_codec_video_decoder_);
int height =
Java_MediaCodecVideoDecoder_getHeight(jni, j_media_codec_video_decoder_);
rtc::scoped_refptr<VideoFrameBuffer> frame_buffer;
int64_t presentation_timestamps_ms = 0;
int64_t output_timestamps_ms = 0;
int64_t output_ntp_timestamps_ms = 0;
int decode_time_ms = 0;
int64_t frame_delayed_ms = 0;
if (use_surface_) {
// Extract data from Java DecodedTextureBuffer.
presentation_timestamps_ms =
Java_DecodedTextureBuffer_getPresentationTimestampMs(
jni, j_decoder_output_buffer);
output_timestamps_ms =
Java_DecodedTextureBuffer_getTimeStampMs(jni, j_decoder_output_buffer);
output_ntp_timestamps_ms = Java_DecodedTextureBuffer_getNtpTimestampMs(
jni, j_decoder_output_buffer);
decode_time_ms =
Java_DecodedTextureBuffer_getDecodeTimeMs(jni, j_decoder_output_buffer);
ScopedJavaLocalRef<jobject> j_video_frame_buffer =
Java_DecodedTextureBuffer_getVideoFrameBuffer(jni,
j_decoder_output_buffer);
// |video_frame_buffer| == null represents a dropped frame.
if (!j_video_frame_buffer.is_null()) {
frame_delayed_ms = Java_DecodedTextureBuffer_getFrameDelayMs(
jni, j_decoder_output_buffer);
frame_buffer = AndroidVideoBuffer::Adopt(jni, j_video_frame_buffer);
} else {
EnableFrameLogOnWarning();
}
} else {
// Extract data from Java ByteBuffer and create output yuv420 frame -
// for non surface decoding only.
int stride = Java_MediaCodecVideoDecoder_getStride(
jni, j_media_codec_video_decoder_);
const int slice_height = Java_MediaCodecVideoDecoder_getSliceHeight(
jni, j_media_codec_video_decoder_);
const int output_buffer_index =
Java_DecodedOutputBuffer_getIndex(jni, j_decoder_output_buffer);
const int output_buffer_offset =
Java_DecodedOutputBuffer_getOffset(jni, j_decoder_output_buffer);
const int output_buffer_size =
Java_DecodedOutputBuffer_getSize(jni, j_decoder_output_buffer);
presentation_timestamps_ms =
Java_DecodedOutputBuffer_getPresentationTimestampMs(
jni, j_decoder_output_buffer);
output_timestamps_ms =
Java_DecodedOutputBuffer_getTimestampMs(jni, j_decoder_output_buffer);
output_ntp_timestamps_ms = Java_DecodedOutputBuffer_getNtpTimestampMs(
jni, j_decoder_output_buffer);
decode_time_ms =
Java_DecodedOutputBuffer_getDecodeTimeMs(jni, j_decoder_output_buffer);
RTC_CHECK_GE(slice_height, height);
if (output_buffer_size < width * height * 3 / 2) {
ALOGE << "Insufficient output buffer size: " << output_buffer_size;
return false;
}
if (output_buffer_size < stride * height * 3 / 2 &&
slice_height == height && stride > width) {
// Some codecs (Exynos) incorrectly report stride information for
// output byte buffer, so actual stride value need to be corrected.
stride = output_buffer_size * 2 / (height * 3);
}
ScopedJavaLocalRef<jobjectArray> output_buffers =
Java_MediaCodecVideoDecoder_getOutputBuffers(
jni, j_media_codec_video_decoder_);
jobject output_buffer =
jni->GetObjectArrayElement(output_buffers.obj(), output_buffer_index);
uint8_t* payload =
reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(output_buffer));
if (CheckException(jni)) {
return false;
}
payload += output_buffer_offset;
// Create yuv420 frame.
rtc::scoped_refptr<I420Buffer> i420_buffer =
decoded_frame_pool_.CreateBuffer(width, height);
if (color_format == COLOR_FormatYUV420Planar) {
RTC_CHECK_EQ(0, stride % 2);
const int uv_stride = stride / 2;
const uint8_t* y_ptr = payload;
const uint8_t* u_ptr = y_ptr + stride * slice_height;
// Note that the case with odd |slice_height| is handled in a special way.
// The chroma height contained in the payload is rounded down instead of
// up, making it one row less than what we expect in WebRTC. Therefore, we
// have to duplicate the last chroma rows for this case. Also, the offset
// between the Y plane and the U plane is unintuitive for this case. See
// http://bugs.webrtc.org/6651 for more info.
const int chroma_width = (width + 1) / 2;
const int chroma_height =
(slice_height % 2 == 0) ? (height + 1) / 2 : height / 2;
const int u_offset = uv_stride * slice_height / 2;
const uint8_t* v_ptr = u_ptr + u_offset;
libyuv::CopyPlane(y_ptr, stride, i420_buffer->MutableDataY(),
i420_buffer->StrideY(), width, height);
libyuv::CopyPlane(u_ptr, uv_stride, i420_buffer->MutableDataU(),
i420_buffer->StrideU(), chroma_width, chroma_height);
libyuv::CopyPlane(v_ptr, uv_stride, i420_buffer->MutableDataV(),
i420_buffer->StrideV(), chroma_width, chroma_height);
if (slice_height % 2 == 1) {
RTC_CHECK_EQ(height, slice_height);
// Duplicate the last chroma rows.
uint8_t* u_last_row_ptr = i420_buffer->MutableDataU() +
chroma_height * i420_buffer->StrideU();
memcpy(u_last_row_ptr, u_last_row_ptr - i420_buffer->StrideU(),
i420_buffer->StrideU());
uint8_t* v_last_row_ptr = i420_buffer->MutableDataV() +
chroma_height * i420_buffer->StrideV();
memcpy(v_last_row_ptr, v_last_row_ptr - i420_buffer->StrideV(),
i420_buffer->StrideV());
}
} else {
// All other supported formats are nv12.
const uint8_t* y_ptr = payload;
const uint8_t* uv_ptr = y_ptr + stride * slice_height;
libyuv::NV12ToI420(y_ptr, stride, uv_ptr, stride,
i420_buffer->MutableDataY(), i420_buffer->StrideY(),
i420_buffer->MutableDataU(), i420_buffer->StrideU(),
i420_buffer->MutableDataV(), i420_buffer->StrideV(),
width, height);
}
frame_buffer = i420_buffer;
// Return output byte buffer back to codec.
Java_MediaCodecVideoDecoder_returnDecodedOutputBuffer(
jni, j_media_codec_video_decoder_, output_buffer_index);
if (CheckException(jni)) {
ALOGE << "returnDecodedOutputBuffer error";
return false;
}
}
if (frames_decoded_ < frames_decoded_logged_) {
ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width << " x "
<< height << ". Color: " << color_format
<< ". TS: " << presentation_timestamps_ms
<< ". DecTime: " << static_cast<int>(decode_time_ms)
<< ". DelayTime: " << static_cast<int>(frame_delayed_ms);
}
// Calculate and print decoding statistics - every 3 seconds.
frames_decoded_++;
current_frames_++;
current_decoding_time_ms_ += decode_time_ms;
current_delay_time_ms_ += frame_delayed_ms;
int statistic_time_ms = rtc::TimeMillis() - start_time_ms_;
if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
current_frames_ > 0) {
int current_bitrate = current_bytes_ * 8 / statistic_time_ms;
int current_fps =
(current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms;
ALOGD << "Frames decoded: " << frames_decoded_
<< ". Received: " << frames_received_
<< ". Bitrate: " << current_bitrate << " kbps"
<< ". Fps: " << current_fps
<< ". DecTime: " << (current_decoding_time_ms_ / current_frames_)
<< ". DelayTime: " << (current_delay_time_ms_ / current_frames_)
<< " for last " << statistic_time_ms << " ms.";
start_time_ms_ = rtc::TimeMillis();
current_frames_ = 0;
current_bytes_ = 0;
current_decoding_time_ms_ = 0;
current_delay_time_ms_ = 0;
}
// If the frame was dropped, frame_buffer is left as nullptr.
if (frame_buffer) {
VideoFrame decoded_frame(frame_buffer, 0, 0, kVideoRotation_0);
decoded_frame.set_timestamp(output_timestamps_ms);
decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms);
absl::optional<uint8_t> qp = pending_frame_qps_.front();
pending_frame_qps_.pop_front();
callback_->Decoded(decoded_frame, decode_time_ms, qp);
}
return true;
}
int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
if (!inited_) {
return;
}
// We only ever send one message to |this| directly (not through a Bind()'d
// functor), so expect no ID/data.
RTC_CHECK(!msg->message_id) << "Unexpected message!";
RTC_CHECK(!msg->pdata) << "Unexpected message!";
CheckOnCodecThread();
if (!DeliverPendingOutputs(jni, 0)) {
ALOGE << "OnMessage: DeliverPendingOutputs error";
ProcessHWErrorOnCodecThread();
return;
}
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
}
MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory()
: egl_context_(nullptr) {
ALOGD << "MediaCodecVideoDecoderFactory ctor";
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
supported_formats_.clear();
if (Java_MediaCodecVideoDecoder_isVp8HwSupported(jni) &&
!CheckException(jni)) {
ALOGD << "VP8 HW Decoder supported.";
supported_formats_.push_back(SdpVideoFormat(cricket::kVp8CodecName));
}
if (Java_MediaCodecVideoDecoder_isVp9HwSupported(jni) &&
!CheckException(jni)) {
ALOGD << "VP9 HW Decoder supported.";
supported_formats_.push_back(SdpVideoFormat(cricket::kVp9CodecName));
}
if (Java_MediaCodecVideoDecoder_isH264HwSupported(jni) &&
!CheckException(jni)) {
ALOGD << "H264 HW Decoder supported.";
supported_formats_.push_back(SdpVideoFormat(cricket::kH264CodecName));
}
}
MediaCodecVideoDecoderFactory::~MediaCodecVideoDecoderFactory() {
ALOGD << "MediaCodecVideoDecoderFactory dtor";
if (egl_context_) {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
jni->DeleteGlobalRef(egl_context_);
}
}
void MediaCodecVideoDecoderFactory::SetEGLContext(JNIEnv* jni,
jobject egl_context) {
ALOGD << "MediaCodecVideoDecoderFactory::SetEGLContext";
if (egl_context_) {
jni->DeleteGlobalRef(egl_context_);
egl_context_ = nullptr;
}
egl_context_ = jni->NewGlobalRef(egl_context);
if (CheckException(jni)) {
ALOGE << "error calling NewGlobalRef for EGL Context.";
}
}
std::vector<SdpVideoFormat> MediaCodecVideoDecoderFactory::GetSupportedFormats()
const {
return supported_formats_;
}
std::unique_ptr<VideoDecoder> MediaCodecVideoDecoderFactory::CreateVideoDecoder(
const SdpVideoFormat& format) {
if (supported_formats_.empty()) {
ALOGW << "No HW video decoder for type " << format.name;
return nullptr;
}
for (SdpVideoFormat supported_format : supported_formats_) {
if (supported_format == format) {
ALOGD << "Create HW video decoder for type " << format.name;
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
return rtc::MakeUnique<MediaCodecVideoDecoder>(
jni, PayloadStringToCodecType(format.name), egl_context_);
}
}
ALOGW << "Can not find HW video decoder for type " << format.name;
return nullptr;
}
bool MediaCodecVideoDecoderFactory::IsH264HighProfileSupported(JNIEnv* env) {
return Java_MediaCodecVideoDecoder_isH264HighProfileHwSupported(env);
}
const char* MediaCodecVideoDecoder::ImplementationName() const {
return "MediaCodec";
}
} // namespace jni
} // namespace webrtc
Reference in a new issue View git blame Copy permalink