webrtc/modules/video_coding/generic_decoder.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/video_coding/generic_decoder.h"

#include <stddef.h>

#include <algorithm>
#include <cmath>

#include "api/video/video_timing.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/thread.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/clock.h"
#include "system_wrappers/include/field_trial.h"

namespace webrtc {

VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing,
                                                 Clock* clock)
    : _clock(clock),
      _timing(timing),
      _timestampMap(kDecoderFrameMemoryLength),
      _extra_decode_time("t", absl::nullopt),
      low_latency_renderer_enabled_("enabled", true),
      low_latency_renderer_include_predecode_buffer_("include_predecode_buffer",
                                                     true) {
  ntp_offset_ =
      _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();

  ParseFieldTrial({&_extra_decode_time},
                  field_trial::FindFullName("WebRTC-SlowDownDecoder"));
  ParseFieldTrial({&low_latency_renderer_enabled_,
                   &low_latency_renderer_include_predecode_buffer_},
                  field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
}

VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {}

void VCMDecodedFrameCallback::SetUserReceiveCallback(
    VCMReceiveCallback* receiveCallback) {
  RTC_DCHECK(construction_thread_.IsCurrent());
  RTC_DCHECK((!_receiveCallback && receiveCallback) ||
             (_receiveCallback && !receiveCallback));
  _receiveCallback = receiveCallback;
}

VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
  // Called on the decode thread via VCMCodecDataBase::GetDecoder.
  // The callback must always have been set before this happens.
  RTC_DCHECK(_receiveCallback);
  return _receiveCallback;
}

int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) {
  // This function may be called on the decode TaskQueue, but may also be called
  // on an OS provided queue such as on iOS (see e.g. b/153465112).
  return Decoded(decodedImage, -1);
}

int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
                                         int64_t decode_time_ms) {
  Decoded(decodedImage,
          decode_time_ms >= 0 ? absl::optional<int32_t>(decode_time_ms)
                              : absl::nullopt,
          absl::nullopt);
  return WEBRTC_VIDEO_CODEC_OK;
}

void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
                                      absl::optional<int32_t> decode_time_ms,
                                      absl::optional<uint8_t> qp) {
  // Wait some extra time to simulate a slow decoder.
  if (_extra_decode_time) {
    rtc::Thread::SleepMs(_extra_decode_time->ms());
  }

  RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
  TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
                       "timestamp", decodedImage.timestamp());
  // TODO(holmer): We should improve this so that we can handle multiple
  // callbacks from one call to Decode().
  absl::optional<VCMFrameInformation> frameInfo;
  int timestamp_map_size = 0;
  int dropped_frames = 0;
  {
    MutexLock lock(&lock_);
    int initial_timestamp_map_size = _timestampMap.Size();
    frameInfo = _timestampMap.Pop(decodedImage.timestamp());
    timestamp_map_size = _timestampMap.Size();
    // _timestampMap.Pop() erases all frame upto the specified timestamp and
    // return the frame info for this timestamp if it exists. Thus, the
    // difference in the _timestampMap size before and after Pop() will show
    // internally dropped frames.
    dropped_frames =
        initial_timestamp_map_size - timestamp_map_size - (frameInfo ? 1 : 0);
  }

  if (dropped_frames > 0) {
    _receiveCallback->OnDroppedFrames(dropped_frames);
  }

  if (!frameInfo) {
    RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
                           "this one.";
    return;
  }

  decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
  decodedImage.set_packet_infos(frameInfo->packet_infos);
  decodedImage.set_rotation(frameInfo->rotation);

  if (low_latency_renderer_enabled_) {
    absl::optional<int> max_composition_delay_in_frames =
        _timing->MaxCompositionDelayInFrames();
    if (max_composition_delay_in_frames) {
      // Subtract frames that are in flight.
      if (low_latency_renderer_include_predecode_buffer_) {
        *max_composition_delay_in_frames -= timestamp_map_size;
        *max_composition_delay_in_frames =
            std::max(0, *max_composition_delay_in_frames);
      }
      decodedImage.set_max_composition_delay_in_frames(
          max_composition_delay_in_frames);
    }
  }

  RTC_DCHECK(frameInfo->decodeStart);
  const Timestamp now = _clock->CurrentTime();
  const TimeDelta decode_time = decode_time_ms
                                    ? TimeDelta::Millis(*decode_time_ms)
                                    : now - *frameInfo->decodeStart;
  _timing->StopDecodeTimer(decode_time.ms(), now.ms());
  decodedImage.set_processing_time(
      {*frameInfo->decodeStart, *frameInfo->decodeStart + decode_time});

  // Report timing information.
  TimingFrameInfo timing_frame_info;
  if (frameInfo->timing.flags != VideoSendTiming::kInvalid) {
    int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
    // Convert remote timestamps to local time from ntp timestamps.
    frameInfo->timing.encode_start_ms -= ntp_offset_;
    frameInfo->timing.encode_finish_ms -= ntp_offset_;
    frameInfo->timing.packetization_finish_ms -= ntp_offset_;
    frameInfo->timing.pacer_exit_ms -= ntp_offset_;
    frameInfo->timing.network_timestamp_ms -= ntp_offset_;
    frameInfo->timing.network2_timestamp_ms -= ntp_offset_;

    int64_t sender_delta_ms = 0;
    if (decodedImage.ntp_time_ms() < 0) {
      // Sender clock is not estimated yet. Make sure that sender times are all
      // negative to indicate that. Yet they still should be relatively correct.
      sender_delta_ms =
          std::max({capture_time_ms, frameInfo->timing.encode_start_ms,
                    frameInfo->timing.encode_finish_ms,
                    frameInfo->timing.packetization_finish_ms,
                    frameInfo->timing.pacer_exit_ms,
                    frameInfo->timing.network_timestamp_ms,
                    frameInfo->timing.network2_timestamp_ms}) +
          1;
    }

    timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms;
    timing_frame_info.encode_start_ms =
        frameInfo->timing.encode_start_ms - sender_delta_ms;
    timing_frame_info.encode_finish_ms =
        frameInfo->timing.encode_finish_ms - sender_delta_ms;
    timing_frame_info.packetization_finish_ms =
        frameInfo->timing.packetization_finish_ms - sender_delta_ms;
    timing_frame_info.pacer_exit_ms =
        frameInfo->timing.pacer_exit_ms - sender_delta_ms;
    timing_frame_info.network_timestamp_ms =
        frameInfo->timing.network_timestamp_ms - sender_delta_ms;
    timing_frame_info.network2_timestamp_ms =
        frameInfo->timing.network2_timestamp_ms - sender_delta_ms;
  }

  timing_frame_info.flags = frameInfo->timing.flags;
  timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms();
  timing_frame_info.decode_finish_ms = now.ms();
  timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
  timing_frame_info.rtp_timestamp = decodedImage.timestamp();
  timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
  timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms;
  _timing->SetTimingFrameInfo(timing_frame_info);

  decodedImage.set_timestamp_us(frameInfo->renderTimeMs *
                                rtc::kNumMicrosecsPerMillisec);
  _receiveCallback->FrameToRender(decodedImage, qp, decode_time.ms(),
                                  frameInfo->content_type);
}

void VCMDecodedFrameCallback::OnDecoderImplementationName(
    const char* implementation_name) {
  _receiveCallback->OnDecoderImplementationName(implementation_name);
}

void VCMDecodedFrameCallback::Map(uint32_t timestamp,
                                  const VCMFrameInformation& frameInfo) {
  int dropped_frames = 0;
  {
    MutexLock lock(&lock_);
    int initial_size = _timestampMap.Size();
    _timestampMap.Add(timestamp, frameInfo);
    // If no frame is dropped, the new size should be |initial_size| + 1
    dropped_frames = (initial_size + 1) - _timestampMap.Size();
  }
  if (dropped_frames > 0) {
    _receiveCallback->OnDroppedFrames(dropped_frames);
  }
}

void VCMDecodedFrameCallback::ClearTimestampMap() {
  int dropped_frames = 0;
  {
    MutexLock lock(&lock_);
    dropped_frames = _timestampMap.Size();
    _timestampMap.Clear();
  }
  if (dropped_frames > 0) {
    _receiveCallback->OnDroppedFrames(dropped_frames);
  }
}

VCMGenericDecoder::VCMGenericDecoder(std::unique_ptr<VideoDecoder> decoder)
    : VCMGenericDecoder(decoder.release(), false /* isExternal */) {}

VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal)
    : _callback(NULL),
      decoder_(decoder),
      _codecType(kVideoCodecGeneric),
      _isExternal(isExternal),
      _last_keyframe_content_type(VideoContentType::UNSPECIFIED) {
  RTC_DCHECK(decoder_);
}

VCMGenericDecoder::~VCMGenericDecoder() {
  decoder_->Release();
  if (_isExternal)
    decoder_.release();
  RTC_DCHECK(_isExternal || decoder_);
}

int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings,
                                      int32_t numberOfCores) {
  TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode");
  _codecType = settings->codecType;

  int err = decoder_->InitDecode(settings, numberOfCores);
  decoder_info_ = decoder_->GetDecoderInfo();
  RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString();
  if (_callback) {
    _callback->OnDecoderImplementationName(
        decoder_info_.implementation_name.c_str());
  }
  return err;
}

int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
  TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
               frame.Timestamp());
  VCMFrameInformation frame_info;
  frame_info.decodeStart = now;
  frame_info.renderTimeMs = frame.RenderTimeMs();
  frame_info.rotation = frame.rotation();
  frame_info.timing = frame.video_timing();
  frame_info.ntp_time_ms = frame.EncodedImage().ntp_time_ms_;
  frame_info.packet_infos = frame.PacketInfos();

  // Set correctly only for key frames. Thus, use latest key frame
  // content type. If the corresponding key frame was lost, decode will fail
  // and content type will be ignored.
  if (frame.FrameType() == VideoFrameType::kVideoFrameKey) {
    frame_info.content_type = frame.contentType();
    _last_keyframe_content_type = frame.contentType();
  } else {
    frame_info.content_type = _last_keyframe_content_type;
  }
  _callback->Map(frame.Timestamp(), frame_info);

  int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
                                 frame.RenderTimeMs());
  VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo();
  if (decoder_info != decoder_info_) {
    RTC_LOG(LS_INFO) << "Changed decoder implementation to: "
                     << decoder_info.ToString();
    decoder_info_ = decoder_info;
    _callback->OnDecoderImplementationName(
        decoder_info.implementation_name.empty()
            ? "unknown"
            : decoder_info.implementation_name.c_str());
  }
  if (ret < WEBRTC_VIDEO_CODEC_OK) {
    RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp "
                        << frame.Timestamp() << ", error code: " << ret;
    _callback->ClearTimestampMap();
  } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) {
    // No output.
    _callback->ClearTimestampMap();
  }
  return ret;
}

int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
    VCMDecodedFrameCallback* callback) {
  _callback = callback;
  int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
  if (callback && !decoder_info_.implementation_name.empty()) {
    callback->OnDecoderImplementationName(
        decoder_info_.implementation_name.c_str());
  }
  return ret;
}

}  // namespace webrtc