mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove multiplex codec.

Browse source 
The feature isn't in use by Google and has proven to contain security
issues. It's time to remove it.

Bug: b/324864439
Change-Id: I80344eb2f2060469d2d69a54dc4519fdd02ab4ea
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/340324
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41808}
This commit is contained in:
Markus Handell 2024-02-23 12:12:54 +01:00 committed by WebRTC LUCI CQ
parent db2f52ba88
commit 97df932ecc
54 changed files with 5 additions and 3922 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/video/video_codec_type.h
View file

@ -21,7 +21,6 @@ enum VideoCodecType {
kVideoCodecVP9,
kVideoCodecAV1,
kVideoCodecH264,
kVideoCodecMultiplex,
kVideoCodecH265,
};

5
api/video_codecs/video_codec.cc
View file

@ -28,7 +28,6 @@ constexpr char kPayloadNameAv1[] = "AV1";
constexpr char kPayloadNameAv1x[] = "AV1X";
constexpr char kPayloadNameH264[] = "H264";
constexpr char kPayloadNameGeneric[] = "Generic";
constexpr char kPayloadNameMultiplex[] = "Multiplex";
constexpr char kPayloadNameH265[] = "H265";
} // namespace
@ -153,8 +152,6 @@ const char* CodecTypeToPayloadString(VideoCodecType type) {
return kPayloadNameAv1;
case kVideoCodecH264:
return kPayloadNameH264;
case kVideoCodecMultiplex:
return kPayloadNameMultiplex;
case kVideoCodecGeneric:
return kPayloadNameGeneric;
case kVideoCodecH265:
@ -173,8 +170,6 @@ VideoCodecType PayloadStringToCodecType(const std::string& name) {
return kVideoCodecAV1;
if (absl::EqualsIgnoreCase(name, kPayloadNameH264))
return kVideoCodecH264;
if (absl::EqualsIgnoreCase(name, kPayloadNameMultiplex))
return kVideoCodecMultiplex;
if (absl::EqualsIgnoreCase(name, kPayloadNameH265))
return kVideoCodecH265;
return kVideoCodecGeneric;

4
api/video_codecs/video_decoder_software_fallback_wrapper.cc
View file

@ -171,10 +171,6 @@ void VideoDecoderSoftwareFallbackWrapper::UpdateFallbackDecoderHistograms() {
RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "H264",
hw_decoded_frames_since_last_fallback_);
break;
case kVideoCodecMultiplex:
RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Multiplex",
hw_decoded_frames_since_last_fallback_);
break;
case kVideoCodecH265:
RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "H265",
hw_decoded_frames_since_last_fallback_);

4
call/rtp_payload_params.cc
View file

@ -96,7 +96,6 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
info.codecSpecific.H264.packetization_mode;
return;
}
case kVideoCodecMultiplex:
case kVideoCodecGeneric:
rtp->codec = kVideoCodecGeneric;
return;
@ -340,8 +339,6 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info,
is_keyframe, rtp_video_header);
}
return;
case VideoCodecType::kVideoCodecMultiplex:
return;
case VideoCodecType::kVideoCodecH265:
// TODO(bugs.webrtc.org/13485): Implement H265 to generic descriptor.
return;
@ -407,7 +404,6 @@ absl::optional<FrameDependencyStructure> RtpPayloadParams::GenericStructure(
case VideoCodecType::kVideoCodecAV1:
case VideoCodecType::kVideoCodecH264:
case VideoCodecType::kVideoCodecH265:
case VideoCodecType::kVideoCodecMultiplex:
return absl::nullopt;
}
RTC_DCHECK_NOTREACHED() << "Unsupported codec.";

91
examples/BUILD.gn
View file

@ -60,10 +60,6 @@ group("examples") {
deps += [ ":peerconnection_client" ]
}
}
if (is_android || is_win) {
deps += [ ":webrtc_unity_plugin" ]
}
}
rtc_library("read_auth_file") {
@ -838,94 +834,7 @@ if (is_linux || is_chromeos || is_win) {
}
}
if (is_win || is_android) {
rtc_shared_library("webrtc_unity_plugin") {
testonly = true
sources = [
"unityplugin/simple_peer_connection.cc",
"unityplugin/simple_peer_connection.h",
"unityplugin/unity_plugin_apis.cc",
"unityplugin/unity_plugin_apis.h",
"unityplugin/video_observer.cc",
"unityplugin/video_observer.h",
]
if (is_android) {
sources += [
"unityplugin/class_reference_holder.cc",
"unityplugin/class_reference_holder.h",
"unityplugin/jni_onload.cc",
]
suppressed_configs += [ "//build/config/android:hide_all_but_jni_onload" ]
}
if (is_win) {
configs += [ "//build/config/win:windowed" ]
}
deps = [
"../api:create_peerconnection_factory",
"../api:libjingle_peerconnection_api",
"../api:media_stream_interface",
"../api/audio_codecs:builtin_audio_decoder_factory",
"../api/audio_codecs:builtin_audio_encoder_factory",
"../api/video:video_frame",
"../api/video:video_rtp_headers",
"../media:rtc_audio_video",
"../media:rtc_internal_video_codecs",
"../media:rtc_media",
"../media:rtc_media_base",
"../modules/audio_device",
"../modules/audio_processing",
"../modules/audio_processing:api",
"../modules/video_capture:video_capture_module",
"../pc:libjingle_peerconnection",
"../pc:video_track_source",
"../rtc_base:ssl",
"../test:platform_video_capturer",
"../test:video_test_common",
"//third_party/abseil-cpp/absl/memory",
]
if (is_android) {
deps += [
"../modules/utility",
"../sdk/android:libjingle_peerconnection_jni",
"../sdk/android:native_api_jni",
]
}
}
}
if (is_android) {
rtc_android_library("webrtc_unity_java") {
sources = [ "unityplugin/java/src/org/webrtc/UnityUtility.java" ]
deps = [
"../rtc_base:base_java",
"../sdk/android:camera_java",
"../sdk/android:libjingle_peerconnection_java",
"../sdk/android:peerconnection_java",
"../sdk/android:video_api_java",
"../sdk/android:video_java",
"//third_party/androidx:androidx_annotation_annotation_java",
]
}
# TODO(https://bugs.webrtc.org/15095) - Fix or remove this target.
#dist_jar("libwebrtc_unity") {
# _target_dir_name = get_label_info(":$target_name", "dir")
# output = "${root_out_dir}/lib.java${_target_dir_name}/${target_name}.jar"
# direct_deps_only = false
# use_interface_jars = false
# use_unprocessed_jars = false
# requires_android = true
# deps = [
# ":webrtc_unity_java",
# "../rtc_base:base_java",
# "../sdk/android:libjingle_peerconnection_java",
# "../sdk/android:libjingle_peerconnection_metrics_default_java",
# "//third_party/androidx:androidx_annotation_annotation_java",
# ]
#}
robolectric_binary("android_examples_junit_tests") {
sources = [
"androidjunit/src/org/appspot/apprtc/BluetoothManagerTest.java",

33
examples/unityplugin/ANDROID_INSTRUCTION
View file

@ -1,33 +0,0 @@
Instruction of running webrtc_unity_plugin on Android Unity
1. On Linux machine, compile target webrtc_unity_plugin.
Checkout WebRTC codebase: fetch --nohooks webrtc_android
If you already have a checkout for linux, add target_os=”android” into .gclient file.
Run gclient sync
Run gn args out/Android, and again set target_os=”android” in the args.gn
Run ninja -C out/Android webrtc_unity_plugin
2. On Linux machine, build target libwebrtc_unity under webrtc checkout. This is the java code for webrtc to work on Android.
3. Copy libwebrtc_unity.jar and libwebrtc_unity_plugin.so into Unity project folder, under Assets/Plugins/Android folder.
4. Rename libwebrtc_unity_plugin.so to libjingle_peerconnection_so.so. This is hacky, and the purpose is to let the java code in libwebrtc_unity.jar to find their JNI implementations. Simultaneously, in your C# wrapper script for the native plugin libjingle_peerconnection_so.so, the dll_path should be set to “jingle_peerconnection_so”.
5. In the Unity Main Scene’s Start method, write the following code to initialize the Java environment for webrtc (otherwise, webrtc will not be able to access audio device or camera from C++ code):
#if UNITY_ANDROID
AndroidJavaClass playerClass = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
AndroidJavaObject activity = playerClass.GetStatic<AndroidJavaObject>("currentActivity");
AndroidJavaClass utilityClass = new AndroidJavaClass("org.webrtc.UnityUtility");
utilityClass.CallStatic("InitializePeerConncectionFactory", new object[1] { activity });
#endif
6. Compile the unity project into an APK, and decompile the apk using apktool that you can download from https://ibotpeaches.github.io/Apktool/
Run apktool d apkname.apk.
Then copy the AndroidManifest.xml in the decompiled folder to the Assets/Plugins/Android folder, and add two lines:
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.CAMERA" />
The purpose of using apktool is to get a well-written android manifest xml file. If you know how to write manifest file from scratch, you can skip using apktool.
7. Compile the unity project into an APK again and deploy it to an android device.

4
examples/unityplugin/DEPS
View file

@ -1,4 +0,0 @@
include_rules = [
"+modules/utility",
"+sdk",
]

309
examples/unityplugin/README
View file

@ -1,309 +0,0 @@
This directory contains an example Unity native plugin for Windows OS and Android.
The APIs use Platform Invoke (P/Invoke) technology as required by Unity native plugin.
This plugin dll can also be used by Windows C# applications other than Unity.
For detailed build instruction on Android, see ANDROID_INSTRUCTION
An example of wrapping native plugin into a C# managed class in Unity is given as following:
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
namespace SimplePeerConnectionM {
// A class for ice candidate.
public class IceCandidate {
public IceCandidate(string candidate, int sdpMlineIndex, string sdpMid) {
mCandidate = candidate;
mSdpMlineIndex = sdpMlineIndex;
mSdpMid = sdpMid;
}
string mCandidate;
int mSdpMlineIndex;
string mSdpMid;
public string Candidate {
get { return mCandidate; }
set { mCandidate = value; }
}
public int SdpMlineIndex {
get { return mSdpMlineIndex; }
set { mSdpMlineIndex = value; }
}
public string SdpMid {
get { return mSdpMid; }
set { mSdpMid = value; }
}
}
// A managed wrapper up class for the native c style peer connection APIs.
public class PeerConnectionM {
private const string dllPath = "webrtc_unity_plugin";
//create a peerconnection with turn servers
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern int CreatePeerConnection(string[] turnUrls, int noOfUrls,
string username, string credential);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool ClosePeerConnection(int peerConnectionId);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool AddStream(int peerConnectionId, bool audioOnly);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool AddDataChannel(int peerConnectionId);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool CreateOffer(int peerConnectionId);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool CreateAnswer(int peerConnectionId);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool SendDataViaDataChannel(int peerConnectionId, string data);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool SetAudioControl(int peerConnectionId, bool isMute, bool isRecord);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void LocalDataChannelReadyInternalDelegate();
public delegate void LocalDataChannelReadyDelegate(int id);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnLocalDataChannelReady(
int peerConnectionId, LocalDataChannelReadyInternalDelegate callback);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void DataFromDataChannelReadyInternalDelegate(string s);
public delegate void DataFromDataChannelReadyDelegate(int id, string s);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnDataFromDataChannelReady(
int peerConnectionId, DataFromDataChannelReadyInternalDelegate callback);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void FailureMessageInternalDelegate(string msg);
public delegate void FailureMessageDelegate(int id, string msg);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnFailure(int peerConnectionId,
FailureMessageInternalDelegate callback);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void AudioBusReadyInternalDelegate(IntPtr data, int bitsPerSample,
int sampleRate, int numberOfChannels, int numberOfFrames);
public delegate void AudioBusReadyDelegate(int id, IntPtr data, int bitsPerSample,
int sampleRate, int numberOfChannels, int numberOfFrames);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnAudioBusReady(int peerConnectionId,
AudioBusReadyInternalDelegate callback);
// Video callbacks.
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void I420FrameReadyInternalDelegate(
IntPtr dataY, IntPtr dataU, IntPtr dataV,
int strideY, int strideU, int strideV,
uint width, uint height);
public delegate void I420FrameReadyDelegate(int id,
IntPtr dataY, IntPtr dataU, IntPtr dataV,
int strideY, int strideU, int strideV,
uint width, uint height);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnLocalI420FrameReady(int peerConnectionId,
I420FrameReadyInternalDelegate callback);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnRemoteI420FrameReady(int peerConnectionId,
I420FrameReadyInternalDelegate callback);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void LocalSdpReadytoSendInternalDelegate(string type, string sdp);
public delegate void LocalSdpReadytoSendDelegate(int id, string type, string sdp);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnLocalSdpReadytoSend(int peerConnectionId,
LocalSdpReadytoSendInternalDelegate callback);
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate void IceCandidateReadytoSendInternalDelegate(
string candidate, int sdpMlineIndex, string sdpMid);
public delegate void IceCandidateReadytoSendDelegate(
int id, string candidate, int sdpMlineIndex, string sdpMid);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RegisterOnIceCandidateReadytoSend(
int peerConnectionId, IceCandidateReadytoSendInternalDelegate callback);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool SetRemoteDescription(int peerConnectionId, string type, string sdp);
[DllImport(dllPath, CallingConvention = CallingConvention.Cdecl)]
private static extern bool AddIceCandidate(int peerConnectionId, string sdp,
int sdpMlineindex, string sdpMid);
public PeerConnectionM(List<string> turnUrls, string username, string credential) {
string[] urls = turnUrls != null ? turnUrls.ToArray() : null;
int length = turnUrls != null ? turnUrls.Count : 0;
mPeerConnectionId = CreatePeerConnection(urls, length, username, credential);
RegisterCallbacks();
}
public void ClosePeerConnection() {
ClosePeerConnection(mPeerConnectionId);
mPeerConnectionId = -1;
}
// Return -1 if Peerconnection is not available.
public int GetUniqueId() {
return mPeerConnectionId;
}
public void AddStream(bool audioOnly) {
AddStream(mPeerConnectionId, audioOnly);
}
public void AddDataChannel() {
AddDataChannel(mPeerConnectionId);
}
public void CreateOffer() {
CreateOffer(mPeerConnectionId);
}
public void CreateAnswer() {
CreateAnswer(mPeerConnectionId);
}
public void SendDataViaDataChannel(string data) {
SendDataViaDataChannel(mPeerConnectionId, data);
}
public void SetAudioControl(bool isMute, bool isRecord) {
SetAudioControl(mPeerConnectionId, isMute, isRecord);
}
public void SetRemoteDescription(string type, string sdp) {
SetRemoteDescription(mPeerConnectionId, type, sdp);
}
public void AddIceCandidate(string candidate, int sdpMlineindex, string sdpMid) {
AddIceCandidate(mPeerConnectionId, candidate, sdpMlineindex, sdpMid);
}
private void RegisterCallbacks() {
localDataChannelReadyDelegate = new LocalDataChannelReadyInternalDelegate(
RaiseLocalDataChannelReady);
RegisterOnLocalDataChannelReady(mPeerConnectionId, localDataChannelReadyDelegate);
dataFromDataChannelReadyDelegate = new DataFromDataChannelReadyInternalDelegate(
RaiseDataFromDataChannelReady);
RegisterOnDataFromDataChannelReady(mPeerConnectionId, dataFromDataChannelReadyDelegate);
failureMessageDelegate = new FailureMessageInternalDelegate(RaiseFailureMessage);
RegisterOnFailure(mPeerConnectionId, failureMessageDelegate);
audioBusReadyDelegate = new AudioBusReadyInternalDelegate(RaiseAudioBusReady);
RegisterOnAudioBusReady(mPeerConnectionId, audioBusReadyDelegate);
localI420FrameReadyDelegate = new I420FrameReadyInternalDelegate(
RaiseLocalVideoFrameReady);
RegisterOnLocalI420FrameReady(mPeerConnectionId, localI420FrameReadyDelegate);
remoteI420FrameReadyDelegate = new I420FrameReadyInternalDelegate(
RaiseRemoteVideoFrameReady);
RegisterOnRemoteI420FrameReady(mPeerConnectionId, remoteI420FrameReadyDelegate);
localSdpReadytoSendDelegate = new LocalSdpReadytoSendInternalDelegate(
RaiseLocalSdpReadytoSend);
RegisterOnLocalSdpReadytoSend(mPeerConnectionId, localSdpReadytoSendDelegate);
iceCandidateReadytoSendDelegate =
new IceCandidateReadytoSendInternalDelegate(RaiseIceCandidateReadytoSend);
RegisterOnIceCandidateReadytoSend(
mPeerConnectionId, iceCandidateReadytoSendDelegate);
}
private void RaiseLocalDataChannelReady() {
if (OnLocalDataChannelReady != null)
OnLocalDataChannelReady(mPeerConnectionId);
}
private void RaiseDataFromDataChannelReady(string data) {
if (OnDataFromDataChannelReady != null)
OnDataFromDataChannelReady(mPeerConnectionId, data);
}
private void RaiseFailureMessage(string msg) {
if (OnFailureMessage != null)
OnFailureMessage(mPeerConnectionId, msg);
}
private void RaiseAudioBusReady(IntPtr data, int bitsPerSample,
int sampleRate, int numberOfChannels, int numberOfFrames) {
if (OnAudioBusReady != null)
OnAudioBusReady(mPeerConnectionId, data, bitsPerSample, sampleRate,
numberOfChannels, numberOfFrames);
}
private void RaiseLocalVideoFrameReady(
IntPtr dataY, IntPtr dataU, IntPtr dataV,
int strideY, int strideU, int strideV,
uint width, uint height) {
if (OnLocalVideoFrameReady != null)
OnLocalVideoFrameReady(mPeerConnectionId, dataY, dataU, dataV, strideY, strideU, strideV,
width, height);
}
private void RaiseRemoteVideoFrameReady(
IntPtr dataY, IntPtr dataU, IntPtr dataV,
int strideY, int strideU, int strideV,
uint width, uint height) {
if (OnRemoteVideoFrameReady != null)
OnRemoteVideoFrameReady(mPeerConnectionId, dataY, dataU, dataV, strideY, strideU, strideV,
width, height);
}
private void RaiseLocalSdpReadytoSend(string type, string sdp) {
if (OnLocalSdpReadytoSend != null)
OnLocalSdpReadytoSend(mPeerConnectionId, type, sdp);
}
private void RaiseIceCandidateReadytoSend(string candidate, int sdpMlineIndex, string sdpMid) {
if (OnIceCandidateReadytoSend != null)
OnIceCandidateReadytoSend(mPeerConnectionId, candidate, sdpMlineIndex, sdpMid);
}
public void AddQueuedIceCandidate(List<IceCandidate> iceCandidateQueue) {
if (iceCandidateQueue != null) {
foreach (IceCandidate ic in iceCandidateQueue) {
AddIceCandidate(mPeerConnectionId, ic.Candidate, ic.SdpMlineIndex, ic.SdpMid);
}
}
}
private LocalDataChannelReadyInternalDelegate localDataChannelReadyDelegate = null;
public event LocalDataChannelReadyDelegate OnLocalDataChannelReady;
private DataFromDataChannelReadyInternalDelegate dataFromDataChannelReadyDelegate = null;
public event DataFromDataChannelReadyDelegate OnDataFromDataChannelReady;
private FailureMessageInternalDelegate failureMessageDelegate = null;
public event FailureMessageDelegate OnFailureMessage;
private AudioBusReadyInternalDelegate audioBusReadyDelegate = null;
public event AudioBusReadyDelegate OnAudioBusReady;
private I420FrameReadyInternalDelegate localI420FrameReadyDelegate = null;
public event I420FrameReadyDelegate OnLocalVideoFrameReady;
private I420FrameReadyInternalDelegate remoteI420FrameReadyDelegate = null;
public event I420FrameReadyDelegate OnRemoteVideoFrameReady;
private LocalSdpReadytoSendInternalDelegate localSdpReadytoSendDelegate = null;
public event LocalSdpReadytoSendDelegate OnLocalSdpReadytoSend;
private IceCandidateReadytoSendInternalDelegate iceCandidateReadytoSendDelegate = null;
public event IceCandidateReadytoSendDelegate OnIceCandidateReadytoSend;
private int mPeerConnectionId = -1;
}
}

88
examples/unityplugin/class_reference_holder.cc
View file

@ -1,88 +0,0 @@
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "examples/unityplugin/class_reference_holder.h"
#include <utility>
#include "sdk/android/src/jni/jni_helpers.h"
namespace unity_plugin {
// ClassReferenceHolder holds global reference to Java classes in app/webrtc.
class ClassReferenceHolder {
public:
explicit ClassReferenceHolder(JNIEnv* jni);
~ClassReferenceHolder();
void FreeReferences(JNIEnv* jni);
jclass GetClass(const std::string& name);
void LoadClass(JNIEnv* jni, const std::string& name);
private:
std::map<std::string, jclass> classes_;
};
// Allocated in LoadGlobalClassReferenceHolder(),
// freed in FreeGlobalClassReferenceHolder().
static ClassReferenceHolder* g_class_reference_holder = nullptr;
void LoadGlobalClassReferenceHolder() {
RTC_CHECK(g_class_reference_holder == nullptr);
g_class_reference_holder = new ClassReferenceHolder(webrtc::jni::GetEnv());
}
void FreeGlobalClassReferenceHolder() {
g_class_reference_holder->FreeReferences(
webrtc::jni::AttachCurrentThreadIfNeeded());
delete g_class_reference_holder;
g_class_reference_holder = nullptr;
}
ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
LoadClass(jni, "org/webrtc/UnityUtility");
}
ClassReferenceHolder::~ClassReferenceHolder() {
RTC_CHECK(classes_.empty()) << "Must call FreeReferences() before dtor!";
}
void ClassReferenceHolder::FreeReferences(JNIEnv* jni) {
for (std::map<std::string, jclass>::const_iterator it = classes_.begin();
it != classes_.end(); ++it) {
jni->DeleteGlobalRef(it->second);
}
classes_.clear();
}
jclass ClassReferenceHolder::GetClass(const std::string& name) {
std::map<std::string, jclass>::iterator it = classes_.find(name);
RTC_CHECK(it != classes_.end()) << "Unexpected GetClass() call for: " << name;
return it->second;
}
void ClassReferenceHolder::LoadClass(JNIEnv* jni, const std::string& name) {
jclass localRef = jni->FindClass(name.c_str());
CHECK_EXCEPTION(jni) << "error during FindClass: " << name;
RTC_CHECK(localRef) << name;
jclass globalRef = reinterpret_cast<jclass>(jni->NewGlobalRef(localRef));
CHECK_EXCEPTION(jni) << "error during NewGlobalRef: " << name;
RTC_CHECK(globalRef) << name;
bool inserted = classes_.insert(std::make_pair(name, globalRef)).second;
RTC_CHECK(inserted) << "Duplicate class name: " << name;
}
// Returns a global reference guaranteed to be valid for the lifetime of the
// process.
jclass FindClass(JNIEnv* jni, const char* name) {
return g_class_reference_holder->GetClass(name);
}
} // namespace unity_plugin

38
examples/unityplugin/class_reference_holder.h
View file

@ -1,38 +0,0 @@
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// This is a supplement of webrtc::jni::ClassReferenceHolder.
// The purpose of this ClassReferenceHolder is to load the example
// specific java class into JNI c++ side, so that our c++ code can
// call those java functions.
#ifndef EXAMPLES_UNITYPLUGIN_CLASS_REFERENCE_HOLDER_H_
#define EXAMPLES_UNITYPLUGIN_CLASS_REFERENCE_HOLDER_H_
#include <jni.h>
#include <map>
#include <string>
#include <vector>
namespace unity_plugin {
// LoadGlobalClassReferenceHolder must be called in JNI_OnLoad.
void LoadGlobalClassReferenceHolder();
// FreeGlobalClassReferenceHolder must be called in JNI_UnLoad.
void FreeGlobalClassReferenceHolder();
// Returns a global reference guaranteed to be valid for the lifetime of the
// process.
jclass FindClass(JNIEnv* jni, const char* name);
} // namespace unity_plugin
#endif // EXAMPLES_UNITYPLUGIN_CLASS_REFERENCE_HOLDER_H_

68
examples/unityplugin/java/src/org/webrtc/UnityUtility.java
View file

@ -1,68 +0,0 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import android.content.Context;
import androidx.annotation.Nullable;
public class UnityUtility {
private static final String VIDEO_CAPTURER_THREAD_NAME = "VideoCapturerThread";
public static SurfaceTextureHelper LoadSurfaceTextureHelper() {
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create(VIDEO_CAPTURER_THREAD_NAME, null);
return surfaceTextureHelper;
}
private static boolean useCamera2() {
return Camera2Enumerator.isSupported(ContextUtils.getApplicationContext());
}
private static @Nullable VideoCapturer createCameraCapturer(CameraEnumerator enumerator) {
final String[] deviceNames = enumerator.getDeviceNames();
for (String deviceName : deviceNames) {
if (enumerator.isFrontFacing(deviceName)) {
VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null);
if (videoCapturer != null) {
return videoCapturer;
}
}
}
return null;
}
public static VideoCapturer LinkCamera(
long nativeTrackSource, SurfaceTextureHelper surfaceTextureHelper) {
VideoCapturer capturer =
createCameraCapturer(new Camera2Enumerator(ContextUtils.getApplicationContext()));
VideoSource videoSource = new VideoSource(nativeTrackSource);
capturer.initialize(surfaceTextureHelper, ContextUtils.getApplicationContext(),
videoSource.getCapturerObserver());
capturer.startCapture(720, 480, 30);
return capturer;
}
public static void StopCamera(VideoCapturer camera) throws InterruptedException {
camera.stopCapture();
camera.dispose();
}
public static void InitializePeerConncectionFactory(Context context) throws InterruptedException {
PeerConnectionFactory.initialize(
PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions());
}
}

42
examples/unityplugin/jni_onload.cc
View file

@ -1,42 +0,0 @@
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <jni.h>
#undef JNIEXPORT
#define JNIEXPORT __attribute__((visibility("default")))
#include "examples/unityplugin/class_reference_holder.h"
#include "rtc_base/ssl_adapter.h"
#include "sdk/android/native_api/jni/class_loader.h"
#include "sdk/android/src/jni/jni_helpers.h"
namespace webrtc {
namespace jni {
extern "C" jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM* jvm, void* reserved) {
jint ret = InitGlobalJniVariables(jvm);
RTC_DCHECK_GE(ret, 0);
if (ret < 0)
return -1;
RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()";
webrtc::InitClassLoader(GetEnv());
unity_plugin::LoadGlobalClassReferenceHolder();
return ret;
}
extern "C" void JNIEXPORT JNICALL JNI_OnUnLoad(JavaVM* jvm, void* reserved) {
unity_plugin::FreeGlobalClassReferenceHolder();
RTC_CHECK(rtc::CleanupSSL()) << "Failed to CleanupSSL()";
}
} // namespace jni
} // namespace webrtc

586
examples/unityplugin/simple_peer_connection.cc
View file

@ -1,586 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "examples/unityplugin/simple_peer_connection.h"
#include <utility>
#include "absl/memory/memory.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
#include "api/create_peerconnection_factory.h"
#include "media/engine/internal_decoder_factory.h"
#include "media/engine/internal_encoder_factory.h"
#include "media/engine/multiplex_codec_factory.h"
#include "modules/audio_device/include/audio_device.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/video_capture/video_capture_factory.h"
#include "pc/video_track_source.h"
#include "test/vcm_capturer.h"
#if defined(WEBRTC_ANDROID)
#include "examples/unityplugin/class_reference_holder.h"
#include "modules/utility/include/helpers_android.h"
#include "sdk/android/src/jni/android_video_track_source.h"
#include "sdk/android/src/jni/jni_helpers.h"
#endif
// Names used for media stream ids.
const char kAudioLabel[] = "audio_label";
const char kVideoLabel[] = "video_label";
const char kStreamId[] = "stream_id";
namespace {
static int g_peer_count = 0;
static std::unique_ptr<rtc::Thread> g_worker_thread;
static std::unique_ptr<rtc::Thread> g_signaling_thread;
static rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
g_peer_connection_factory;
#if defined(WEBRTC_ANDROID)
// Android case: the video track does not own the capturer, and it
// relies on the app to dispose the capturer when the peerconnection
// shuts down.
static jobject g_camera = nullptr;
#else
class CapturerTrackSource : public webrtc::VideoTrackSource {
public:
static rtc::scoped_refptr<CapturerTrackSource> Create() {
const size_t kWidth = 640;
const size_t kHeight = 480;
const size_t kFps = 30;
const size_t kDeviceIndex = 0;
std::unique_ptr<webrtc::test::VcmCapturer> capturer = absl::WrapUnique(
webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, kDeviceIndex));
if (!capturer) {
return nullptr;
}
return rtc::make_ref_counted<CapturerTrackSource>(std::move(capturer));
}
protected:
explicit CapturerTrackSource(
std::unique_ptr<webrtc::test::VcmCapturer> capturer)
: VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
private:
rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
return capturer_.get();
}
std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
};
#endif
std::string GetEnvVarOrDefault(const char* env_var_name,
const char* default_value) {
std::string value;
const char* env_var = getenv(env_var_name);
if (env_var)
value = env_var;
if (value.empty())
value = default_value;
return value;
}
std::string GetPeerConnectionString() {
return GetEnvVarOrDefault("WEBRTC_CONNECT", "stun:stun.l.google.com:19302");
}
class DummySetSessionDescriptionObserver
: public webrtc::SetSessionDescriptionObserver {
public:
static rtc::scoped_refptr<DummySetSessionDescriptionObserver> Create() {
return rtc::make_ref_counted<DummySetSessionDescriptionObserver>();
}
virtual void OnSuccess() { RTC_LOG(LS_INFO) << __FUNCTION__; }
virtual void OnFailure(webrtc::RTCError error) {
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << ToString(error.type()) << ": "
<< error.message();
}
protected:
DummySetSessionDescriptionObserver() {}
~DummySetSessionDescriptionObserver() {}
};
} // namespace
bool SimplePeerConnection::InitializePeerConnection(const char** turn_urls,
const int no_of_urls,
const char* username,
const char* credential,
bool is_receiver) {
RTC_DCHECK(peer_connection_.get() == nullptr);
if (g_peer_connection_factory == nullptr) {
g_worker_thread = rtc::Thread::Create();
g_worker_thread->Start();
g_signaling_thread = rtc::Thread::Create();
g_signaling_thread->Start();
g_peer_connection_factory = webrtc::CreatePeerConnectionFactory(
g_worker_thread.get(), g_worker_thread.get(), g_signaling_thread.get(),
nullptr, webrtc::CreateBuiltinAudioEncoderFactory(),
webrtc::CreateBuiltinAudioDecoderFactory(),
std::unique_ptr<webrtc::VideoEncoderFactory>(
new webrtc::MultiplexEncoderFactory(
std::make_unique<webrtc::InternalEncoderFactory>())),
std::unique_ptr<webrtc::VideoDecoderFactory>(
new webrtc::MultiplexDecoderFactory(
std::make_unique<webrtc::InternalDecoderFactory>())),
nullptr, nullptr);
}
if (!g_peer_connection_factory.get()) {
DeletePeerConnection();
return false;
}
g_peer_count++;
if (!CreatePeerConnection(turn_urls, no_of_urls, username, credential)) {
DeletePeerConnection();
return false;
}
mandatory_receive_ = is_receiver;
return peer_connection_.get() != nullptr;
}
bool SimplePeerConnection::CreatePeerConnection(const char** turn_urls,
const int no_of_urls,
const char* username,
const char* credential) {
RTC_DCHECK(g_peer_connection_factory.get() != nullptr);
RTC_DCHECK(peer_connection_.get() == nullptr);
local_video_observer_.reset(new VideoObserver());
remote_video_observer_.reset(new VideoObserver());
// Add the turn server.
if (turn_urls != nullptr) {
if (no_of_urls > 0) {
webrtc::PeerConnectionInterface::IceServer turn_server;
for (int i = 0; i < no_of_urls; i++) {
std::string url(turn_urls[i]);
if (url.length() > 0)
turn_server.urls.push_back(turn_urls[i]);
}
std::string user_name(username);
if (user_name.length() > 0)
turn_server.username = username;
std::string password(credential);
if (password.length() > 0)
turn_server.password = credential;
config_.servers.push_back(turn_server);
}
}
// Add the stun server.
webrtc::PeerConnectionInterface::IceServer stun_server;
stun_server.uri = GetPeerConnectionString();
config_.servers.push_back(stun_server);
auto result = g_peer_connection_factory->CreatePeerConnectionOrError(
config_, webrtc::PeerConnectionDependencies(this));
if (!result.ok()) {
peer_connection_ = nullptr;
return false;
}
peer_connection_ = result.MoveValue();
return true;
}
void SimplePeerConnection::DeletePeerConnection() {
g_peer_count--;
#if defined(WEBRTC_ANDROID)
if (g_camera) {
JNIEnv* env = webrtc::jni::GetEnv();
jclass pc_factory_class =
unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
jmethodID stop_camera_method = webrtc::GetStaticMethodID(
env, pc_factory_class, "StopCamera", "(Lorg/webrtc/VideoCapturer;)V");
env->CallStaticVoidMethod(pc_factory_class, stop_camera_method, g_camera);
CHECK_EXCEPTION(env);
g_camera = nullptr;
}
#endif
CloseDataChannel();
peer_connection_ = nullptr;
active_streams_.clear();
if (g_peer_count == 0) {
g_peer_connection_factory = nullptr;
g_signaling_thread.reset();
g_worker_thread.reset();
}
}
bool SimplePeerConnection::CreateOffer() {
if (!peer_connection_.get())
return false;
webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options;
if (mandatory_receive_) {
options.offer_to_receive_audio = true;
options.offer_to_receive_video = true;
}
peer_connection_->CreateOffer(this, options);
return true;
}
bool SimplePeerConnection::CreateAnswer() {
if (!peer_connection_.get())
return false;
webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options;
if (mandatory_receive_) {
options.offer_to_receive_audio = true;
options.offer_to_receive_video = true;
}
peer_connection_->CreateAnswer(this, options);
return true;
}
void SimplePeerConnection::OnSuccess(
webrtc::SessionDescriptionInterface* desc) {
peer_connection_->SetLocalDescription(
DummySetSessionDescriptionObserver::Create().get(), desc);
std::string sdp;
desc->ToString(&sdp);
if (OnLocalSdpReady)
OnLocalSdpReady(desc->type().c_str(), sdp.c_str());
}
void SimplePeerConnection::OnFailure(webrtc::RTCError error) {
RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message();
// TODO(hta): include error.type in the message
if (OnFailureMessage)
OnFailureMessage(error.message());
}
void SimplePeerConnection::OnIceCandidate(
const webrtc::IceCandidateInterface* candidate) {
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
std::string sdp;
if (!candidate->ToString(&sdp)) {
RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
return;
}
if (OnIceCandidateReady)
OnIceCandidateReady(sdp.c_str(), candidate->sdp_mline_index(),
candidate->sdp_mid().c_str());
}
void SimplePeerConnection::RegisterOnLocalI420FrameReady(
I420FRAMEREADY_CALLBACK callback) {
if (local_video_observer_)
local_video_observer_->SetVideoCallback(callback);
}
void SimplePeerConnection::RegisterOnRemoteI420FrameReady(
I420FRAMEREADY_CALLBACK callback) {
if (remote_video_observer_)
remote_video_observer_->SetVideoCallback(callback);
}
void SimplePeerConnection::RegisterOnLocalDataChannelReady(
LOCALDATACHANNELREADY_CALLBACK callback) {
OnLocalDataChannelReady = callback;
}
void SimplePeerConnection::RegisterOnDataFromDataChannelReady(
DATAFROMEDATECHANNELREADY_CALLBACK callback) {
OnDataFromDataChannelReady = callback;
}
void SimplePeerConnection::RegisterOnFailure(FAILURE_CALLBACK callback) {
OnFailureMessage = callback;
}
void SimplePeerConnection::RegisterOnAudioBusReady(
AUDIOBUSREADY_CALLBACK callback) {
OnAudioReady = callback;
}
void SimplePeerConnection::RegisterOnLocalSdpReadytoSend(
LOCALSDPREADYTOSEND_CALLBACK callback) {
OnLocalSdpReady = callback;
}
void SimplePeerConnection::RegisterOnIceCandidateReadytoSend(
ICECANDIDATEREADYTOSEND_CALLBACK callback) {
OnIceCandidateReady = callback;
}
bool SimplePeerConnection::SetRemoteDescription(const char* type,
const char* sdp) {
if (!peer_connection_)
return false;
std::string remote_desc(sdp);
std::string desc_type(type);
webrtc::SdpParseError error;
webrtc::SessionDescriptionInterface* session_description(
webrtc::CreateSessionDescription(desc_type, remote_desc, &error));
if (!session_description) {
RTC_LOG(LS_WARNING) << "Can't parse received session description message. "
"SdpParseError was: "
<< error.description;
return false;
}
RTC_LOG(LS_INFO) << " Received session description :" << remote_desc;
peer_connection_->SetRemoteDescription(
DummySetSessionDescriptionObserver::Create().get(), session_description);
return true;
}
bool SimplePeerConnection::AddIceCandidate(const char* candidate,
const int sdp_mlineindex,
const char* sdp_mid) {
if (!peer_connection_)
return false;
webrtc::SdpParseError error;
std::unique_ptr<webrtc::IceCandidateInterface> ice_candidate(
webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, candidate, &error));
if (!ice_candidate.get()) {
RTC_LOG(LS_WARNING) << "Can't parse received candidate message. "
"SdpParseError was: "
<< error.description;
return false;
}
if (!peer_connection_->AddIceCandidate(ice_candidate.get())) {
RTC_LOG(LS_WARNING) << "Failed to apply the received candidate";
return false;
}
RTC_LOG(LS_INFO) << " Received candidate :" << candidate;
return true;
}
void SimplePeerConnection::SetAudioControl(bool is_mute, bool is_record) {
is_mute_audio_ = is_mute;
is_record_audio_ = is_record;
SetAudioControl();
}
void SimplePeerConnection::SetAudioControl() {
if (!remote_stream_)
return;
webrtc::AudioTrackVector tracks = remote_stream_->GetAudioTracks();
if (tracks.empty())
return;
rtc::scoped_refptr<webrtc::AudioTrackInterface>& audio_track = tracks[0];
if (is_record_audio_)
audio_track->AddSink(this);
else
audio_track->RemoveSink(this);
for (auto& track : tracks) {
if (is_mute_audio_)
track->set_enabled(false);
else
track->set_enabled(true);
}
}
void SimplePeerConnection::OnAddStream(
rtc::scoped_refptr<webrtc::MediaStreamInterface> stream) {
RTC_LOG(LS_INFO) << __FUNCTION__ << " " << stream->id();
remote_stream_ = stream;
if (remote_video_observer_ && !remote_stream_->GetVideoTracks().empty()) {
remote_stream_->GetVideoTracks()[0]->AddOrUpdateSink(
remote_video_observer_.get(), rtc::VideoSinkWants());
}
SetAudioControl();
}
void SimplePeerConnection::AddStreams(bool audio_only) {
if (active_streams_.find(kStreamId) != active_streams_.end())
return; // Already added.
rtc::scoped_refptr<webrtc::MediaStreamInterface> stream =
g_peer_connection_factory->CreateLocalMediaStream(kStreamId);
rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(
g_peer_connection_factory->CreateAudioTrack(
kAudioLabel,
g_peer_connection_factory->CreateAudioSource(cricket::AudioOptions())
.get()));
stream->AddTrack(audio_track);
if (!audio_only) {
#if defined(WEBRTC_ANDROID)
JNIEnv* env = webrtc::jni::GetEnv();
jclass pc_factory_class =
unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
jmethodID load_texture_helper_method = webrtc::GetStaticMethodID(
env, pc_factory_class, "LoadSurfaceTextureHelper",
"()Lorg/webrtc/SurfaceTextureHelper;");
jobject texture_helper = env->CallStaticObjectMethod(
pc_factory_class, load_texture_helper_method);
CHECK_EXCEPTION(env);
RTC_DCHECK(texture_helper != nullptr)
<< "Cannot get the Surface Texture Helper.";
auto source = rtc::make_ref_counted<webrtc::jni::AndroidVideoTrackSource>(
g_signaling_thread.get(), env, /*is_screencast=*/false,
/*align_timestamps=*/true);
// link with VideoCapturer (Camera);
jmethodID link_camera_method = webrtc::GetStaticMethodID(
env, pc_factory_class, "LinkCamera",
"(JLorg/webrtc/SurfaceTextureHelper;)Lorg/webrtc/VideoCapturer;");
jobject camera_tmp =
env->CallStaticObjectMethod(pc_factory_class, link_camera_method,
(jlong)source.get(), texture_helper);
CHECK_EXCEPTION(env);
g_camera = (jobject)env->NewGlobalRef(camera_tmp);
rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
g_peer_connection_factory->CreateVideoTrack(source, kVideoLabel));
stream->AddTrack(video_track);
#else
rtc::scoped_refptr<CapturerTrackSource> video_device =
CapturerTrackSource::Create();
if (video_device) {
rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
g_peer_connection_factory->CreateVideoTrack(video_device,
kVideoLabel));
stream->AddTrack(video_track);
}
#endif
if (local_video_observer_ && !stream->GetVideoTracks().empty()) {
stream->GetVideoTracks()[0]->AddOrUpdateSink(local_video_observer_.get(),
rtc::VideoSinkWants());
}
}
if (!peer_connection_->AddStream(stream.get())) {
RTC_LOG(LS_ERROR) << "Adding stream to PeerConnection failed";
}
typedef std::pair<std::string,
rtc::scoped_refptr<webrtc::MediaStreamInterface>>
MediaStreamPair;
active_streams_.insert(MediaStreamPair(stream->id(), stream));
}
bool SimplePeerConnection::CreateDataChannel() {
struct webrtc::DataChannelInit init;
init.ordered = true;
init.reliable = true;
auto result = peer_connection_->CreateDataChannelOrError("Hello", &init);
if (result.ok()) {
data_channel_ = result.MoveValue();
data_channel_->RegisterObserver(this);
RTC_LOG(LS_INFO) << "Succeeds to create data channel";
return true;
} else {
RTC_LOG(LS_INFO) << "Fails to create data channel";
return false;
}
}
void SimplePeerConnection::CloseDataChannel() {
if (data_channel_.get()) {
data_channel_->UnregisterObserver();
data_channel_->Close();
}
data_channel_ = nullptr;
}
bool SimplePeerConnection::SendDataViaDataChannel(const std::string& data) {
if (!data_channel_.get()) {
RTC_LOG(LS_INFO) << "Data channel is not established";
return false;
}
webrtc::DataBuffer buffer(data);
data_channel_->Send(buffer);
return true;
}
// Peerconnection observer
void SimplePeerConnection::OnDataChannel(
rtc::scoped_refptr<webrtc::DataChannelInterface> channel) {
channel->RegisterObserver(this);
}
void SimplePeerConnection::OnStateChange() {
if (data_channel_) {
webrtc::DataChannelInterface::DataState state = data_channel_->state();
if (state == webrtc::DataChannelInterface::kOpen) {
if (OnLocalDataChannelReady)
OnLocalDataChannelReady();
RTC_LOG(LS_INFO) << "Data channel is open";
}
}
}
// A data buffer was successfully received.
void SimplePeerConnection::OnMessage(const webrtc::DataBuffer& buffer) {
size_t size = buffer.data.size();
char* msg = new char[size + 1];
memcpy(msg, buffer.data.data(), size);
msg[size] = 0;
if (OnDataFromDataChannelReady)
OnDataFromDataChannelReady(msg);
delete[] msg;
}
// AudioTrackSinkInterface implementation.
void SimplePeerConnection::OnData(const void* audio_data,
int bits_per_sample,
int sample_rate,
size_t number_of_channels,
size_t number_of_frames) {
if (OnAudioReady)
OnAudioReady(audio_data, bits_per_sample, sample_rate,
static_cast<int>(number_of_channels),
static_cast<int>(number_of_frames));
}
std::vector<uint32_t> SimplePeerConnection::GetRemoteAudioTrackSsrcs() {
std::vector<rtc::scoped_refptr<webrtc::RtpReceiverInterface>> receivers =
peer_connection_->GetReceivers();
std::vector<uint32_t> ssrcs;
for (const auto& receiver : receivers) {
if (receiver->media_type() != cricket::MEDIA_TYPE_AUDIO)
continue;
std::vector<webrtc::RtpEncodingParameters> params =
receiver->GetParameters().encodings;
for (const auto& param : params) {
uint32_t ssrc = param.ssrc.value_or(0);
if (ssrc > 0)
ssrcs.push_back(ssrc);
}
}
return ssrcs;
}

135
examples/unityplugin/simple_peer_connection.h
View file

@ -1,135 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef EXAMPLES_UNITYPLUGIN_SIMPLE_PEER_CONNECTION_H_
#define EXAMPLES_UNITYPLUGIN_SIMPLE_PEER_CONNECTION_H_
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "api/data_channel_interface.h"
#include "api/media_stream_interface.h"
#include "api/peer_connection_interface.h"
#include "examples/unityplugin/unity_plugin_apis.h"
#include "examples/unityplugin/video_observer.h"
class SimplePeerConnection : public webrtc::PeerConnectionObserver,
public webrtc::CreateSessionDescriptionObserver,
public webrtc::DataChannelObserver,
public webrtc::AudioTrackSinkInterface {
public:
SimplePeerConnection() {}
~SimplePeerConnection() {}
bool InitializePeerConnection(const char** turn_urls,
int no_of_urls,
const char* username,
const char* credential,
bool is_receiver);
void DeletePeerConnection();
void AddStreams(bool audio_only);
bool CreateDataChannel();
bool CreateOffer();
bool CreateAnswer();
bool SendDataViaDataChannel(const std::string& data);
void SetAudioControl(bool is_mute, bool is_record);
// Register callback functions.
void RegisterOnLocalI420FrameReady(I420FRAMEREADY_CALLBACK callback);
void RegisterOnRemoteI420FrameReady(I420FRAMEREADY_CALLBACK callback);
void RegisterOnLocalDataChannelReady(LOCALDATACHANNELREADY_CALLBACK callback);
void RegisterOnDataFromDataChannelReady(
DATAFROMEDATECHANNELREADY_CALLBACK callback);
void RegisterOnFailure(FAILURE_CALLBACK callback);
void RegisterOnAudioBusReady(AUDIOBUSREADY_CALLBACK callback);
void RegisterOnLocalSdpReadytoSend(LOCALSDPREADYTOSEND_CALLBACK callback);
void RegisterOnIceCandidateReadytoSend(
ICECANDIDATEREADYTOSEND_CALLBACK callback);
bool SetRemoteDescription(const char* type, const char* sdp);
bool AddIceCandidate(const char* sdp,
int sdp_mlineindex,
const char* sdp_mid);
protected:
// create a peerconneciton and add the turn servers info to the configuration.
bool CreatePeerConnection(const char** turn_urls,
int no_of_urls,
const char* username,
const char* credential);
void CloseDataChannel();
void SetAudioControl();
// PeerConnectionObserver implementation.
void OnSignalingChange(
webrtc::PeerConnectionInterface::SignalingState new_state) override {}
void OnAddStream(
rtc::scoped_refptr<webrtc::MediaStreamInterface> stream) override;
void OnRemoveStream(
rtc::scoped_refptr<webrtc::MediaStreamInterface> stream) override {}
void OnDataChannel(
rtc::scoped_refptr<webrtc::DataChannelInterface> channel) override;
void OnRenegotiationNeeded() override {}
void OnIceConnectionChange(
webrtc::PeerConnectionInterface::IceConnectionState new_state) override {}
void OnIceGatheringChange(
webrtc::PeerConnectionInterface::IceGatheringState new_state) override {}
void OnIceCandidate(const webrtc::IceCandidateInterface* candidate) override;
void OnIceConnectionReceivingChange(bool receiving) override {}
// CreateSessionDescriptionObserver implementation.
void OnSuccess(webrtc::SessionDescriptionInterface* desc) override;
void OnFailure(webrtc::RTCError error) override;
// DataChannelObserver implementation.
void OnStateChange() override;
void OnMessage(const webrtc::DataBuffer& buffer) override;
// AudioTrackSinkInterface implementation.
void OnData(const void* audio_data,
int bits_per_sample,
int sample_rate,
size_t number_of_channels,
size_t number_of_frames) override;
// Get remote audio tracks ssrcs.
std::vector<uint32_t> GetRemoteAudioTrackSsrcs();
private:
rtc::scoped_refptr<webrtc::PeerConnectionInterface> peer_connection_;
rtc::scoped_refptr<webrtc::DataChannelInterface> data_channel_;
std::map<std::string, rtc::scoped_refptr<webrtc::MediaStreamInterface> >
active_streams_;
std::unique_ptr<VideoObserver> local_video_observer_;
std::unique_ptr<VideoObserver> remote_video_observer_;
rtc::scoped_refptr<webrtc::MediaStreamInterface> remote_stream_ = nullptr;
webrtc::PeerConnectionInterface::RTCConfiguration config_;
LOCALDATACHANNELREADY_CALLBACK OnLocalDataChannelReady = nullptr;
DATAFROMEDATECHANNELREADY_CALLBACK OnDataFromDataChannelReady = nullptr;
FAILURE_CALLBACK OnFailureMessage = nullptr;
AUDIOBUSREADY_CALLBACK OnAudioReady = nullptr;
LOCALSDPREADYTOSEND_CALLBACK OnLocalSdpReady = nullptr;
ICECANDIDATEREADYTOSEND_CALLBACK OnIceCandidateReady = nullptr;
bool is_mute_audio_ = false;
bool is_record_audio_ = false;
bool mandatory_receive_ = false;
// disallow copy-and-assign
SimplePeerConnection(const SimplePeerConnection&) = delete;
SimplePeerConnection& operator=(const SimplePeerConnection&) = delete;
};
#endif // EXAMPLES_UNITYPLUGIN_SIMPLE_PEER_CONNECTION_H_

196
examples/unityplugin/unity_plugin_apis.cc
View file

@ -1,196 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "examples/unityplugin/unity_plugin_apis.h"
#include <map>
#include <string>
#include "examples/unityplugin/simple_peer_connection.h"
namespace {
static int g_peer_connection_id = 1;
static std::map<int, rtc::scoped_refptr<SimplePeerConnection>>
g_peer_connection_map;
} // namespace
int CreatePeerConnection(const char** turn_urls,
const int no_of_urls,
const char* username,
const char* credential,
bool mandatory_receive_video) {
g_peer_connection_map[g_peer_connection_id] =
rtc::make_ref_counted<SimplePeerConnection>();
if (!g_peer_connection_map[g_peer_connection_id]->InitializePeerConnection(
turn_urls, no_of_urls, username, credential, mandatory_receive_video))
return -1;
return g_peer_connection_id++;
}
bool ClosePeerConnection(int peer_connection_id) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->DeletePeerConnection();
g_peer_connection_map.erase(peer_connection_id);
return true;
}
bool AddStream(int peer_connection_id, bool audio_only) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->AddStreams(audio_only);
return true;
}
bool AddDataChannel(int peer_connection_id) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
return g_peer_connection_map[peer_connection_id]->CreateDataChannel();
}
bool CreateOffer(int peer_connection_id) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
return g_peer_connection_map[peer_connection_id]->CreateOffer();
}
bool CreateAnswer(int peer_connection_id) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
return g_peer_connection_map[peer_connection_id]->CreateAnswer();
}
bool SendDataViaDataChannel(int peer_connection_id, const char* data) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
std::string s(data);
g_peer_connection_map[peer_connection_id]->SendDataViaDataChannel(s);
return true;
}
bool SetAudioControl(int peer_connection_id, bool is_mute, bool is_record) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->SetAudioControl(is_mute,
is_record);
return true;
}
bool SetRemoteDescription(int peer_connection_id,
const char* type,
const char* sdp) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
return g_peer_connection_map[peer_connection_id]->SetRemoteDescription(type,
sdp);
}
bool AddIceCandidate(const int peer_connection_id,
const char* candidate,
const int sdp_mlineindex,
const char* sdp_mid) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
return g_peer_connection_map[peer_connection_id]->AddIceCandidate(
candidate, sdp_mlineindex, sdp_mid);
}
// Register callback functions.
bool RegisterOnLocalI420FrameReady(int peer_connection_id,
I420FRAMEREADY_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnLocalI420FrameReady(
callback);
return true;
}
bool RegisterOnRemoteI420FrameReady(int peer_connection_id,
I420FRAMEREADY_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnRemoteI420FrameReady(
callback);
return true;
}
bool RegisterOnLocalDataChannelReady(int peer_connection_id,
LOCALDATACHANNELREADY_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnLocalDataChannelReady(
callback);
return true;
}
bool RegisterOnDataFromDataChannelReady(
int peer_connection_id,
DATAFROMEDATECHANNELREADY_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnDataFromDataChannelReady(
callback);
return true;
}
bool RegisterOnFailure(int peer_connection_id, FAILURE_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnFailure(callback);
return true;
}
bool RegisterOnAudioBusReady(int peer_connection_id,
AUDIOBUSREADY_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnAudioBusReady(callback);
return true;
}
// Singnaling channel related functions.
bool RegisterOnLocalSdpReadytoSend(int peer_connection_id,
LOCALSDPREADYTOSEND_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnLocalSdpReadytoSend(
callback);
return true;
}
bool RegisterOnIceCandidateReadytoSend(
int peer_connection_id,
ICECANDIDATEREADYTOSEND_CALLBACK callback) {
if (!g_peer_connection_map.count(peer_connection_id))
return false;
g_peer_connection_map[peer_connection_id]->RegisterOnIceCandidateReadytoSend(
callback);
return true;
}

108
examples/unityplugin/unity_plugin_apis.h
View file

@ -1,108 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// This file provides an example of unity native plugin APIs.
#ifndef EXAMPLES_UNITYPLUGIN_UNITY_PLUGIN_APIS_H_
#define EXAMPLES_UNITYPLUGIN_UNITY_PLUGIN_APIS_H_
#include <stdint.h>
// Definitions of callback functions.
typedef void (*I420FRAMEREADY_CALLBACK)(const uint8_t* data_y,
const uint8_t* data_u,
const uint8_t* data_v,
const uint8_t* data_a,
int stride_y,
int stride_u,
int stride_v,
int stride_a,
uint32_t width,
uint32_t height);
typedef void (*LOCALDATACHANNELREADY_CALLBACK)();
typedef void (*DATAFROMEDATECHANNELREADY_CALLBACK)(const char* msg);
typedef void (*FAILURE_CALLBACK)(const char* msg);
typedef void (*LOCALSDPREADYTOSEND_CALLBACK)(const char* type, const char* sdp);
typedef void (*ICECANDIDATEREADYTOSEND_CALLBACK)(const char* candidate,
int sdp_mline_index,
const char* sdp_mid);
typedef void (*AUDIOBUSREADY_CALLBACK)(const void* audio_data,
int bits_per_sample,
int sample_rate,
int number_of_channels,
int number_of_frames);
#if defined(WEBRTC_WIN)
#define WEBRTC_PLUGIN_API __declspec(dllexport)
#elif defined(WEBRTC_ANDROID)
#define WEBRTC_PLUGIN_API __attribute__((visibility("default")))
#endif
extern "C" {
// Create a peerconnection and return a unique peer connection id.
WEBRTC_PLUGIN_API int CreatePeerConnection(const char** turn_urls,
int no_of_urls,
const char* username,
const char* credential,
bool mandatory_receive_video);
// Close a peerconnection.
WEBRTC_PLUGIN_API bool ClosePeerConnection(int peer_connection_id);
// Add a audio stream. If audio_only is true, the stream only has an audio
// track and no video track.
WEBRTC_PLUGIN_API bool AddStream(int peer_connection_id, bool audio_only);
// Add a data channel to peer connection.
WEBRTC_PLUGIN_API bool AddDataChannel(int peer_connection_id);
// Create a peer connection offer.
WEBRTC_PLUGIN_API bool CreateOffer(int peer_connection_id);
// Create a peer connection answer.
WEBRTC_PLUGIN_API bool CreateAnswer(int peer_connection_id);
// Send data through data channel.
WEBRTC_PLUGIN_API bool SendDataViaDataChannel(int peer_connection_id,
const char* data);
// Set audio control. If is_mute=true, no audio will playout. If is_record=true,
// AUDIOBUSREADY_CALLBACK will be called every 10 ms.
WEBRTC_PLUGIN_API bool SetAudioControl(int peer_connection_id,
bool is_mute,
bool is_record);
// Set remote sdp.
WEBRTC_PLUGIN_API bool SetRemoteDescription(int peer_connection_id,
const char* type,
const char* sdp);
// Add ice candidate.
WEBRTC_PLUGIN_API bool AddIceCandidate(int peer_connection_id,
const char* candidate,
int sdp_mlineindex,
const char* sdp_mid);
// Register callback functions.
WEBRTC_PLUGIN_API bool RegisterOnLocalI420FrameReady(
int peer_connection_id,
I420FRAMEREADY_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnRemoteI420FrameReady(
int peer_connection_id,
I420FRAMEREADY_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnLocalDataChannelReady(
int peer_connection_id,
LOCALDATACHANNELREADY_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnDataFromDataChannelReady(
int peer_connection_id,
DATAFROMEDATECHANNELREADY_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnFailure(int peer_connection_id,
FAILURE_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnAudioBusReady(int peer_connection_id,
AUDIOBUSREADY_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnLocalSdpReadytoSend(
int peer_connection_id,
LOCALSDPREADYTOSEND_CALLBACK callback);
WEBRTC_PLUGIN_API bool RegisterOnIceCandidateReadytoSend(
int peer_connection_id,
ICECANDIDATEREADYTOSEND_CALLBACK callback);
}
#endif // EXAMPLES_UNITYPLUGIN_UNITY_PLUGIN_APIS_H_

44
examples/unityplugin/video_observer.cc
View file

@ -1,44 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "examples/unityplugin/video_observer.h"
void VideoObserver::SetVideoCallback(I420FRAMEREADY_CALLBACK callback) {
std::lock_guard<std::mutex> lock(mutex);
OnI420FrameReady = callback;
}
void VideoObserver::OnFrame(const webrtc::VideoFrame& frame) {
std::unique_lock<std::mutex> lock(mutex);
if (!OnI420FrameReady)
return;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer(
frame.video_frame_buffer());
if (buffer->type() != webrtc::VideoFrameBuffer::Type::kI420A) {
rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer =
buffer->ToI420();
OnI420FrameReady(i420_buffer->DataY(), i420_buffer->DataU(),
i420_buffer->DataV(), nullptr, i420_buffer->StrideY(),
i420_buffer->StrideU(), i420_buffer->StrideV(), 0,
frame.width(), frame.height());
} else {
// The buffer has alpha channel.
const webrtc::I420ABufferInterface* i420a_buffer = buffer->GetI420A();
OnI420FrameReady(i420a_buffer->DataY(), i420a_buffer->DataU(),
i420a_buffer->DataV(), i420a_buffer->DataA(),
i420a_buffer->StrideY(), i420a_buffer->StrideU(),
i420a_buffer->StrideV(), i420a_buffer->StrideA(),
frame.width(), frame.height());
}
}

35
examples/unityplugin/video_observer.h
View file

@ -1,35 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef EXAMPLES_UNITYPLUGIN_VIDEO_OBSERVER_H_
#define EXAMPLES_UNITYPLUGIN_VIDEO_OBSERVER_H_
#include <mutex>
#include "api/media_stream_interface.h"
#include "api/video/video_sink_interface.h"
#include "examples/unityplugin/unity_plugin_apis.h"
class VideoObserver : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
public:
VideoObserver() {}
~VideoObserver() {}
void SetVideoCallback(I420FRAMEREADY_CALLBACK callback);
protected:
// VideoSinkInterface implementation
void OnFrame(const webrtc::VideoFrame& frame) override;
private:
I420FRAMEREADY_CALLBACK OnI420FrameReady = nullptr;
std::mutex mutex;
};
#endif // EXAMPLES_UNITYPLUGIN_VIDEO_OBSERVER_H_

3
logging/rtc_event_log/encoder/rtc_event_log_encoder_new_format.cc
View file

@ -106,9 +106,6 @@ rtclog2::FrameDecodedEvents::Codec ConvertToProtoFormat(VideoCodecType codec) {
return rtclog2::FrameDecodedEvents::CODEC_AV1;
case VideoCodecType::kVideoCodecH264:
return rtclog2::FrameDecodedEvents::CODEC_H264;
case VideoCodecType::kVideoCodecMultiplex:
// This codec type is afaik not used.
return rtclog2::FrameDecodedEvents::CODEC_UNKNOWN;
case VideoCodecType::kVideoCodecH265:
return rtclog2::FrameDecodedEvents::CODEC_H265;
}

7
logging/rtc_event_log/rtc_event_log_parser.cc
View file

@ -271,12 +271,11 @@ VideoCodecType GetRuntimeCodecType(rtclog2::FrameDecodedEvents::Codec codec) {
case rtclog2::FrameDecodedEvents::CODEC_H265:
return VideoCodecType::kVideoCodecH265;
case rtclog2::FrameDecodedEvents::CODEC_UNKNOWN:
RTC_LOG(LS_ERROR) << "Unknown codec type. Assuming "
"VideoCodecType::kVideoCodecMultiplex";
return VideoCodecType::kVideoCodecMultiplex;
RTC_LOG(LS_ERROR) << "Unknown codec type. Returning generic.";
return VideoCodecType::kVideoCodecGeneric;
}
RTC_DCHECK_NOTREACHED();
return VideoCodecType::kVideoCodecMultiplex;
return VideoCodecType::kVideoCodecGeneric;
}
ParsedRtcEventLog::ParseStatus GetHeaderExtensions(

4
media/BUILD.gn
View file

@ -396,7 +396,6 @@ rtc_library("rtc_internal_video_codecs") {
"../call:video_stream_api",
"../modules/video_coding:video_codec_interface",
"../modules/video_coding:webrtc_h264",
"../modules/video_coding:webrtc_multiplex",
"../modules/video_coding:webrtc_vp8",
"../modules/video_coding:webrtc_vp9",
"../rtc_base:checks",
@ -427,8 +426,6 @@ rtc_library("rtc_internal_video_codecs") {
"engine/internal_decoder_factory.h",
"engine/internal_encoder_factory.cc",
"engine/internal_encoder_factory.h",
"engine/multiplex_codec_factory.cc",
"engine/multiplex_codec_factory.h",
]
}
@ -897,7 +894,6 @@ if (rtc_include_tests) {
"base/video_common_unittest.cc",
"engine/internal_decoder_factory_unittest.cc",
"engine/internal_encoder_factory_unittest.cc",
"engine/multiplex_codec_factory_unittest.cc",
"engine/null_webrtc_video_engine_unittest.cc",
"engine/payload_type_mapper_unittest.cc",
"engine/simulcast_encoder_adapter_unittest.cc",

1
media/base/media_constants.cc
View file

@ -24,7 +24,6 @@ const float kProcessCpuThreshold = 0.10f;
const char kRedCodecName[] = "red";
const char kUlpfecCodecName[] = "ulpfec";
const char kMultiplexCodecName[] = "multiplex";
// TODO(brandtr): Change this to 'flexfec' when we are confident that the
// header format is not changing anymore.

117
media/engine/multiplex_codec_factory.cc
View file

@ -1,117 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "media/engine/multiplex_codec_factory.h"
#include <map>
#include <memory>
#include <string>
#include <utility>
#include "absl/strings/match.h"
#include "api/environment/environment.h"
#include "api/video_codecs/sdp_video_format.h"
#include "media/base/codec.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include "rtc_base/logging.h"
namespace {
bool IsMultiplexCodec(const cricket::VideoCodec& codec) {
return absl::EqualsIgnoreCase(codec.name.c_str(),
cricket::kMultiplexCodecName);
}
} // anonymous namespace
namespace webrtc {
constexpr const char* kMultiplexAssociatedCodecName = cricket::kVp9CodecName;
MultiplexEncoderFactory::MultiplexEncoderFactory(
std::unique_ptr<VideoEncoderFactory> factory,
bool supports_augmenting_data)
: factory_(std::move(factory)),
supports_augmenting_data_(supports_augmenting_data) {}
std::vector<SdpVideoFormat> MultiplexEncoderFactory::GetSupportedFormats()
const {
std::vector<SdpVideoFormat> formats = factory_->GetSupportedFormats();
for (const auto& format : formats) {
if (absl::EqualsIgnoreCase(format.name, kMultiplexAssociatedCodecName)) {
SdpVideoFormat multiplex_format = format;
multiplex_format.parameters[cricket::kCodecParamAssociatedCodecName] =
format.name;
multiplex_format.name = cricket::kMultiplexCodecName;
formats.push_back(multiplex_format);
break;
}
}
return formats;
}
std::unique_ptr<VideoEncoder> MultiplexEncoderFactory::CreateVideoEncoder(
const SdpVideoFormat& format) {
if (!IsMultiplexCodec(cricket::CreateVideoCodec(format)))
return factory_->CreateVideoEncoder(format);
const auto& it =
format.parameters.find(cricket::kCodecParamAssociatedCodecName);
if (it == format.parameters.end()) {
RTC_LOG(LS_ERROR) << "No assicated codec for multiplex.";
return nullptr;
}
SdpVideoFormat associated_format = format;
associated_format.name = it->second;
return std::unique_ptr<VideoEncoder>(new MultiplexEncoderAdapter(
factory_.get(), associated_format, supports_augmenting_data_));
}
MultiplexDecoderFactory::MultiplexDecoderFactory(
std::unique_ptr<VideoDecoderFactory> factory,
bool supports_augmenting_data)
: factory_(std::move(factory)),
supports_augmenting_data_(supports_augmenting_data) {}
std::vector<SdpVideoFormat> MultiplexDecoderFactory::GetSupportedFormats()
const {
std::vector<SdpVideoFormat> formats = factory_->GetSupportedFormats();
std::vector<SdpVideoFormat> augmented_formats = formats;
for (const auto& format : formats) {
if (absl::EqualsIgnoreCase(format.name, kMultiplexAssociatedCodecName)) {
SdpVideoFormat multiplex_format = format;
multiplex_format.parameters[cricket::kCodecParamAssociatedCodecName] =
format.name;
multiplex_format.name = cricket::kMultiplexCodecName;
augmented_formats.push_back(multiplex_format);
}
}
return augmented_formats;
}
std::unique_ptr<VideoDecoder> MultiplexDecoderFactory::Create(
const Environment& env,
const SdpVideoFormat& format) {
if (!IsMultiplexCodec(cricket::CreateVideoCodec(format))) {
return factory_->Create(env, format);
}
auto it = format.parameters.find(cricket::kCodecParamAssociatedCodecName);
if (it == format.parameters.end()) {
RTC_LOG(LS_ERROR) << "No assicated codec for multiplex.";
return nullptr;
}
SdpVideoFormat associated_format = format;
associated_format.name = it->second;
return std::make_unique<MultiplexDecoderAdapter>(
env, factory_.get(), associated_format, supports_augmenting_data_);
}
} // namespace webrtc

80
media/engine/multiplex_codec_factory.h
View file

@ -1,80 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MEDIA_ENGINE_MULTIPLEX_CODEC_FACTORY_H_
#define MEDIA_ENGINE_MULTIPLEX_CODEC_FACTORY_H_
#include <memory>
#include <vector>
#include "api/environment/environment.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
// Multiplex codec is a completely modular/optional codec that allows users to
// send more than a frame's opaque content(RGB/YUV) over video channels.
// - Allows sending Alpha channel over the wire iff input is
// I420ABufferInterface. Users can expect to receive I420ABufferInterface as the
// decoded video frame buffer. I420A data is split into YUV/AXX portions,
// encoded/decoded seperately and bitstreams are concatanated.
// - Allows sending augmenting data over the wire attached to the frame. This
// attached data portion is not encoded in any way and sent as it is. Users can
// input AugmentedVideoFrameBuffer and can expect the same interface as the
// decoded video frame buffer.
// - Showcases an example of how to add a custom codec in webrtc video channel.
// How to use it end-to-end:
// - Wrap your existing VideoEncoderFactory implemention with
// MultiplexEncoderFactory and VideoDecoderFactory implemention with
// MultiplexDecoderFactory below. For actual coding, multiplex creates encoder
// and decoder instance(s) using these factories.
// - Use Multiplex*coderFactory classes in CreatePeerConnectionFactory() calls.
// - Select "multiplex" codec in SDP negotiation.
class RTC_EXPORT MultiplexEncoderFactory : public VideoEncoderFactory {
public:
// `supports_augmenting_data` defines if the encoder would support augmenting
// data. If set, the encoder expects to receive video frame buffers of type
// AugmentedVideoFrameBuffer.
MultiplexEncoderFactory(std::unique_ptr<VideoEncoderFactory> factory,
bool supports_augmenting_data = false);
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
std::unique_ptr<VideoEncoder> CreateVideoEncoder(
const SdpVideoFormat& format) override;
private:
std::unique_ptr<VideoEncoderFactory> factory_;
const bool supports_augmenting_data_;
};
class RTC_EXPORT MultiplexDecoderFactory : public VideoDecoderFactory {
public:
// `supports_augmenting_data` defines if the decoder would support augmenting
// data. If set, the decoder is expected to output video frame buffers of type
// AugmentedVideoFrameBuffer.
MultiplexDecoderFactory(std::unique_ptr<VideoDecoderFactory> factory,
bool supports_augmenting_data = false);
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
std::unique_ptr<VideoDecoder> Create(const Environment& env,
const SdpVideoFormat& format) override;
private:
std::unique_ptr<VideoDecoderFactory> factory_;
const bool supports_augmenting_data_;
};
} // namespace webrtc
#endif // MEDIA_ENGINE_MULTIPLEX_CODEC_FACTORY_H_

50
media/engine/multiplex_codec_factory_unittest.cc
View file

@ -1,50 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "media/engine/multiplex_codec_factory.h"
#include <memory>
#include <utility>
#include "api/environment/environment_factory.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "media/base/media_constants.h"
#include "media/engine/internal_decoder_factory.h"
#include "media/engine/internal_encoder_factory.h"
#include "test/gtest.h"
namespace webrtc {
TEST(MultiplexDecoderFactoryTest, CreateVideoDecoder) {
std::unique_ptr<VideoDecoderFactory> internal_factory =
std::make_unique<InternalDecoderFactory>();
MultiplexDecoderFactory factory(std::move(internal_factory));
std::unique_ptr<VideoDecoder> decoder = factory.Create(
CreateEnvironment(),
SdpVideoFormat(
cricket::kMultiplexCodecName,
{{cricket::kCodecParamAssociatedCodecName, cricket::kVp9CodecName}}));
EXPECT_TRUE(decoder);
}
TEST(MultiplexEncoderFactory, CreateVideoEncoder) {
std::unique_ptr<VideoEncoderFactory> internal_factory(
new InternalEncoderFactory());
MultiplexEncoderFactory factory(std::move(internal_factory));
std::unique_ptr<VideoEncoder> encoder =
factory.CreateVideoEncoder(SdpVideoFormat(
cricket::kMultiplexCodecName,
{{cricket::kCodecParamAssociatedCodecName, cricket::kVp9CodecName}}));
EXPECT_TRUE(encoder);
}
} // namespace webrtc

1
media/engine/webrtc_video_engine.cc
View file

@ -2225,7 +2225,6 @@ WebRtcVideoSendChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig(
case webrtc::kVideoCodecVP9:
case webrtc::kVideoCodecAV1:
case webrtc::kVideoCodecGeneric:
case webrtc::kVideoCodecMultiplex:
max_qp = kDefaultVideoMaxQpVpx;
break;
}

1
modules/rtp_rtcp/source/create_video_rtp_depacketizer.cc
View file

@ -43,7 +43,6 @@ std::unique_ptr<VideoRtpDepacketizer> CreateVideoRtpDepacketizer(
return nullptr;
#endif
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
return std::make_unique<VideoRtpDepacketizerGeneric>();
}
RTC_CHECK_NOTREACHED();

33
modules/video_coding/BUILD.gn
View file

@ -512,37 +512,6 @@ rtc_library("webrtc_h264") {
}
}
rtc_library("webrtc_multiplex") {
sources = [
"codecs/multiplex/augmented_video_frame_buffer.cc",
"codecs/multiplex/include/augmented_video_frame_buffer.h",
"codecs/multiplex/include/multiplex_decoder_adapter.h",
"codecs/multiplex/include/multiplex_encoder_adapter.h",
"codecs/multiplex/multiplex_decoder_adapter.cc",
"codecs/multiplex/multiplex_encoded_image_packer.cc",
"codecs/multiplex/multiplex_encoded_image_packer.h",
"codecs/multiplex/multiplex_encoder_adapter.cc",
]
deps = [
":video_codec_interface",
":video_coding_utility",
"../../api:fec_controller_api",
"../../api:scoped_refptr",
"../../api/environment",
"../../api/video:encoded_image",
"../../api/video:video_frame",
"../../api/video:video_rtp_headers",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
"../../media:rtc_media_base",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base/synchronization:mutex",
"../rtp_rtcp:rtp_rtcp_format",
]
}
# This target defines a bare-bones interface towards libvpx, used by the
# VP8 and VP9 wrappers below.
rtc_library("webrtc_libvpx_interface") {
@ -1054,7 +1023,6 @@ if (rtc_include_tests) {
sources = [
"codecs/h264/test/h264_impl_unittest.cc",
"codecs/multiplex/test/multiplex_adapter_unittest.cc",
"codecs/test/video_encoder_decoder_instantiation_tests.cc",
"codecs/test/videocodec_test_av1.cc",
"codecs/test/videocodec_test_libvpx.cc",
@ -1078,7 +1046,6 @@ if (rtc_include_tests) {
":videocodec_test_impl",
":webrtc_h264",
":webrtc_libvpx_interface",
":webrtc_multiplex",
":webrtc_vp8",
":webrtc_vp9",
":webrtc_vp9_helpers",

65
modules/video_coding/codecs/multiplex/augmented_video_frame_buffer.cc
View file

@ -1,65 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include <stdint.h>
#include <utility>
#include "api/video/video_frame_buffer.h"
namespace webrtc {
AugmentedVideoFrameBuffer::AugmentedVideoFrameBuffer(
const rtc::scoped_refptr<VideoFrameBuffer>& video_frame_buffer,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size)
: augmenting_data_size_(augmenting_data_size),
augmenting_data_(std::move(augmenting_data)),
video_frame_buffer_(video_frame_buffer) {}
rtc::scoped_refptr<VideoFrameBuffer>
AugmentedVideoFrameBuffer::GetVideoFrameBuffer() const {
return video_frame_buffer_;
}
uint8_t* AugmentedVideoFrameBuffer::GetAugmentingData() const {
return augmenting_data_.get();
}
uint16_t AugmentedVideoFrameBuffer::GetAugmentingDataSize() const {
return augmenting_data_size_;
}
VideoFrameBuffer::Type AugmentedVideoFrameBuffer::type() const {
return video_frame_buffer_->type();
}
int AugmentedVideoFrameBuffer::width() const {
return video_frame_buffer_->width();
}
int AugmentedVideoFrameBuffer::height() const {
return video_frame_buffer_->height();
}
rtc::scoped_refptr<I420BufferInterface> AugmentedVideoFrameBuffer::ToI420() {
return video_frame_buffer_->ToI420();
}
const I420BufferInterface* AugmentedVideoFrameBuffer::GetI420() const {
// TODO(https://crbug.com/webrtc/12021): When AugmentedVideoFrameBuffer is
// updated to implement the buffer interfaces of relevant
// VideoFrameBuffer::Types, stop overriding GetI420() as a workaround to
// AugmentedVideoFrameBuffer not being the type that is returned by type().
return video_frame_buffer_->GetI420();
}
} // namespace webrtc

62
modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h
View file

@ -1,62 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_
#include <cstdint>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/video/video_frame_buffer.h"
namespace webrtc {
class AugmentedVideoFrameBuffer : public VideoFrameBuffer {
public:
AugmentedVideoFrameBuffer(
const rtc::scoped_refptr<VideoFrameBuffer>& video_frame_buffer,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size);
// Retrieves the underlying VideoFrameBuffer without the augmented data
rtc::scoped_refptr<VideoFrameBuffer> GetVideoFrameBuffer() const;
// Gets a pointer to the augmenting data and moves ownership to the caller
uint8_t* GetAugmentingData() const;
// Get the size of the augmenting data
uint16_t GetAugmentingDataSize() const;
// Returns the type of the underlying VideoFrameBuffer
Type type() const final;
// Returns the width of the underlying VideoFrameBuffer
int width() const final;
// Returns the height of the underlying VideoFrameBuffer
int height() const final;
// Get the I140 Buffer from the underlying frame buffer
rtc::scoped_refptr<I420BufferInterface> ToI420() final;
// Returns GetI420() of the underlying VideoFrameBuffer.
// TODO(hbos): AugmentedVideoFrameBuffer should not return a type (such as
// kI420) without also implementing that type's interface (i.e.
// I420BufferInterface). Either implement all possible Type's interfaces or
// return kNative.
const I420BufferInterface* GetI420() const final;
private:
uint16_t augmenting_data_size_;
std::unique_ptr<uint8_t[]> augmenting_data_;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_

82
modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h
View file

@ -1,82 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_DECODER_ADAPTER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_DECODER_ADAPTER_H_
#include <map>
#include <memory>
#include <vector>
#include "api/environment/environment.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
namespace webrtc {
class MultiplexDecoderAdapter : public VideoDecoder {
public:
// `factory` is not owned and expected to outlive this class.
MultiplexDecoderAdapter(const Environment& env,
VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data = false);
virtual ~MultiplexDecoderAdapter();
// Implements VideoDecoder
bool Configure(const Settings& settings) override;
int32_t Decode(const EncodedImage& input_image,
int64_t render_time_ms) override;
int32_t RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) override;
int32_t Release() override;
void Decoded(AlphaCodecStream stream_idx,
VideoFrame* decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp);
private:
// Wrapper class that redirects Decoded() calls.
class AdapterDecodedImageCallback;
// Holds the decoded image output of a frame.
struct DecodedImageData;
// Holds the augmenting data of an image
struct AugmentingData;
void MergeAlphaImages(VideoFrame* decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp,
VideoFrame* multiplex_decoded_image,
const absl::optional<int32_t>& multiplex_decode_time_ms,
const absl::optional<uint8_t>& multiplex_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length);
const Environment env_;
VideoDecoderFactory* const factory_;
const SdpVideoFormat associated_format_;
std::vector<std::unique_ptr<VideoDecoder>> decoders_;
std::vector<std::unique_ptr<AdapterDecodedImageCallback>> adapter_callbacks_;
DecodedImageCallback* decoded_complete_callback_;
// Holds YUV or AXX decode output of a frame that is identified by timestamp.
std::map<uint32_t /* timestamp */, DecodedImageData> decoded_data_;
std::map<uint32_t /* timestamp */, AugmentingData> decoded_augmenting_data_;
const bool supports_augmenting_data_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_DECODER_ADAPTER_H_

91
modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h
View file

@ -1,91 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODER_ADAPTER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODER_ADAPTER_H_
#include <map>
#include <memory>
#include <vector>
#include "api/fec_controller_override.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/synchronization/mutex.h"
namespace webrtc {
enum AlphaCodecStream {
kYUVStream = 0,
kAXXStream = 1,
kAlphaCodecStreams = 2,
};
class MultiplexEncoderAdapter : public VideoEncoder {
public:
// `factory` is not owned and expected to outlive this class.
MultiplexEncoderAdapter(VideoEncoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data = false);
virtual ~MultiplexEncoderAdapter();
// Implements VideoEncoder
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int InitEncode(const VideoCodec* inst,
const VideoEncoder::Settings& settings) override;
int Encode(const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) override;
int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
void SetRates(const RateControlParameters& parameters) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(const LossNotification& loss_notification) override;
int Release() override;
EncoderInfo GetEncoderInfo() const override;
EncodedImageCallback::Result OnEncodedImage(
AlphaCodecStream stream_idx,
const EncodedImage& encodedImage,
const CodecSpecificInfo* codecSpecificInfo);
private:
// Wrapper class that redirects OnEncodedImage() calls.
class AdapterEncodedImageCallback;
VideoEncoderFactory* const factory_;
const SdpVideoFormat associated_format_;
std::vector<std::unique_ptr<VideoEncoder>> encoders_;
std::vector<std::unique_ptr<AdapterEncodedImageCallback>> adapter_callbacks_;
EncodedImageCallback* encoded_complete_callback_;
std::map<uint32_t /* timestamp */, MultiplexImage> stashed_images_
RTC_GUARDED_BY(mutex_);
uint16_t picture_index_ = 0;
std::vector<uint8_t> multiplex_dummy_planes_;
int key_frame_interval_;
EncodedImage combined_image_;
Mutex mutex_;
const bool supports_augmented_data_;
int augmenting_data_size_ = 0;
EncoderInfo encoder_info_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODER_ADAPTER_H_

267
modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc
View file

@ -1,267 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "api/environment/environment.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame_buffer.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "rtc_base/logging.h"
namespace webrtc {
class MultiplexDecoderAdapter::AdapterDecodedImageCallback
: public webrtc::DecodedImageCallback {
public:
AdapterDecodedImageCallback(webrtc::MultiplexDecoderAdapter* adapter,
AlphaCodecStream stream_idx)
: adapter_(adapter), stream_idx_(stream_idx) {}
void Decoded(VideoFrame& decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override {
if (!adapter_)
return;
adapter_->Decoded(stream_idx_, &decoded_image, decode_time_ms, qp);
}
int32_t Decoded(VideoFrame& decoded_image) override {
RTC_DCHECK_NOTREACHED();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override {
RTC_DCHECK_NOTREACHED();
return WEBRTC_VIDEO_CODEC_OK;
}
private:
MultiplexDecoderAdapter* adapter_;
const AlphaCodecStream stream_idx_;
};
struct MultiplexDecoderAdapter::DecodedImageData {
explicit DecodedImageData(AlphaCodecStream stream_idx)
: stream_idx_(stream_idx),
decoded_image_(
VideoFrame::Builder()
.set_video_frame_buffer(
I420Buffer::Create(1 /* width */, 1 /* height */))
.set_timestamp_rtp(0)
.set_timestamp_us(0)
.set_rotation(kVideoRotation_0)
.build()) {
RTC_DCHECK_EQ(kAXXStream, stream_idx);
}
DecodedImageData(AlphaCodecStream stream_idx,
const VideoFrame& decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp)
: stream_idx_(stream_idx),
decoded_image_(decoded_image),
decode_time_ms_(decode_time_ms),
qp_(qp) {}
DecodedImageData() = delete;
DecodedImageData(const DecodedImageData&) = delete;
DecodedImageData& operator=(const DecodedImageData&) = delete;
const AlphaCodecStream stream_idx_;
VideoFrame decoded_image_;
const absl::optional<int32_t> decode_time_ms_;
const absl::optional<uint8_t> qp_;
};
struct MultiplexDecoderAdapter::AugmentingData {
AugmentingData(std::unique_ptr<uint8_t[]> augmenting_data, uint16_t data_size)
: data_(std::move(augmenting_data)), size_(data_size) {}
AugmentingData() = delete;
AugmentingData(const AugmentingData&) = delete;
AugmentingData& operator=(const AugmentingData&) = delete;
std::unique_ptr<uint8_t[]> data_;
const uint16_t size_;
};
MultiplexDecoderAdapter::MultiplexDecoderAdapter(
const Environment& env,
VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmenting_data)
: env_(env),
factory_(factory),
associated_format_(associated_format),
supports_augmenting_data_(supports_augmenting_data) {}
MultiplexDecoderAdapter::~MultiplexDecoderAdapter() {
Release();
}
bool MultiplexDecoderAdapter::Configure(const Settings& settings) {
RTC_DCHECK_EQ(settings.codec_type(), kVideoCodecMultiplex);
Settings associated_settings = settings;
associated_settings.set_codec_type(
PayloadStringToCodecType(associated_format_.name));
for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
std::unique_ptr<VideoDecoder> decoder =
factory_->Create(env_, associated_format_);
if (!decoder->Configure(associated_settings)) {
return false;
}
adapter_callbacks_.emplace_back(
new MultiplexDecoderAdapter::AdapterDecodedImageCallback(
this, static_cast<AlphaCodecStream>(i)));
decoder->RegisterDecodeCompleteCallback(adapter_callbacks_.back().get());
decoders_.emplace_back(std::move(decoder));
}
return true;
}
int32_t MultiplexDecoderAdapter::Decode(const EncodedImage& input_image,
int64_t render_time_ms) {
MultiplexImage image = MultiplexEncodedImagePacker::Unpack(input_image);
if (supports_augmenting_data_) {
RTC_DCHECK(decoded_augmenting_data_.find(input_image.RtpTimestamp()) ==
decoded_augmenting_data_.end());
decoded_augmenting_data_.emplace(
std::piecewise_construct,
std::forward_as_tuple(input_image.RtpTimestamp()),
std::forward_as_tuple(std::move(image.augmenting_data),
image.augmenting_data_size));
}
if (image.component_count == 1) {
RTC_DCHECK(decoded_data_.find(input_image.RtpTimestamp()) ==
decoded_data_.end());
decoded_data_.emplace(std::piecewise_construct,
std::forward_as_tuple(input_image.RtpTimestamp()),
std::forward_as_tuple(kAXXStream));
}
int32_t rv = 0;
for (size_t i = 0; i < image.image_components.size(); i++) {
rv = decoders_[image.image_components[i].component_index]->Decode(
image.image_components[i].encoded_image, render_time_ms);
if (rv != WEBRTC_VIDEO_CODEC_OK)
return rv;
}
return rv;
}
int32_t MultiplexDecoderAdapter::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
decoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MultiplexDecoderAdapter::Release() {
for (auto& decoder : decoders_) {
const int32_t rv = decoder->Release();
if (rv)
return rv;
}
decoders_.clear();
adapter_callbacks_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
void MultiplexDecoderAdapter::Decoded(AlphaCodecStream stream_idx,
VideoFrame* decoded_image,
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) {
const auto& other_decoded_data_it =
decoded_data_.find(decoded_image->timestamp());
const auto& augmenting_data_it =
decoded_augmenting_data_.find(decoded_image->timestamp());
const bool has_augmenting_data =
augmenting_data_it != decoded_augmenting_data_.end();
if (other_decoded_data_it != decoded_data_.end()) {
uint16_t augmenting_data_size =
has_augmenting_data ? augmenting_data_it->second.size_ : 0;
std::unique_ptr<uint8_t[]> augmenting_data =
has_augmenting_data ? std::move(augmenting_data_it->second.data_)
: nullptr;
auto& other_image_data = other_decoded_data_it->second;
if (stream_idx == kYUVStream) {
RTC_DCHECK_EQ(kAXXStream, other_image_data.stream_idx_);
MergeAlphaImages(decoded_image, decode_time_ms, qp,
&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
std::move(augmenting_data), augmenting_data_size);
} else {
RTC_DCHECK_EQ(kYUVStream, other_image_data.stream_idx_);
RTC_DCHECK_EQ(kAXXStream, stream_idx);
MergeAlphaImages(&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
decoded_image, decode_time_ms, qp,
std::move(augmenting_data), augmenting_data_size);
}
decoded_data_.erase(decoded_data_.begin(), other_decoded_data_it);
if (has_augmenting_data) {
decoded_augmenting_data_.erase(decoded_augmenting_data_.begin(),
augmenting_data_it);
}
return;
}
RTC_DCHECK(decoded_data_.find(decoded_image->timestamp()) ==
decoded_data_.end());
decoded_data_.emplace(
std::piecewise_construct,
std::forward_as_tuple(decoded_image->timestamp()),
std::forward_as_tuple(stream_idx, *decoded_image, decode_time_ms, qp));
}
void MultiplexDecoderAdapter::MergeAlphaImages(
VideoFrame* decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp,
VideoFrame* alpha_decoded_image,
const absl::optional<int32_t>& alpha_decode_time_ms,
const absl::optional<uint8_t>& alpha_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length) {
rtc::scoped_refptr<VideoFrameBuffer> merged_buffer;
if (!alpha_decoded_image->timestamp()) {
merged_buffer = decoded_image->video_frame_buffer();
} else {
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
decoded_image->video_frame_buffer()->ToI420();
rtc::scoped_refptr<webrtc::I420BufferInterface> alpha_buffer =
alpha_decoded_image->video_frame_buffer()->ToI420();
RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width());
RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height());
merged_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(),
alpha_buffer->StrideY(),
// To keep references alive.
[yuv_buffer, alpha_buffer] {});
}
if (supports_augmenting_data_) {
merged_buffer = rtc::make_ref_counted<AugmentedVideoFrameBuffer>(
merged_buffer, std::move(augmenting_data), augmenting_data_length);
}
VideoFrame merged_image = VideoFrame::Builder()
.set_video_frame_buffer(merged_buffer)
.set_timestamp_rtp(decoded_image->timestamp())
.set_timestamp_us(0)
.set_rotation(decoded_image->rotation())
.set_id(decoded_image->id())
.set_packet_infos(decoded_image->packet_infos())
.build();
decoded_complete_callback_->Decoded(merged_image, decode_time_ms, qp);
}
} // namespace webrtc

277
modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.cc
View file

@ -1,277 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include <cstring>
#include <utility>
#include "modules/rtp_rtcp/source/byte_io.h"
#include "rtc_base/checks.h"
namespace webrtc {
int PackHeader(uint8_t* buffer, MultiplexImageHeader header) {
int offset = 0;
ByteWriter<uint8_t>::WriteBigEndian(buffer + offset, header.component_count);
offset += sizeof(uint8_t);
ByteWriter<uint16_t>::WriteBigEndian(buffer + offset, header.image_index);
offset += sizeof(uint16_t);
ByteWriter<uint16_t>::WriteBigEndian(buffer + offset,
header.augmenting_data_size);
offset += sizeof(uint16_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
header.augmenting_data_offset);
offset += sizeof(uint32_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
header.first_component_header_offset);
offset += sizeof(uint32_t);
RTC_DCHECK_EQ(offset, kMultiplexImageHeaderSize);
return offset;
}
MultiplexImageHeader UnpackHeader(const uint8_t* buffer) {
MultiplexImageHeader header;
int offset = 0;
header.component_count = ByteReader<uint8_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint8_t);
header.image_index = ByteReader<uint16_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint16_t);
header.augmenting_data_size =
ByteReader<uint16_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint16_t);
header.augmenting_data_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
header.first_component_header_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
RTC_DCHECK_EQ(offset, kMultiplexImageHeaderSize);
return header;
}
int PackFrameHeader(uint8_t* buffer,
MultiplexImageComponentHeader frame_header) {
int offset = 0;
ByteWriter<uint32_t>::WriteBigEndian(
buffer + offset, frame_header.next_component_header_offset);
offset += sizeof(uint32_t);
ByteWriter<uint8_t>::WriteBigEndian(buffer + offset,
frame_header.component_index);
offset += sizeof(uint8_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
frame_header.bitstream_offset);
offset += sizeof(uint32_t);
ByteWriter<uint32_t>::WriteBigEndian(buffer + offset,
frame_header.bitstream_length);
offset += sizeof(uint32_t);
ByteWriter<uint8_t>::WriteBigEndian(buffer + offset, frame_header.codec_type);
offset += sizeof(uint8_t);
ByteWriter<uint8_t>::WriteBigEndian(
buffer + offset, static_cast<uint8_t>(frame_header.frame_type));
offset += sizeof(uint8_t);
RTC_DCHECK_EQ(offset, kMultiplexImageComponentHeaderSize);
return offset;
}
MultiplexImageComponentHeader UnpackFrameHeader(const uint8_t* buffer) {
MultiplexImageComponentHeader frame_header;
int offset = 0;
frame_header.next_component_header_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
frame_header.component_index =
ByteReader<uint8_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint8_t);
frame_header.bitstream_offset =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
frame_header.bitstream_length =
ByteReader<uint32_t>::ReadBigEndian(buffer + offset);
offset += sizeof(uint32_t);
// This makes the wire format depend on the numeric values of the
// VideoCodecType and VideoFrameType enum constants.
frame_header.codec_type = static_cast<VideoCodecType>(
ByteReader<uint8_t>::ReadBigEndian(buffer + offset));
offset += sizeof(uint8_t);
frame_header.frame_type = static_cast<VideoFrameType>(
ByteReader<uint8_t>::ReadBigEndian(buffer + offset));
offset += sizeof(uint8_t);
RTC_DCHECK_EQ(offset, kMultiplexImageComponentHeaderSize);
return frame_header;
}
void PackBitstream(uint8_t* buffer, MultiplexImageComponent image) {
memcpy(buffer, image.encoded_image.data(), image.encoded_image.size());
}
MultiplexImage::MultiplexImage(uint16_t picture_index,
uint8_t frame_count,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size)
: image_index(picture_index),
component_count(frame_count),
augmenting_data_size(augmenting_data_size),
augmenting_data(std::move(augmenting_data)) {}
EncodedImage MultiplexEncodedImagePacker::PackAndRelease(
const MultiplexImage& multiplex_image) {
MultiplexImageHeader header;
std::vector<MultiplexImageComponentHeader> frame_headers;
header.component_count = multiplex_image.component_count;
header.image_index = multiplex_image.image_index;
int header_offset = kMultiplexImageHeaderSize;
header.first_component_header_offset = header_offset;
header.augmenting_data_offset =
header_offset +
kMultiplexImageComponentHeaderSize * header.component_count;
header.augmenting_data_size = multiplex_image.augmenting_data_size;
int bitstream_offset =
header.augmenting_data_offset + header.augmenting_data_size;
const std::vector<MultiplexImageComponent>& images =
multiplex_image.image_components;
EncodedImage combined_image = images[0].encoded_image;
for (size_t i = 0; i < images.size(); i++) {
MultiplexImageComponentHeader frame_header;
header_offset += kMultiplexImageComponentHeaderSize;
frame_header.next_component_header_offset =
(i == images.size() - 1) ? 0 : header_offset;
frame_header.component_index = images[i].component_index;
frame_header.bitstream_offset = bitstream_offset;
frame_header.bitstream_length =
static_cast<uint32_t>(images[i].encoded_image.size());
bitstream_offset += frame_header.bitstream_length;
frame_header.codec_type = images[i].codec_type;
frame_header.frame_type = images[i].encoded_image._frameType;
// As long as one component is delta frame, we have to mark the combined
// frame as delta frame, because it is necessary for all components to be
// key frame so as to decode the whole image without previous frame data.
// Thus only when all components are key frames, we can mark the combined
// frame as key frame.
if (frame_header.frame_type == VideoFrameType::kVideoFrameDelta) {
combined_image._frameType = VideoFrameType::kVideoFrameDelta;
}
frame_headers.push_back(frame_header);
}
auto buffer = EncodedImageBuffer::Create(bitstream_offset);
combined_image.SetEncodedData(buffer);
// header
header_offset = PackHeader(buffer->data(), header);
RTC_DCHECK_EQ(header.first_component_header_offset,
kMultiplexImageHeaderSize);
// Frame Header
for (size_t i = 0; i < images.size(); i++) {
int relative_offset =
PackFrameHeader(buffer->data() + header_offset, frame_headers[i]);
RTC_DCHECK_EQ(relative_offset, kMultiplexImageComponentHeaderSize);
header_offset = frame_headers[i].next_component_header_offset;
RTC_DCHECK_EQ(header_offset,
(i == images.size() - 1)
? 0
: (kMultiplexImageHeaderSize +
kMultiplexImageComponentHeaderSize * (i + 1)));
}
// Augmenting Data
if (multiplex_image.augmenting_data_size != 0) {
memcpy(buffer->data() + header.augmenting_data_offset,
multiplex_image.augmenting_data.get(),
multiplex_image.augmenting_data_size);
}
// Bitstreams
for (size_t i = 0; i < images.size(); i++) {
PackBitstream(buffer->data() + frame_headers[i].bitstream_offset,
images[i]);
}
return combined_image;
}
MultiplexImage MultiplexEncodedImagePacker::Unpack(
const EncodedImage& combined_image) {
const MultiplexImageHeader& header = UnpackHeader(combined_image.data());
std::vector<MultiplexImageComponentHeader> frame_headers;
int header_offset = header.first_component_header_offset;
while (header_offset > 0) {
frame_headers.push_back(
UnpackFrameHeader(combined_image.data() + header_offset));
header_offset = frame_headers.back().next_component_header_offset;
}
RTC_DCHECK_LE(frame_headers.size(), header.component_count);
std::unique_ptr<uint8_t[]> augmenting_data = nullptr;
if (header.augmenting_data_size != 0) {
augmenting_data =
std::unique_ptr<uint8_t[]>(new uint8_t[header.augmenting_data_size]);
memcpy(augmenting_data.get(),
combined_image.data() + header.augmenting_data_offset,
header.augmenting_data_size);
}
MultiplexImage multiplex_image(header.image_index, header.component_count,
std::move(augmenting_data),
header.augmenting_data_size);
for (size_t i = 0; i < frame_headers.size(); i++) {
MultiplexImageComponent image_component;
image_component.component_index = frame_headers[i].component_index;
image_component.codec_type = frame_headers[i].codec_type;
EncodedImage encoded_image = combined_image;
encoded_image.SetRtpTimestamp(combined_image.RtpTimestamp());
encoded_image._frameType = frame_headers[i].frame_type;
encoded_image.SetEncodedData(EncodedImageBuffer::Create(
combined_image.data() + frame_headers[i].bitstream_offset,
frame_headers[i].bitstream_length));
image_component.encoded_image = encoded_image;
multiplex_image.image_components.push_back(image_component);
}
return multiplex_image;
}
} // namespace webrtc

120
modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h
View file

@ -1,120 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_MULTIPLEX_ENCODED_IMAGE_PACKER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_MULTIPLEX_ENCODED_IMAGE_PACKER_H_
#include <cstdint>
#include <memory>
#include <vector>
#include "api/video/encoded_image.h"
#include "api/video_codecs/video_codec.h"
namespace webrtc {
// Struct describing the whole bundle of multiple frames of an image.
// This struct is expected to be the set in the beginning of a picture's
// bitstream data.
struct MultiplexImageHeader {
// The number of frame components making up the complete picture data.
// For example, `frame_count` = 2 for the case of YUV frame with Alpha frame.
uint8_t component_count;
// The increasing image ID given by the encoder. For different components
// of a single picture, they have the same `picture_index`.
uint16_t image_index;
// The location of the first MultiplexImageComponentHeader in the bitstream,
// in terms of byte from the beginning of the bitstream.
uint32_t first_component_header_offset;
// The location of the augmenting data in the bitstream, in terms of bytes
// from the beginning of the bitstream
uint32_t augmenting_data_offset;
// The size of the augmenting data in the bitstream it terms of byte
uint16_t augmenting_data_size;
};
const int kMultiplexImageHeaderSize =
sizeof(uint8_t) + 2 * sizeof(uint16_t) + 2 * sizeof(uint32_t);
// Struct describing the individual image component's content.
struct MultiplexImageComponentHeader {
// The location of the next MultiplexImageComponentHeader in the bitstream,
// in terms of the byte from the beginning of the bitstream;
uint32_t next_component_header_offset;
// Identifies which component this frame represent, i.e. YUV frame vs Alpha
// frame.
uint8_t component_index;
// The location of the real encoded image data of the frame in the bitstream,
// in terms of byte from the beginning of the bitstream.
uint32_t bitstream_offset;
// Indicates the number of bytes of the encoded image data.
uint32_t bitstream_length;
// Indicated the underlying VideoCodecType of the frame, i.e. VP9 or VP8 etc.
VideoCodecType codec_type;
// Indicated the underlying frame is a key frame or delta frame.
VideoFrameType frame_type;
};
const int kMultiplexImageComponentHeaderSize =
sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t) + sizeof(uint32_t) +
sizeof(uint8_t) + sizeof(uint8_t);
// Struct holding the encoded image for one component.
struct MultiplexImageComponent {
// Indicated the underlying VideoCodecType of the frame, i.e. VP9 or VP8 etc.
VideoCodecType codec_type;
// Identifies which component this frame represent, i.e. YUV frame vs Alpha
// frame.
uint8_t component_index;
// Stores the actual frame data of the encoded image.
EncodedImage encoded_image;
};
// Struct holding the whole frame bundle of components of an image.
struct MultiplexImage {
uint16_t image_index;
uint8_t component_count;
uint16_t augmenting_data_size;
std::unique_ptr<uint8_t[]> augmenting_data;
std::vector<MultiplexImageComponent> image_components;
MultiplexImage(uint16_t picture_index,
uint8_t component_count,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size);
};
// A utility class providing conversion between two representations of a
// multiplex image frame:
// 1. Packed version is just one encoded image, we pack all necessary metadata
// in the bitstream as headers.
// 2. Unpacked version is essentially a list of encoded images, one for one
// component.
class MultiplexEncodedImagePacker {
public:
// Note: It is caller responsibility to release the buffer of the result.
static EncodedImage PackAndRelease(const MultiplexImage& image);
// Note: The image components just share the memory with `combined_image`.
static MultiplexImage Unpack(const EncodedImage& combined_image);
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_MULTIPLEX_ENCODED_IMAGE_PACKER_H_

356
modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc
View file

@ -1,356 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include <cstring>
#include "api/video/encoded_image.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/video_common.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "rtc_base/logging.h"
namespace webrtc {
// Callback wrapper that helps distinguish returned results from `encoders_`
// instances.
class MultiplexEncoderAdapter::AdapterEncodedImageCallback
: public webrtc::EncodedImageCallback {
public:
AdapterEncodedImageCallback(webrtc::MultiplexEncoderAdapter* adapter,
AlphaCodecStream stream_idx)
: adapter_(adapter), stream_idx_(stream_idx) {}
EncodedImageCallback::Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override {
if (!adapter_)
return Result(Result::OK);
return adapter_->OnEncodedImage(stream_idx_, encoded_image,
codec_specific_info);
}
private:
MultiplexEncoderAdapter* adapter_;
const AlphaCodecStream stream_idx_;
};
MultiplexEncoderAdapter::MultiplexEncoderAdapter(
VideoEncoderFactory* factory,
const SdpVideoFormat& associated_format,
bool supports_augmented_data)
: factory_(factory),
associated_format_(associated_format),
encoded_complete_callback_(nullptr),
key_frame_interval_(0),
supports_augmented_data_(supports_augmented_data) {}
MultiplexEncoderAdapter::~MultiplexEncoderAdapter() {
Release();
}
void MultiplexEncoderAdapter::SetFecControllerOverride(
FecControllerOverride* fec_controller_override) {
// Ignored.
}
int MultiplexEncoderAdapter::InitEncode(
const VideoCodec* inst,
const VideoEncoder::Settings& settings) {
const size_t buffer_size =
CalcBufferSize(VideoType::kI420, inst->width, inst->height);
multiplex_dummy_planes_.resize(buffer_size);
// It is more expensive to encode 0x00, so use 0x80 instead.
std::fill(multiplex_dummy_planes_.begin(), multiplex_dummy_planes_.end(),
0x80);
RTC_DCHECK_EQ(kVideoCodecMultiplex, inst->codecType);
VideoCodec video_codec = *inst;
video_codec.codecType = PayloadStringToCodecType(associated_format_.name);
// Take over the key frame interval at adapter level, because we have to
// sync the key frames for both sub-encoders.
switch (video_codec.codecType) {
case kVideoCodecVP8:
key_frame_interval_ = video_codec.VP8()->keyFrameInterval;
video_codec.VP8()->keyFrameInterval = 0;
break;
case kVideoCodecVP9:
key_frame_interval_ = video_codec.VP9()->keyFrameInterval;
video_codec.VP9()->keyFrameInterval = 0;
break;
case kVideoCodecH264:
key_frame_interval_ = video_codec.H264()->keyFrameInterval;
video_codec.H264()->keyFrameInterval = 0;
break;
case kVideoCodecH265:
// TODO(bugs.webrtc.org/13485)
break;
default:
break;
}
encoder_info_ = EncoderInfo();
encoder_info_.implementation_name = "MultiplexEncoderAdapter (";
encoder_info_.requested_resolution_alignment = 1;
encoder_info_.apply_alignment_to_all_simulcast_layers = false;
// This needs to be false so that we can do the split in Encode().
encoder_info_.supports_native_handle = false;
for (size_t i = 0; i < kAlphaCodecStreams; ++i) {
std::unique_ptr<VideoEncoder> encoder =
factory_->CreateVideoEncoder(associated_format_);
const int rv = encoder->InitEncode(&video_codec, settings);
if (rv) {
RTC_LOG(LS_ERROR) << "Failed to create multiplex codec index " << i;
return rv;
}
adapter_callbacks_.emplace_back(new AdapterEncodedImageCallback(
this, static_cast<AlphaCodecStream>(i)));
encoder->RegisterEncodeCompleteCallback(adapter_callbacks_.back().get());
const EncoderInfo& encoder_impl_info = encoder->GetEncoderInfo();
encoder_info_.implementation_name += encoder_impl_info.implementation_name;
if (i != kAlphaCodecStreams - 1) {
encoder_info_.implementation_name += ", ";
}
// Uses hardware support if any of the encoders uses it.
// For example, if we are having issues with down-scaling due to
// pipelining delay in HW encoders we need higher encoder usage
// thresholds in CPU adaptation.
if (i == 0) {
encoder_info_.is_hardware_accelerated =
encoder_impl_info.is_hardware_accelerated;
} else {
encoder_info_.is_hardware_accelerated |=
encoder_impl_info.is_hardware_accelerated;
}
encoder_info_.requested_resolution_alignment = cricket::LeastCommonMultiple(
encoder_info_.requested_resolution_alignment,
encoder_impl_info.requested_resolution_alignment);
if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
encoder_info_.apply_alignment_to_all_simulcast_layers = true;
}
encoders_.emplace_back(std::move(encoder));
}
encoder_info_.implementation_name += ")";
return WEBRTC_VIDEO_CODEC_OK;
}
int MultiplexEncoderAdapter::Encode(
const VideoFrame& input_image,
const std::vector<VideoFrameType>* frame_types) {
if (!encoded_complete_callback_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
// The input image is forwarded as-is, unless it is a native buffer and
// `supports_augmented_data_` is true in which case we need to map it in order
// to access the underlying AugmentedVideoFrameBuffer.
VideoFrame forwarded_image = input_image;
if (supports_augmented_data_ &&
forwarded_image.video_frame_buffer()->type() ==
VideoFrameBuffer::Type::kNative) {
auto info = GetEncoderInfo();
rtc::scoped_refptr<VideoFrameBuffer> mapped_buffer =
forwarded_image.video_frame_buffer()->GetMappedFrameBuffer(
info.preferred_pixel_formats);
if (!mapped_buffer) {
// Unable to map the buffer.
return WEBRTC_VIDEO_CODEC_ERROR;
}
forwarded_image.set_video_frame_buffer(std::move(mapped_buffer));
}
std::vector<VideoFrameType> adjusted_frame_types;
if (key_frame_interval_ > 0 && picture_index_ % key_frame_interval_ == 0) {
adjusted_frame_types.push_back(VideoFrameType::kVideoFrameKey);
} else {
adjusted_frame_types.push_back(VideoFrameType::kVideoFrameDelta);
}
const bool has_alpha = forwarded_image.video_frame_buffer()->type() ==
VideoFrameBuffer::Type::kI420A;
std::unique_ptr<uint8_t[]> augmenting_data = nullptr;
uint16_t augmenting_data_length = 0;
AugmentedVideoFrameBuffer* augmented_video_frame_buffer = nullptr;
if (supports_augmented_data_) {
augmented_video_frame_buffer = static_cast<AugmentedVideoFrameBuffer*>(
forwarded_image.video_frame_buffer().get());
augmenting_data_length =
augmented_video_frame_buffer->GetAugmentingDataSize();
augmenting_data =
std::unique_ptr<uint8_t[]>(new uint8_t[augmenting_data_length]);
memcpy(augmenting_data.get(),
augmented_video_frame_buffer->GetAugmentingData(),
augmenting_data_length);
augmenting_data_size_ = augmenting_data_length;
}
{
MutexLock lock(&mutex_);
stashed_images_.emplace(
std::piecewise_construct,
std::forward_as_tuple(forwarded_image.timestamp()),
std::forward_as_tuple(
picture_index_, has_alpha ? kAlphaCodecStreams : 1,
std::move(augmenting_data), augmenting_data_length));
}
++picture_index_;
// Encode YUV
int rv =
encoders_[kYUVStream]->Encode(forwarded_image, &adjusted_frame_types);
// If we do not receive an alpha frame, we send a single frame for this
// `picture_index_`. The receiver will receive `frame_count` as 1 which
// specifies this case.
if (rv || !has_alpha)
return rv;
// Encode AXX
rtc::scoped_refptr<VideoFrameBuffer> frame_buffer =
supports_augmented_data_
? augmented_video_frame_buffer->GetVideoFrameBuffer()
: forwarded_image.video_frame_buffer();
const I420ABufferInterface* yuva_buffer = frame_buffer->GetI420A();
rtc::scoped_refptr<I420BufferInterface> alpha_buffer =
WrapI420Buffer(forwarded_image.width(), forwarded_image.height(),
yuva_buffer->DataA(), yuva_buffer->StrideA(),
multiplex_dummy_planes_.data(), yuva_buffer->StrideU(),
multiplex_dummy_planes_.data(), yuva_buffer->StrideV(),
// To keep reference alive.
[frame_buffer] {});
VideoFrame alpha_image =
VideoFrame::Builder()
.set_video_frame_buffer(alpha_buffer)
.set_timestamp_rtp(forwarded_image.timestamp())
.set_timestamp_ms(forwarded_image.render_time_ms())
.set_rotation(forwarded_image.rotation())
.set_id(forwarded_image.id())
.set_packet_infos(forwarded_image.packet_infos())
.build();
rv = encoders_[kAXXStream]->Encode(alpha_image, &adjusted_frame_types);
return rv;
}
int MultiplexEncoderAdapter::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
encoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
void MultiplexEncoderAdapter::SetRates(
const RateControlParameters& parameters) {
VideoBitrateAllocation bitrate_allocation(parameters.bitrate);
bitrate_allocation.SetBitrate(
0, 0, parameters.bitrate.GetBitrate(0, 0) - augmenting_data_size_);
for (auto& encoder : encoders_) {
// TODO(emircan): `framerate` is used to calculate duration in encoder
// instances. We report the total frame rate to keep real time for now.
// Remove this after refactoring duration logic.
encoder->SetRates(RateControlParameters(
bitrate_allocation,
static_cast<uint32_t>(encoders_.size() * parameters.framerate_fps),
parameters.bandwidth_allocation -
DataRate::BitsPerSec(augmenting_data_size_)));
}
}
void MultiplexEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
for (auto& encoder : encoders_) {
encoder->OnPacketLossRateUpdate(packet_loss_rate);
}
}
void MultiplexEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
for (auto& encoder : encoders_) {
encoder->OnRttUpdate(rtt_ms);
}
}
void MultiplexEncoderAdapter::OnLossNotification(
const LossNotification& loss_notification) {
for (auto& encoder : encoders_) {
encoder->OnLossNotification(loss_notification);
}
}
int MultiplexEncoderAdapter::Release() {
for (auto& encoder : encoders_) {
const int rv = encoder->Release();
if (rv)
return rv;
}
encoders_.clear();
adapter_callbacks_.clear();
MutexLock lock(&mutex_);
stashed_images_.clear();
return WEBRTC_VIDEO_CODEC_OK;
}
VideoEncoder::EncoderInfo MultiplexEncoderAdapter::GetEncoderInfo() const {
return encoder_info_;
}
EncodedImageCallback::Result MultiplexEncoderAdapter::OnEncodedImage(
AlphaCodecStream stream_idx,
const EncodedImage& encodedImage,
const CodecSpecificInfo* codecSpecificInfo) {
// Save the image
MultiplexImageComponent image_component;
image_component.component_index = stream_idx;
image_component.codec_type =
PayloadStringToCodecType(associated_format_.name);
image_component.encoded_image = encodedImage;
MutexLock lock(&mutex_);
const auto& stashed_image_itr =
stashed_images_.find(encodedImage.RtpTimestamp());
const auto& stashed_image_next_itr = std::next(stashed_image_itr, 1);
RTC_DCHECK(stashed_image_itr != stashed_images_.end());
MultiplexImage& stashed_image = stashed_image_itr->second;
const uint8_t frame_count = stashed_image.component_count;
stashed_image.image_components.push_back(image_component);
if (stashed_image.image_components.size() == frame_count) {
// Complete case
for (auto iter = stashed_images_.begin();
iter != stashed_images_.end() && iter != stashed_image_next_itr;
iter++) {
// No image at all, skip.
if (iter->second.image_components.size() == 0)
continue;
// We have to send out those stashed frames, otherwise the delta frame
// dependency chain is broken.
combined_image_ =
MultiplexEncodedImagePacker::PackAndRelease(iter->second);
CodecSpecificInfo codec_info = *codecSpecificInfo;
codec_info.codecType = kVideoCodecMultiplex;
encoded_complete_callback_->OnEncodedImage(combined_image_, &codec_info);
}
stashed_images_.erase(stashed_images_.begin(), stashed_image_next_itr);
}
return EncodedImageCallback::Result(EncodedImageCallback::Result::OK);
}
} // namespace webrtc

323
modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc
View file

@ -1,323 +0,0 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stddef.h>
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/environment/environment.h"
#include "api/environment/environment_factory.h"
#include "api/scoped_refptr.h"
#include "api/test/mock_video_decoder_factory.h"
#include "api/test/mock_video_encoder_factory.h"
#include "api/video/encoded_image.h"
#include "api/video/video_frame.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/media_constants.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/video_codec_settings.h"
using ::testing::_;
using ::testing::Return;
namespace webrtc {
constexpr const char* kMultiplexAssociatedCodecName = cricket::kVp9CodecName;
const VideoCodecType kMultiplexAssociatedCodecType =
PayloadStringToCodecType(kMultiplexAssociatedCodecName);
class TestMultiplexAdapter : public VideoCodecUnitTest,
public ::testing::WithParamInterface<
bool /* supports_augmenting_data */> {
public:
TestMultiplexAdapter()
: decoder_factory_(new webrtc::MockVideoDecoderFactory),
encoder_factory_(new webrtc::MockVideoEncoderFactory),
supports_augmenting_data_(GetParam()) {}
protected:
std::unique_ptr<VideoDecoder> CreateDecoder() override {
return std::make_unique<MultiplexDecoderAdapter>(
env_, decoder_factory_.get(),
SdpVideoFormat(kMultiplexAssociatedCodecName),
supports_augmenting_data_);
}
std::unique_ptr<VideoEncoder> CreateEncoder() override {
return std::make_unique<MultiplexEncoderAdapter>(
encoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName),
supports_augmenting_data_);
}
void ModifyCodecSettings(VideoCodec* codec_settings) override {
webrtc::test::CodecSettings(kMultiplexAssociatedCodecType, codec_settings);
codec_settings->VP9()->numberOfTemporalLayers = 1;
codec_settings->VP9()->numberOfSpatialLayers = 1;
codec_settings->codecType = webrtc::kVideoCodecMultiplex;
}
std::unique_ptr<VideoFrame> CreateDataAugmentedInputFrame(
VideoFrame* video_frame) {
rtc::scoped_refptr<VideoFrameBuffer> video_buffer =
video_frame->video_frame_buffer();
std::unique_ptr<uint8_t[]> data =
std::unique_ptr<uint8_t[]>(new uint8_t[16]);
for (int i = 0; i < 16; i++) {
data[i] = i;
}
auto augmented_video_frame_buffer =
rtc::make_ref_counted<AugmentedVideoFrameBuffer>(video_buffer,
std::move(data), 16);
return std::make_unique<VideoFrame>(
VideoFrame::Builder()
.set_video_frame_buffer(augmented_video_frame_buffer)
.set_timestamp_rtp(video_frame->timestamp())
.set_timestamp_ms(video_frame->render_time_ms())
.set_rotation(video_frame->rotation())
.set_id(video_frame->id())
.build());
}
std::unique_ptr<VideoFrame> CreateI420AInputFrame() {
VideoFrame input_frame = NextInputFrame();
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
input_frame.video_frame_buffer()->ToI420();
rtc::scoped_refptr<I420ABufferInterface> yuva_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), yuv_buffer->DataY(),
yuv_buffer->StrideY(),
// To keep reference alive.
[yuv_buffer] {});
return std::make_unique<VideoFrame>(VideoFrame::Builder()
.set_video_frame_buffer(yuva_buffer)
.set_timestamp_rtp(123)
.set_timestamp_ms(345)
.set_rotation(kVideoRotation_0)
.build());
}
std::unique_ptr<VideoFrame> CreateInputFrame(bool contains_alpha) {
std::unique_ptr<VideoFrame> video_frame;
if (contains_alpha) {
video_frame = CreateI420AInputFrame();
} else {
VideoFrame next_frame = NextInputFrame();
video_frame = std::make_unique<VideoFrame>(
VideoFrame::Builder()
.set_video_frame_buffer(next_frame.video_frame_buffer())
.set_timestamp_rtp(next_frame.timestamp())
.set_timestamp_ms(next_frame.render_time_ms())
.set_rotation(next_frame.rotation())
.set_id(next_frame.id())
.build());
}
if (supports_augmenting_data_) {
video_frame = CreateDataAugmentedInputFrame(video_frame.get());
}
return video_frame;
}
void CheckData(rtc::scoped_refptr<VideoFrameBuffer> video_frame_buffer) {
if (!supports_augmenting_data_) {
return;
}
AugmentedVideoFrameBuffer* augmented_buffer =
static_cast<AugmentedVideoFrameBuffer*>(video_frame_buffer.get());
EXPECT_EQ(augmented_buffer->GetAugmentingDataSize(), 16);
uint8_t* data = augmented_buffer->GetAugmentingData();
for (int i = 0; i < 16; i++) {
EXPECT_EQ(data[i], i);
}
}
std::unique_ptr<VideoFrame> ExtractAXXFrame(const VideoFrame& video_frame) {
rtc::scoped_refptr<VideoFrameBuffer> video_frame_buffer =
video_frame.video_frame_buffer();
if (supports_augmenting_data_) {
AugmentedVideoFrameBuffer* augmentedBuffer =
static_cast<AugmentedVideoFrameBuffer*>(video_frame_buffer.get());
video_frame_buffer = augmentedBuffer->GetVideoFrameBuffer();
}
const I420ABufferInterface* yuva_buffer = video_frame_buffer->GetI420A();
rtc::scoped_refptr<I420BufferInterface> axx_buffer = WrapI420Buffer(
yuva_buffer->width(), yuva_buffer->height(), yuva_buffer->DataA(),
yuva_buffer->StrideA(), yuva_buffer->DataU(), yuva_buffer->StrideU(),
yuva_buffer->DataV(), yuva_buffer->StrideV(), [video_frame_buffer] {});
return std::make_unique<VideoFrame>(VideoFrame::Builder()
.set_video_frame_buffer(axx_buffer)
.set_timestamp_rtp(123)
.set_timestamp_ms(345)
.set_rotation(kVideoRotation_0)
.build());
}
private:
void SetUp() override {
EXPECT_CALL(*decoder_factory_, Die);
// The decoders/encoders will be owned by the caller of
// CreateVideoDecoder()/CreateVideoEncoder().
EXPECT_CALL(*decoder_factory_, Create).Times(2).WillRepeatedly([] {
return VP9Decoder::Create();
});
EXPECT_CALL(*encoder_factory_, Die);
EXPECT_CALL(*encoder_factory_, CreateVideoEncoder)
.Times(2)
.WillRepeatedly([] { return VP9Encoder::Create(); });
VideoCodecUnitTest::SetUp();
}
const Environment env_ = CreateEnvironment();
const std::unique_ptr<webrtc::MockVideoDecoderFactory> decoder_factory_;
const std::unique_ptr<webrtc::MockVideoEncoderFactory> encoder_factory_;
const bool supports_augmenting_data_;
};
// TODO(emircan): Currently VideoCodecUnitTest tests do a complete setup
// step that goes beyond constructing `decoder_`. Simplify these tests to do
// less.
TEST_P(TestMultiplexAdapter, ConstructAndDestructDecoder) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
}
TEST_P(TestMultiplexAdapter, ConstructAndDestructEncoder) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
}
TEST_P(TestMultiplexAdapter, EncodeDecodeI420Frame) {
std::unique_ptr<VideoFrame> input_frame = CreateInputFrame(false);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*input_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, -1));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame.get(), decoded_frame.get()), 36);
CheckData(decoded_frame->video_frame_buffer());
}
TEST_P(TestMultiplexAdapter, EncodeDecodeI420AFrame) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*yuva_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, 0));
std::unique_ptr<VideoFrame> decoded_frame;
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(yuva_frame.get(), decoded_frame.get()), 36);
// Find PSNR for AXX bits.
std::unique_ptr<VideoFrame> input_axx_frame = ExtractAXXFrame(*yuva_frame);
std::unique_ptr<VideoFrame> output_axx_frame =
ExtractAXXFrame(*decoded_frame);
EXPECT_GT(I420PSNR(input_axx_frame.get(), output_axx_frame.get()), 47);
CheckData(decoded_frame->video_frame_buffer());
}
TEST_P(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) {
std::unique_ptr<VideoFrame> input_frame = CreateInputFrame(false);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*input_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_FALSE(encoded_frame.SpatialIndex());
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
EXPECT_EQ(0, unpacked_frame.image_index);
EXPECT_EQ(1, unpacked_frame.component_count);
const MultiplexImageComponent& component = unpacked_frame.image_components[0];
EXPECT_EQ(0, component.component_index);
EXPECT_NE(nullptr, component.encoded_image.data());
EXPECT_EQ(VideoFrameType::kVideoFrameKey, component.encoded_image._frameType);
}
TEST_P(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*yuva_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(kVideoCodecMultiplex, codec_specific_info.codecType);
EXPECT_FALSE(encoded_frame.SpatialIndex());
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
EXPECT_EQ(0, unpacked_frame.image_index);
EXPECT_EQ(2, unpacked_frame.component_count);
EXPECT_EQ(unpacked_frame.image_components.size(),
unpacked_frame.component_count);
for (int i = 0; i < unpacked_frame.component_count; ++i) {
const MultiplexImageComponent& component =
unpacked_frame.image_components[i];
EXPECT_EQ(i, component.component_index);
EXPECT_NE(nullptr, component.encoded_image.data());
EXPECT_EQ(VideoFrameType::kVideoFrameKey,
component.encoded_image._frameType);
}
}
TEST_P(TestMultiplexAdapter, ImageIndexIncreases) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
const size_t expected_num_encoded_frames = 3;
for (size_t i = 0; i < expected_num_encoded_frames; ++i) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*yuva_frame, nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
const MultiplexImage& unpacked_frame =
MultiplexEncodedImagePacker::Unpack(encoded_frame);
EXPECT_EQ(i, unpacked_frame.image_index);
EXPECT_EQ(
i ? VideoFrameType::kVideoFrameDelta : VideoFrameType::kVideoFrameKey,
encoded_frame._frameType);
}
}
INSTANTIATE_TEST_SUITE_P(TestMultiplexAdapter,
TestMultiplexAdapter,
::testing::Bool());
} // namespace webrtc

11
modules/video_coding/video_codec_initializer.cc
View file

@ -34,17 +34,6 @@ namespace webrtc {
bool VideoCodecInitializer::SetupCodec(const VideoEncoderConfig& config,
const std::vector<VideoStream>& streams,
VideoCodec* codec) {
if (config.codec_type == kVideoCodecMultiplex) {
VideoEncoderConfig associated_config = config.Copy();
associated_config.codec_type = kVideoCodecVP9;
if (!SetupCodec(associated_config, streams, codec)) {
RTC_LOG(LS_ERROR) << "Failed to create stereo encoder configuration.";
return false;
}
codec->codecType = kVideoCodecMultiplex;
return true;
}
*codec = VideoEncoderConfigToVideoCodec(config, streams);
return true;
}

10
modules/video_coding/video_codec_initializer_unittest.cc
View file

@ -86,8 +86,6 @@ class VideoCodecInitializerTest : public ::testing::Test {
vp9_settings.numberOfTemporalLayers = num_temporal_streams;
config_.encoder_specific_settings = rtc::make_ref_counted<
webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
} else if (type != VideoCodecType::kVideoCodecMultiplex) {
ADD_FAILURE() << "Unexpected codec type: " << type;
}
}
@ -100,8 +98,6 @@ class VideoCodecInitializerTest : public ::testing::Test {
bitrate_allocator_ = CreateBuiltinVideoBitrateAllocatorFactory()
->CreateVideoBitrateAllocator(codec_out_);
RTC_CHECK(bitrate_allocator_);
if (codec_out_.codecType == VideoCodecType::kVideoCodecMultiplex)
return true;
// Make sure temporal layers instances have been created.
if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) {
@ -288,12 +284,6 @@ TEST_F(VideoCodecInitializerTest, HighFpsSimulcastVp8Screenshare) {
bitrate_allocation.GetBitrate(1, 1));
}
TEST_F(VideoCodecInitializerTest, SingleStreamMultiplexCodec) {
SetUpFor(VideoCodecType::kVideoCodecMultiplex, absl::nullopt, 1, 1, true);
streams_.push_back(DefaultStream());
EXPECT_TRUE(InitializeCodec());
}
TEST_F(VideoCodecInitializerTest, Vp9SvcDefaultLayering) {
SetUpFor(VideoCodecType::kVideoCodecVP9, absl::nullopt, 3, 3, false);
VideoStream stream = DefaultStream();

4
rtc_base/experiments/balanced_degradation_settings_unittest.cc
View file

@ -70,7 +70,6 @@ TEST(BalancedDegradationSettings, GetsDefaultConfigIfNoList) {
EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecH264, 1));
EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecAV1, 1));
EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecGeneric, 1));
EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecMultiplex, 1));
}
TEST(BalancedDegradationSettings, GetsConfig) {
@ -399,7 +398,6 @@ TEST(BalancedDegradationSettings, CanAdaptUpWithCodecType) {
EXPECT_TRUE(s.CanAdaptUp(kVideoCodecAV1, 1000, 77000));
EXPECT_FALSE(s.CanAdaptUp(kVideoCodecGeneric, 1000, 24000));
EXPECT_TRUE(s.CanAdaptUp(kVideoCodecGeneric, 1000, 25000));
EXPECT_TRUE(s.CanAdaptUp(kVideoCodecMultiplex, 1000, 1)); // Not configured.
}
TEST(BalancedDegradationSettings, CanAdaptUpResolution) {
@ -435,8 +433,6 @@ TEST(BalancedDegradationSettings, CanAdaptUpResolutionWithCodecType) {
EXPECT_TRUE(s.CanAdaptUpResolution(kVideoCodecAV1, 1000, 77000));
EXPECT_FALSE(s.CanAdaptUpResolution(kVideoCodecGeneric, 1000, 24000));
EXPECT_TRUE(s.CanAdaptUpResolution(kVideoCodecGeneric, 1000, 25000));
EXPECT_TRUE(s.CanAdaptUpResolution(kVideoCodecMultiplex, 1000,
1)); // Not configured.
}
TEST(BalancedDegradationSettings, GetsFpsDiff) {

1
rtc_base/experiments/min_video_bitrate_experiment.cc
View file

@ -103,7 +103,6 @@ absl::optional<DataRate> GetExperimentalMinVideoBitrate(VideoCodecType type) {
case kVideoCodecH264:
return min_bitrate_h264.GetOptional();
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
return absl::nullopt;
}

21
rtc_base/experiments/min_video_bitrate_experiment_unittest.cc
View file

@ -31,9 +31,6 @@ TEST(GetExperimentalMinVideoBitrateTest,
absl::nullopt);
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::nullopt);
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::nullopt);
}
TEST(GetExperimentalMinVideoBitrateTest,
@ -49,9 +46,6 @@ TEST(GetExperimentalMinVideoBitrateTest,
absl::nullopt);
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::nullopt);
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::nullopt);
}
TEST(GetExperimentalMinVideoBitrateTest, BrForAllCodecsIfDefined) {
@ -66,9 +60,6 @@ TEST(GetExperimentalMinVideoBitrateTest, BrForAllCodecsIfDefined) {
absl::make_optional(DataRate::KilobitsPerSec(123)));
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::make_optional(DataRate::KilobitsPerSec(123)));
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::make_optional(DataRate::KilobitsPerSec(123)));
}
TEST(GetExperimentalMinVideoBitrateTest, BrTrumpsSpecificCodecConfigs) {
@ -84,9 +75,6 @@ TEST(GetExperimentalMinVideoBitrateTest, BrTrumpsSpecificCodecConfigs) {
absl::make_optional(DataRate::KilobitsPerSec(123)));
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::make_optional(DataRate::KilobitsPerSec(123)));
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::make_optional(DataRate::KilobitsPerSec(123)));
}
TEST(GetExperimentalMinVideoBitrateTest,
@ -103,9 +91,6 @@ TEST(GetExperimentalMinVideoBitrateTest,
absl::nullopt);
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::nullopt);
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::nullopt);
}
TEST(GetExperimentalMinVideoBitrateTest, SpecificCodecConfigsUsedIfExpEnabled) {
@ -121,9 +106,6 @@ TEST(GetExperimentalMinVideoBitrateTest, SpecificCodecConfigsUsedIfExpEnabled) {
absl::make_optional(DataRate::KilobitsPerSec(200)));
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::make_optional(DataRate::KilobitsPerSec(300)));
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::nullopt);
}
TEST(GetExperimentalMinVideoBitrateTest,
@ -152,9 +134,6 @@ TEST(GetExperimentalMinVideoBitrateTest,
absl::make_optional(DataRate::KilobitsPerSec(200)));
EXPECT_EQ(GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecH264),
absl::make_optional(DataRate::KilobitsPerSec(300)));
EXPECT_EQ(
GetExperimentalMinVideoBitrate(VideoCodecType::kVideoCodecMultiplex),
absl::nullopt);
}
} // namespace

1
rtc_tools/rtc_event_log_to_text/converter.cc
View file

@ -428,7 +428,6 @@ bool Convert(std::string inputfile,
{VideoCodecType::kVideoCodecVP9, "VP9"},
{VideoCodecType::kVideoCodecAV1, "AV1"},
{VideoCodecType::kVideoCodecH264, "H264"},
{VideoCodecType::kVideoCodecMultiplex, "MULTIPLEX"},
{VideoCodecType::kVideoCodecH265, "H265"}};
fprintf(output,

1
test/scenario/BUILD.gn
View file

@ -128,7 +128,6 @@ if (rtc_include_tests && !build_with_chromium) {
"../../modules/video_coding:video_codec_interface",
"../../modules/video_coding:video_coding_utility",
"../../modules/video_coding:webrtc_h264",
"../../modules/video_coding:webrtc_multiplex",
"../../modules/video_coding:webrtc_vp8",
"../../modules/video_coding:webrtc_vp9",
"../../modules/video_coding/svc:scalability_mode_util",

3
test/scenario/video_stream.cc
View file

@ -208,9 +208,6 @@ CreateEncoderSpecificSettings(VideoStreamConfig config) {
case Codec::kVideoCodecAV1:
case Codec::kVideoCodecH265:
return nullptr;
case Codec::kVideoCodecMultiplex:
RTC_DCHECK_NOTREACHED();
return nullptr;
}
}

4
test/video_codec_tester.cc
View file

@ -1070,7 +1070,6 @@ class Encoder : public EncodedImageCallback {
vc.qpMax = cricket::kDefaultVideoMaxQpH26x;
break;
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
RTC_CHECK_NOTREACHED();
break;
}
@ -1278,7 +1277,6 @@ void SetDefaultCodecSpecificSettings(VideoCodec* vc, int num_temporal_layers) {
case kVideoCodecH265:
break;
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
RTC_CHECK_NOTREACHED();
}
}
@ -1357,7 +1355,6 @@ SplitBitrateAndUpdateScalabilityMode(std::string codec_type,
}
break;
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
RTC_CHECK_NOTREACHED();
}
} else {
@ -1380,7 +1377,6 @@ SplitBitrateAndUpdateScalabilityMode(std::string codec_type,
RTC_CHECK(result) << "SetAv1SvcConfig failed";
} break;
case kVideoCodecGeneric:
case kVideoCodecMultiplex:
RTC_CHECK_NOTREACHED();
}

2
video/BUILD.gn
View file

@ -549,7 +549,6 @@ if (rtc_include_tests) {
"../modules/video_coding",
"../modules/video_coding:video_coding_utility",
"../modules/video_coding:webrtc_h264",
"../modules/video_coding:webrtc_multiplex",
"../modules/video_coding:webrtc_vp8",
"../modules/video_coding:webrtc_vp9",
"../rtc_base:macromagic",
@ -908,7 +907,6 @@ if (rtc_include_tests) {
"../modules/video_coding:video_codec_interface",
"../modules/video_coding:video_coding_utility",
"../modules/video_coding:webrtc_h264",
"../modules/video_coding:webrtc_multiplex",
"../modules/video_coding:webrtc_vp8",
"../modules/video_coding:webrtc_vp9",
"../modules/video_coding:webrtc_vp9_helpers",

1
video/encoder_overshoot_detector.cc
View file

@ -273,7 +273,6 @@ void EncoderOvershootDetector::UpdateHistograms() {
average_overshoot_percent);
break;
case VideoCodecType::kVideoCodecGeneric:
case VideoCodecType::kVideoCodecMultiplex:
break;
}
}

2
video/encoder_overshoot_detector_unittest.cc
View file

@ -39,8 +39,6 @@ static std::string CodecTypeToHistogramSuffix(VideoCodecType codec) {
return "H265";
case kVideoCodecGeneric:
return "Generic";
case kVideoCodecMultiplex:
return "Multiplex";
}
}

44
video/end_to_end_tests/codec_tests.cc
View file

@ -21,8 +21,6 @@
#include "media/engine/internal_decoder_factory.h"
#include "media/engine/internal_encoder_factory.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "test/call_test.h"
@ -191,48 +189,6 @@ TEST_F(CodecEndToEndTest,
RunBaseTest(&test);
}
// Mutiplex tests are using VP9 as the underlying implementation.
TEST_F(CodecEndToEndTest, SendsAndReceivesMultiplex) {
InternalEncoderFactory internal_encoder_factory;
InternalDecoderFactory internal_decoder_factory;
test::FunctionVideoEncoderFactory encoder_factory(
[&internal_encoder_factory]() {
return std::make_unique<MultiplexEncoderAdapter>(
&internal_encoder_factory, SdpVideoFormat(cricket::kVp9CodecName));
});
test::FunctionVideoDecoderFactory decoder_factory(
[&internal_decoder_factory](const Environment& env,
const SdpVideoFormat& /*format*/) {
return std::make_unique<MultiplexDecoderAdapter>(
env, &internal_decoder_factory,
SdpVideoFormat(cricket::kVp9CodecName));
});
CodecObserver test(5, kVideoRotation_0, absl::nullopt, "multiplex",
&encoder_factory, &decoder_factory);
RunBaseTest(&test);
}
TEST_F(CodecEndToEndTest, SendsAndReceivesMultiplexVideoRotation90) {
InternalEncoderFactory internal_encoder_factory;
InternalDecoderFactory internal_decoder_factory;
test::FunctionVideoEncoderFactory encoder_factory(
[&internal_encoder_factory]() {
return std::make_unique<MultiplexEncoderAdapter>(
&internal_encoder_factory, SdpVideoFormat(cricket::kVp9CodecName));
});
test::FunctionVideoDecoderFactory decoder_factory(
[&internal_decoder_factory](const Environment& env,
const SdpVideoFormat& /*format*/) {
return std::make_unique<MultiplexDecoderAdapter>(
env, &internal_decoder_factory,
SdpVideoFormat(cricket::kVp9CodecName));
});
CodecObserver test(5, kVideoRotation_90, absl::nullopt, "multiplex",
&encoder_factory, &decoder_factory);
RunBaseTest(&test);
}
#endif // defined(RTC_ENABLE_VP9)
#if defined(WEBRTC_USE_H264)

27
video/full_stack_tests.cc
View file

@ -147,33 +147,6 @@ TEST(FullStackTest, Generator_Net_Delay_0_0_Plr_0_VP9Profile2) {
fixture->RunWithAnalyzer(generator);
}
TEST(FullStackTest, Foreman_Cif_Net_Delay_0_0_Plr_0_Multiplex) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging foreman_cif;
foreman_cif.call.send_side_bwe = true;
foreman_cif.video[0] = {
true, 352, 288, 30,
700000, 700000, 700000, false,
"multiplex", 1, 0, 0,
false, false, false, ClipNameToClipPath("foreman_cif")};
foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_Multiplex", 0.0, 0.0,
kFullStackTestDurationSecs};
fixture->RunWithAnalyzer(foreman_cif);
}
TEST(FullStackTest, Generator_Net_Delay_0_0_Plr_0_Multiplex) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging generator;
generator.call.send_side_bwe = true;
generator.video[0] = {
true, 352, 288, 30, 700000, 700000, 700000, false,
"multiplex", 1, 0, 0, false, false, false, "GeneratorI420A"};
generator.analyzer = {"generator_net_delay_0_0_plr_0_Multiplex", 0.0, 0.0,
kFullStackTestDurationSecs};
fixture->RunWithAnalyzer(generator);
}
#endif // defined(RTC_ENABLE_VP9)
#if defined(WEBRTC_LINUX)

30
video/pc_full_stack_tests.cc
View file

@ -217,36 +217,6 @@ TEST(PCFullStackTest, MAYBE_Pc_Generator_Net_Delay_0_0_Plr_0_VP9Profile2) {
fixture->Run(RunParams(TimeDelta::Seconds(kTestDurationSec)));
}
/*
// TODO(bugs.webrtc.org/10639) migrate commented out test, when required
// functionality will be supported in PeerConnection level framework.
TEST(PCFullStackTest, ForemanCifWithoutPacketLossMultiplexI420Frame) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging foreman_cif;
foreman_cif.call.send_side_bwe = true;
foreman_cif.video[0] = {
true, 352, 288, 30,
700000, 700000, 700000, false,
"multiplex", 1, 0, 0,
false, false, false, ClipNameToClipPath("foreman_cif")};
foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_Multiplex", 0.0, 0.0,
kTestDurationSec};
fixture->RunWithAnalyzer(foreman_cif);
}
TEST(PCFullStackTest, GeneratorWithoutPacketLossMultiplexI420AFrame) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging generator;
generator.call.send_side_bwe = true;
generator.video[0] = {
true, 352, 288, 30, 700000, 700000, 700000, false,
"multiplex", 1, 0, 0, false, false, false, "GeneratorI420A"};
generator.analyzer = {"generator_net_delay_0_0_plr_0_Multiplex", 0.0, 0.0,
kTestDurationSec};
fixture->RunWithAnalyzer(generator);
}
*/
#endif // defined(RTC_ENABLE_VP9)
TEST(PCFullStackTest, Pc_Net_Delay_0_0_Plr_0) {

12
video/video_quality_test.cc
View file

@ -40,8 +40,6 @@
#include "modules/audio_device/include/audio_device.h"
#include "modules/audio_mixer/audio_mixer_impl.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/utility/ivf_file_writer.h"
@ -297,10 +295,7 @@ std::unique_ptr<VideoDecoder> VideoQualityTest::CreateVideoDecoder(
const Environment& env,
const SdpVideoFormat& format) {
std::unique_ptr<VideoDecoder> decoder;
if (format.name == "multiplex") {
decoder = std::make_unique<MultiplexDecoderAdapter>(
env, decoder_factory_.get(), SdpVideoFormat(cricket::kVp9CodecName));
} else if (format.name == "FakeCodec") {
if (format.name == "FakeCodec") {
decoder = webrtc::FakeVideoDecoderFactory::CreateVideoDecoder();
} else {
decoder = decoder_factory_->Create(env, format);
@ -323,9 +318,6 @@ std::unique_ptr<VideoEncoder> VideoQualityTest::CreateVideoEncoder(
if (format.name == "VP8") {
encoder = std::make_unique<SimulcastEncoderAdapter>(encoder_factory_.get(),
format);
} else if (format.name == "multiplex") {
encoder = std::make_unique<MultiplexEncoderAdapter>(
encoder_factory_.get(), SdpVideoFormat(cricket::kVp9CodecName));
} else if (format.name == "FakeCodec") {
encoder = webrtc::FakeVideoEncoderFactory::CreateVideoEncoder();
} else {
@ -724,8 +716,6 @@ void VideoQualityTest::SetupVideo(Transport* send_transport,
payload_type = test::VideoTestConstants::kPayloadTypeVP8;
} else if (params_.video[video_idx].codec == "VP9") {
payload_type = test::VideoTestConstants::kPayloadTypeVP9;
} else if (params_.video[video_idx].codec == "multiplex") {
payload_type = test::VideoTestConstants::kPayloadTypeVP9;
} else if (params_.video[video_idx].codec == "FakeCodec") {
payload_type = test::VideoTestConstants::kFakeVideoSendPayloadType;
} else {

22
video/video_stream_encoder_unittest.cc
View file

@ -45,7 +45,6 @@
#include "media/engine/webrtc_video_engine.h"
#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
@ -8817,16 +8816,6 @@ class VideoStreamEncoderWithRealEncoderTest
case kVideoCodecH264:
encoder = H264Encoder::Create();
break;
case kVideoCodecMultiplex:
mock_encoder_factory_for_multiplex_ =
std::make_unique<MockVideoEncoderFactory>();
EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
.WillRepeatedly([] { return VP8Encoder::Create(); });
encoder = std::make_unique<MultiplexEncoderAdapter>(
mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
false);
break;
case kVideoCodecH265:
// TODO(bugs.webrtc.org/13485): Use a fake encoder
break;
@ -8908,11 +8897,6 @@ TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
}
TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
if (codec_type_ == kVideoCodecMultiplex) {
// Multiplex codec here uses wrapped mock codecs, ignore for this test.
return;
}
const size_t kNumSpatialLayers = 3u;
const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
const int kFrameWidth = 1280;
@ -9045,8 +9029,6 @@ constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
#if defined(WEBRTC_USE_H264)
constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
@ -9060,7 +9042,6 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values(kVP8DisallowConversion,
kVP9DisallowConversion,
kAV1AllowConversion,
kMultiplexDisallowConversion,
kH264AllowConversion),
TestParametersVideoCodecAndAllowI420ConversionToString);
#else
@ -9069,8 +9050,7 @@ INSTANTIATE_TEST_SUITE_P(
VideoStreamEncoderWithRealEncoderTest,
::testing::Values(kVP8DisallowConversion,
kVP9DisallowConversion,
kAV1AllowConversion,
kMultiplexDisallowConversion),
kAV1AllowConversion),
TestParametersVideoCodecAndAllowI420ConversionToString);
#endif