Add Java support for AudioProcessing and PostProcessing injection

This allows injection of a user-defined post processing module from
the Android layer.

Bug: webrtc:8163
Change-Id: If3a6b4726c34c5f82d186b8cf95373c283cbd3f6
Reviewed-on: https://webrtc-review.googlesource.com/7610
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Taylor Brandstetter <deadbeef@webrtc.org>
Commit-Queue: Sam Zackrisson <saza@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20367}

sdk/android/BUILD.gn

@@ -67,12 +67,15 @@ rtc_source_set("base_jni") {
 rtc_static_library("audio_jni") {
   sources = [
     "src/jni/pc/audio_jni.cc",
+    "src/jni/pc/defaultaudioprocessingfactory_jni.cc",
   ]
 
   deps = [
     ":base_jni",
     "../../api/audio_codecs:builtin_audio_decoder_factory",
     "../../api/audio_codecs:builtin_audio_encoder_factory",
+    "../../modules/audio_processing:audio_processing",
+    "../../rtc_base:rtc_base_approved",
     "../../voice_engine:voice_engine",
   ]
 }
@@ -374,6 +377,7 @@ android_library("libjingle_peerconnection_java") {
   java_files = [
     "api/org/webrtc/AudioSource.java",
     "api/org/webrtc/AudioTrack.java",
+    "api/org/webrtc/AudioProcessingFactory.java",
     "api/org/webrtc/CallSessionFileRotatingLogSink.java",
     "api/org/webrtc/Camera1Capturer.java",
     "api/org/webrtc/Camera1Enumerator.java",
@@ -383,6 +387,7 @@ android_library("libjingle_peerconnection_java") {
     "api/org/webrtc/CameraEnumerator.java",
     "api/org/webrtc/CameraVideoCapturer.java",
     "api/org/webrtc/DataChannel.java",
+    "api/org/webrtc/DefaultAudioProcessingFactory.java",
     "api/org/webrtc/DtmfSender.java",
     "api/org/webrtc/EglBase.java",
     "api/org/webrtc/EglRenderer.java",
@@ -407,6 +412,7 @@ android_library("libjingle_peerconnection_java") {
     "api/org/webrtc/NetworkMonitorAutoDetect.java",
     "api/org/webrtc/PeerConnection.java",
     "api/org/webrtc/PeerConnectionFactory.java",
+    "api/org/webrtc/PostProcessingFactory.java",
     "api/org/webrtc/RendererCommon.java",
     "api/org/webrtc/RTCStats.java",
     "api/org/webrtc/RTCStatsCollectorCallback.java",
@@ -490,6 +496,7 @@ if (rtc_include_tests) {
     android_manifest = "instrumentationtests/AndroidManifest.xml"
 
     java_files = [
+      "instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java",
       "instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java",
       "instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java",
       "instrumentationtests/src/org/webrtc/Camera2CapturerTest.java",

sdk/android/api/org/webrtc/AudioProcessingFactory.java

@@ -0,0 +1,20 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/** Factory for creating webrtc::AudioProcessing instances. */
+public interface AudioProcessingFactory {
+  /**
+   * Dynamically allocates a webrtc::AudioProcessing instance and returns a pointer to it.
+   * The caller takes ownership of the object.
+   */
+  public long createNative();
+}

sdk/android/api/org/webrtc/DefaultAudioProcessingFactory.java

@@ -0,0 +1,47 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/** Factory for instantiating the default webrtc::AudioProcessing implementation. */
+public class DefaultAudioProcessingFactory implements AudioProcessingFactory {
+  public DefaultAudioProcessingFactory() {
+    this(null /* postProcessingFactory */);
+  }
+
+  /**
+   * Allows injecting a PostProcessingFactory. A null PostProcessingFactory creates a
+   * webrtc::AudioProcessing with nullptr webrtc::postProcessing.
+   */
+  public DefaultAudioProcessingFactory(PostProcessingFactory postProcessingFactory) {
+    this.postProcessingFactory = postProcessingFactory;
+  }
+
+  /**
+   * Creates a default webrtc::AudioProcessing module, which takes ownership of objects created by
+   * its factories.
+   */
+  @Override
+  public long createNative() {
+    long nativePostProcessor = 0;
+    if (postProcessingFactory != null) {
+      nativePostProcessor = postProcessingFactory.createNative();
+      if (nativePostProcessor == 0) {
+        throw new NullPointerException(
+            "PostProcessingFactory.createNative() may not return 0 (nullptr).");
+      }
+    }
+    return nativeCreateAudioProcessing(nativePostProcessor);
+  }
+
+  private PostProcessingFactory postProcessingFactory;
+
+  private static native long nativeCreateAudioProcessing(long nativePostProcessor);
+}

sdk/android/api/org/webrtc/PeerConnectionFactory.java

@@ -125,6 +125,14 @@ public class PeerConnectionFactory {
     }
   }
 
+  private void checkInitializeHasBeenCalled() {
+    if (!NativeLibrary.isLoaded() || ContextUtils.getApplicationContext() == null) {
+      throw new IllegalStateException(
+          "PeerConnectionFactory.initialize was not called before creating a "
+          + "PeerConnectionFactory.");
+    }
+  }
+
   // Must be called at least once before creating a PeerConnectionFactory
   // (for example, at application startup time).
   private static native void nativeInitializeAndroidGlobals(
@@ -185,17 +193,27 @@ public class PeerConnectionFactory {
 
   public PeerConnectionFactory(
       Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory) {
-    if (!NativeLibrary.isLoaded() || ContextUtils.getApplicationContext() == null) {
+    checkInitializeHasBeenCalled();
-      throw new IllegalStateException(
-          "PeerConnectionFactory.initialize was not called before creating a "
-          + "PeerConnectionFactory.");
-    }
     nativeFactory = nativeCreatePeerConnectionFactory(options, encoderFactory, decoderFactory);
     if (nativeFactory == 0) {
       throw new RuntimeException("Failed to initialize PeerConnectionFactory!");
     }
   }
 
+  public PeerConnectionFactory(Options options, VideoEncoderFactory encoderFactory,
+      VideoDecoderFactory decoderFactory, AudioProcessingFactory audioProcessingFactory) {
+    checkInitializeHasBeenCalled();
+    if (audioProcessingFactory == null) {
+      throw new NullPointerException(
+          "PeerConnectionFactory constructor does not accept a null AudioProcessingFactory.");
+    }
+    nativeFactory = nativeCreatePeerConnectionFactoryWithAudioProcessing(
+        options, encoderFactory, decoderFactory, audioProcessingFactory.createNative());
+    if (nativeFactory == 0) {
+      throw new RuntimeException("Failed to initialize PeerConnectionFactory!");
+    }
+  }
+
   public PeerConnection createPeerConnection(PeerConnection.RTCConfiguration rtcConfig,
       MediaConstraints constraints, PeerConnection.Observer observer) {
     long nativeObserver = nativeCreateObserver(observer);
@@ -337,6 +355,10 @@ public class PeerConnectionFactory {
   private static native long nativeCreatePeerConnectionFactory(
       Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory);
 
+  private static native long nativeCreatePeerConnectionFactoryWithAudioProcessing(Options options,
+      VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory,
+      long nativeAudioProcessor);
+
   private static native long nativeCreateObserver(PeerConnection.Observer observer);
 
   private static native long nativeCreatePeerConnection(long nativeFactory,

sdk/android/api/org/webrtc/PostProcessingFactory.java

@@ -0,0 +1,20 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/** Factory for creating webrtc::PostProcessing instances. */
+public interface PostProcessingFactory {
+  /**
+   * Dynamically allocates a webrtc::PostProcessing instance and returns a pointer to it.
+   * The caller takes ownership of the object.
+   */
+  public long createNative();
+}

sdk/android/instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java

@@ -0,0 +1,47 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import android.support.test.InstrumentationRegistry;
+import android.support.test.filters.MediumTest;
+import android.support.test.filters.SmallTest;
+import org.chromium.base.test.BaseJUnit4ClassRunner;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link DefaultAudioProcessingFactory}. */
+@RunWith(BaseJUnit4ClassRunner.class)
+public final class DefaultAudioProcessingFactoryTest {
+  @Before
+  public void setUp() {
+    PeerConnectionFactory.initialize(
+        PeerConnectionFactory.InitializationOptions.builder(InstrumentationRegistry.getContext())
+            .createInitializationOptions());
+  }
+
+  /**
+   * Tests that a PeerConnectionFactory can be initialized with an AudioProcessingFactory without
+   * crashing.
+   */
+  @Test
+  @MediumTest
+  public void testInitializePeerConnectionFactory() {
+    AudioProcessingFactory audioProcessingFactory = new DefaultAudioProcessingFactory();
+    PeerConnectionFactory peerConnectionFactory = new PeerConnectionFactory(null /* options */,
+        null /* encoderFactory */, null /* decoderFactory */, audioProcessingFactory);
+    peerConnectionFactory.dispose();
+  }
+}

sdk/android/src/jni/pc/audio_jni.cc

@@ -12,6 +12,7 @@
 
 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "modules/audio_processing/include/audio_processing.h"
 
 namespace webrtc {
 namespace jni {
@@ -24,5 +25,9 @@ rtc::scoped_refptr<AudioEncoderFactory> CreateAudioEncoderFactory() {
   return CreateBuiltinAudioEncoderFactory();
 }
 
+rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing() {
+  return AudioProcessing::Create();
+}
+
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/pc/audio_jni.h

@@ -15,6 +15,7 @@
 // We don't want this target depend on audio related targets
 #include "api/audio_codecs/audio_decoder_factory.h"  // nogncheck
 #include "api/audio_codecs/audio_encoder_factory.h"  // nogncheck
+#include "modules/audio_processing/include/audio_processing.h"  // nogncheck
 #include "rtc_base/scoped_ref_ptr.h"
 
 namespace webrtc {
@@ -24,6 +25,8 @@ rtc::scoped_refptr<AudioDecoderFactory> CreateAudioDecoderFactory();
 
 rtc::scoped_refptr<AudioEncoderFactory> CreateAudioEncoderFactory();
 
+rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing();
+
 }  // namespace jni
 }  // namespace webrtc
 

sdk/android/src/jni/pc/defaultaudioprocessingfactory_jni.cc

@@ -0,0 +1,36 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "modules/audio_processing/include/audio_processing.h"
+#include "rtc_base/scoped_ref_ptr.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+namespace webrtc {
+namespace jni {
+
+JNI_FUNCTION_DECLARATION(
+    jlong,
+    DefaultAudioProcessingFactory_nativeCreateAudioProcessing,
+    JNIEnv*,
+    jclass,
+    jlong native_post_processor) {
+  std::unique_ptr<PostProcessing> post_processor(
+      reinterpret_cast<PostProcessing*>(native_post_processor));
+  rtc::scoped_refptr<AudioProcessing> audio_processing =
+      AudioProcessing::Create(webrtc::Config(), std::move(post_processor),
+                              nullptr /* echo_control_factory */,
+                              nullptr /* beamformer */);
+  return jlongFromPointer(audio_processing.release());
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/pc/media_jni.cc

@@ -31,10 +31,11 @@ cricket::MediaEngineInterface* CreateMediaEngine(
     const rtc::scoped_refptr<AudioDecoderFactory>& audio_decoder_factory,
     cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
     cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
-    rtc::scoped_refptr<AudioMixer> audio_mixer) {
+    rtc::scoped_refptr<AudioMixer> audio_mixer,
+    rtc::scoped_refptr<AudioProcessing> audio_processor) {
   return cricket::WebRtcMediaEngineFactory::Create(
       adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory,
-      video_decoder_factory, audio_mixer, AudioProcessing::Create());
+      video_decoder_factory, audio_mixer, audio_processor);
 }
 
 }  // namespace jni

sdk/android/src/jni/pc/media_jni.h

@@ -20,6 +20,7 @@ class AudioEncoderFactory;
 class AudioDecoderFactory;
 class RtcEventLogFactoryInterface;
 class AudioMixer;
+class AudioProcessing;
 }  // namespace webrtc
 
 namespace cricket {
@@ -40,7 +41,8 @@ cricket::MediaEngineInterface* CreateMediaEngine(
     const rtc::scoped_refptr<AudioDecoderFactory>& audio_decoder_factory,
     cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
     cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
-    rtc::scoped_refptr<AudioMixer> audio_mixer);
+    rtc::scoped_refptr<AudioMixer> audio_mixer,
+    rtc::scoped_refptr<AudioProcessing> audio_processor);
 
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/pc/null_audio_jni.cc

@@ -21,5 +21,9 @@ rtc::scoped_refptr<AudioEncoderFactory> CreateAudioEncoderFactory() {
   return nullptr;
 }
 
+rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing() {
+  return nullptr;
+}
+
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/pc/null_media_jni.cc

@@ -27,7 +27,8 @@ cricket::MediaEngineInterface* CreateMediaEngine(
     const rtc::scoped_refptr<AudioDecoderFactory>& audio_decoder_factory,
     cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
     cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
-    rtc::scoped_refptr<AudioMixer> audio_mixer) {
+    rtc::scoped_refptr<AudioMixer> audio_mixer,
+    rtc::scoped_refptr<AudioProcessing> audio_processor) {
   return nullptr;
 }
 

sdk/android/src/jni/pc/peerconnectionfactory_jni.cc

@@ -14,6 +14,9 @@
 #include "api/peerconnectioninterface.h"
 #include "media/base/mediaengine.h"
 #include "modules/utility/include/jvm_android.h"
+// We don't depend on the audio processing module implementation.
+// The user may pass in a nullptr.
+#include "modules/audio_processing/include/audio_processing.h"  // nogncheck
 #include "rtc_base/event_tracer.h"
 #include "rtc_base/stringutils.h"
 #include "rtc_base/thread.h"
@@ -132,14 +135,12 @@ JNI_FUNCTION_DECLARATION(void,
   rtc::tracing::ShutdownInternalTracer();
 }
 
-JNI_FUNCTION_DECLARATION(
+jlong CreatePeerConnectionFactoryForJava(
-    jlong,
-    PeerConnectionFactory_nativeCreatePeerConnectionFactory,
     JNIEnv* jni,
-    jclass,
     jobject joptions,
     jobject jencoder_factory,
-    jobject jdecoder_factory) {
+    jobject jdecoder_factory,
+    rtc::scoped_refptr<AudioProcessing> audio_processor) {
   // talk/ assumes pretty widely that the current Thread is ThreadManager'd, but
   // ThreadManager only WrapCurrentThread()s the thread where it is first
   // created.  Since the semantics around when auto-wrapping happens in
@@ -190,7 +191,7 @@ JNI_FUNCTION_DECLARATION(
       CreateRtcEventLogFactory());
   std::unique_ptr<cricket::MediaEngineInterface> media_engine(CreateMediaEngine(
       adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory,
-      video_decoder_factory, audio_mixer));
+      video_decoder_factory, audio_mixer, audio_processor));
 
   rtc::scoped_refptr<PeerConnectionFactoryInterface> factory(
       CreateModularPeerConnectionFactory(
@@ -212,6 +213,35 @@ JNI_FUNCTION_DECLARATION(
   return jlongFromPointer(owned_factory);
 }
 
+JNI_FUNCTION_DECLARATION(
+    jlong,
+    PeerConnectionFactory_nativeCreatePeerConnectionFactory,
+    JNIEnv* jni,
+    jclass,
+    jobject joptions,
+    jobject jencoder_factory,
+    jobject jdecoder_factory) {
+  return CreatePeerConnectionFactoryForJava(jni, joptions, jencoder_factory,
+                                            jdecoder_factory,
+                                            CreateAudioProcessing());
+}
+
+JNI_FUNCTION_DECLARATION(
+    jlong,
+    PeerConnectionFactory_nativeCreatePeerConnectionFactoryWithAudioProcessing,
+    JNIEnv* jni,
+    jclass,
+    jobject joptions,
+    jobject jencoder_factory,
+    jobject jdecoder_factory,
+    jlong native_audio_processor) {
+  rtc::scoped_refptr<AudioProcessing> audio_processor =
+      reinterpret_cast<AudioProcessing*>(native_audio_processor);
+  RTC_DCHECK(audio_processor);
+  return CreatePeerConnectionFactoryForJava(jni, joptions, jencoder_factory,
+                                            jdecoder_factory, audio_processor);
+}
+
 JNI_FUNCTION_DECLARATION(void,
                          PeerConnectionFactory_nativeFreeFactory,
                          JNIEnv*,