Propagate webrtc::Environment to create VideoDecoder through java wrappers

Bug: webrtc:15791
Change-Id: I25976158ef780211355deb934e83d275d4f881e2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/336980
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41686}

sdk/android/BUILD.gn

@@ -663,6 +663,7 @@ if (current_os == "linux" || is_android) {
       "../../api:libjingle_peerconnection_api",
       "../../api:media_stream_interface",
       "../../api:sequence_checker",
+      "../../api/environment",
       "../../api/task_queue",
       "../../api/video:encoded_image",
       "../../api/video:render_resolution",
@@ -689,7 +690,10 @@ if (current_os == "linux" || is_android) {
       "../../rtc_base/synchronization:mutex",
       "//third_party/libyuv",
     ]
-    absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+    absl_deps = [
+      "//third_party/abseil-cpp/absl/memory",
+      "//third_party/abseil-cpp/absl/types:optional",
+    ]
   }
 
   # Sources here require -lEGL linker flag. It is separated from video_jni
@@ -908,6 +912,7 @@ if (current_os == "linux" || is_android) {
       ":libvpx_vp9_jni",
       ":native_api_jni",
       ":video_jni",
+      "../../api/environment",
       "../../api/video_codecs:builtin_video_decoder_factory",
       "../../api/video_codecs:builtin_video_encoder_factory",
       "../../api/video_codecs:video_codecs_api",

sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java

@@ -11,7 +11,6 @@
 package org.webrtc;
 
 import androidx.annotation.Nullable;
-import java.util.Arrays;
 import java.util.List;
 
 public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
@@ -26,16 +25,19 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
   @Nullable
   @Override
   public VideoDecoder createDecoder(VideoCodecInfo info) {
-    long nativeDecoder = nativeCreateDecoder(nativeFactory, info);
-    if (nativeDecoder == 0) {
+    if (!nativeIsSupported(nativeFactory, info)) {
       Logging.w(TAG, "Trying to create decoder for unsupported format. " + info);
       return null;
     }
-
     return new WrappedNativeVideoDecoder() {
       @Override
       public long createNativeVideoDecoder() {
-        return nativeDecoder;
+        return nativeCreateDecoder(nativeFactory, info);
+      }
+
+      @Override
+      public long createNative(long webrtcEnvRef) {
+        return nativeCreate(nativeFactory, webrtcEnvRef, info);
       }
     };
   }
@@ -49,5 +51,10 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
 
   private static native long nativeCreateDecoder(long factory, VideoCodecInfo videoCodecInfo);
 
+  private static native boolean nativeIsSupported(long factory, VideoCodecInfo info);
+
+  private static native long nativeCreate(
+      long factory, long webrtcEnvRef, VideoCodecInfo info);
+
   private static native List<VideoCodecInfo> nativeGetSupportedCodecs(long factory);
 }

sdk/android/api/org/webrtc/VideoDecoder.java

@@ -56,11 +56,11 @@ public interface VideoDecoder {
    * decoder (e.g., an Android platform decoder), or alternatively 2) a native
    * decoder (e.g., a software decoder or a C++ decoder adapter).
    *
-   * For case 1), createNativeVideoDecoder() should return zero.
+   * For case 1), createNative() should return zero.
    * In this case, we expect the native library to call the decoder through
    * JNI using the Java interface declared below.
    *
-   * For case 2), createNativeVideoDecoder() should return a non-zero value.
+   * For case 2), createNative() should return a non-zero value.
    * In this case, we expect the native library to treat the returned value as
    * a raw pointer of type webrtc::VideoDecoder* (ownership is transferred to
    * the caller). The native library should then directly call the
@@ -69,6 +69,12 @@ public interface VideoDecoder {
    * UnsupportedOperationException.
    */
   @CalledByNative
+  default long createNative(long webrtcEnvRef) {
+    return createNativeVideoDecoder();
+  }
+
+  @CalledByNative
+  @Deprecated
   default long createNativeVideoDecoder() {
     return 0;
   }

sdk/android/api/org/webrtc/VideoDecoderFallback.java

@@ -27,5 +27,13 @@ public class VideoDecoderFallback extends WrappedNativeVideoDecoder {
     return nativeCreateDecoder(fallback, primary);
   }
 
-  private static native long nativeCreateDecoder(VideoDecoder fallback, VideoDecoder primary);
+  @Override
+  public long createNative(long webrtcEnvRef) {
+    return nativeCreate(webrtcEnvRef, fallback, primary);
+  }
+
+  private static native long nativeCreateDecoder(VideoDecoder fallback, VideoDecoder primary);
+
+  private static native long nativeCreate(
+      long webrtcEnvRef, VideoDecoder fallback, VideoDecoder primary);
 }

sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java

@@ -14,7 +14,9 @@ package org.webrtc;
  * Wraps a native webrtc::VideoDecoder.
  */
 public abstract class WrappedNativeVideoDecoder implements VideoDecoder {
-  @Override public abstract long createNativeVideoDecoder();
+  // TODO: bugs.webrtc.org/15791 - uncomment when implemented by all
+  // derived classes instead of the createNativeVideoDecoder
+  // @Override public abstract long createNative(long webrtcEnvRef);
 
   @Override
   public final VideoCodecStatus initDecode(Settings settings, Callback decodeCallback) {

sdk/android/src/jni/software_video_decoder_factory.cc

@@ -8,8 +8,12 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <memory>
+
+#include "api/environment/environment.h"
 #include "api/video_codecs/builtin_video_decoder_factory.h"
 #include "api/video_codecs/video_decoder.h"
+#include "api/video_codecs/video_decoder_factory.h"
 #include "sdk/android/generated_swcodecs_jni/SoftwareVideoDecoderFactory_jni.h"
 #include "sdk/android/native_api/jni/java_types.h"
 #include "sdk/android/src/jni/jni_helpers.h"
@@ -39,6 +43,27 @@ static jlong JNI_SoftwareVideoDecoderFactory_CreateDecoder(
   return webrtc::NativeToJavaPointer(decoder.release());
 }
 
+jboolean JNI_SoftwareVideoDecoderFactory_IsSupported(
+    JNIEnv* env,
+    jlong j_factory,
+    const JavaParamRef<jobject>& j_info) {
+  return VideoCodecInfoToSdpVideoFormat(env, j_info)
+      .IsCodecInList(reinterpret_cast<VideoDecoderFactory*>(j_factory)
+                         ->GetSupportedFormats());
+}
+
+jlong JNI_SoftwareVideoDecoderFactory_Create(
+    JNIEnv* env,
+    jlong j_factory,
+    jlong j_webrtc_env_ref,
+    const JavaParamRef<jobject>& j_info) {
+  return NativeToJavaPointer(
+      reinterpret_cast<VideoDecoderFactory*>(j_factory)
+          ->Create(*reinterpret_cast<const Environment*>(j_webrtc_env_ref),
+                   VideoCodecInfoToSdpVideoFormat(env, j_info))
+          .release());
+}
+
 static webrtc::ScopedJavaLocalRef<jobject>
 JNI_SoftwareVideoDecoderFactory_GetSupportedCodecs(JNIEnv* env,
                                                    jlong j_factory) {

sdk/android/src/jni/video_decoder_factory_wrapper.cc

@@ -10,6 +10,7 @@
 
 #include "sdk/android/src/jni/video_decoder_factory_wrapper.h"
 
+#include "api/environment/environment.h"
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_decoder.h"
 #include "rtc_base/logging.h"
@@ -27,7 +28,8 @@ VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper(
     : decoder_factory_(jni, decoder_factory) {}
 VideoDecoderFactoryWrapper::~VideoDecoderFactoryWrapper() = default;
 
-std::unique_ptr<VideoDecoder> VideoDecoderFactoryWrapper::CreateVideoDecoder(
+std::unique_ptr<VideoDecoder> VideoDecoderFactoryWrapper::Create(
+    const Environment& env,
     const SdpVideoFormat& format) {
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedJavaLocalRef<jobject> j_codec_info =
@@ -36,7 +38,7 @@ std::unique_ptr<VideoDecoder> VideoDecoderFactoryWrapper::CreateVideoDecoder(
       jni, decoder_factory_, j_codec_info);
   if (!decoder.obj())
     return nullptr;
-  return JavaToNativeVideoDecoder(jni, decoder);
+  return JavaToNativeVideoDecoder(jni, decoder, NativeToJavaPointer(&env));
 }
 
 std::vector<SdpVideoFormat> VideoDecoderFactoryWrapper::GetSupportedFormats()

sdk/android/src/jni/video_decoder_factory_wrapper.h

@@ -13,6 +13,7 @@
 
 #include <jni.h>
 
+#include "api/environment/environment.h"
 #include "api/video_codecs/video_decoder_factory.h"
 #include "sdk/android/src/jni/jni_helpers.h"
 
@@ -28,7 +29,7 @@ class VideoDecoderFactoryWrapper : public VideoDecoderFactory {
   ~VideoDecoderFactoryWrapper() override;
 
   std::vector<SdpVideoFormat> GetSupportedFormats() const override;
-  std::unique_ptr<VideoDecoder> CreateVideoDecoder(
+  std::unique_ptr<VideoDecoder> Create(const Environment& env,
                                        const SdpVideoFormat& format) override;
 
  private:

sdk/android/src/jni/video_decoder_fallback.cc

@@ -35,5 +35,23 @@ static jlong JNI_VideoDecoderFallback_CreateDecoder(
   return jlongFromPointer(nativeWrapper);
 }
 
+static jlong JNI_VideoDecoderFallback_Create(
+    JNIEnv* jni,
+    jlong j_webrtc_env_ref,
+    const JavaParamRef<jobject>& j_fallback_decoder,
+    const JavaParamRef<jobject>& j_primary_decoder) {
+  std::unique_ptr<VideoDecoder> fallback_decoder =
+      JavaToNativeVideoDecoder(jni, j_fallback_decoder, j_webrtc_env_ref);
+  std::unique_ptr<VideoDecoder> primary_decoder =
+      JavaToNativeVideoDecoder(jni, j_primary_decoder, j_webrtc_env_ref);
+
+  VideoDecoder* native_wrapper =
+      CreateVideoDecoderSoftwareFallbackWrapper(std::move(fallback_decoder),
+                                                std::move(primary_decoder))
+          .release();
+
+  return NativeToJavaPointer(native_wrapper);
+}
+
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/video_decoder_wrapper.cc

@@ -10,6 +10,8 @@
 
 #include "sdk/android/src/jni/video_decoder_wrapper.h"
 
+#include "absl/memory/memory.h"
+#include "api/environment/environment.h"
 #include "api/video/render_resolution.h"
 #include "api/video/video_frame.h"
 #include "api/video_codecs/video_decoder.h"
@@ -281,5 +283,18 @@ std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
   return std::unique_ptr<VideoDecoder>(decoder);
 }
 
+std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_decoder,
+    jlong webrtcEnvRef) {
+  if (jlong native_decoder =
+          Java_VideoDecoder_createNative(jni, j_decoder, webrtcEnvRef);
+      native_decoder != 0) {
+    return absl::WrapUnique(reinterpret_cast<VideoDecoder*>(native_decoder));
+  } else {
+    return std::make_unique<VideoDecoderWrapper>(jni, j_decoder);
+  }
+}
+
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/video_decoder_wrapper.h

@@ -116,10 +116,17 @@ class VideoDecoderWrapper : public VideoDecoder {
 /* If the j_decoder is a wrapped native decoder, unwrap it. If it is not,
  * wrap it in a VideoDecoderWrapper.
  */
+// TODO: bugs.webrtc.org/15791 - delete variant without the webrtcEnvRef
+// parameter when unused.
 std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
     JNIEnv* jni,
     const JavaRef<jobject>& j_decoder);
 
+std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_decoder,
+    jlong webrtcEnvRef);
+
 }  // namespace jni
 }  // namespace webrtc