Android: Extract Cargo build from gn

bin/build-aar.py

@@ -18,13 +18,11 @@ try:
     import argparse
     import logging
     import subprocess
-    import sys
     import os
-    import zipfile
     import shutil
 
 except ImportError as e:
-    raise ImportError(str(e) + "- required module not found")
+    raise ImportError(str(e) + '- required module not found')
 
 
 DEFAULT_ARCHS = ['arm', 'arm64', 'x86', 'x64']
@@ -51,6 +49,9 @@ def ParseArgs():
     parser.add_argument('-q', '--quiet',
                         action='store_true',
                         help='Quiet output')
+    parser.add_argument('--project-dir',
+                        default=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                        help='Project root directory')
     parser.add_argument('-b', '--build-dir',
                         required=True,
                         help='Build directory')
@@ -83,6 +84,9 @@ def ParseArgs():
                         nargs='*', default=[],
                         help='''Additional GN flags, overriding anything set internally
                                 by this script.''')
+    parser.add_argument('--extra-cargo-flags',
+                        nargs='*', default=[],
+                        help='Additional Cargo arguments')
     parser.add_argument('-j', '--jobs',
                         default=32,
                         help='Number of parallel ninja jobs to run.')
@@ -138,14 +142,7 @@ def RunSdkmanagerLicenses(dry_run):
     if dry_run is False:
         subprocess.check_call(cmd)
 
-def RunGn(dry_run, args):
+def RunCmd(dry_run, cmd):
-    cmd = [ 'gn' ] + args
-    logging.debug('Running: {}'.format(cmd))
-    if dry_run is False:
-        subprocess.check_call(cmd)
-
-def RunNinja(dry_run, args):
-    cmd = [ 'ninja' ] + args
     logging.debug('Running: {}'.format(cmd))
     if dry_run is False:
         subprocess.check_call(cmd)
@@ -172,8 +169,8 @@ def GetOutputDir(build_dir, debug_build):
 def GetGradleBuildDir(build_dir):
     return os.path.join(build_dir, 'gradle')
 
-def BuildArch(dry_run, build_dir, arch, debug_build, extra_gn_args,
+def BuildArch(dry_run, project_dir, build_dir, arch, debug_build, extra_gn_args,
-              extra_gn_flags, extra_ninja_flags, jobs):
+              extra_gn_flags, extra_ninja_flags, extra_cargo_flags, jobs):
 
     logging.info('Building: {} ...'.format(arch))
 
@@ -197,11 +194,56 @@ def BuildArch(dry_run, build_dir, arch, debug_build, extra_gn_args,
     gn_args_string = '--args=' + ' '.join(
         [k + '=' + v for k, v in gn_args.items()] + extra_gn_args)
 
-    gn_total_args = [ 'gen', output_dir, gn_args_string ] + extra_gn_flags
+    gn_total_args = [ 'gn', 'gen', output_dir, gn_args_string ] + extra_gn_flags
-    RunGn(dry_run, gn_total_args)
+    RunCmd(dry_run, gn_total_args)
 
-    ninja_args = [ '-C', output_dir ] + NINJA_TARGETS + [ '-j', jobs ] + extra_ninja_flags
+    ninja_args = [ 'ninja', '-C', output_dir ] + NINJA_TARGETS + [ '-j', jobs ] + extra_ninja_flags
-    RunNinja(dry_run, ninja_args)
+    RunCmd(dry_run, ninja_args)
+
+    # FIXME: Shouldn't hardcode Linux, but eventually this won't use WebRTC's NDK anyway.
+    ndk_toolchain_dir = os.path.join(
+        os.getcwd(),
+        'third_party',
+        'android_ndk',
+        'toolchains',
+        'llvm',
+        'prebuilt',
+        'linux-x86_64'
+    )
+    cargo_args = [
+        'cargo', 'rustc',
+        '--target', GetCargoTarget(arch),
+        '--target-dir', output_dir,
+        '--manifest-path', os.path.join(project_dir, 'src', 'rust', 'Cargo.toml'),
+    ]
+    if not debug_build:
+        cargo_args += ['--release']
+    cargo_args += extra_cargo_flags
+    # Arguments directly for rustc
+    cargo_args += [
+        '--',
+        '-C', 'debuginfo=2',
+        '-C', 'linker={}/bin/{}{}-clang'.format(ndk_toolchain_dir, GetClangTarget(arch), GetAndroidApiLevel(arch)),
+        '-C', 'link-arg=-fuse-ld=lld',
+        '-L', 'native=' + output_dir,
+    ]
+    RunCmd(dry_run, cargo_args)
+
+    if dry_run:
+        return
+
+    # Copy the built library alongside libringrtc_rffi.so.
+    shutil.copyfile(
+        os.path.join(output_dir, GetCargoTarget(arch), 'debug' if debug_build else 'release', 'libringrtc.so'),
+        os.path.join(output_dir, 'lib.unstripped', 'libringrtc.so'))
+    # And strip another copy.
+    strip_args = [
+        '{}/bin/llvm-strip'.format(ndk_toolchain_dir),
+        '-s',
+        os.path.join(output_dir, 'lib.unstripped', 'libringrtc.so'),
+        '-o', os.path.join(output_dir, 'libringrtc.so'),
+    ]
+    RunCmd(dry_run, strip_args)
 
 def GetABI(arch):
     if arch == 'arm':
@@ -215,13 +257,37 @@ def GetABI(arch):
     else:
         raise Exception('Unknown architecture: ' + arch)
 
-def CreateLibs(dry_run, build_dir, archs, output, debug_build, unstripped,
+def GetCargoTarget(arch):
-               extra_gn_args, extra_gn_flags, extra_ninja_flags, jobs,
+    if arch == 'arm':
-               compile_only):
+        return 'armv7-linux-androideabi'
+    elif arch == 'arm64':
+        return 'aarch64-linux-android'
+    elif arch == 'x86':
+        return 'i686-linux-android'
+    elif arch == 'x64':
+        return 'x86_64-linux-android'
+    else:
+        raise Exception('Unknown architecture: ' + arch)
+
+def GetClangTarget(arch):
+    if arch == 'arm':
+        return 'armv7a-linux-androideabi'
+    else:
+        return GetCargoTarget(arch)
+
+def GetAndroidApiLevel(arch):
+    if arch == 'arm' or arch == 'x86':
+        return 19
+    else:
+        return 21
+
+def CreateLibs(dry_run, project_dir, build_dir, archs, output, debug_build, unstripped,
+               extra_gn_args, extra_gn_flags, extra_ninja_flags,
+               extra_cargo_flags, jobs, compile_only):
 
     for arch in archs:
-        BuildArch(dry_run, build_dir, arch, debug_build, extra_gn_args,
+        BuildArch(dry_run, project_dir, build_dir, arch, debug_build, extra_gn_args,
-                  extra_gn_flags, extra_ninja_flags, jobs)
+                  extra_gn_flags, extra_ninja_flags, extra_cargo_flags, jobs)
 
     if compile_only is True:
         return
@@ -246,7 +312,7 @@ def CreateLibs(dry_run, build_dir, archs, output, debug_build, unstripped,
             output_arch_dir = GetArchBuildDir(build_dir, arch, debug_build)
             if unstripped is True:
                 # package the unstripped libraries
-                lib_file = os.path.join("lib.unstripped", lib)
+                lib_file = os.path.join('lib.unstripped', lib)
             else:
                 lib_file = lib
             target_dir = os.path.join(output_dir, GetABI(arch))
@@ -266,9 +332,10 @@ def CreateAar(dry_run, extra_gradle_args, version, gradle_dir,
               sonatype_repo, sonatype_user, sonatype_password,
               signing_keyid, signing_password, signing_secret_keyring,
               compile_only,
-              install_local, install_dir, build_dir, archs,
+              install_local, install_dir, project_dir, build_dir, archs,
               output, debug_build, release_build, unstripped,
-              extra_gn_args, extra_gn_flags, extra_ninja_flags, jobs):
+              extra_gn_args, extra_gn_flags, extra_ninja_flags,
+              extra_cargo_flags, jobs):
 
     build_types = []
     if not (debug_build or release_build):
@@ -322,9 +389,9 @@ def CreateAar(dry_run, extra_gradle_args, version, gradle_dir,
             gradle_args = gradle_args + [
                 "-PreleaseRingrtcLibDirs=['{}']".format(lib_dir),
             ]
-        CreateLibs(dry_run, build_dir, archs, output, build_debug, unstripped,
+        CreateLibs(dry_run, project_dir, build_dir, archs, output, build_debug, unstripped,
-                   extra_gn_args, extra_gn_flags, extra_ninja_flags, jobs,
+                   extra_gn_args, extra_gn_flags, extra_ninja_flags,
-                   compile_only)
+                   extra_cargo_flags, jobs, compile_only)
 
     if compile_only is True:
         return
@@ -392,6 +459,7 @@ def main():
 
     if args.verbose is True:
         args.extra_ninja_flags = args.extra_ninja_flags + ['-v']
+        args.extra_cargo_flags = args.extra_cargo_flags + ['-v']
 
     gradle_dir = os.path.abspath(args.gradle_dir)
     logging.debug('Using gradle directory: {}'.format(gradle_dir))
@@ -429,9 +497,10 @@ def main():
               args.signing_keyid, args.signing_password, args.signing_secret_keyring,
               args.compile_only,
               args.install_local, args.install_dir,
-              build_dir, args.arch, args.output,
+              args.project_dir, build_dir, args.arch, args.output,
               args.debug_build, args.release_build, args.unstripped, args.extra_gn_args,
-              args.extra_gn_flags, args.extra_ninja_flags, str(args.jobs))
+              args.extra_gn_flags, args.extra_ninja_flags, args.extra_cargo_flags,
+              str(args.jobs))
 
     logging.info('''
 Version           : {}

src/BUILD.gn

@@ -11,7 +11,7 @@ if (is_android) {
   group("ringrtc") {
     public_deps = [
       "android",
-      "rust",
+      "rffi:libringrtc_rffi",
     ]
   }
 }

src/rffi/BUILD.gn

@@ -32,7 +32,7 @@ if (is_android) {
 
   rtc_shared_library("libringrtc_rffi") {
 
-    visibility = [ "../rust:libringrtc_lint" ]
+    visibility = [ "//ringrtc:*" ]
 
     android_sdk = "//sdk/android"
     # jni_onload.cc -- taken from webrtc/sdk/android/BUILD.gn.

src/rust/BUILD.gn

@@ -1,183 +0,0 @@
-#
-# Copyright 2019-2021 Signal Messenger, LLC
-# SPDX-License-Identifier: AGPL-3.0-only
-#
-
-if (is_android) {
-  import("//build/config/android/config.gni")
-  import("//build/config/android/rules.gni")
-  import("//webrtc.gni")
-
-  group("rust") {
-    deps = [
-      ":libringrtc"
-    ]
-  }
-
-  libname = "libringrtc.so"
-
-  if (target_cpu == "arm") {
-    cargo_target = "armv7-linux-androideabi"
-    clang_target = "armv7a-linux-androideabi"
-    android_api_level = android32_ndk_api_level
-  } else if (target_cpu == "arm64") {
-    cargo_target = "aarch64-linux-android"
-    clang_target = cargo_target
-    android_api_level = android64_ndk_api_level
-  } else if (target_cpu == "x86") {
-    cargo_target = "i686-linux-android"
-    clang_target = cargo_target
-    android_api_level = android32_ndk_api_level
-  } else if (target_cpu == "x64") {
-    cargo_target = "x86_64-linux-android"
-    clang_target = cargo_target
-    android_api_level = android64_ndk_api_level
-  }
-  assert(defined(cargo_target), "Unsupported target_cpu for cargo: $target_cpu")
-
-  if (is_debug) {
-    cargo_output_dir = "${target_gen_dir}/${cargo_target}/debug"
-  } else {
-    cargo_output_dir = "${target_gen_dir}/${cargo_target}/release"
-  }
-
-  unstripped_lib = "$root_out_dir/lib.unstripped/${libname}"
-  stripped_lib = "$root_out_dir/${libname}"
-
-  rust_sources = [
-    "Cargo.toml",
-    "Cargo.lock",
-    "cbindgen.toml",
-    "scripts/cargo.py",
-    "scripts/clippy.py",
-    "scripts/strip.py",
-    "src/android/api/jni_call_manager.rs",
-    "src/android/android_platform.rs",
-    "src/android/call_manager.rs",
-    "src/android/error.rs",
-    "src/android/jni_util.rs",
-    "src/android/logging.rs",
-    "src/android/webrtc_java_media_stream.rs",
-    "src/android/webrtc_peer_connection_factory.rs",
-    "src/common/actor.rs",
-    "src/common/mod.rs",
-    "src/common/units.rs",
-    "src/core/call_fsm.rs",
-    "src/core/call_manager.rs",
-    "src/core/call_mutex.rs",
-    "src/core/call.rs",
-    "src/core/connection.rs",
-    "src/core/connection_fsm.rs",
-    "src/core/crypto.rs",
-    "src/core/group_call.rs",
-    "src/core/platform.rs",
-    "src/core/signaling.rs",
-    "src/core/util.rs",
-    "src/lite/http.rs",
-    "src/lite/sfu.rs",
-    "src/lite/ffi.rs",
-    "src/error/mod.rs",
-    "src/webrtc/arc.rs",
-    "src/webrtc/ice_gatherer.rs",
-    "src/webrtc/media.rs",
-    "src/webrtc/network.rs",
-    "src/webrtc/peer_connection_factory.rs",
-    "src/webrtc/peer_connection_observer.rs",
-    "src/webrtc/peer_connection.rs",
-    "src/webrtc/rtp.rs",
-    "src/webrtc/sdp_observer.rs",
-    "src/webrtc/stats_observer.rs",
-    "src/webrtc/ffi/ice_gatherer.rs",
-    "src/webrtc/ffi/logging.rs",
-    "src/webrtc/ffi/media.rs",
-    "src/webrtc/ffi/peer_connection_factory.rs",
-    "src/webrtc/ffi/peer_connection_observer.rs",
-    "src/webrtc/ffi/peer_connection.rs",
-    "src/webrtc/ffi/ref_count.rs",
-    "src/webrtc/ffi/sdp_observer.rs",
-    "src/webrtc/ffi/stats_observer.rs",
-    "src/lib.rs",
-    "src/protobuf.rs",
-  ]
-
-  # Lint the Rust code using "cargo clippy"
-  action("libringrtc_lint") {
-    sources = rust_sources
-
-    stamp_file = "${cargo_output_dir}/clippy"
-
-    args = [
-      "${stamp_file}",
-      "--target=${cargo_target}",
-      "--target-dir=" + rebase_path(target_gen_dir),
-      "--manifest-path=" + rebase_path("Cargo.toml"),
-    ]
-
-    if (!is_debug) {
-      args += [ "--release" ]
-    }
-
-    outputs = [ "${stamp_file}" ]
-    script = "scripts/clippy.py"
-
-    deps = [
-      "../rffi:libringrtc_rffi",
-    ]
-  }
-
-  action("libringrtc_cargo") {
-    sources = rust_sources
-
-    args = [ "rustc",
-             "--target=${cargo_target}",
-             "--target-dir=" + rebase_path(target_gen_dir),
-             "--manifest-path=" + rebase_path("Cargo.toml"),
-           ]
-    # add  "--verbose" here (and below) to increase rustc verbosity
-
-    if (!is_debug) {
-      args += [ "--release" ]
-    }
-
-    args += [ "--",
-              "-C", "linker=" + rebase_path(android_toolchain_root) + "/bin/" + clang_target + android_api_level + "-clang",
-              "-C", "link-arg=-fuse-ld=lld",
-              "-C", "link-arg=-Wl,--dynamic-linker,/system/bin/linker",
-              "-L", "native=" + rebase_path(root_build_dir),
-              "-l", "dylib=ringrtc_rffi",
-            ]
-    # add  "--verbose" here (and above) to increase rustc verbosity
-
-    outputs = [ "${cargo_output_dir}/${libname}" ]
-    script = "scripts/cargo.py"
-
-    deps = [
-      ":libringrtc_lint",
-    ]
-
-  }
-
-  copy("libringrtc_unstripped") {
-    sources = [ "${cargo_output_dir}/${libname}" ]
-    outputs = [ "$unstripped_lib" ]
-
-    deps = [ ":libringrtc_cargo" ]
-
-  }
-
-  action("libringrtc") {
-    sources = [ "$unstripped_lib" ]
-    outputs = [ "$stripped_lib" ]
-
-    strip = rebase_path("//buildtools/third_party/eu-strip/bin/eu-strip", root_build_dir)
-    args = [ "--command=${strip}",
-             "--input=" + rebase_path(unstripped_lib),
-             "--output=" + rebase_path(stripped_lib),
-           ]
-
-    script = "scripts/strip.py"
-
-    deps = [ ":libringrtc_unstripped" ]
-
-  }
-}

src/rust/build.rs

@@ -82,6 +82,9 @@ fn main() {
         } else {
             println!("cargo:rustc-link-lib=stdc++");
         }
+    } else if env::var("CARGO_CFG_TARGET_OS").unwrap() == "android" {
+        // Rely on the compile invocation to provide the right search path.
+        println!("cargo:rustc-link-lib=ringrtc_rffi");
     }
 }
 

src/rust/scripts/cargo.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Copyright 2019-2021 Signal Messenger, LLC
-# SPDX-License-Identifier: AGPL-3.0-only
-#
-
-# Simple pass through wrapper for calling cargo
-
-# ------------------------------------------------------------------------------
-#
-# Imports
-#
-
-try:
-    import os
-    import sys
-    import subprocess
-except ImportError as e:
-    raise ImportError(str(e) + "- required module not found")
-
-
-# ------------------------------------------------------------------------------
-#
-# Main
-#
-def main():
-
-    work_dir = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
-    sys.stderr.write("cargo: Entering directory `" + work_dir + "'\n")
-    os.chdir(work_dir)
-    cmd = [ "cargo" ] + sys.argv[1:]
-    sys.stderr.write("cargo command: " + str(cmd))
-    subprocess.check_call(cmd)
-    return 0
-
-
-# --------------------
-#
-# execution check
-#
-if __name__ == '__main__':
-    exit(main())

src/rust/scripts/clippy.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Copyright 2019-2021 Signal Messenger, LLC
-# SPDX-License-Identifier: AGPL-3.0-only
-#
-
-# Simple wrapper for calling "cargo clippy"
-
-# ------------------------------------------------------------------------------
-#
-# Imports
-#
-
-try:
-    import os
-    import sys
-    import subprocess
-except ImportError as e:
-    raise ImportError(str(e) + "- required module not found")
-
-
-# ------------------------------------------------------------------------------
-#
-# Main
-#
-def main():
-
-    work_dir = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
-    sys.stderr.write("cargo: Entering directory `" + work_dir + "'\n")
-    os.chdir(work_dir)
-
-    stamp_file = sys.argv[1]
-
-    cmd = [ "cargo", "clippy" ] + sys.argv[2:]
-    sys.stderr.write("cargo command: " + str(cmd))
-    subprocess.check_call(cmd)
-
-    # touch stamp_file
-    with open(stamp_file, 'a'):
-        os.utime(stamp_file, None)
-
-    return 0
-
-
-# --------------------
-#
-# execution check
-#
-if __name__ == '__main__':
-    exit(main())

src/rust/scripts/strip.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Copyright 2019-2021 Signal Messenger, LLC
-# SPDX-License-Identifier: AGPL-3.0-only
-#
-
-# ------------------------------------------------------------------------------
-#
-# Imports
-#
-
-try:
-    import subprocess
-    import argparse
-except ImportError as e:
-    raise ImportError(str(e) + "- required module not found")
-
-
-# ------------------------------------------------------------------------------
-#
-# Main
-#
-def main():
-
-    parser = argparse.ArgumentParser(
-        description='strip rust shared library')
-    parser.add_argument('-c', '--command',
-                        required=True,
-                        help='strip executable')
-    parser.add_argument('-i', '--input',
-                        required=True,
-                        help='input binary to strip')
-    parser.add_argument('-o', '--output',
-                        required=True,
-                        help='output, stripped binary')
-
-
-    args = parser.parse_args()
-    strip_cmd = [ args.command,
-                  '-o', args.output,
-                  args.input
-    ]
-
-    subprocess.check_call(strip_cmd)
-
-    return 0
-
-
-# --------------------
-#
-# execution check
-#
-if __name__ == '__main__':
-    exit(main())