diff --git a/DEPS b/DEPS index ad3f9191c0..b5e50ff6b1 100644 --- a/DEPS +++ b/DEPS @@ -10,7 +10,7 @@ vars = { # chromium waterfalls. More info at: crbug.com/570091. 'checkout_configuration': 'default', 'checkout_instrumented_libraries': 'checkout_linux and checkout_configuration == "default"', - 'chromium_revision': '5350dd2460b8cf103e701f7fd5d0adeb6eb45acd', + 'chromium_revision': 'd5c79b109a2356e0e4b8883057224b0b2719eabf', # Fetch the prebuilt binaries for llvm-cov and llvm-profdata. Needed to # process the raw profiles produced by instrumented targets (built with @@ -27,7 +27,7 @@ vars = { # By default, download the fuchsia sdk from the public sdk directory. 'fuchsia_sdk_cipd_prefix': 'fuchsia/sdk/core/', - 'fuchsia_version': 'version:19.20240320.0.1', + 'fuchsia_version': 'version:20.20240404.3.1', # By default, download the fuchsia images from the fuchsia GCS bucket. 'fuchsia_images_bucket': 'fuchsia', 'checkout_fuchsia': False, @@ -52,9 +52,9 @@ vars = { deps = { # TODO(kjellander): Move this to be Android-only. 'src/base': - 'https://chromium.googlesource.com/chromium/src/base@218e807167d2719d6caaf82d2df4c861d3372304', + 'https://chromium.googlesource.com/chromium/src/base@dbb0eee8e8238195972062b68b7f46abc5a6f110', 'src/build': - 'https://chromium.googlesource.com/chromium/src/build@04c884cbadf1489a7216cb4a43b77daa33442ce2', + 'https://chromium.googlesource.com/chromium/src/build@6d974542a020b0ed31b0d9909b3fdc615066fbf0', 'src/buildtools': 'https://chromium.googlesource.com/chromium/src/buildtools@8919328651a559f8a974641d40fe712062cc6718', # Gradle 6.6.1. Used for testing Android Studio project generation for WebRTC. @@ -63,19 +63,19 @@ deps = { 'condition': 'checkout_android', }, 'src/ios': { - 'url': 'https://chromium.googlesource.com/chromium/src/ios@77324ec269e07ea82cd8c8a15ffb0dfcb3742d72', + 'url': 'https://chromium.googlesource.com/chromium/src/ios@0bf6089d480e995951dc097c1aeaab5b562787a4', 'condition': 'checkout_ios', }, 'src/testing': - 'https://chromium.googlesource.com/chromium/src/testing@977e41b6f78f22f5f9507d8f86f23ddb5c57ae28', + 'https://chromium.googlesource.com/chromium/src/testing@02d496a8fcd939b4a906808cbb0a5b4ae8496ea1', 'src/third_party': - 'https://chromium.googlesource.com/chromium/src/third_party@d0b81aaaa6d9c29983762fa7d85c4b8d07870688', + 'https://chromium.googlesource.com/chromium/src/third_party@2831ffa216574f0708fe777e50785a421f68b5ca', 'src/buildtools/linux64': { 'packages': [ { 'package': 'gn/gn/linux-${{arch}}', - 'version': 'git_revision:93ee9b91423c1f1f53fb5f6cba7b8eef6247a564', + 'version': 'git_revision:415b3b19e094cd4b6982147693485df65037f942', } ], 'dep_type': 'cipd', @@ -85,7 +85,7 @@ deps = { 'packages': [ { 'package': 'gn/gn/mac-${{arch}}', - 'version': 'git_revision:93ee9b91423c1f1f53fb5f6cba7b8eef6247a564', + 'version': 'git_revision:415b3b19e094cd4b6982147693485df65037f942', } ], 'dep_type': 'cipd', @@ -95,7 +95,7 @@ deps = { 'packages': [ { 'package': 'gn/gn/windows-amd64', - 'version': 'git_revision:93ee9b91423c1f1f53fb5f6cba7b8eef6247a564', + 'version': 'git_revision:415b3b19e094cd4b6982147693485df65037f942', } ], 'dep_type': 'cipd', @@ -119,7 +119,7 @@ deps = { 'src/third_party/libc++/src': 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git@6ddb5cb9499f29383155d7ce6b31f7d1102e6ed9', 'src/third_party/libc++abi/src': - 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git@1317096ef8cf278e8d654b78fba7833237673ee1', + 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git@932d253fedb390a08b17ec3a92469a4553934a6a', 'src/third_party/libunwind/src': 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git@419b03c0b8f20d6da9ddcb0d661a94a97cdd7dad', @@ -187,11 +187,11 @@ deps = { }, 'src/third_party/boringssl/src': - 'https://boringssl.googlesource.com/boringssl.git@368d0d87d0bd00f8227f74ce18e8e4384eaf6afa', + 'https://boringssl.googlesource.com/boringssl.git@f94f3ed3965ea033001fb9ae006084eee408b861', 'src/third_party/breakpad/breakpad': 'https://chromium.googlesource.com/breakpad/breakpad.git@76788faa4ef163081f82273bfca7fae8a734b971', 'src/third_party/catapult': - 'https://chromium.googlesource.com/catapult.git@bb95c350195ce0748de8029855bc71ca1336d5e6', + 'https://chromium.googlesource.com/catapult.git@0b9e7b88303cc20347ca923ea45feeb11860c615', 'src/third_party/ced/src': { 'url': 'https://chromium.googlesource.com/external/github.com/google/compact_enc_det.git@ba412eaaacd3186085babcd901679a48863c7dd5', }, @@ -204,9 +204,9 @@ deps = { 'src/third_party/crc32c/src': 'https://chromium.googlesource.com/external/github.com/google/crc32c.git@fa5ade41ee480003d9c5af6f43567ba22e4e17e6', 'src/third_party/depot_tools': - 'https://chromium.googlesource.com/chromium/tools/depot_tools.git@e545830db2fb823da1217b142cb5be10bf45d575', + 'https://chromium.googlesource.com/chromium/tools/depot_tools.git@246580c7b80617f291c257c0aa10c155574a7a05', 'src/third_party/ffmpeg': - 'https://chromium.googlesource.com/chromium/third_party/ffmpeg.git@52d8ef3799b2f16b66351dd0972bb0bcee1648ac', + 'https://chromium.googlesource.com/chromium/third_party/ffmpeg.git@bdcb0b447f433de3b69f0252732791b9f7e26f37', 'src/third_party/flatbuffers/src': 'https://chromium.googlesource.com/external/github.com/google/flatbuffers.git@bcb9ef187628fe07514e57756d05e6a6296f7dc5', 'src/third_party/grpc/src': { @@ -232,7 +232,7 @@ deps = { 'condition': 'checkout_android', }, 'src/third_party/googletest/src': - 'https://chromium.googlesource.com/external/github.com/google/googletest.git@77afe8e0149c207edd9561c28de6d2226673b51f', + 'https://chromium.googlesource.com/external/github.com/google/googletest.git@f10e11fb27301fba21caa71030bb5024e67aa135', 'src/third_party/icu': { 'url': 'https://chromium.googlesource.com/chromium/deps/icu.git@364118a1d9da24bb5b770ac3d762ac144d6da5a4', }, @@ -280,7 +280,7 @@ deps = { 'packages': [ { 'package': 'chromium/third_party/kotlinc', - 'version': '-kUQ1HWm0wwi5pXKSqIplyfSInHmtRS9cVUzg-2l-Y0C', + 'version': 'Rr02Gf2EkaeSs3EhSUHhPqDHSd1AzimrM6cRYUJCPjQC', }, ], 'condition': 'checkout_android', @@ -304,9 +304,11 @@ deps = { 'condition': 'checkout_android', }, 'src/third_party/perfetto': - 'https://android.googlesource.com/platform/external/perfetto.git@6fd518058c5ce2c2ae99ddb591f7223a290f39ce', + 'https://android.googlesource.com/platform/external/perfetto.git@7609c6712c3a3afc5f2f6b3e4dfe308ee6517f74', + 'src/third_party/protobuf-javascript/src': + Var('chromium_git') + '/external/github.com/protocolbuffers/protobuf-javascript' + '@' + 'e34549db516f8712f678fcd4bc411613b5cc5295', 'src/third_party/libvpx/source/libvpx': - 'https://chromium.googlesource.com/webm/libvpx.git@d790001fd56edbc39482d596868ed83ee7355368', + 'https://chromium.googlesource.com/webm/libvpx.git@6445da1b40da7967ccaee23d1ac46d5c3981b89c', 'src/third_party/libyuv': 'https://chromium.googlesource.com/libyuv/libyuv.git@a6a2ec654b1be1166b376476a7555c89eca0c275', 'src/third_party/lss': { @@ -333,7 +335,7 @@ deps = { 'packages': [ { 'package': 'chromium/third_party/r8', - 'version': 'eHemH-tzLR3jqxqGYiQu6AYGLAPyFYG7klrqbvu1mcQC', + 'version': 'kPZJoj3X2XUBofNCeOoUj0P8XChVFgtiNQqMnTwhcOwC', }, ], 'condition': 'checkout_android', @@ -357,7 +359,7 @@ deps = { 'condition': 'checkout_android', }, 'src/tools': - 'https://chromium.googlesource.com/chromium/src/tools@09b9b5615bc591b4c8f08ed0c3e0826df769f045', + 'https://chromium.googlesource.com/chromium/src/tools@7dad49cbe0ac07edc648e3671e044e16782652d6', 'src/third_party/espresso': { 'packages': [ @@ -396,7 +398,7 @@ deps = { 'packages': [ { 'package': 'chromium/third_party/androidx', - 'version': 'piz2tht912VQfctH5Z23YCOpLUBoypzE5ymRqB3vgLkC', + 'version': '1qBFaGY_bIR5f12dL7zKpePRlN4UxIDze8xDv_a0sQEC', }, ], 'condition': 'checkout_android', @@ -407,7 +409,7 @@ deps = { 'packages': [ { 'package': 'chromium/third_party/android_build_tools/manifest_merger', - 'version': 'HxnrwdWmIAhi90brIHiGZ4zmnmgKxP4PD0ZsJX6j-mUC', + 'version': 'wtRWPCJVk_NA2GQp0fI-1i-JaPzYJwp6w3udjEhgni4C', }, ], 'condition': 'checkout_android', @@ -500,11 +502,11 @@ deps = { 'packages': [ { 'package': 'infra/tools/luci/isolate/${{platform}}', - 'version': 'git_revision:a84377ac0800e2330d02c3dcbf7b4b74a06d6a5b', + 'version': 'git_revision:06dc7a1f2eeb1d095f7876799458328a44438df1', }, { 'package': 'infra/tools/luci/swarming/${{platform}}', - 'version': 'git_revision:a84377ac0800e2330d02c3dcbf7b4b74a06d6a5b', + 'version': 'git_revision:06dc7a1f2eeb1d095f7876799458328a44438df1', }, ], 'dep_type': 'cipd', @@ -1595,11 +1597,33 @@ deps = { 'dep_type': 'cipd', }, + 'src/third_party/android_deps/libs/com_squareup_moshi_moshi': { + 'packages': [ + { + 'package': 'chromium/third_party/android_deps/libs/com_squareup_moshi_moshi', + 'version': 'version:2@1.15.0.cr1', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, + + 'src/third_party/android_deps/libs/com_squareup_moshi_moshi_adapters': { + 'packages': [ + { + 'package': 'chromium/third_party/android_deps/libs/com_squareup_moshi_moshi_adapters', + 'version': 'version:2@1.15.0.cr1', + }, + ], + 'condition': 'checkout_android', + 'dep_type': 'cipd', + }, + 'src/third_party/android_deps/libs/com_squareup_okio_okio_jvm': { 'packages': [ { 'package': 'chromium/third_party/android_deps/libs/com_squareup_okio_okio_jvm', - 'version': 'version:2@3.3.0.cr1', + 'version': 'version:2@3.7.0.cr1', }, ], 'condition': 'checkout_android', @@ -1610,7 +1634,7 @@ deps = { 'packages': [ { 'package': 'chromium/third_party/android_deps/libs/com_squareup_wire_wire_runtime_jvm', - 'version': 'version:2@4.7.0.cr1', + 'version': 'version:2@4.9.7.cr1', }, ], 'condition': 'checkout_android', diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 5c9c88dc8b..0f9ac8aa57 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -50,6 +50,8 @@ CPPLINT_EXCEPTIONS = [ PYLINT_OLD_STYLE = [ "PRESUBMIT.py", "tools_webrtc/autoroller/roll_deps.py", + "tools_webrtc/android/build_aar.py", + "tools_webrtc/ios/build_ios_libs.py", ] # These filters will always be removed, even if the caller specifies a filter diff --git a/tools_webrtc/android/build_aar.py b/tools_webrtc/android/build_aar.py index d910b39a7c..b442358283 100755 --- a/tools_webrtc/android/build_aar.py +++ b/tools_webrtc/android/build_aar.py @@ -51,178 +51,180 @@ import find_depot_tools def _ParseArgs(): - parser = argparse.ArgumentParser(description='libwebrtc.aar generator.') - parser.add_argument( - '--build-dir', - type=os.path.abspath, - help='Build dir. By default will create and use temporary dir.') - parser.add_argument('--output', - default='libwebrtc.aar', - type=os.path.abspath, - help='Output file of the script.') - parser.add_argument('--arch', - default=DEFAULT_ARCHS, - nargs='*', - help='Architectures to build. Defaults to %(default)s.') - parser.add_argument('--use-goma', - action='store_true', - default=False, - help='Use goma.') - parser.add_argument('--use-remoteexec', - action='store_true', - default=False, - help='Use RBE.') - parser.add_argument('--use-unstripped-libs', - action='store_true', - default=False, - help='Use unstripped .so files within libwebrtc.aar') - parser.add_argument('--verbose', - action='store_true', - default=False, - help='Debug logging.') - parser.add_argument( - '--extra-gn-args', - default=[], - nargs='*', - help="""Additional GN arguments to be used during Ninja generation. + parser = argparse.ArgumentParser(description='libwebrtc.aar generator.') + parser.add_argument( + '--build-dir', + type=os.path.abspath, + help='Build dir. By default will create and use temporary dir.') + parser.add_argument('--output', + default='libwebrtc.aar', + type=os.path.abspath, + help='Output file of the script.') + parser.add_argument( + '--arch', + default=DEFAULT_ARCHS, + nargs='*', + help='Architectures to build. Defaults to %(default)s.') + parser.add_argument('--use-goma', + action='store_true', + default=False, + help='Use goma.') + parser.add_argument('--use-remoteexec', + action='store_true', + default=False, + help='Use RBE.') + parser.add_argument('--use-unstripped-libs', + action='store_true', + default=False, + help='Use unstripped .so files within libwebrtc.aar') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + parser.add_argument( + '--extra-gn-args', + default=[], + nargs='*', + help="""Additional GN arguments to be used during Ninja generation. These are passed to gn inside `--args` switch and applied after any other arguments and will override any values defined by the script. Example of building debug aar file: build_aar.py --extra-gn-args='is_debug=true'""") - parser.add_argument( - '--extra-ninja-switches', - default=[], - nargs='*', - help="""Additional Ninja switches to be used during compilation. + parser.add_argument( + '--extra-ninja-switches', + default=[], + nargs='*', + help="""Additional Ninja switches to be used during compilation. These are applied after any other Ninja switches. Example of enabling verbose Ninja output: build_aar.py --extra-ninja-switches='-v'""") - parser.add_argument( - '--extra-gn-switches', - default=[], - nargs='*', - help="""Additional GN switches to be used during compilation. + parser.add_argument( + '--extra-gn-switches', + default=[], + nargs='*', + help="""Additional GN switches to be used during compilation. These are applied after any other GN switches. Example of enabling verbose GN output: build_aar.py --extra-gn-switches='-v'""") - return parser.parse_args() + return parser.parse_args() def _RunGN(args): - cmd = [ - sys.executable, - os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') - ] - cmd.extend(args) - logging.debug('Running: %r', cmd) - subprocess.check_call(cmd) + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') + ] + cmd.extend(args) + logging.debug('Running: %r', cmd) + subprocess.check_call(cmd) def _RunNinja(output_directory, args): - cmd = [ - os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja'), '-C', - output_directory - ] - cmd.extend(args) - logging.debug('Running: %r', cmd) - subprocess.check_call(cmd) + cmd = [ + os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja'), '-C', + output_directory + ] + cmd.extend(args) + logging.debug('Running: %r', cmd) + subprocess.check_call(cmd) def _EncodeForGN(value): - """Encodes value as a GN literal.""" - if isinstance(value, str): - return '"' + value + '"' - if isinstance(value, bool): - return repr(value).lower() - return repr(value) + """Encodes value as a GN literal.""" + if isinstance(value, str): + return '"' + value + '"' + if isinstance(value, bool): + return repr(value).lower() + return repr(value) def _GetOutputDirectory(build_dir, arch): - """Returns the GN output directory for the target architecture.""" - return os.path.join(build_dir, arch) + """Returns the GN output directory for the target architecture.""" + return os.path.join(build_dir, arch) def _GetTargetCpu(arch): - """Returns target_cpu for the GN build with the given architecture.""" - if arch in ['armeabi', 'armeabi-v7a']: - return 'arm' - if arch == 'arm64-v8a': - return 'arm64' - if arch == 'x86': - return 'x86' - if arch == 'x86_64': - return 'x64' - raise Exception('Unknown arch: ' + arch) + """Returns target_cpu for the GN build with the given architecture.""" + if arch in ['armeabi', 'armeabi-v7a']: + return 'arm' + if arch == 'arm64-v8a': + return 'arm64' + if arch == 'x86': + return 'x86' + if arch == 'x86_64': + return 'x64' + raise Exception('Unknown arch: ' + arch) def _GetArmVersion(arch): - """Returns arm_version for the GN build with the given architecture.""" - if arch == 'armeabi': - return 6 - if arch == 'armeabi-v7a': - return 7 - if arch in ['arm64-v8a', 'x86', 'x86_64']: - return None - raise Exception('Unknown arch: ' + arch) + """Returns arm_version for the GN build with the given architecture.""" + if arch == 'armeabi': + return 6 + if arch == 'armeabi-v7a': + return 7 + if arch in ['arm64-v8a', 'x86', 'x86_64']: + return None + raise Exception('Unknown arch: ' + arch) def Build(build_dir, arch, use_goma, use_remoteexec, extra_gn_args, extra_gn_switches, extra_ninja_switches): - """Generates target architecture using GN and builds it using ninja.""" - logging.info('Building: %s', arch) - output_directory = _GetOutputDirectory(build_dir, arch) - gn_args = { - 'target_os': 'android', - 'is_debug': False, - 'is_component_build': False, - 'rtc_include_tests': False, - 'target_cpu': _GetTargetCpu(arch), - 'use_goma': use_goma, - 'use_remoteexec': use_remoteexec, - } - arm_version = _GetArmVersion(arch) - if arm_version: - gn_args['arm_version'] = arm_version - gn_args_str = '--args=' + ' '.join( - [k + '=' + _EncodeForGN(v) for k, v in gn_args.items()] + extra_gn_args) + """Generates target architecture using GN and builds it using ninja.""" + logging.info('Building: %s', arch) + output_directory = _GetOutputDirectory(build_dir, arch) + gn_args = { + 'target_os': 'android', + 'is_debug': False, + 'is_component_build': False, + 'rtc_include_tests': False, + 'target_cpu': _GetTargetCpu(arch), + 'use_goma': use_goma, + 'use_remoteexec': use_remoteexec, + } + arm_version = _GetArmVersion(arch) + if arm_version: + gn_args['arm_version'] = arm_version + gn_args_str = '--args=' + ' '.join( + [k + '=' + _EncodeForGN(v) + for k, v in gn_args.items()] + extra_gn_args) - gn_args_list = ['gen', output_directory, gn_args_str] - gn_args_list.extend(extra_gn_switches) - _RunGN(gn_args_list) + gn_args_list = ['gen', output_directory, gn_args_str] + gn_args_list.extend(extra_gn_switches) + _RunGN(gn_args_list) - ninja_args = TARGETS[:] - if use_goma or use_remoteexec: - ninja_args.extend(['-j', '200']) - ninja_args.extend(extra_ninja_switches) - _RunNinja(output_directory, ninja_args) + ninja_args = TARGETS[:] + if use_goma or use_remoteexec: + ninja_args.extend(['-j', '200']) + ninja_args.extend(extra_ninja_switches) + _RunNinja(output_directory, ninja_args) def CollectCommon(aar_file, build_dir, arch): - """Collects architecture independent files into the .aar-archive.""" - logging.info('Collecting common files.') - output_directory = _GetOutputDirectory(build_dir, arch) - aar_file.write(MANIFEST_FILE, 'AndroidManifest.xml') - aar_file.write(os.path.join(output_directory, JAR_FILE), 'classes.jar') + """Collects architecture independent files into the .aar-archive.""" + logging.info('Collecting common files.') + output_directory = _GetOutputDirectory(build_dir, arch) + aar_file.write(MANIFEST_FILE, 'AndroidManifest.xml') + aar_file.write(os.path.join(output_directory, JAR_FILE), 'classes.jar') def Collect(aar_file, build_dir, arch, unstripped): - """Collects architecture specific files into the .aar-archive.""" - logging.info('Collecting: %s', arch) - output_directory = _GetOutputDirectory(build_dir, arch) + """Collects architecture specific files into the .aar-archive.""" + logging.info('Collecting: %s', arch) + output_directory = _GetOutputDirectory(build_dir, arch) - abi_dir = os.path.join('jni', arch) - for so_file in NEEDED_SO_FILES: - source_so_file = os.path.join("lib.unstripped", - so_file) if unstripped else so_file - aar_file.write(os.path.join(output_directory, source_so_file), - os.path.join(abi_dir, so_file)) + abi_dir = os.path.join('jni', arch) + for so_file in NEEDED_SO_FILES: + source_so_file = os.path.join("lib.unstripped", + so_file) if unstripped else so_file + aar_file.write(os.path.join(output_directory, source_so_file), + os.path.join(abi_dir, so_file)) def GenerateLicenses(output_dir, build_dir, archs): - builder = LicenseBuilder( - [_GetOutputDirectory(build_dir, arch) for arch in archs], TARGETS) - builder.GenerateLicenseText(output_dir) + builder = LicenseBuilder( + [_GetOutputDirectory(build_dir, arch) for arch in archs], TARGETS) + builder.generate_license_text(output_dir) def BuildAar(archs, @@ -234,36 +236,36 @@ def BuildAar(archs, extra_gn_switches=None, extra_ninja_switches=None, unstripped=False): - extra_gn_args = extra_gn_args or [] - extra_gn_switches = extra_gn_switches or [] - extra_ninja_switches = extra_ninja_switches or [] - build_dir = ext_build_dir if ext_build_dir else tempfile.mkdtemp() + extra_gn_args = extra_gn_args or [] + extra_gn_switches = extra_gn_switches or [] + extra_ninja_switches = extra_ninja_switches or [] + build_dir = ext_build_dir if ext_build_dir else tempfile.mkdtemp() - for arch in archs: - Build(build_dir, arch, use_goma, use_remoteexec, extra_gn_args, - extra_gn_switches, extra_ninja_switches) - - with zipfile.ZipFile(output_file, 'w') as aar_file: - # Architecture doesn't matter here, arbitrarily using the first one. - CollectCommon(aar_file, build_dir, archs[0]) for arch in archs: - Collect(aar_file, build_dir, arch, unstripped) + Build(build_dir, arch, use_goma, use_remoteexec, extra_gn_args, + extra_gn_switches, extra_ninja_switches) - license_dir = os.path.dirname(os.path.realpath(output_file)) - GenerateLicenses(license_dir, build_dir, archs) + with zipfile.ZipFile(output_file, 'w') as aar_file: + # Architecture doesn't matter here, arbitrarily using the first one. + CollectCommon(aar_file, build_dir, archs[0]) + for arch in archs: + Collect(aar_file, build_dir, arch, unstripped) - if not ext_build_dir: - shutil.rmtree(build_dir, True) + license_dir = os.path.dirname(os.path.realpath(output_file)) + GenerateLicenses(license_dir, build_dir, archs) + + if not ext_build_dir: + shutil.rmtree(build_dir, True) def main(): - args = _ParseArgs() - logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + args = _ParseArgs() + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) - BuildAar(args.arch, args.output, args.use_goma, args.use_remoteexec, - args.extra_gn_args, args.build_dir, args.extra_gn_switches, - args.extra_ninja_switches, args.use_unstripped_libs) + BuildAar(args.arch, args.output, args.use_goma, args.use_remoteexec, + args.extra_gn_args, args.build_dir, args.extra_gn_switches, + args.extra_ninja_switches, args.use_unstripped_libs) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/tools_webrtc/ios/build_ios_libs.py b/tools_webrtc/ios/build_ios_libs.py index 3ab0cbe953..2f4e997ada 100755 --- a/tools_webrtc/ios/build_ios_libs.py +++ b/tools_webrtc/ios/build_ios_libs.py @@ -51,308 +51,315 @@ from generate_licenses import LicenseBuilder def _ParseArgs(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--build_config', - default='release', - choices=['debug', 'release'], - help='The build config. Can be "debug" or "release". ' - 'Defaults to "release".') - parser.add_argument('--arch', - nargs='+', - default=DEFAULT_ARCHS, - choices=ENABLED_ARCHS, - help='Architectures to build. Defaults to %(default)s.') - parser.add_argument( - '-c', - '--clean', - action='store_true', - default=False, - help='Removes the previously generated build output, if any.') - parser.add_argument('-p', - '--purify', - action='store_true', - default=False, - help='Purifies the previously generated build output by ' - 'removing the temporary results used when (re)building.') - parser.add_argument( - '-o', - '--output-dir', - type=os.path.abspath, - default=SDK_OUTPUT_DIR, - help='Specifies a directory to output the build artifacts to. ' - 'If specified together with -c, deletes the dir.') - parser.add_argument( - '-r', - '--revision', - type=int, - default=0, - help='Specifies a revision number to embed if building the framework.') - parser.add_argument('--verbose', - action='store_true', - default=False, - help='Debug logging.') - parser.add_argument('--use-goma', - action='store_true', - default=False, - help='Use goma to build.') - parser.add_argument('--use-remoteexec', - action='store_true', - default=False, - help='Use RBE to build.') - parser.add_argument('--deployment-target', - default=IOS_MINIMUM_DEPLOYMENT_TARGET['device'], - help='Raise the minimum deployment target to build for. ' - 'Cannot be lowered below 12.0 for iOS/iPadOS ' - 'and 14.0 for Catalyst.') - parser.add_argument( - '--extra-gn-args', - default=[], - nargs='*', - help='Additional GN args to be used during Ninja generation.') + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--build_config', + default='release', + choices=['debug', 'release'], + help='The build config. Can be "debug" or "release". ' + 'Defaults to "release".') + parser.add_argument( + '--arch', + nargs='+', + default=DEFAULT_ARCHS, + choices=ENABLED_ARCHS, + help='Architectures to build. Defaults to %(default)s.') + parser.add_argument( + '-c', + '--clean', + action='store_true', + default=False, + help='Removes the previously generated build output, if any.') + parser.add_argument( + '-p', + '--purify', + action='store_true', + default=False, + help='Purifies the previously generated build output by ' + 'removing the temporary results used when (re)building.') + parser.add_argument( + '-o', + '--output-dir', + type=os.path.abspath, + default=SDK_OUTPUT_DIR, + help='Specifies a directory to output the build artifacts to. ' + 'If specified together with -c, deletes the dir.') + parser.add_argument( + '-r', + '--revision', + type=int, + default=0, + help='Specifies a revision number to embed if building the framework.') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + parser.add_argument('--use-goma', + action='store_true', + default=False, + help='Use goma to build.') + parser.add_argument('--use-remoteexec', + action='store_true', + default=False, + help='Use RBE to build.') + parser.add_argument( + '--deployment-target', + default=IOS_MINIMUM_DEPLOYMENT_TARGET['device'], + help='Raise the minimum deployment target to build for. ' + 'Cannot be lowered below 12.0 for iOS/iPadOS ' + 'and 14.0 for Catalyst.') + parser.add_argument( + '--extra-gn-args', + default=[], + nargs='*', + help='Additional GN args to be used during Ninja generation.') - return parser.parse_args() + return parser.parse_args() def _RunCommand(cmd): - logging.debug('Running: %r', cmd) - subprocess.check_call(cmd, cwd=SRC_DIR) + logging.debug('Running: %r', cmd) + subprocess.check_call(cmd, cwd=SRC_DIR) def _CleanArtifacts(output_dir): - if os.path.isdir(output_dir): - logging.info('Deleting %s', output_dir) - shutil.rmtree(output_dir) + if os.path.isdir(output_dir): + logging.info('Deleting %s', output_dir) + shutil.rmtree(output_dir) def _CleanTemporary(output_dir, architectures): - if os.path.isdir(output_dir): - logging.info('Removing temporary build files.') - for arch in architectures: - arch_lib_path = os.path.join(output_dir, arch) - if os.path.isdir(arch_lib_path): - shutil.rmtree(arch_lib_path) + if os.path.isdir(output_dir): + logging.info('Removing temporary build files.') + for arch in architectures: + arch_lib_path = os.path.join(output_dir, arch) + if os.path.isdir(arch_lib_path): + shutil.rmtree(arch_lib_path) def _ParseArchitecture(architectures): - result = dict() - for arch in architectures: - if ":" in arch: - target_environment, target_cpu = arch.split(":") - else: - logging.warning('The environment for build is not specified.') - logging.warning('It is assumed based on cpu type.') - logging.warning('See crbug.com/1138425 for more details.') - if arch == "x64": - target_environment = "simulator" - else: - target_environment = "device" - target_cpu = arch - archs = result.get(target_environment) - if archs is None: - result[target_environment] = {target_cpu} - else: - archs.add(target_cpu) + result = dict() + for arch in architectures: + if ":" in arch: + target_environment, target_cpu = arch.split(":") + else: + logging.warning('The environment for build is not specified.') + logging.warning('It is assumed based on cpu type.') + logging.warning('See crbug.com/1138425 for more details.') + if arch == "x64": + target_environment = "simulator" + else: + target_environment = "device" + target_cpu = arch + archs = result.get(target_environment) + if archs is None: + result[target_environment] = {target_cpu} + else: + archs.add(target_cpu) - return result + return result def _VersionMax(*versions): - return max( - *versions, - key=lambda version: [int(component) for component in version.split('.')]) + return max(*versions, + key=lambda version: + [int(component) for component in version.split('.')]) def BuildWebRTC(output_dir, target_environment, target_arch, flavor, gn_target_name, ios_deployment_target, libvpx_build_vp9, use_goma, use_remoteexec, extra_gn_args): - gn_args = [ - 'target_os="ios"', - 'ios_enable_code_signing=false', - 'is_component_build=false', - 'rtc_include_tests=false', - ] + gn_args = [ + 'target_os="ios"', + 'ios_enable_code_signing=false', + 'is_component_build=false', + 'rtc_include_tests=false', + ] - # Add flavor option. - if flavor == 'debug': - gn_args.append('is_debug=true') - elif flavor == 'release': - gn_args.append('is_debug=false') - else: - raise ValueError('Unexpected flavor type: %s' % flavor) + # Add flavor option. + if flavor == 'debug': + gn_args.append('is_debug=true') + elif flavor == 'release': + gn_args.append('is_debug=false') + else: + raise ValueError('Unexpected flavor type: %s' % flavor) - gn_args.append('target_environment="%s"' % target_environment) + gn_args.append('target_environment="%s"' % target_environment) - gn_args.append('target_cpu="%s"' % target_arch) + gn_args.append('target_cpu="%s"' % target_arch) - gn_args.append('ios_deployment_target="%s"' % ios_deployment_target) + gn_args.append('ios_deployment_target="%s"' % ios_deployment_target) - gn_args.append('rtc_libvpx_build_vp9=' + - ('true' if libvpx_build_vp9 else 'false')) + gn_args.append('rtc_libvpx_build_vp9=' + + ('true' if libvpx_build_vp9 else 'false')) - gn_args.append('use_lld=true') - gn_args.append('use_goma=' + ('true' if use_goma else 'false')) - gn_args.append('use_remoteexec=' + ('true' if use_remoteexec else 'false')) - gn_args.append('rtc_enable_objc_symbol_export=true') + gn_args.append('use_lld=true') + gn_args.append('use_goma=' + ('true' if use_goma else 'false')) + gn_args.append('use_remoteexec=' + ('true' if use_remoteexec else 'false')) + gn_args.append('rtc_enable_objc_symbol_export=true') - args_string = ' '.join(gn_args + extra_gn_args) - logging.info('Building WebRTC with args: %s', args_string) + args_string = ' '.join(gn_args + extra_gn_args) + logging.info('Building WebRTC with args: %s', args_string) - cmd = [ - sys.executable, - os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), - 'gen', - output_dir, - '--args=' + args_string, - ] - _RunCommand(cmd) - logging.info('Building target: %s', gn_target_name) + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), + 'gen', + output_dir, + '--args=' + args_string, + ] + _RunCommand(cmd) + logging.info('Building target: %s', gn_target_name) - cmd = [ - os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja'), - '-C', - output_dir, - gn_target_name, - ] - if use_goma or use_remoteexec: - cmd.extend(['-j', '200']) - _RunCommand(cmd) + cmd = [ + os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja'), + '-C', + output_dir, + gn_target_name, + ] + if use_goma or use_remoteexec: + cmd.extend(['-j', '200']) + _RunCommand(cmd) def main(): - args = _ParseArgs() + args = _ParseArgs() - logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) - if args.clean: - _CleanArtifacts(args.output_dir) - return 0 + if args.clean: + _CleanArtifacts(args.output_dir) + return 0 - # architectures is typed as Dict[str, Set[str]], - # where key is for the environment (device or simulator) - # and value is for the cpu type. - architectures = _ParseArchitecture(args.arch) - gn_args = args.extra_gn_args + # architectures is typed as Dict[str, Set[str]], + # where key is for the environment (device or simulator) + # and value is for the cpu type. + architectures = _ParseArchitecture(args.arch) + gn_args = args.extra_gn_args - if args.purify: - _CleanTemporary(args.output_dir, list(architectures.keys())) - return 0 + if args.purify: + _CleanTemporary(args.output_dir, list(architectures.keys())) + return 0 - gn_target_name = 'framework_objc' - gn_args.append('enable_dsyms=true') - gn_args.append('enable_stripping=true') + gn_target_name = 'framework_objc' + gn_args.append('enable_dsyms=true') + gn_args.append('enable_stripping=true') - # Build all architectures. - framework_paths = [] - all_lib_paths = [] - for (environment, archs) in list(architectures.items()): - ios_deployment_target = _VersionMax( - args.deployment_target, IOS_MINIMUM_DEPLOYMENT_TARGET[environment]) - framework_path = os.path.join(args.output_dir, environment) - framework_paths.append(framework_path) - lib_paths = [] - for arch in archs: - lib_path = os.path.join(framework_path, arch + '_libs') - lib_paths.append(lib_path) - BuildWebRTC(lib_path, environment, arch, args.build_config, - gn_target_name, ios_deployment_target, LIBVPX_BUILD_VP9, - args.use_goma, args.use_remoteexec, gn_args) - all_lib_paths.extend(lib_paths) + # Build all architectures. + framework_paths = [] + all_lib_paths = [] + for (environment, archs) in list(architectures.items()): + ios_deployment_target = _VersionMax( + args.deployment_target, IOS_MINIMUM_DEPLOYMENT_TARGET[environment]) + framework_path = os.path.join(args.output_dir, environment) + framework_paths.append(framework_path) + lib_paths = [] + for arch in archs: + lib_path = os.path.join(framework_path, arch + '_libs') + lib_paths.append(lib_path) + BuildWebRTC(lib_path, environment, arch, args.build_config, + gn_target_name, ios_deployment_target, + LIBVPX_BUILD_VP9, args.use_goma, args.use_remoteexec, + gn_args) + all_lib_paths.extend(lib_paths) + + # Combine the slices. + dylib_path = os.path.join(SDK_FRAMEWORK_NAME, 'WebRTC') + # Dylibs will be combined, all other files are the same across archs. + shutil.rmtree(os.path.join(framework_path, SDK_FRAMEWORK_NAME), + ignore_errors=True) + shutil.copytree(os.path.join(lib_paths[0], SDK_FRAMEWORK_NAME), + os.path.join(framework_path, SDK_FRAMEWORK_NAME), + symlinks=True) + logging.info('Merging framework slices for %s.', environment) + dylib_paths = [os.path.join(path, dylib_path) for path in lib_paths] + out_dylib_path = os.path.join(framework_path, dylib_path) + if os.path.islink(out_dylib_path): + out_dylib_path = os.path.join(os.path.dirname(out_dylib_path), + os.readlink(out_dylib_path)) + try: + os.remove(out_dylib_path) + except OSError: + pass + cmd = ['lipo'] + dylib_paths + ['-create', '-output', out_dylib_path] + _RunCommand(cmd) + + # Merge the dSYM slices. + lib_dsym_dir_path = os.path.join(lib_paths[0], SDK_DSYM_NAME) + if os.path.isdir(lib_dsym_dir_path): + shutil.rmtree(os.path.join(framework_path, SDK_DSYM_NAME), + ignore_errors=True) + shutil.copytree(lib_dsym_dir_path, + os.path.join(framework_path, SDK_DSYM_NAME)) + logging.info('Merging dSYM slices.') + dsym_path = os.path.join(SDK_DSYM_NAME, 'Contents', 'Resources', + 'DWARF', 'WebRTC') + lib_dsym_paths = [ + os.path.join(path, dsym_path) for path in lib_paths + ] + out_dsym_path = os.path.join(framework_path, dsym_path) + try: + os.remove(out_dsym_path) + except OSError: + pass + cmd = ['lipo' + ] + lib_dsym_paths + ['-create', '-output', out_dsym_path] + _RunCommand(cmd) + + # Check for Mac-style WebRTC.framework/Resources/ (for Catalyst)... + resources_dir = os.path.join(framework_path, SDK_FRAMEWORK_NAME, + 'Resources') + if not os.path.exists(resources_dir): + # ...then fall back to iOS-style WebRTC.framework/ + resources_dir = os.path.dirname(resources_dir) + + # Modify the version number. + # Format should be ... + # e.g. 55.0.14986 means + # branch cut 55, no hotfixes, and revision 14986. + infoplist_path = os.path.join(resources_dir, 'Info.plist') + cmd = [ + 'PlistBuddy', '-c', 'Print :CFBundleShortVersionString', + infoplist_path + ] + major_minor = subprocess.check_output(cmd).decode('utf-8').strip() + version_number = '%s.%s' % (major_minor, args.revision) + logging.info('Substituting revision number: %s', version_number) + cmd = [ + 'PlistBuddy', '-c', 'Set :CFBundleVersion ' + version_number, + infoplist_path + ] + _RunCommand(cmd) + _RunCommand(['plutil', '-convert', 'binary1', infoplist_path]) + + xcframework_dir = os.path.join(args.output_dir, SDK_XCFRAMEWORK_NAME) + if os.path.isdir(xcframework_dir): + shutil.rmtree(xcframework_dir) + + logging.info('Creating xcframework.') + cmd = ['xcodebuild', '-create-xcframework', '-output', xcframework_dir] + + # Apparently, xcodebuild needs absolute paths for input arguments + for framework_path in framework_paths: + cmd += [ + '-framework', + os.path.abspath(os.path.join(framework_path, SDK_FRAMEWORK_NAME)), + ] + dsym_full_path = os.path.join(framework_path, SDK_DSYM_NAME) + if os.path.exists(dsym_full_path): + cmd += ['-debug-symbols', os.path.abspath(dsym_full_path)] - # Combine the slices. - dylib_path = os.path.join(SDK_FRAMEWORK_NAME, 'WebRTC') - # Dylibs will be combined, all other files are the same across archs. - shutil.rmtree(os.path.join(framework_path, SDK_FRAMEWORK_NAME), - ignore_errors=True) - shutil.copytree(os.path.join(lib_paths[0], SDK_FRAMEWORK_NAME), - os.path.join(framework_path, SDK_FRAMEWORK_NAME), - symlinks=True) - logging.info('Merging framework slices for %s.', environment) - dylib_paths = [os.path.join(path, dylib_path) for path in lib_paths] - out_dylib_path = os.path.join(framework_path, dylib_path) - if os.path.islink(out_dylib_path): - out_dylib_path = os.path.join(os.path.dirname(out_dylib_path), - os.readlink(out_dylib_path)) - try: - os.remove(out_dylib_path) - except OSError: - pass - cmd = ['lipo'] + dylib_paths + ['-create', '-output', out_dylib_path] _RunCommand(cmd) - # Merge the dSYM slices. - lib_dsym_dir_path = os.path.join(lib_paths[0], SDK_DSYM_NAME) - if os.path.isdir(lib_dsym_dir_path): - shutil.rmtree(os.path.join(framework_path, SDK_DSYM_NAME), - ignore_errors=True) - shutil.copytree(lib_dsym_dir_path, - os.path.join(framework_path, SDK_DSYM_NAME)) - logging.info('Merging dSYM slices.') - dsym_path = os.path.join(SDK_DSYM_NAME, 'Contents', 'Resources', 'DWARF', - 'WebRTC') - lib_dsym_paths = [os.path.join(path, dsym_path) for path in lib_paths] - out_dsym_path = os.path.join(framework_path, dsym_path) - try: - os.remove(out_dsym_path) - except OSError: - pass - cmd = ['lipo'] + lib_dsym_paths + ['-create', '-output', out_dsym_path] - _RunCommand(cmd) + # Generate the license file. + logging.info('Generate license file.') + gn_target_full_name = '//sdk:' + gn_target_name + builder = LicenseBuilder(all_lib_paths, [gn_target_full_name]) + builder.generate_license_text( + os.path.join(args.output_dir, SDK_XCFRAMEWORK_NAME)) - # Check for Mac-style WebRTC.framework/Resources/ (for Catalyst)... - resources_dir = os.path.join(framework_path, SDK_FRAMEWORK_NAME, - 'Resources') - if not os.path.exists(resources_dir): - # ...then fall back to iOS-style WebRTC.framework/ - resources_dir = os.path.dirname(resources_dir) - - # Modify the version number. - # Format should be ... - # e.g. 55.0.14986 means - # branch cut 55, no hotfixes, and revision 14986. - infoplist_path = os.path.join(resources_dir, 'Info.plist') - cmd = [ - 'PlistBuddy', '-c', 'Print :CFBundleShortVersionString', - infoplist_path - ] - major_minor = subprocess.check_output(cmd).decode('utf-8').strip() - version_number = '%s.%s' % (major_minor, args.revision) - logging.info('Substituting revision number: %s', version_number) - cmd = [ - 'PlistBuddy', '-c', 'Set :CFBundleVersion ' + version_number, - infoplist_path - ] - _RunCommand(cmd) - _RunCommand(['plutil', '-convert', 'binary1', infoplist_path]) - - xcframework_dir = os.path.join(args.output_dir, SDK_XCFRAMEWORK_NAME) - if os.path.isdir(xcframework_dir): - shutil.rmtree(xcframework_dir) - - logging.info('Creating xcframework.') - cmd = ['xcodebuild', '-create-xcframework', '-output', xcframework_dir] - - # Apparently, xcodebuild needs absolute paths for input arguments - for framework_path in framework_paths: - cmd += [ - '-framework', - os.path.abspath(os.path.join(framework_path, SDK_FRAMEWORK_NAME)), - ] - dsym_full_path = os.path.join(framework_path, SDK_DSYM_NAME) - if os.path.exists(dsym_full_path): - cmd += ['-debug-symbols', os.path.abspath(dsym_full_path)] - - _RunCommand(cmd) - - # Generate the license file. - logging.info('Generate license file.') - gn_target_full_name = '//sdk:' + gn_target_name - builder = LicenseBuilder(all_lib_paths, [gn_target_full_name]) - builder.GenerateLicenseText( - os.path.join(args.output_dir, SDK_XCFRAMEWORK_NAME)) - - logging.info('Done.') - return 0 + logging.info('Done.') + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/tools_webrtc/libs/generate_licenses.py b/tools_webrtc/libs/generate_licenses.py index feff4281f7..7d34ec6e82 100755 --- a/tools_webrtc/libs/generate_licenses.py +++ b/tools_webrtc/libs/generate_licenses.py @@ -78,8 +78,9 @@ LIB_TO_LICENSES_DICT = { 'spl_sqrt_floor': ['common_audio/third_party/spl_sqrt_floor/LICENSE'], 'kotlin_stdlib': ['third_party/kotlin_stdlib/LICENSE'], 'jni_zero': ['third_party/jni_zero/LICENSE'], + 'protobuf-javascript': ['third_party/protobuf-javascript/LICENSE'], # TODO(bugs.webrtc.org/1110): Remove this hack. This is not a lib. - # For some reason it is listed as so in _GetThirdPartyLibraries. + # For some reason it is listed as so in _get_third_party_libraries. 'android_deps': [], # This is not a library but a collection of libraries. 'androidx': [], @@ -97,7 +98,7 @@ LIB_REGEX_TO_LICENSES_DICT = { 'com_android_support_support_annotations/LICENSE' ], - # Internal dependencies, licenses are already included by other dependencies + # Internal dependencies, licenses are already included by other deps. 'android_deps:com_android_support_support_annotations.*': [], } @@ -111,7 +112,7 @@ SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) # tools we need are *actually* in their build folder, thus we need to move up # to the *true* source root, when we're embedded like this. if SRC_DIR.endswith(os.path.join('third_party', 'webrtc')): - SRC_DIR = os.path.abspath(os.path.join(SRC_DIR, os.pardir, os.pardir)) + SRC_DIR = os.path.abspath(os.path.join(SRC_DIR, os.pardir, os.pardir)) sys.path.append(os.path.join(SRC_DIR, 'build')) import find_depot_tools @@ -120,28 +121,29 @@ THIRD_PARTY_LIB_REGEX_TEMPLATE = r'^.*/third_party/%s$' class LicenseBuilder: - def __init__(self, - buildfile_dirs, - targets, - lib_to_licenses_dict=None, - lib_regex_to_licenses_dict=None): - if lib_to_licenses_dict is None: - lib_to_licenses_dict = LIB_TO_LICENSES_DICT - if lib_regex_to_licenses_dict is None: - lib_regex_to_licenses_dict = LIB_REGEX_TO_LICENSES_DICT + def __init__(self, + buildfile_dirs, + targets, + lib_to_licenses_dict=None, + lib_regex_to_licenses_dict=None): + if lib_to_licenses_dict is None: + lib_to_licenses_dict = LIB_TO_LICENSES_DICT - self.buildfile_dirs = buildfile_dirs - self.targets = targets - self.lib_to_licenses_dict = lib_to_licenses_dict - self.lib_regex_to_licenses_dict = lib_regex_to_licenses_dict + if lib_regex_to_licenses_dict is None: + lib_regex_to_licenses_dict = LIB_REGEX_TO_LICENSES_DICT - self.common_licenses_dict = self.lib_to_licenses_dict.copy() - self.common_licenses_dict.update(self.lib_regex_to_licenses_dict) + self.buildfile_dirs = buildfile_dirs + self.targets = targets + self.lib_to_licenses_dict = lib_to_licenses_dict + self.lib_regex_to_licenses_dict = lib_regex_to_licenses_dict - @staticmethod - def _ParseLibraryName(dep): - """Returns library name after third_party + self.common_licenses_dict = self.lib_to_licenses_dict.copy() + self.common_licenses_dict.update(self.lib_regex_to_licenses_dict) + + @staticmethod + def _parse_library_name(dep): + """Returns library name after third_party Input one of: //a/b/third_party/libname:c @@ -150,11 +152,11 @@ class LicenseBuilder: Outputs libname or None if this is not a third_party dependency. """ - groups = re.match(THIRD_PARTY_LIB_SIMPLE_NAME_REGEX, dep) - return groups.group(1) if groups else None + groups = re.match(THIRD_PARTY_LIB_SIMPLE_NAME_REGEX, dep) + return groups.group(1) if groups else None - def _ParseLibrary(self, dep): - """Returns library simple or regex name that matches `dep` after third_party + def _parse_library(self, dep): + """Return library simple or regex name matching `dep` after third_party This method matches `dep` dependency against simple names in LIB_TO_LICENSES_DICT and regular expression names in @@ -162,104 +164,110 @@ class LicenseBuilder: Outputs matched dict key or None if this is not a third_party dependency. """ - libname = LicenseBuilder._ParseLibraryName(dep) + libname = LicenseBuilder._parse_library_name(dep) - for lib_regex in self.lib_regex_to_licenses_dict: - if re.match(THIRD_PARTY_LIB_REGEX_TEMPLATE % lib_regex, dep): - return lib_regex + for lib_regex in self.lib_regex_to_licenses_dict: + if re.match(THIRD_PARTY_LIB_REGEX_TEMPLATE % lib_regex, dep): + return lib_regex - return libname + return libname - @staticmethod - def _RunGN(buildfile_dir, target): - cmd = [ - sys.executable, - os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), - 'desc', - '--all', - '--format=json', - os.path.abspath(buildfile_dir), - target, - ] - logging.debug('Running: %r', cmd) - output_json = subprocess.check_output(cmd, cwd=WEBRTC_ROOT).decode('UTF-8') - logging.debug('Output: %s', output_json) - return output_json + @staticmethod + def _run_gn(buildfile_dir, target): + cmd = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), + 'desc', + '--all', + '--format=json', + os.path.abspath(buildfile_dir), + target, + ] + logging.debug('Running: %r', cmd) + output_json = subprocess.check_output(cmd, + cwd=WEBRTC_ROOT).decode('UTF-8') + logging.debug('Output: %s', output_json) + return output_json - def _GetThirdPartyLibraries(self, buildfile_dir, target): - output = json.loads(LicenseBuilder._RunGN(buildfile_dir, target)) - libraries = set() - for described_target in list(output.values()): - third_party_libs = (self._ParseLibrary(dep) - for dep in described_target['deps']) - libraries |= set(lib for lib in third_party_libs if lib) - return libraries + def _get_third_party_libraries(self, buildfile_dir, target): + output = json.loads(LicenseBuilder._run_gn(buildfile_dir, target)) + libraries = set() + for described_target in list(output.values()): + third_party_libs = (self._parse_library(dep) + for dep in described_target['deps']) + libraries |= set(lib for lib in third_party_libs if lib) + return libraries - def GenerateLicenseText(self, output_dir): - # Get a list of third_party libs from gn. For fat libraries we must consider - # all architectures, hence the multiple buildfile directories. - third_party_libs = set() - for buildfile in self.buildfile_dirs: - for target in self.targets: - third_party_libs |= self._GetThirdPartyLibraries(buildfile, target) - assert len(third_party_libs) > 0 + def generate_license_text(self, output_dir): + # Get a list of third_party libs from gn. For fat libraries we must + # consider all architectures, hence the multiple buildfile directories. + third_party_libs = set() + for buildfile in self.buildfile_dirs: + for target in self.targets: + third_party_libs |= self._get_third_party_libraries( + buildfile, target) + assert len(third_party_libs) > 0 - missing_licenses = third_party_libs - set(self.common_licenses_dict.keys()) - if missing_licenses: - error_msg = 'Missing licenses for following third_party targets: %s' % \ - ', '.join(sorted(missing_licenses)) - logging.error(error_msg) - raise Exception(error_msg) + missing_licenses = third_party_libs - set( + self.common_licenses_dict.keys()) + if missing_licenses: + error_msg = 'Missing licenses for third_party targets: %s' % \ + ', '.join(sorted(missing_licenses)) + logging.error(error_msg) + raise Exception(error_msg) - # Put webrtc at the front of the list. - license_libs = sorted(third_party_libs) - license_libs.insert(0, 'webrtc') + # Put webrtc at the front of the list. + license_libs = sorted(third_party_libs) + license_libs.insert(0, 'webrtc') - logging.info('List of licenses: %s', ', '.join(license_libs)) + logging.info('List of licenses: %s', ', '.join(license_libs)) - # Generate markdown. - output_license_file = open(os.path.join(output_dir, 'LICENSE.md'), 'w+') - for license_lib in license_libs: - if len(self.common_licenses_dict[license_lib]) == 0: - logging.info('Skipping compile time or internal dependency: %s', - license_lib) - continue # Compile time dependency + # Generate markdown. + output_license_file = open(os.path.join(output_dir, 'LICENSE.md'), + 'w+') + for license_lib in license_libs: + if len(self.common_licenses_dict[license_lib]) == 0: + logging.info( + 'Skipping compile time or internal dependency: %s', + license_lib) + continue # Compile time dependency - output_license_file.write('# %s\n' % license_lib) - output_license_file.write('```\n') - for path in self.common_licenses_dict[license_lib]: - license_path = os.path.join(WEBRTC_ROOT, path) - with open(license_path, 'r') as license_file: - license_text = escape(license_file.read(), quote=True) - output_license_file.write(license_text) - output_license_file.write('\n') - output_license_file.write('```\n\n') + output_license_file.write('# %s\n' % license_lib) + output_license_file.write('```\n') + for path in self.common_licenses_dict[license_lib]: + license_path = os.path.join(WEBRTC_ROOT, path) + with open(license_path, 'r') as license_file: + license_text = escape(license_file.read(), quote=True) + output_license_file.write(license_text) + output_license_file.write('\n') + output_license_file.write('```\n\n') - output_license_file.close() + output_license_file.close() def main(): - parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md') - parser.add_argument('--verbose', - action='store_true', - default=False, - help='Debug logging.') - parser.add_argument('--target', - required=True, - action='append', - default=[], - help='Name of the GN target to generate a license for') - parser.add_argument('output_dir', help='Directory to output LICENSE.md to.') - parser.add_argument('buildfile_dirs', - nargs='+', - help='Directories containing gn generated ninja files') - args = parser.parse_args() + parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Debug logging.') + parser.add_argument('--target', + required=True, + action='append', + default=[], + help='Name of the GN target to generate a license for') + parser.add_argument('output_dir', + help='Directory to output LICENSE.md to.') + parser.add_argument('buildfile_dirs', + nargs='+', + help='Directories containing gn generated ninja files') + args = parser.parse_args() - logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) - builder = LicenseBuilder(args.buildfile_dirs, args.target) - builder.GenerateLicenseText(args.output_dir) + builder = LicenseBuilder(args.buildfile_dirs, args.target) + builder.generate_license_text(args.output_dir) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/tools_webrtc/libs/generate_licenses_test.py b/tools_webrtc/libs/generate_licenses_test.py index 6dfd8f3e22..4f200e92c5 100755 --- a/tools_webrtc/libs/generate_licenses_test.py +++ b/tools_webrtc/libs/generate_licenses_test.py @@ -17,9 +17,10 @@ from generate_licenses import LicenseBuilder class TestLicenseBuilder(unittest.TestCase): - @staticmethod - def _FakeRunGN(buildfile_dir, target): - return """ + + @staticmethod + def _fake_run_gn(buildfile_dir, target): + return """ { "target1": { "deps": [ @@ -32,89 +33,94 @@ class TestLicenseBuilder(unittest.TestCase): } """ - def testParseLibraryName(self): - self.assertEqual( - LicenseBuilder._ParseLibraryName('//a/b/third_party/libname1:c'), - 'libname1') - self.assertEqual( - LicenseBuilder._ParseLibraryName('//a/b/third_party/libname2:c(d)'), - 'libname2') - self.assertEqual( - LicenseBuilder._ParseLibraryName('//a/b/third_party/libname3/c:d(e)'), - 'libname3') - self.assertEqual( - LicenseBuilder._ParseLibraryName('//a/b/not_third_party/c'), None) + def test_parse_library_name(self): + self.assertEqual( + LicenseBuilder._parse_library_name('//a/b/third_party/libname1:c'), + 'libname1') + self.assertEqual( + LicenseBuilder._parse_library_name( + '//a/b/third_party/libname2:c(d)'), 'libname2') + self.assertEqual( + LicenseBuilder._parse_library_name( + '//a/b/third_party/libname3/c:d(e)'), 'libname3') + self.assertEqual( + LicenseBuilder._parse_library_name('//a/b/not_third_party/c'), + None) - def testParseLibrarySimpleMatch(self): - builder = LicenseBuilder([], [], {}, {}) - self.assertEqual(builder._ParseLibrary('//a/b/third_party/libname:c'), - 'libname') + def test_parse_library_simple_match(self): + builder = LicenseBuilder([], [], {}, {}) + self.assertEqual(builder._parse_library('//a/b/third_party/libname:c'), + 'libname') - def testParseLibraryRegExNoMatchFallbacksToDefaultLibname(self): - lib_dict = { - 'libname:foo.*': ['path/to/LICENSE'], - } - builder = LicenseBuilder([], [], lib_dict, {}) - self.assertEqual( - builder._ParseLibrary('//a/b/third_party/libname:bar_java'), 'libname') + def test_parse_library_regex_no_match_fallbacks_to_default_libname(self): + lib_dict = { + 'libname:foo.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], lib_dict, {}) + self.assertEqual( + builder._parse_library('//a/b/third_party/libname:bar_java'), + 'libname') - def testParseLibraryRegExMatch(self): - lib_regex_dict = { - 'libname:foo.*': ['path/to/LICENSE'], - } - builder = LicenseBuilder([], [], {}, lib_regex_dict) - self.assertEqual( - builder._ParseLibrary('//a/b/third_party/libname:foo_bar_java'), - 'libname:foo.*') + def test_parse_library_regex_match(self): + lib_regex_dict = { + 'libname:foo.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._parse_library('//a/b/third_party/libname:foo_bar_java'), + 'libname:foo.*') - def testParseLibraryRegExMatchWithSubDirectory(self): - lib_regex_dict = { - 'libname/foo:bar.*': ['path/to/LICENSE'], - } - builder = LicenseBuilder([], [], {}, lib_regex_dict) - self.assertEqual( - builder._ParseLibrary('//a/b/third_party/libname/foo:bar_java'), - 'libname/foo:bar.*') + def test_parse_library_regex_match_with_sub_directory(self): + lib_regex_dict = { + 'libname/foo:bar.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._parse_library('//a/b/third_party/libname/foo:bar_java'), + 'libname/foo:bar.*') - def testParseLibraryRegExMatchWithStarInside(self): - lib_regex_dict = { - 'libname/foo.*bar.*': ['path/to/LICENSE'], - } - builder = LicenseBuilder([], [], {}, lib_regex_dict) - self.assertEqual( - builder._ParseLibrary('//a/b/third_party/libname/fooHAHA:bar_java'), - 'libname/foo.*bar.*') + def test_parse_library_regex_match_with_star_inside(self): + lib_regex_dict = { + 'libname/foo.*bar.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._parse_library( + '//a/b/third_party/libname/fooHAHA:bar_java'), + 'libname/foo.*bar.*') - @patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) - def testGetThirdPartyLibrariesWithoutRegex(self): - builder = LicenseBuilder([], [], {}, {}) - self.assertEqual(builder._GetThirdPartyLibraries('out/arm', 'target1'), - set(['libname1', 'libname2', 'libname3'])) + @patch('generate_licenses.LicenseBuilder._run_gn', _fake_run_gn) + def test_get_third_party_libraries_without_regex(self): + builder = LicenseBuilder([], [], {}, {}) + self.assertEqual( + builder._get_third_party_libraries('out/arm', 'target1'), + set(['libname1', 'libname2', 'libname3'])) - @patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) - def testGetThirdPartyLibrariesWithRegex(self): - lib_regex_dict = { - 'libname2:c.*': ['path/to/LICENSE'], - } - builder = LicenseBuilder([], [], {}, lib_regex_dict) - self.assertEqual(builder._GetThirdPartyLibraries('out/arm', 'target1'), - set(['libname1', 'libname2:c.*', 'libname3'])) + @patch('generate_licenses.LicenseBuilder._run_gn', _fake_run_gn) + def test_get_third_party_libraries_with_regex(self): + lib_regex_dict = { + 'libname2:c.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEqual( + builder._get_third_party_libraries('out/arm', 'target1'), + set(['libname1', 'libname2:c.*', 'libname3'])) - @patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) - def testGenerateLicenseTextFailIfUnknownLibrary(self): - lib_dict = { - 'simple_library': ['path/to/LICENSE'], - } - builder = LicenseBuilder(['dummy_dir'], ['dummy_target'], lib_dict, {}) + @patch('generate_licenses.LicenseBuilder._run_gn', _fake_run_gn) + def test_generate_license_text_fail_if_unknown_library(self): + lib_dict = { + 'simple_library': ['path/to/LICENSE'], + } + builder = LicenseBuilder(['dummy_dir'], ['dummy_target'], lib_dict, {}) - with self.assertRaises(Exception) as context: - builder.GenerateLicenseText('dummy/dir') + with self.assertRaises(Exception) as context: + builder.generate_license_text('dummy/dir') - self.assertEqual( - context.exception.args[0], - 'Missing licenses for following third_party targets: ' - 'libname1, libname2, libname3') + self.assertEqual( + context.exception.args[0], + 'Missing licenses for third_party targets: ' + 'libname1, libname2, libname3') if __name__ == '__main__': - unittest.main() + unittest.main()