DEPS: Sync Git subtree mirrors instead of symlinking into chromium/src

This changes the way we pull in dependencies WebRTC shares with
Chromium. The base, build, tools and third_party directories from
Chromium are now synced as Git subtree mirrors in the DEPS file.

All symlinks to directories that were previously created by the
setup_links.py are replaced with proper DEPS entries.

One downside with this solution is that we get a lot of directories
in tools/ and third_party/ that we currently don't use. Going forward
it might be possible to improve this but as long as the BUILD.gn files
are stored in the Chromium repo rather at each dependency's repo,
this will be very cumbersome.

The DEPS file will be kept auto-rolled by the script in
https://chromium.googlesource.com/external/webrtc/+/master/tools-webrtc/autoroller/roll_deps.py
which is periodically executed by a bot.

This change brings back the Google Play Services download for Android,
which displays a license confirmation dialog to the user at the first sync.
By running it as a proper hook instead of inside sync_chromium.py, the
problems with that the interactive prompt gets hidden/stuck should be
fixed (now the behavior is identical to Chromium).

Some measurements on the size savings for a clean, newly created checkout:
Linux: 15GB -> 6.4GB (-8.6GB)
Linux (with Android): 25 GB -> 16 GB (-9GB). 8.4GB of this is Android SDK+NDK.
Mac (with iOS): 14 GB -> 5.6GB (-8.4GB)

Note that for all of the above, 1GB is occupied by the resources/ dir.

BUG=webrtc:5006, webrtc:5578
NOTRY=True
R=agable@chromium.org, henrika@webrtc.org, iannucci@chromium.org

Review-Url: https://codereview.webrtc.org/1414343008 .
Cr-Commit-Position: refs/heads/master@{#15754}
This commit is contained in:
kjellander@webrtc.org 2016-12-22 10:40:28 +01:00
parent 526248779a
commit 177567c518
11 changed files with 157 additions and 945 deletions

107
.gitignore vendored
View file

@ -40,121 +40,18 @@
/base
/build
/buildtools
/chromium/.gclient.tmp
/chromium/.gclient.tmp_entries
/chromium/.last_sync_chromium
/chromium/_bad_scm
/chromium/src
/links
/links.db
/mojo
/out
/testing
/third_party/BUILD.gn
/third_party/WebKit
/third_party/WebKit/Tools/Scripts
/third_party/accessibility_test_framework
/third_party/afl
/third_party/android_platform
/third_party/android_support_test_runner
/third_party/android_tools
/third_party/apache_velocity
/third_party/appurify-python
/third_party/asan
/third_party/ashmem
/third_party/binutils
/third_party/boringssl
/third_party/bouncycastle
/third_party/byte_buddy
/third_party/catapult
/third_party/ced
/third_party/clang_format
/third_party/class-dump
/third_party/closure_compiler
/third_party/colorama
/third_party/cygwin
/third_party/directxsdk
/third_party/espresso
/third_party/expat
/third_party/ffmpeg
/third_party/gaeunit
/third_party/gflags
/third_party/google-visualization-python
/third_party/gtest-parallel
/third_party/guava
/third_party/hamcrest
/third_party/icu
/third_party/icu4j
/third_party/ijar
/third_party/instrumented_libraries
/third_party/intellij
/third_party/jsoncpp
/third_party/javax_inject
/third_party/jsr-305
/third_party/junit
/third_party/libFuzzer
/third_party/libc++
/third_party/libc++-static
/third_party/libc++abi
/third_party/libjingle
/third_party/libjpeg
/third_party/libjpeg_turbo
/third_party/libsrtp
/third_party/libvpx
/third_party/libxml
/third_party/libyuv
/third_party/llvm
/third_party/llvm-build
/third_party/lss
/third_party/mockito
/third_party/modp_b64
/third_party/oauth2
/third_party/objenesis
/third_party/ocmock
/third_party/openh264
/third_party/openmax_dl
/third_party/opus
/third_party/ow2_asm
/third_party/proguard
/third_party/protobuf
/third_party/requests
/third_party/robolectric
/third_party/sqlite
/third_party/sqlite4java
/third_party/syzygy
/third_party/tcmalloc
/third_party/usrsctp
/third_party/valgrind
/third_party/winsdk_samples
/third_party/yasm
/third_party/zlib
/tools/android
/tools/clang
/tools/clang_format_merge_driver
/tools/determinism
/tools/generate_library_loader
/tools/generate_stubs
/tools/gn
/tools/grit
/tools/gyp
/tools/isolate_driver.py
/tools/luci-go
/tools/memory
/tools/protoc_wrapper
/tools/python
/tools/sanitizer_options
/tools/swarming_client
/tools/telemetry
/tools/tsan_suppressions
/tools/valgrind
/third_party
/tools
/tools-webrtc/video_quality_toolchain/linux/ffmpeg
/tools-webrtc/video_quality_toolchain/linux/zxing
/tools-webrtc/video_quality_toolchain/mac/ffmpeg
/tools-webrtc/video_quality_toolchain/mac/zxing
/tools-webrtc/video_quality_toolchain/win/*.dll
/tools-webrtc/video_quality_toolchain/win/*.exe
/tools/vim
/tools/win
/tools/whitespace.txt
/x86-generic_out/
/xcodebuild

153
DEPS
View file

@ -1,14 +1,97 @@
# This file contains dependencies for WebRTC that are not shared with Chromium.
# If you wish to add a dependency that is present in Chromium's src/DEPS or a
# directory from the Chromium checkout, you should add it to setup_links.py
# instead.
# This file contains dependencies for WebRTC.
vars = {
'chromium_git': 'https://chromium.googlesource.com',
'chromium_revision': 'e6b29f780581dedef421548119586efeb5825d2c',
'boringssl_git': 'https://boringssl.googlesource.com',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling swarming_client
# and whatever else without interference from each other.
'swarming_revision': 'ebc8dab6f8b8d79ec221c94de39a921145abd404',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling openmax_dl
# and whatever else without interference from each other.
'openmax_dl_revision': '7acede9c039ea5d14cf326f44aad1245b9e674a7',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling BoringSSL
# and whatever else without interference from each other.
'boringssl_revision': '33b1d4f575543b42b9c3406b7f8259daf11945dc',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling nss
# and whatever else without interference from each other.
'nss_revision': 'db89506ac363956b4b253db881219eb5d6246781',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling lss
# and whatever else without interference from each other.
'lss_revision': '3f6478ac95edf86cd3da300c2c0d34a438f5dbeb',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling catapult
# and whatever else without interference from each other.
'catapult_revision': '70f42a7c55ca69cdeb9aa6ec7e40ff3f155040b9',
# Three lines of non-changing comments so that
# the commit queue can handle CLs rolling libFuzzer
# and whatever else without interference from each other.
'libfuzzer_revision': '2d19afdbadb1f20d4254f8a5447fceb30f616157',
}
deps = {
# TODO(kjellander): Move this to be Android-only once the libevent dependency
# in base/third_party/libevent is solved.
'src/base':
Var('chromium_git') + '/chromium/src/base' + '@' + '426683655fbb37d58a331d97121fe5d754d3ea02',
'src/build':
Var('chromium_git') + '/chromium/src/build' + '@' + '53a8a4bbfa7ecf84bba943aa0fdaceb2bd542c85',
'src/buildtools':
Var('chromium_git') + '/chromium/buildtools.git' + '@' + '0ef801087682b271e9ace93cfa93e9d3dea98079',
'src/testing':
Var('chromium_git') + '/chromium/src/testing' + '@' + '97282a50ebdaf10a2844481566c574bcb5776272',
'src/testing/gtest':
Var('chromium_git') + '/external/github.com/google/googletest.git' + '@' + '6f8a66431cb592dad629028a50b3dd418a408c87',
'src/testing/gmock':
Var('chromium_git') + '/external/googlemock.git' + '@' + '0421b6f358139f02e102c9c332ce19a33faf75be', # from svn revision 566
'src/third_party':
Var('chromium_git') + '/chromium/src/third_party' + '@' + '905ae0c419397358564b01df29260c136a0c4aff',
'src/third_party/boringssl/src':
Var('boringssl_git') + '/boringssl.git' + '@' + Var('boringssl_revision'),
'src/third_party/catapult':
Var('chromium_git') + '/external/github.com/catapult-project/catapult.git' + '@' + Var('catapult_revision'),
'src/third_party/colorama/src':
Var('chromium_git') + '/external/colorama.git' + '@' + '799604a1041e9b3bc5d2789ecbd7e8db2e18e6b8',
'src/third_party/ffmpeg':
Var('chromium_git') + '/chromium/third_party/ffmpeg.git' + '@' + 'f309edd7828e3ea500c2891187d15926690ddd27',
'src/third_party/jsoncpp/source':
Var('chromium_git') + '/external/github.com/open-source-parsers/jsoncpp.git' + '@' + 'f572e8e42e22cfcf5ab0aea26574f408943edfa4', # from svn 248
# Used for building libFuzzers (only supports Linux).
'src/third_party/libFuzzer/src':
Var('chromium_git') + '/chromium/llvm-project/llvm/lib/Fuzzer.git' + '@' + Var('libfuzzer_revision'),
'src/third_party/libjpeg_turbo':
Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git' + '@' + '7260e4d8b8e1e40b17f03fafdf1cd83296900f76',
'src/third_party/libsrtp':
Var('chromium_git') + '/chromium/deps/libsrtp.git' + '@' + '0e0936f3013fe5884eac82f95e370c8d460a179f',
'src/third_party/libvpx/source/libvpx':
Var('chromium_git') + '/webm/libvpx.git' + '@' + 'd1eca240fb04fae3039ac28510ba0992eb03248d',
'src/third_party/libyuv':
Var('chromium_git') + '/libyuv/libyuv.git' + '@' + 'b18fd21d3c27fce69b5c1ed44b89131dedc87284',
'src/third_party/nss':
Var('chromium_git') + '/chromium/deps/nss.git' + '@' + Var('nss_revision'),
'src/third_party/openh264/src':
Var('chromium_git') + '/external/github.com/cisco/openh264' + '@' + '0fd88df93c5dcaf858c57eb7892bd27763f0f0ac',
'src/third_party/openmax_dl':
Var('chromium_git') + '/external/webrtc/deps/third_party/openmax.git' + '@' + Var('openmax_dl_revision'),
'src/third_party/usrsctp/usrsctplib':
Var('chromium_git') + '/external/github.com/sctplab/usrsctp' + '@' + '7f9228152ab3d70e6848cc9c67389a0d4218740e',
'src/third_party/yasm/source/patched-yasm':
Var('chromium_git') + '/chromium/deps/yasm/patched-yasm.git' + '@' + '7da28c6c7c6a1387217352ce02b31754deb54d2a',
'src/tools':
Var('chromium_git') + '/chromium/src/tools' + '@' + '4b7f075b7070bd3e422f1da2c526934bc14a7834',
'src/tools/gyp':
Var('chromium_git') + '/external/gyp.git' + '@' + 'e7079f0e0e14108ab0dba58728ff219637458563',
'src/tools/swarming_client':
Var('chromium_git') + '/external/swarming.client.git' + '@' + Var('swarming_revision'),
# WebRTC-only dependencies (not present in Chromium).
'src/third_party/gflags':
Var('chromium_git') + '/external/webrtc/deps/third_party/gflags' + '@' + '892576179b45861b53e04a112996a738309cf364',
'src/third_party/gflags/src':
@ -18,7 +101,38 @@ deps = {
}
deps_os = {
'android': {
'src/third_party/android_tools':
Var('chromium_git') + '/android_tools.git' + '@' + 'b43a6a289a7588b1769814f04dd6c7d7176974cc',
'src/third_party/ced/src':
Var('chromium_git') + '/external/github.com/google/compact_enc_det.git' + '@' + 'e57cdc44bd541d10669312a6fdc59fc4bf52d2b9',
'src/third_party/icu':
Var('chromium_git') + '/chromium/deps/icu.git' + '@' + '9cd2828740572ba6f694b9365236a8356fd06147',
'src/third_party/jsr-305/src':
Var('chromium_git') + '/external/jsr-305.git' + '@' + '642c508235471f7220af6d5df2d3210e3bfc0919',
'src/third_party/junit/src':
Var('chromium_git') + '/external/junit.git' + '@' + '64155f8a9babcfcf4263cf4d08253a1556e75481',
'src/third_party/lss':
Var('chromium_git') + '/linux-syscall-support.git' + '@' + Var('lss_revision'),
'src/third_party/mockito/src':
Var('chromium_git') + '/external/mockito/mockito.git' + '@' + 'de83ad4598ad4cf5ea53c69a8a8053780b04b850',
'src/third_party/requests/src':
Var('chromium_git') + '/external/github.com/kennethreitz/requests.git' + '@' + 'f172b30356d821d180fa4ecfa3e71c7274a32de4',
'src/third_party/robolectric/robolectric':
Var('chromium_git') + '/external/robolectric.git' + '@' + 'e38b49a12fdfa17a94f0382cc8ffaf69132fd09b',
'src/third_party/ub-uiautomator/lib':
Var('chromium_git') + '/chromium/third_party/ub-uiautomator.git' + '@' + '00270549ce3161ae72ceb24712618ea28b4f9434',
},
'unix': {
'src/third_party/lss':
Var('chromium_git') + '/linux-syscall-support.git' + '@' + Var('lss_revision'),
},
'win': {
# Dependencies used by libjpeg-turbo
'src/third_party/yasm/binaries':
Var('chromium_git') + '/chromium/deps/yasm/binaries.git' + '@' + '52f9b3f4b0aa06da24ef8b123058bb61ee468881',
# WebRTC-only dependency (not present in Chromium).
'src/third_party/winsdk_samples':
Var('chromium_git') + '/external/webrtc/deps/third_party/winsdk_samples_v71' + '@' + '6e4f93cc0c2f597547449cbaa80a6af629215a63',
},
@ -35,19 +149,6 @@ hooks = [
'_ = os.system("%s %s" % (sys.executable,script)) '
'if os.path.exists(script) else 0')],
},
{
# Clone chromium and its deps.
'name': 'sync chromium',
'pattern': '.',
'action': ['python', '-u', 'src/sync_chromium.py',
'--target-revision', Var('chromium_revision')],
},
{
# Create links to shared dependencies in Chromium.
'name': 'setup_links',
'pattern': '.',
'action': ['python', 'src/setup_links.py'],
},
{
# This clobbers when necessary (based on get_landmines.py). It should be
# an early hook but it will need to be run after syncing Chromium and
@ -63,15 +164,21 @@ hooks = [
'src',
],
},
{
# Download Google Play Services SDK (without license prompt).
'name': 'google_play_services_download',
'pattern': '.',
'action': ['python', 'src/webrtc/build/google_play_services_download.py'],
},
# Android dependencies. Many are downloaded using Google Storage these days.
# They're copied from https://cs.chromium.org/chromium/src/DEPS for all
# such dependencies we share with Chromium.
{
# This downloads SDK extras and puts them in the
# third_party/android_tools/sdk/extras directory.
'name': 'sdkextras',
'pattern': '.',
# When adding a new sdk extras package to download, add the package
# directory and zip file to .gitignore in third_party/android_tools.
'action': ['python',
'src/build/android/play_services/update.py',
'download'
],
},
{
'name': 'intellij',
'pattern': '.',

View file

@ -434,30 +434,13 @@ def _CommonChecks(input_api, output_api):
results.extend(_CheckApprovedFilesLintClean(
input_api, output_api, source_file_filter))
results.extend(input_api.canned_checks.RunPylint(input_api, output_api,
black_list=(r'^.*gviz_api\.py$',
r'^.*gaeunit\.py$',
# Embedded shell-script fakes out pylint.
black_list=(r'^base[\\\/].*\.py$',
r'^build[\\\/].*\.py$',
r'^buildtools[\\\/].*\.py$',
r'^chromium[\\\/].*\.py$',
r'^mojo.*[\\\/].*\.py$',
r'^out.*[\\\/].*\.py$',
r'^testing[\\\/].*\.py$',
r'^third_party[\\\/].*\.py$',
r'^tools[\\\/]clang[\\\/].*\.py$',
r'^tools[\\\/]generate_library_loader[\\\/].*\.py$',
r'^tools[\\\/]generate_stubs[\\\/].*\.py$',
r'^tools[\\\/]gn[\\\/].*\.py$',
r'^tools[\\\/]isolate_driver.py$',
r'^tools[\\\/]mb[\\\/].*\.py$',
r'^tools[\\\/]protoc_wrapper[\\\/].*\.py$',
r'^tools[\\\/]python[\\\/].*\.py$',
r'^tools[\\\/]python_charts[\\\/]data[\\\/].*\.py$',
r'^tools[\\\/]refactoring[\\\/].*\.py$',
r'^tools[\\\/]swarming_client[\\\/].*\.py$',
r'^tools[\\\/]vim[\\\/].*\.py$',
r'^tools[\\\/]valgrind[\\\/].*\.py$',
r'^tools[\\\/]win[\\\/].*\.py$',
r'^tools[\\\/].*\.py$',
# TODO(phoglund): should arguably be checked.
r'^tools-webrtc[\\\/]mb[\\\/].*\.py$',
r'^tools-webrtc[\\\/]valgrind[\\\/].*\.py$',

View file

@ -1,23 +0,0 @@
solutions = [{
'name': 'src',
'url': 'https://chromium.googlesource.com/chromium/src.git',
'deps_file': '.DEPS.git',
'managed': False,
'custom_deps': {
# Skip syncing some large dependencies WebRTC will never need.
'src/chrome/tools/test/reference_build/chrome_linux': None,
'src/chrome/tools/test/reference_build/chrome_mac': None,
'src/chrome/tools/test/reference_build/chrome_win': None,
'src/native_client': None,
'src/third_party/cld_2/src': None,
'src/third_party/hunspell_dictionaries': None,
'src/third_party/liblouis/src': None,
'src/third_party/pdfium': None,
'src/third_party/skia': None,
'src/third_party/trace-viewer': None,
'src/third_party/webrtc': None,
},
'safesync_url': ''
}]
cache_dir = None

View file

@ -1 +0,0 @@
kjellander@webrtc.org

View file

@ -1,5 +0,0 @@
This .gclient file is used to do download a copy of Chromium.
WebRTC uses the Chromium build toolchain and a number of shared
dependencies by creating symlinks to folders in this checkout,
using the ../setup_links.py script.

View file

@ -1,517 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Setup links to a Chromium checkout for WebRTC.
WebRTC standalone shares a lot of dependencies and build tools with Chromium.
To do this, many of the paths of a Chromium checkout is emulated by creating
symlinks to files and directories. This script handles the setup of symlinks to
achieve this.
"""
import ctypes
import errno
import logging
import optparse
import os
import shelve
import shutil
import subprocess
import sys
import textwrap
DIRECTORIES = [
'base',
'build',
'buildtools',
'testing',
'third_party/afl',
'third_party/binutils',
'third_party/boringssl',
'third_party/catapult',
'third_party/closure_compiler',
'third_party/colorama',
'third_party/expat',
'third_party/ffmpeg',
'third_party/instrumented_libraries',
'third_party/jsoncpp',
'third_party/libFuzzer',
'third_party/libjpeg',
'third_party/libjpeg_turbo',
'third_party/libsrtp',
'third_party/libvpx',
'third_party/libyuv',
'third_party/lss',
'third_party/ocmock',
'third_party/openh264',
'third_party/openmax_dl',
'third_party/opus',
'third_party/proguard',
'third_party/protobuf',
'third_party/sqlite',
'third_party/usrsctp',
'third_party/yasm',
'third_party/zlib',
'tools/clang',
'tools/clang_format_merge_driver',
'tools/determinism',
'tools/generate_library_loader',
'tools/generate_stubs',
'tools/gn',
'tools/grit',
'tools/gyp',
'tools/luci-go',
'tools/memory',
'tools/protoc_wrapper',
'tools/python',
'tools/swarming_client',
'tools/valgrind',
'tools/vim',
'tools/win',
]
from sync_chromium import get_target_os_list
target_os = get_target_os_list()
if 'android' in target_os:
DIRECTORIES += [
'third_party/accessibility_test_framework',
'third_party/android_platform',
'third_party/android_support_test_runner',
'third_party/android_tools',
'third_party/apache_velocity',
'third_party/ashmem',
'third_party/bouncycastle',
'third_party/byte_buddy',
'third_party/ced',
'third_party/espresso',
'third_party/guava',
'third_party/hamcrest',
'third_party/icu',
'third_party/icu4j',
'third_party/ijar',
'third_party/intellij',
'third_party/javax_inject',
'third_party/jsr-305',
'third_party/junit',
'third_party/libxml',
'third_party/mockito',
'third_party/modp_b64',
'third_party/objenesis',
'third_party/ow2_asm',
'third_party/requests',
'third_party/robolectric',
'third_party/sqlite4java',
'third_party/tcmalloc',
'tools/android',
]
FILES = {
'third_party/BUILD.gn': None,
}
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
LINKS_DB = 'links'
# Version management to make future upgrades/downgrades easier to support.
SCHEMA_VERSION = 1
def query_yes_no(question, default=False):
"""Ask a yes/no question via raw_input() and return their answer.
Modified from http://stackoverflow.com/a/3041990.
"""
prompt = " [%s/%%s]: "
prompt = prompt % ('Y' if default is True else 'y')
prompt = prompt % ('N' if default is False else 'n')
if default is None:
default = 'INVALID'
while True:
sys.stdout.write(question + prompt)
choice = raw_input().lower()
if choice == '' and default != 'INVALID':
return default
if 'yes'.startswith(choice):
return True
elif 'no'.startswith(choice):
return False
print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
# Actions
class Action(object):
def __init__(self, dangerous):
self.dangerous = dangerous
def announce(self, planning):
"""Log a description of this action.
Args:
planning - True iff we're in the planning stage, False if we're in the
doit stage.
"""
pass
def doit(self, links_db):
"""Execute the action, recording what we did to links_db, if necessary."""
pass
class Remove(Action):
def __init__(self, path, dangerous):
super(Remove, self).__init__(dangerous)
self._priority = 0
self._path = path
def announce(self, planning):
log = logging.warn
filesystem_type = 'file'
if not self.dangerous:
log = logging.info
filesystem_type = 'link'
if planning:
log('Planning to remove %s: %s', filesystem_type, self._path)
else:
log('Removing %s: %s', filesystem_type, self._path)
def doit(self, _):
os.remove(self._path)
class Rmtree(Action):
def __init__(self, path):
super(Rmtree, self).__init__(dangerous=True)
self._priority = 0
self._path = path
def announce(self, planning):
if planning:
logging.warn('Planning to remove directory: %s', self._path)
else:
logging.warn('Removing directory: %s', self._path)
def doit(self, _):
if sys.platform.startswith('win'):
# shutil.rmtree() doesn't work on Windows if any of the directories are
# read-only, which svn repositories are.
subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
else:
shutil.rmtree(self._path)
class Makedirs(Action):
def __init__(self, path):
super(Makedirs, self).__init__(dangerous=False)
self._priority = 1
self._path = path
def doit(self, _):
try:
os.makedirs(self._path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
class Symlink(Action):
def __init__(self, source_path, link_path):
super(Symlink, self).__init__(dangerous=False)
self._priority = 2
self._source_path = source_path
self._link_path = link_path
def announce(self, planning):
if planning:
logging.info(
'Planning to create link from %s to %s', self._link_path,
self._source_path)
else:
logging.debug(
'Linking from %s to %s', self._link_path, self._source_path)
def doit(self, links_db):
# Files not in the root directory need relative path calculation.
# On Windows, use absolute paths instead since NTFS doesn't seem to support
# relative paths for symlinks.
if sys.platform.startswith('win'):
source_path = os.path.abspath(self._source_path)
else:
if os.path.dirname(self._link_path) != self._link_path:
source_path = os.path.relpath(self._source_path,
os.path.dirname(self._link_path))
os.symlink(source_path, os.path.abspath(self._link_path))
links_db[self._source_path] = self._link_path
class LinkError(IOError):
"""Failed to create a link."""
pass
# Use junctions instead of symlinks on the Windows platform.
if sys.platform.startswith('win'):
def symlink(source_path, link_path):
if os.path.isdir(source_path):
subprocess.check_call(['cmd.exe', '/c', 'mklink', '/J', link_path,
source_path])
else:
# Don't create symlinks to files on Windows, just copy the file instead
# (there's no way to create a link without administrator's privileges).
shutil.copy(source_path, link_path)
os.symlink = symlink
class WebRTCLinkSetup(object):
def __init__(self, links_db, force=False, dry_run=False, prompt=False):
self._force = force
self._dry_run = dry_run
self._prompt = prompt
self._links_db = links_db
def CreateLinks(self, on_bot):
logging.debug('CreateLinks')
# First, make a plan of action
actions = []
for source_path, link_path in FILES.iteritems():
actions += self._ActionForPath(
source_path, link_path, check_fn=os.path.isfile, check_msg='files')
for source_dir in DIRECTORIES:
actions += self._ActionForPath(
source_dir, None, check_fn=os.path.isdir,
check_msg='directories')
if not on_bot and self._force:
# When making the manual switch from legacy SVN checkouts to the new
# Git-based Chromium DEPS, the .gclient_entries file that contains cached
# URLs for all DEPS entries must be removed to avoid future sync problems.
entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
if os.path.exists(entries_file):
actions.append(Remove(entries_file, dangerous=True))
actions.sort()
if self._dry_run:
for action in actions:
action.announce(planning=True)
logging.info('Not doing anything because dry-run was specified.')
sys.exit(0)
if any(a.dangerous for a in actions):
logging.warn('Dangerous actions:')
for action in (a for a in actions if a.dangerous):
action.announce(planning=True)
print
if not self._force:
logging.error(textwrap.dedent("""\
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
A C T I O N R E Q I R E D
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
Setting up the checkout requires creating symlinks to directories in the
Chromium checkout inside chromium/src.
To avoid disrupting developers, we've chosen to not delete directories
forcibly, in case you have some work in progress in one of them :)
ACTION REQUIRED:
Before running `gclient sync|runhooks` again, you must run:
%s%s --force
Which will replace all directories which now must be symlinks, after
prompting with a summary of the work-to-be-done.
"""), 'python ' if sys.platform.startswith('win') else '', __file__)
sys.exit(1)
elif self._prompt:
if not query_yes_no('Would you like to perform the above plan?'):
sys.exit(1)
for action in actions:
action.announce(planning=False)
action.doit(self._links_db)
if not on_bot and self._force:
logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
'let the remaining hooks (that probably were interrupted) '
'execute.')
def CleanupLinks(self):
logging.debug('CleanupLinks')
for source, link_path in self._links_db.iteritems():
if source == 'SCHEMA_VERSION':
continue
if os.path.islink(link_path) or sys.platform.startswith('win'):
# os.path.islink() always returns false on Windows
# See http://bugs.python.org/issue13143.
logging.debug('Removing link to %s at %s', source, link_path)
if not self._dry_run:
if os.path.exists(link_path):
if sys.platform.startswith('win') and os.path.isdir(link_path):
subprocess.check_call(['rmdir', '/q', '/s', link_path],
shell=True)
else:
os.remove(link_path)
del self._links_db[source]
@staticmethod
def _ActionForPath(source_path, link_path=None, check_fn=None,
check_msg=None):
"""Create zero or more Actions to link to a file or directory.
This will be a symlink on POSIX platforms. On Windows it will result in:
* a junction for directories
* a copied file for single files.
Args:
source_path: Path relative to the Chromium checkout root.
For readability, the path may contain slashes, which will
automatically be converted to the right path delimiter on Windows.
link_path: The location for the link to create. If omitted it will be the
same path as source_path.
check_fn: A function returning true if the type of filesystem object is
correct for the attempted call. Otherwise an error message with
check_msg will be printed.
check_msg: String used to inform the user of an invalid attempt to create
a file.
Returns:
A list of Action objects.
"""
def fix_separators(path):
if sys.platform.startswith('win'):
return path.replace(os.altsep, os.sep)
else:
return path
assert check_fn
assert check_msg
link_path = link_path or source_path
link_path = fix_separators(link_path)
source_path = fix_separators(source_path)
source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
if os.path.exists(source_path) and not check_fn:
raise LinkError('Can only to link to %s: tried to link to: %s' %
(check_msg, source_path))
if not os.path.exists(source_path):
logging.debug('Silently ignoring missing source: %s. This is to avoid '
'errors on platform-specific dependencies.', source_path)
return []
actions = []
if os.path.exists(link_path) or os.path.islink(link_path):
if os.path.islink(link_path):
actions.append(Remove(link_path, dangerous=False))
elif os.path.isfile(link_path):
actions.append(Remove(link_path, dangerous=True))
elif os.path.isdir(link_path):
actions.append(Rmtree(link_path))
else:
raise LinkError('Don\'t know how to plan: %s' % link_path)
# Create parent directories to the target link if needed.
target_parent_dirs = os.path.dirname(link_path)
if (target_parent_dirs and
target_parent_dirs != link_path and
not os.path.exists(target_parent_dirs)):
actions.append(Makedirs(target_parent_dirs))
actions.append(Symlink(source_path, link_path))
return actions
def _initialize_database(filename):
links_database = shelve.open(filename)
# Wipe the database if this version of the script ends up looking at a
# newer (future) version of the links db, just to be sure.
version = links_database.get('SCHEMA_VERSION')
if version and version != SCHEMA_VERSION:
logging.info('Found database with schema version %s while this script only '
'supports %s. Wiping previous database contents.', version,
SCHEMA_VERSION)
links_database.clear()
links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
return links_database
def main():
on_bot = os.environ.get('CHROME_HEADLESS') == '1'
parser = optparse.OptionParser()
parser.add_option('-d', '--dry-run', action='store_true', default=False,
help='Print what would be done, but don\'t perform any '
'operations. This will automatically set logging to '
'verbose.')
parser.add_option('-c', '--clean-only', action='store_true', default=False,
help='Only clean previously created links, don\'t create '
'new ones. This will automatically set logging to '
'verbose.')
parser.add_option('-f', '--force', action='store_true', default=on_bot,
help='Force link creation. CAUTION: This deletes existing '
'folders and files in the locations where links are '
'about to be created.')
parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
default=(not on_bot),
help='Prompt if we\'re planning to do a dangerous action')
parser.add_option('-v', '--verbose', action='store_const',
const=logging.DEBUG, default=logging.INFO,
help='Print verbose output for debugging.')
options, _ = parser.parse_args()
if options.dry_run or options.force or options.clean_only:
options.verbose = logging.DEBUG
logging.basicConfig(format='%(message)s', level=options.verbose)
# Work from the root directory of the checkout.
script_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(script_dir)
if sys.platform.startswith('win'):
def is_admin():
try:
return os.getuid() == 0
except AttributeError:
return ctypes.windll.shell32.IsUserAnAdmin() != 0
if is_admin():
logging.warning('WARNING: On Windows, you no longer need run as '
'administrator. Please run with user account privileges.')
if not os.path.exists(CHROMIUM_CHECKOUT):
logging.warning('Cannot find a Chromium checkout at %s. Did you run '
'"gclient sync" before running this script?',
CHROMIUM_CHECKOUT)
return 0
links_database = _initialize_database(LINKS_DB)
try:
symlink_creator = WebRTCLinkSetup(links_database, options.force,
options.dry_run, options.prompt)
symlink_creator.CleanupLinks()
if not options.clean_only:
symlink_creator.CreateLinks(on_bot)
except LinkError as e:
print >> sys.stderr, e.message
return 3
finally:
links_database.close()
return 0
if __name__ == '__main__':
sys.exit(main())

View file

@ -1,191 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Script to download a Chromium checkout into the workspace.
The script downloads a full Chromium Git clone and its DEPS.
The following environment variable can be used to alter the behavior:
* CHROMIUM_NO_HISTORY - If set to 1, a Git checkout with no history will be
downloaded. This is consumes less bandwidth and disk space but is known to be
slower in general if you have a high-speed connection.
After a successful sync has completed, a .last_sync_chromium file is written to
the chromium directory. While it exists, no more gclient sync operations will be
performed until the --target-revision changes or the SCRIPT_VERSION constant is
incremented. The file can be removed manually to force a new sync.
"""
import argparse
import os
import shutil
import subprocess
import sys
import textwrap
# Bump this whenever the algorithm changes and you need bots/devs to re-sync,
# ignoring the .last_sync_chromium file
SCRIPT_VERSION = 8
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
CHROMIUM_NO_HISTORY = 'CHROMIUM_NO_HISTORY'
# Duplicated from depot_tools/gclient.py since we cannot depend on that:
DEPS_OS_CHOICES = {
"win32": "win",
"win": "win",
"cygwin": "win",
"darwin": "mac",
"mac": "mac",
"unix": "unix",
"linux": "unix",
"linux2": "unix",
"linux3": "unix",
"android": "android",
}
def _parse_gclient_dict():
gclient_dict = {}
try:
main_gclient = os.path.join(os.path.dirname(ROOT_DIR), '.gclient')
with open(main_gclient, 'rb') as deps_content:
exec(deps_content, gclient_dict)
except Exception as e:
print >> sys.stderr, 'error while parsing .gclient:', e
return gclient_dict
def get_cache_dir():
return _parse_gclient_dict().get('cache_dir')
def get_target_os_list():
# Always add the currently running OS since the --deps option will override
# that if specified:
target_os_list = [DEPS_OS_CHOICES.get(sys.platform, 'unix')]
# Add any target_os entries from .gclient.
target_os_list += _parse_gclient_dict().get('target_os', [])
return ','.join(target_os_list)
def main():
CR_DIR = os.path.join(ROOT_DIR, 'chromium')
p = argparse.ArgumentParser()
p.add_argument('--target-revision', required=True,
help='The target chromium git revision [REQUIRED]')
p.add_argument('--chromium-dir', default=CR_DIR,
help=('The path to the chromium directory to sync '
'(default: %(default)r)'))
opts = p.parse_args()
opts.chromium_dir = os.path.abspath(opts.chromium_dir)
target_os_list = get_target_os_list()
# Do a quick check to see if we were successful last time to make runhooks
# sooper fast.
flag_file = os.path.join(opts.chromium_dir, '.last_sync_chromium')
flag_file_content = '\n'.join([
str(SCRIPT_VERSION),
opts.target_revision,
repr(target_os_list),
])
if (os.path.exists(os.path.join(opts.chromium_dir, 'src')) and
os.path.exists(flag_file)):
with open(flag_file, 'r') as f:
if f.read() == flag_file_content:
print 'Chromium already up to date: ', opts.target_revision
return 0
os.unlink(flag_file)
# Workaround to avoid sync failure due move in
# https://codereview.chromium.org/1155743013
# TODO(kjellander): Remove this after the summer of 2015.
freetype_src = os.path.join(CR_DIR, 'src', 'third_party', 'freetype-android',
'src')
if os.path.isdir(freetype_src):
shutil.rmtree(freetype_src)
# Avoid downloading NaCl toolchain as part of the Chromium hooks.
gclient_cmd = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
args = [
gclient_cmd, 'sync', '--force', '--nohooks', '--revision',
'src@' + opts.target_revision
]
if os.environ.get('CHROME_HEADLESS') == '1':
# Running on a buildbot.
args.append('-vvv')
if sys.platform.startswith('win'):
cache_path = os.path.join(os.path.splitdrive(ROOT_DIR)[0] + os.path.sep,
'b', 'git-cache')
else:
cache_path = '/b/git-cache'
else:
# Verbose, but not as verbose as on the buildbots.
args.append('-v')
# Support developers setting the cache_dir in .gclient.
cache_path = get_cache_dir()
# Allow for users with poor internet connections to download a Git clone
# without history (saves several gigs but is generally slower and doesn't work
# with the Git cache).
if os.environ.get(CHROMIUM_NO_HISTORY) == '1':
if cache_path:
print >> sys.stderr, (
'You cannot use "no-history" mode for syncing Chrome (i.e. set the '
'%s environment variable to 1) when you have cache_dir configured in '
'your .gclient.' % CHROMIUM_NO_HISTORY)
return 1
args.append('--no-history')
gclient_entries_file = os.path.join(opts.chromium_dir, '.gclient_entries')
else:
# Write a temporary .gclient file that has the cache_dir variable added.
gclientfile = os.path.join(opts.chromium_dir, '.gclient')
with open(gclientfile, 'rb') as spec:
spec = spec.read().splitlines()
spec[-1] = 'cache_dir = %r' % (cache_path,)
with open(gclientfile + '.tmp', 'wb') as f:
f.write('\n'.join(spec))
args += [
'--gclientfile', '.gclient.tmp',
'--delete_unversioned_trees', '--reset', '--upstream'
]
gclient_entries_file = os.path.join(opts.chromium_dir,
'.gclient.tmp_entries')
# To avoid gclient sync problems when DEPS entries have been removed we must
# wipe the gclient's entries file that contains cached URLs for all DEPS.
if os.path.exists(gclient_entries_file):
os.unlink(gclient_entries_file)
if target_os_list:
args += ['--deps=' + target_os_list]
print textwrap.dedent("""\
+---------------------------------------------------------------------+
| NOTICE: This sync of Chromium will take a long time as several |
| gigabytes of data are downloaded. If this is your initial |
| sync and it's interrupted, try running 'gclient sync' again.|
| If that fails, wipe everything clean and start over again. |
+---------------------------------------------------------------------+""")
print 'Running "%s" in %s' % (' '.join(args), opts.chromium_dir)
ret = subprocess.call(args, cwd=opts.chromium_dir)
if ret == 0:
with open(flag_file, 'wb') as f:
f.write(flag_file_content)
return ret
if __name__ == '__main__':
sys.exit(main())

View file

@ -23,7 +23,7 @@ import urllib
# Skip these dependencies (list without solution name prefix).
DONT_AUTOROLL_THESE = [
'src/third_party/gflags/src',
'src/third_party/winsdk_samples/src',
'src/third_party/winsdk_samples',
]
WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
@ -40,10 +40,6 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
os.pardir))
CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src')
sys.path.append(CHECKOUT_SRC_DIR)
import setup_links
sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
import find_depot_tools
@ -51,8 +47,8 @@ find_depot_tools.add_depot_tools_to_path()
from gclient import GClientKeywords
CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
'update.py')
CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools',
'clang', 'scripts', 'update.py')
DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
ChangedDep = collections.namedtuple('ChangedDep',
@ -217,10 +213,10 @@ def BuildDepsentryDict(deps_dict):
return result
def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
def CalculateChangedDeps(webrtc_deps, new_cr_deps):
"""
Calculate changed deps entries based on:
1. Entries defined in the WebRTC DEPS file:
Calculate changed deps entries based on entries defined in the WebRTC DEPS
file:
- If a shared dependency with the Chromium DEPS file: roll it to the same
revision as Chromium (i.e. entry in the new_cr_deps dict)
- If it's a Chromium sub-directory, roll it to the HEAD revision (notice
@ -228,18 +224,10 @@ def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
should be close).
- If it's another DEPS entry (not shared with Chromium), roll it to HEAD
unless it's configured to be skipped.
2. Entries present in the setup_links.py file. If the dir has changed between
old_cr_deps and new_cr_deps, it is considered changed and updated to the
revision for the entry in the new_cr_deps dict.
Returns:
A list of ChangedDep objects representing the changed deps.
"""
return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) +
CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps))
def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
result = []
webrtc_entries = BuildDepsentryDict(webrtc_deps)
new_cr_entries = BuildDepsentryDict(new_cr_deps)
@ -264,29 +252,7 @@ def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
logging.debug('Roll dependency %s to %s', path, new_rev)
result.append(ChangedDep(path, webrtc_deps_entry.url,
webrtc_deps_entry.revision, new_rev))
return result
def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps):
result = []
new_cr_entries = BuildDepsentryDict(new_cr_deps)
old_cr_entries = BuildDepsentryDict(old_cr_deps)
all_deps_dirs = setup_links.DIRECTORIES
for deps_dir in all_deps_dirs:
# All deps have 'src' prepended to the path in the Chromium DEPS file.
dir_path = 'src/%s' % deps_dir
for entry in GetMatchingDepsEntries(old_cr_entries, dir_path):
new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path)
assert len(new_matching_entries) <= 1, (
'Should never find more than one entry matching %s in %s, found %d' %
(entry.path, new_cr_entries, len(new_matching_entries)))
if not new_matching_entries:
result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
elif entry != new_matching_entries[0]:
result.append(ChangedDep(entry.path, entry.url, entry.revision,
new_matching_entries[0].revision))
return result
return sorted(result)
def CalculateChangedClang(new_cr_rev):
@ -297,9 +263,7 @@ def CalculateChangedClang(new_cr_rev):
return match.group(1)
raise RollError('Could not parse Clang revision!')
chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR,
CLANG_UPDATE_SCRIPT_LOCAL_PATH)
with open(chromium_src_path, 'rb') as f:
with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f:
current_lines = f.readlines()
current_rev = GetClangRev(current_lines)
@ -366,6 +330,15 @@ def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
# Update each individual DEPS entry.
for dep in changed_deps:
local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path)
if not os.path.isdir(local_dep_dir):
raise RollError(
'Cannot find local directory %s. Either run\n'
'gclient sync --deps=all\n'
'or make sure the .gclient file for your solution contains all '
'platforms in the target_os list, i.e.\n'
'target_os = ["android", "unix", "mac", "ios", "win"];\n'
'Then run "gclient sync" again.' % local_dep_dir)
_, stderr = _RunCommand(
['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
@ -488,10 +461,8 @@ def main():
current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps)
changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
clang_change = CalculateChangedClang(new_cr_rev)
commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
current_commit_pos, new_commit_pos,

View file

@ -19,7 +19,7 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
sys.path.append(PARENT_DIR)
import roll_deps
from roll_deps import CalculateChangedDepsProper, GetMatchingDepsEntries, \
from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \
ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile
@ -119,12 +119,12 @@ class TestRollChromiumRevision(unittest.TestCase):
self.assertEquals(len(entries), 1)
self.assertEquals(entries[0], DEPS_ENTRIES['src/build'])
def testCalculateChangedDepsProper(self):
def testCalculateChangedDeps(self):
_SetupGitLsRemoteCall(self.fake,
'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV)
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
changed_deps = CalculateChangedDepsProper(webrtc_deps, new_cr_deps)
changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
self.assertEquals(len(changed_deps), 2)
self.assertEquals(changed_deps[0].path, 'src/build')
self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV)
@ -134,15 +134,6 @@ class TestRollChromiumRevision(unittest.TestCase):
self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
def testCalculateChangedDepsLegacy(self):
old_cr_deps = ParseLocalDepsFile(self._old_cr_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
changed_deps = CalculateChangedDepsProper(old_cr_deps, new_cr_deps)
self.assertEquals(len(changed_deps), 1)
self.assertEquals(changed_deps[0].path, 'src/buildtools')
self.assertEquals(changed_deps[0].current_rev, BUILDTOOLS_OLD_REV)
self.assertEquals(changed_deps[0].new_rev, BUILDTOOLS_NEW_REV)
def _SetupGitLsRemoteCall(cmd_fake, url, revision):
cmd = ['git', 'ls-remote', url, revision]

View file

@ -22,7 +22,7 @@ deps = {
deps_os = {
# Entry only present in WebRTC, not Chromium.
'win': {
'src/third_party/winsdk_samples/src':
'src/third_party/winsdk_samples':
Var('chromium_git') + '/external/webrtc/deps/third_party/winsdk_samples_v71@e71b549167a665d7424d6f1dadfbff4b4aad1589',
},
}