mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Introduce GCS dependencies support in DEPS autoroller

Browse source 
While GCS dependencies aren't currently used, their support is required
to prevent the autoroller from breaking when encountering GCS dep types.

Bug: None
Change-Id: I58601e9eaeb8372058da4d4ee02cd2ca589e02c7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/349740
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42222}
This commit is contained in:
Byoungchan Lee 2024-05-03 19:39:55 +09:00 committed by WebRTC LUCI CQ
parent 605d00bd6f
commit b1a71aa7c3
4 changed files with 315 additions and 245 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
tools_webrtc/autoroller/roll_deps.py
View file

@ -90,6 +90,7 @@ DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
ChangedDep = collections.namedtuple('ChangedDep', ChangedDep = collections.namedtuple('ChangedDep',
'path url current_rev new_rev') 'path url current_rev new_rev')
CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages')
GcsDepsEntry = collections.namedtuple('GcsDepsEntry', 'path bucket objects')
VersionEntry = collections.namedtuple('VersionEntry', 'version') VersionEntry = collections.namedtuple('VersionEntry', 'version')
ChangedCipdPackage = collections.namedtuple( ChangedCipdPackage = collections.namedtuple(
'ChangedCipdPackage', 'path package current_version new_version') 'ChangedCipdPackage', 'path package current_version new_version')
@ -279,6 +280,9 @@ def BuildDepsentryDict(deps_dict):
dep = {'url': dep} dep = {'url': dep}
if dep.get('dep_type') == 'cipd': if dep.get('dep_type') == 'cipd':
result[path] = CipdDepsEntry(path, dep['packages']) result[path] = CipdDepsEntry(path, dep['packages'])
elif dep.get('dep_type') == 'gcs':
result[path] = GcsDepsEntry(path, dep['bucket'],
dep['objects'])
else: else:
if '@' not in dep['url']: if '@' not in dep['url']:
url, revision = dep['url'], 'HEAD' url, revision = dep['url'], 'HEAD'
@ -441,6 +445,15 @@ def CalculateChangedDeps(webrtc_deps, new_cr_deps):
cr_deps_entry.packages)) cr_deps_entry.packages))
continue continue
if isinstance(cr_deps_entry, GcsDepsEntry):
result.extend(
_FindChangedVars(
path, ','.join(x['object_name']
for x in webrtc_deps_entry.objects),
','.join(x['object_name']
for x in cr_deps_entry.objects)))
continue
if isinstance(cr_deps_entry, VersionEntry): if isinstance(cr_deps_entry, VersionEntry):
result.extend( result.extend(
_FindChangedVars(path, webrtc_deps_entry.version, _FindChangedVars(path, webrtc_deps_entry.version,

79
tools_webrtc/autoroller/unittests/roll_deps_test.py
View file

@ -8,6 +8,7 @@
# in the file PATENTS. All contributing project authors may # in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree. # be found in the AUTHORS file in the root of the source tree.
# pylint: disable=invalid-name
import glob import glob
import os import os
@ -53,6 +54,7 @@ class TestError(Exception):
class FakeCmd: class FakeCmd:
def __init__(self): def __init__(self):
self.expectations = [] self.expectations = []
@ -68,7 +70,8 @@ class FakeCmd:
for item in ignores: for item in ignores:
kwargs.pop(item, None) kwargs.pop(item, None)
if args != exp_args or kwargs != exp_kwargs: if args != exp_args or kwargs != exp_kwargs:
message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) message = ('Expected:\n args: %s\n kwargs: %s\n' %
(exp_args, exp_kwargs))
message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
raise TestError(message) raise TestError(message)
return exp_returns return exp_returns
@ -83,6 +86,7 @@ class NullCmd:
class TestRollChromiumRevision(unittest.TestCase): class TestRollChromiumRevision(unittest.TestCase):
def setUp(self): def setUp(self):
self._output_dir = tempfile.mkdtemp() self._output_dir = tempfile.mkdtemp()
test_data_dir = os.path.join(SCRIPT_DIR, 'testdata', 'roll_deps') test_data_dir = os.path.join(SCRIPT_DIR, 'testdata', 'roll_deps')
@ -116,7 +120,8 @@ class TestRollChromiumRevision(unittest.TestCase):
new_cr_contents) new_cr_contents)
with open(self._webrtc_depsfile, 'rb') as deps_file: with open(self._webrtc_depsfile, 'rb') as deps_file:
deps_contents = deps_file.read().decode('utf-8') deps_contents = deps_file.read().decode('utf-8')
self.assertTrue(new_rev in deps_contents, self.assertTrue(
new_rev in deps_contents,
'Failed to find %s in\n%s' % (new_rev, deps_contents)) 'Failed to find %s in\n%s' % (new_rev, deps_contents))
def _UpdateDepsSetup(self): def _UpdateDepsSetup(self):
@ -129,8 +134,9 @@ class TestRollChromiumRevision(unittest.TestCase):
changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
with mock.patch('roll_deps._RunCommand', NullCmd()): with mock.patch('roll_deps._RunCommand', NullCmd()):
UpdateDepsFile(self._webrtc_depsfile_android, NO_CHROMIUM_REVISION_UPDATE, UpdateDepsFile(self._webrtc_depsfile_android,
changed_deps, new_cr_contents) NO_CHROMIUM_REVISION_UPDATE, changed_deps,
new_cr_contents)
with open(self._webrtc_depsfile_android, 'rb') as deps_file: with open(self._webrtc_depsfile_android, 'rb') as deps_file:
updated_contents = deps_file.read().decode('utf-8') updated_contents = deps_file.read().decode('utf-8')
@ -155,6 +161,7 @@ class TestRollChromiumRevision(unittest.TestCase):
def testRemoveAndroidGeneratedDeps(self): def testRemoveAndroidGeneratedDeps(self):
webrtc_contents, updated_contents = self._UpdateDepsSetup() webrtc_contents, updated_contents = self._UpdateDepsSetup()
# pylint: disable=line-too-long
removed = 'third_party/android_deps/libs/android_arch_lifecycle_runtime' removed = 'third_party/android_deps/libs/android_arch_lifecycle_runtime'
self.assertTrue(removed in webrtc_contents) self.assertTrue(removed in webrtc_contents)
self.assertFalse(removed in updated_contents) self.assertFalse(removed in updated_contents)
@ -166,11 +173,12 @@ class TestRollChromiumRevision(unittest.TestCase):
vars_dict = local_scope['vars'] vars_dict = local_scope['vars']
def AssertVar(variable_name): def AssertVar(variable_name):
self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) self.assertEqual(vars_dict[variable_name],
TEST_DATA_VARS[variable_name])
AssertVar('chromium_git') AssertVar('chromium_git')
AssertVar('chromium_revision') AssertVar('chromium_revision')
self.assertEqual(len(local_scope['deps']), 3) self.assertEqual(len(local_scope['deps']), 4)
self.assertEqual(len(local_scope['deps_os']), 1) self.assertEqual(len(local_scope['deps_os']), 1)
def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
@ -191,14 +199,17 @@ class TestRollChromiumRevision(unittest.TestCase):
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
with mock.patch('roll_deps._RunCommand', self.fake): with mock.patch('roll_deps._RunCommand', self.fake):
_SetupGitLsRemoteCall( _SetupGitLsRemoteCall(
self.fake, 'https://chromium.googlesource.com/chromium/src/build', self.fake,
'https://chromium.googlesource.com/chromium/src/build',
BUILD_NEW_REV) BUILD_NEW_REV)
changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
self.assertEqual(len(changed_deps), 4) self.assertEqual(len(changed_deps), 5)
self.assertEqual(changed_deps[0].path, 'fuchsia') self.assertEqual(changed_deps[0].path, 'fuchsia')
self.assertEqual(changed_deps[0].current_version, 'version:10.20221201.3.1') self.assertEqual(changed_deps[0].current_version,
self.assertEqual(changed_deps[0].new_version, 'version:11.20230207.1.1') 'version:10.20221201.3.1')
self.assertEqual(changed_deps[0].new_version,
'version:11.20230207.1.1')
self.assertEqual(changed_deps[1].path, 'src/build') self.assertEqual(changed_deps[1].path, 'src/build')
self.assertEqual(changed_deps[1].current_rev, BUILD_OLD_REV) self.assertEqual(changed_deps[1].current_rev, BUILD_OLD_REV)
@ -206,14 +217,25 @@ class TestRollChromiumRevision(unittest.TestCase):
self.assertEqual(changed_deps[2].path, 'src/buildtools/linux64') self.assertEqual(changed_deps[2].path, 'src/buildtools/linux64')
self.assertEqual(changed_deps[2].package, 'gn/gn/linux-amd64') self.assertEqual(changed_deps[2].package, 'gn/gn/linux-amd64')
self.assertEqual(changed_deps[2].current_version, self.assertEqual(
changed_deps[2].current_version,
'git_revision:69ec4fca1fa69ddadae13f9e6b7507efa0675263') 'git_revision:69ec4fca1fa69ddadae13f9e6b7507efa0675263')
self.assertEqual(changed_deps[2].new_version, 'git_revision:new-revision') self.assertEqual(changed_deps[2].new_version,
'git_revision:new-revision')
self.assertEqual(changed_deps[3].path, 'src/third_party/depot_tools') self.assertEqual(changed_deps[3].path, 'src/third_party/depot_tools')
self.assertEqual(changed_deps[3].current_rev, DEPOTTOOLS_OLD_REV) self.assertEqual(changed_deps[3].current_rev, DEPOTTOOLS_OLD_REV)
self.assertEqual(changed_deps[3].new_rev, DEPOTTOOLS_NEW_REV) self.assertEqual(changed_deps[3].new_rev, DEPOTTOOLS_NEW_REV)
self.assertEqual(changed_deps[4].path,
'src/third_party/js_code_coverage')
self.assertEqual(
changed_deps[4].current_version,
'js_code_coverage/d538975c93eefc7bafd599b50f867e90c1ef17f3')
self.assertEqual(
changed_deps[4].new_version,
'js_code_coverage/d538975c93eefc7bafd599b50f867e90c1ef17f4')
def testWithDistinctDeps(self): def testWithDistinctDeps(self):
"""Check CalculateChangedDeps works when deps are added/removed.""" """Check CalculateChangedDeps works when deps are added/removed."""
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android) webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android)
@ -232,10 +254,11 @@ class TestRollChromiumRevision(unittest.TestCase):
def testFindAddedDeps(self): def testFindAddedDeps(self):
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android) webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile_android)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android)
added_android_paths, other_paths = FindAddedDeps(webrtc_deps, new_cr_deps) added_android_paths, other_paths = FindAddedDeps(
self.assertEqual( webrtc_deps, new_cr_deps)
added_android_paths, self.assertEqual(added_android_paths, [
['src/third_party/android_deps/libs/android_arch_lifecycle_common']) 'src/third_party/android_deps/libs/android_arch_lifecycle_common'
])
self.assertEqual(other_paths, []) self.assertEqual(other_paths, [])
def testFindRemovedDeps(self): def testFindRemovedDeps(self):
@ -243,25 +266,28 @@ class TestRollChromiumRevision(unittest.TestCase):
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android)
removed_android_paths, other_paths = FindRemovedDeps( removed_android_paths, other_paths = FindRemovedDeps(
webrtc_deps, new_cr_deps) webrtc_deps, new_cr_deps)
self.assertEqual( self.assertEqual(removed_android_paths, [
removed_android_paths, 'src/third_party/android_deps/libs/android_arch_lifecycle_runtime'
['src/third_party/android_deps/libs/android_arch_lifecycle_runtime']) ])
self.assertEqual(other_paths, []) self.assertEqual(other_paths, [])
def testMissingDepsIsDetected(self): def testMissingDepsIsDetected(self):
"""Check error is reported when deps cannot be automatically removed.""" """Check error is reported when deps cannot be automatically removed.
"""
# The situation at test is the following: # The situation at test is the following:
# * A WebRTC DEPS entry is missing from Chromium. # * A WebRTC DEPS entry is missing from Chromium.
# * The dependency isn't an android_deps (those are supported). # * The dependency isn't an android_deps (those are supported).
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile) webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android)
_, other_paths = FindRemovedDeps(webrtc_deps, new_cr_deps) _, other_paths = FindRemovedDeps(webrtc_deps, new_cr_deps)
self.assertEqual( self.assertEqual(other_paths, [
other_paths, 'fuchsia', 'src/buildtools/linux64', 'src/third_party/depot_tools',
['fuchsia', 'src/buildtools/linux64', 'src/third_party/depot_tools']) 'src/third_party/js_code_coverage'
])
def testExpectedDepsIsNotReportedMissing(self): def testExpectedDepsIsNotReportedMissing(self):
"""Some deps musn't be seen as missing, even if absent from Chromium.""" """Some deps musn't be seen as missing, even if absent from Chromium.
"""
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile) webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android) new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile_android)
removed_android_paths, other_paths = FindRemovedDeps( removed_android_paths, other_paths = FindRemovedDeps(
@ -282,7 +308,8 @@ class TestRollChromiumRevision(unittest.TestCase):
commit_msg = GenerateCommitMessage(NO_CHROMIUM_REVISION_UPDATE, commit_msg = GenerateCommitMessage(NO_CHROMIUM_REVISION_UPDATE,
current_commit_pos, new_commit_pos, current_commit_pos, new_commit_pos,
changed_deps, added_paths, removed_paths) changed_deps, added_paths,
removed_paths)
return [l.strip() for l in commit_msg.split('\n')] return [l.strip() for l in commit_msg.split('\n')]
@ -318,6 +345,7 @@ class TestRollChromiumRevision(unittest.TestCase):
class TestChooseCQMode(unittest.TestCase): class TestChooseCQMode(unittest.TestCase):
def testSkip(self): def testSkip(self):
self.assertEqual(ChooseCQMode(True, 99, 500000, 500100), 0) self.assertEqual(ChooseCQMode(True, 99, 500000, 500100), 0)
@ -329,6 +357,7 @@ class TestChooseCQMode(unittest.TestCase):
class TestReadUrlContent(unittest.TestCase): class TestReadUrlContent(unittest.TestCase):
def setUp(self): def setUp(self):
self.url = 'http://localhost+?format=TEXT' self.url = 'http://localhost+?format=TEXT'

14
tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS vendored
View file

@ -30,6 +30,20 @@ deps = {
# Script expects to find these markers. # Script expects to find these markers.
# === ANDROID_DEPS Generated Code Start === # === ANDROID_DEPS Generated Code Start ===
# === ANDROID_DEPS Generated Code End === # === ANDROID_DEPS Generated Code End ===
# Entry that uses GCS
'src/third_party/js_code_coverage': {
'dep_type': 'gcs',
'bucket': 'chromium-nodejs',
'objects': [
{
'object_name': 'js_code_coverage/d538975c93eefc7bafd599b50f867e90c1ef17f3',
'sha256sum': '646bb00ced0a930b2eb1e4dbcfac18ebbb8f889bb80599e0254d9d6505427914',
'size_bytes': 1469185,
'generation': 1657780123604338,
},
],
},
} }
deps_os = { deps_os = {

14
tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.new vendored
View file

@ -26,4 +26,18 @@ deps = {
# Script expects to find these markers. # Script expects to find these markers.
# === ANDROID_DEPS Generated Code Start === # === ANDROID_DEPS Generated Code Start ===
# === ANDROID_DEPS Generated Code End === # === ANDROID_DEPS Generated Code End ===
# Entry that uses GCS
'src/third_party/js_code_coverage': {
'dep_type': 'gcs',
'bucket': 'chromium-nodejs',
'objects': [
{
'object_name': 'js_code_coverage/d538975c93eefc7bafd599b50f867e90c1ef17f4',
'sha256sum': '646bb00ced0a930b2eb1e4dbcfac18ebbb8f889bb80599e0254d9d6505427915',
'size_bytes': 1469186,
'generation': 1657780123604339,
},
],
},
} }