diff --git a/PRESUBMIT.py b/PRESUBMIT.py index d9c32be23e..276db1e83b 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -437,6 +437,7 @@ def _CommonChecks(input_api, output_api): black_list=(r'^base[\\\/].*\.py$', r'^build[\\\/].*\.py$', r'^buildtools[\\\/].*\.py$', + r'^ios[\\\/].*\.py$', r'^out.*[\\\/].*\.py$', r'^testing[\\\/].*\.py$', r'^third_party[\\\/].*\.py$', diff --git a/ios/README b/ios/README new file mode 100644 index 0000000000..be0afa5d60 --- /dev/null +++ b/ios/README @@ -0,0 +1,2 @@ +This directory is only temporary, it will be removed +when https://bugs.chromium.org/p/webrtc/issues/detail?id=6934 has made progress. diff --git a/ios/build/bots/scripts/OWNERS b/ios/build/bots/scripts/OWNERS new file mode 100644 index 0000000000..49eb0a2b1a --- /dev/null +++ b/ios/build/bots/scripts/OWNERS @@ -0,0 +1,2 @@ +set noparent +smut@google.com diff --git a/ios/build/bots/scripts/find_xcode.py b/ios/build/bots/scripts/find_xcode.py new file mode 100755 index 0000000000..f988fbb412 --- /dev/null +++ b/ios/build/bots/scripts/find_xcode.py @@ -0,0 +1,195 @@ +#!/usr/bin/python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Finds Xcode installations, optionally switching to a desired version. + +Usage: + ./find_xcode.py -j /tmp/out.json -v 6.0.1 + + Finds Xcode 6.0.1 and switches to it. Writes a summary to /tmp/out.json + that includes the Xcode installations that were found, the Xcode version + that was active before running this script, and the Xcode version that + is active after running this script. + + e.g. { + "installations": { + "/Applications/Xcode5.app": "5.1.1 (5B1008)", + "/Applications/Xcode6.app": "6.0 (6A313)", + "/Applications/Xcode6.0.1.app": "6.0.1 (6A317)", + "/Applications/Xcode6.1.app": "6.1 (6A1046a)", + }, + "matches": { + "/Applications/Xcode6.0.1.app": "6.0.1 (6A317)", + }, + "previous version": { + "path": "/Application/Xcode5.app", + "version": "5.1.1", + "build": "(5B1008)", + }, + "current version": { + "path": "/Applications/Xcode6.0.1.app", + "version": "6.0.1", + "build": "6A317", + }, + "found": true, + } +""" + +import argparse +import json +import os +import subprocess +import sys + + +def get_xcodebuild_path(xcode_app): + """Returns the path to xcodebuild under the given Xcode app. + + Args: + xcode_app: The path to an installed Xcode.app. e.g. /Applications/Xcode.app. + + Returns: + The absolute path to the xcodebuild binary under the given Xcode app. + """ + return os.path.join( + xcode_app, + 'Contents', + 'Developer', + 'usr', + 'bin', + 'xcodebuild', + ) + + +def get_xcode_version(xcodebuild): + """Returns the Xcode version and build version. + + Args: + xcodebuild: The absolute path to the xcodebuild binary. + + Returns: + A tuple of (version string, build version string). + e.g. ("6.0.1", "6A317") + """ + # Sample output: + # Xcode 6.0.1 + # Build version 6A317 + out = subprocess.check_output([xcodebuild, '-version']).splitlines() + return out[0].split(' ')[-1], out[1].split(' ')[-1] + + +def get_current_xcode_info(): + """Returns the current Xcode path, version, and build number. + + Returns: + A dict with 'path', 'version', and 'build' keys. + 'path': The absolute path to the Xcode installation. + 'version': The Xcode version. + 'build': The Xcode build version. + """ + version, build_version = get_xcode_version('xcodebuild') + + return { + 'path': subprocess.check_output(['xcode-select', '--print-path']).rstrip(), + 'version': version, + 'build': build_version, + } + + +def find_xcode(target_version=None): + """Finds all Xcode versions, switching to the given Xcode version. + + Args: + target_version: The version of Xcode to switch to, or None if the + Xcode version should not be switched. + + Returns: + A summary dict as described in the usage section above. + """ + xcode_info = { + 'installations': { + }, + 'current version': { + }, + } + + if target_version: + xcode_info['found'] = False + xcode_info['matches'] = {} + xcode_info['previous version'] = get_current_xcode_info() + + if xcode_info['previous version']['version'] == target_version: + xcode_info['found'] = True + + for app in os.listdir(os.path.join('/', 'Applications')): + if app.startswith('Xcode'): + installation_path = os.path.join('/', 'Applications', app) + xcodebuild = get_xcodebuild_path(installation_path) + + if os.path.exists(xcodebuild): + version, build_version = get_xcode_version(xcodebuild) + + xcode_info['installations'][installation_path] = "%s (%s)" % ( + version, + build_version, + ) + + if target_version and version == target_version: + xcode_info['matches'][installation_path] = "%s (%s)" % ( + version, + build_version, + ) + + # If this is the first match, switch to it. + if not xcode_info['found']: + subprocess.check_call([ + 'sudo', + 'xcode-select', + '-switch', + installation_path, + ]) + + xcode_info['found'] = True + + xcode_info['current version'] = get_current_xcode_info() + + if target_version and not xcode_info['found']: + # Flush buffers to ensure correct output ordering for buildbot. + sys.stdout.flush() + sys.stderr.write('Target Xcode version not found: %s\n' % target_version) + sys.stderr.flush() + + return xcode_info + + +def main(args): + xcode_info = find_xcode(args.version) + + if args.json_file: + with open(args.json_file, 'w') as json_file: + json.dump(xcode_info, json_file) + + if args.version and not xcode_info['found']: + return 1 + + return 0 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-j', + '--json-file', + help='Location to write a JSON summary.', + metavar='file', + ) + parser.add_argument( + '-v', + '--version', + help='Xcode version to find and switch to.', + metavar='ver', + ) + + sys.exit(main(parser.parse_args())) diff --git a/ios/build/bots/scripts/gtest_utils.py b/ios/build/bots/scripts/gtest_utils.py new file mode 100644 index 0000000000..048929f681 --- /dev/null +++ b/ios/build/bots/scripts/gtest_utils.py @@ -0,0 +1,473 @@ +# Copyright (c) 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import collections +import copy +import re + + +# These labels should match the ones output by gtest's JSON. +TEST_UNKNOWN_LABEL = 'UNKNOWN' +TEST_SUCCESS_LABEL = 'SUCCESS' +TEST_FAILURE_LABEL = 'FAILURE' +TEST_TIMEOUT_LABEL = 'TIMEOUT' +TEST_WARNING_LABEL = 'WARNING' + + +class GTestResult(object): + """A result of gtest. + + Properties: + command: The command argv. + crashed: Whether or not the test crashed. + crashed_test: The name of the test during which execution crashed, or + None if a particular test didn't crash. + failed_tests: A dict mapping the names of failed tests to a list of + lines of output from those tests. + flaked_tests: A dict mapping the names of failed flaky tests to a list + of lines of output from those tests. + passed_tests: A list of passed tests. + perf_links: A dict mapping the names of perf data points collected + to links to view those graphs. + return_code: The return code of the command. + success: Whether or not this run of the command was considered a + successful GTest execution. + """ + @property + def crashed(self): + return self._crashed + + @property + def crashed_test(self): + return self._crashed_test + + @property + def command(self): + return self._command + + @property + def failed_tests(self): + if self.__finalized: + return copy.deepcopy(self._failed_tests) + return self._failed_tests + + @property + def flaked_tests(self): + if self.__finalized: + return copy.deepcopy(self._flaked_tests) + return self._flaked_tests + + @property + def passed_tests(self): + if self.__finalized: + return copy.deepcopy(self._passed_tests) + return self._passed_tests + + @property + def perf_links(self): + if self.__finalized: + return copy.deepcopy(self._perf_links) + return self._perf_links + + @property + def return_code(self): + return self._return_code + + @property + def success(self): + return self._success + + def __init__(self, command): + if not isinstance(command, collections.Iterable): + raise ValueError('Expected an iterable of command arguments.', command) + + if not command: + raise ValueError('Expected a non-empty command.', command) + + self._command = tuple(command) + self._crashed = False + self._crashed_test = None + self._failed_tests = collections.OrderedDict() + self._flaked_tests = collections.OrderedDict() + self._passed_tests = [] + self._perf_links = collections.OrderedDict() + self._return_code = None + self._success = None + self.__finalized = False + + def finalize(self, return_code, success): + self._return_code = return_code + self._success = success + + # If the test was not considered to be a GTest success, but had no + # failing tests, conclude that it must have crashed. + if not self._success and not self._failed_tests and not self._flaked_tests: + self._crashed = True + + # At most one test can crash the entire app in a given parsing. + for test, log_lines in self._failed_tests.iteritems(): + # A test with no output would have crashed. No output is replaced + # by the GTestLogParser by a sentence indicating non-completion. + if 'Did not complete.' in log_lines: + self._crashed = True + self._crashed_test = test + + # A test marked as flaky may also have crashed the app. + for test, log_lines in self._flaked_tests.iteritems(): + if 'Did not complete.' in log_lines: + self._crashed = True + self._crashed_test = test + + self.__finalized = True + + +class GTestLogParser(object): + """This helper class process GTest test output.""" + + def __init__(self): + # State tracking for log parsing + self.completed = False + self._current_test = '' + self._failure_description = [] + self._parsing_failures = False + + # Line number currently being processed. + self._line_number = 0 + + # List of parsing errors, as human-readable strings. + self._internal_error_lines = [] + + # Tests are stored here as 'test.name': (status, [description]). + # The status should be one of ('started', 'OK', 'failed', 'timeout', + # 'warning'). Warning indicates that a test did not pass when run in + # parallel with other tests but passed when run alone. The description is + # a list of lines detailing the test's error, as reported in the log. + self._test_status = {} + + # This may be either text or a number. It will be used in the phrase + # '%s disabled' or '%s flaky' on the waterfall display. + self._disabled_tests = 0 + self._flaky_tests = 0 + + # Regular expressions for parsing GTest logs. Test names look like + # "x.y", with 0 or more "w/" prefixes and 0 or more "/z" suffixes. + # e.g.: + # SomeName/SomeTestCase.SomeTest/1 + # SomeName/SomeTestCase/1.SomeTest + # SomeName/SomeTestCase/1.SomeTest/SomeModifider + test_name_regexp = r'((\w+/)*\w+\.\w+(/\w+)*)' + + self._master_name_re = re.compile(r'\[Running for master: "([^"]*)"') + self.master_name = '' + + self._test_name = re.compile(test_name_regexp) + self._test_start = re.compile(r'\[\s+RUN\s+\] ' + test_name_regexp) + self._test_ok = re.compile(r'\[\s+OK\s+\] ' + test_name_regexp) + self._test_fail = re.compile(r'\[\s+FAILED\s+\] ' + test_name_regexp) + self._test_passed = re.compile(r'\[\s+PASSED\s+\] \d+ tests?.') + self._run_test_cases_line = re.compile( + r'\[\s*\d+\/\d+\]\s+[0-9\.]+s ' + test_name_regexp + ' .+') + self._test_timeout = re.compile( + r'Test timeout \([0-9]+ ms\) exceeded for ' + test_name_regexp) + self._disabled = re.compile(r'\s*YOU HAVE (\d+) DISABLED TEST') + self._flaky = re.compile(r'\s*YOU HAVE (\d+) FLAKY TEST') + + self._retry_message = re.compile('RETRYING FAILED TESTS:') + self.retrying_failed = False + + self.TEST_STATUS_MAP = { + 'OK': TEST_SUCCESS_LABEL, + 'failed': TEST_FAILURE_LABEL, + 'timeout': TEST_TIMEOUT_LABEL, + 'warning': TEST_WARNING_LABEL + } + + def GetCurrentTest(self): + return self._current_test + + def _StatusOfTest(self, test): + """Returns the status code for the given test, or 'not known'.""" + test_status = self._test_status.get(test, ('not known', [])) + return test_status[0] + + def _TestsByStatus(self, status, include_fails, include_flaky): + """Returns list of tests with the given status. + + Args: + include_fails: If False, tests containing 'FAILS_' anywhere in their + names will be excluded from the list. + include_flaky: If False, tests containing 'FLAKY_' anywhere in their + names will be excluded from the list. + """ + test_list = [x[0] for x in self._test_status.items() + if self._StatusOfTest(x[0]) == status] + + if not include_fails: + test_list = [x for x in test_list if x.find('FAILS_') == -1] + if not include_flaky: + test_list = [x for x in test_list if x.find('FLAKY_') == -1] + + return test_list + + def _RecordError(self, line, reason): + """Record a log line that produced a parsing error. + + Args: + line: text of the line at which the error occurred + reason: a string describing the error + """ + self._internal_error_lines.append('%s: %s [%s]' % + (self._line_number, line.strip(), reason)) + + def RunningTests(self): + """Returns list of tests that appear to be currently running.""" + return self._TestsByStatus('started', True, True) + + def ParsingErrors(self): + """Returns a list of lines that have caused parsing errors.""" + return self._internal_error_lines + + def ClearParsingErrors(self): + """Clears the currently stored parsing errors.""" + self._internal_error_lines = ['Cleared.'] + + def PassedTests(self, include_fails=False, include_flaky=False): + """Returns list of tests that passed.""" + return self._TestsByStatus('OK', include_fails, include_flaky) + + def FailedTests(self, include_fails=False, include_flaky=False): + """Returns list of tests that failed, timed out, or didn't finish + (crashed). + + This list will be incorrect until the complete log has been processed, + because it will show currently running tests as having failed. + + Args: + include_fails: If true, all failing tests with FAILS_ in their names will + be included. Otherwise, they will only be included if they crashed or + timed out. + include_flaky: If true, all failing tests with FLAKY_ in their names will + be included. Otherwise, they will only be included if they crashed or + timed out. + + """ + return (self._TestsByStatus('failed', include_fails, include_flaky) + + self._TestsByStatus('timeout', True, True) + + self._TestsByStatus('warning', include_fails, include_flaky) + + self.RunningTests()) + + def TriesForTest(self, test): + """Returns a list containing the state for all tries of the given test. + This parser doesn't support retries so a single result is returned.""" + return [self.TEST_STATUS_MAP.get(self._StatusOfTest(test), + TEST_UNKNOWN_LABEL)] + + def DisabledTests(self): + """Returns the name of the disabled test (if there is only 1) or the number + of disabled tests. + """ + return self._disabled_tests + + def FlakyTests(self): + """Returns the name of the flaky test (if there is only 1) or the number + of flaky tests. + """ + return self._flaky_tests + + def FailureDescription(self, test): + """Returns a list containing the failure description for the given test. + + If the test didn't fail or timeout, returns []. + """ + test_status = self._test_status.get(test, ('', [])) + return ['%s: ' % test] + test_status[1] + + def CompletedWithoutFailure(self): + """Returns True if all tests completed and no tests failed unexpectedly.""" + return self.completed and not self.FailedTests() + + def ProcessLine(self, line): + """This is called once with each line of the test log.""" + + # Track line number for error messages. + self._line_number += 1 + + # Some tests (net_unittests in particular) run subprocesses which can write + # stuff to shared stdout buffer. Sometimes such output appears between new + # line and gtest directives ('[ RUN ]', etc) which breaks the parser. + # Code below tries to detect such cases and recognize a mixed line as two + # separate lines. + + # List of regexps that parses expects to find at the start of a line but + # which can be somewhere in the middle. + gtest_regexps = [ + self._test_start, + self._test_ok, + self._test_fail, + self._test_passed, + ] + + for regexp in gtest_regexps: + match = regexp.search(line) + if match: + break + + if not match or match.start() == 0: + self._ProcessLine(line) + else: + self._ProcessLine(line[:match.start()]) + self._ProcessLine(line[match.start():]) + + def _ProcessLine(self, line): + """Parses the line and changes the state of parsed tests accordingly. + + Will recognize newly started tests, OK or FAILED statuses, timeouts, etc. + """ + + # Note: When sharding, the number of disabled and flaky tests will be read + # multiple times, so this will only show the most recent values (but they + # should all be the same anyway). + + # Is it a line listing the master name? + if not self.master_name: + results = self._master_name_re.match(line) + if results: + self.master_name = results.group(1) + + results = self._run_test_cases_line.match(line) + if results: + # A run_test_cases.py output. + if self._current_test: + if self._test_status[self._current_test][0] == 'started': + self._test_status[self._current_test] = ( + 'timeout', self._failure_description) + self._current_test = '' + self._failure_description = [] + return + + # Is it a line declaring all tests passed? + results = self._test_passed.match(line) + if results: + self.completed = True + self._current_test = '' + return + + # Is it a line reporting disabled tests? + results = self._disabled.match(line) + if results: + try: + disabled = int(results.group(1)) + except ValueError: + disabled = 0 + if disabled > 0 and isinstance(self._disabled_tests, int): + self._disabled_tests = disabled + else: + # If we can't parse the line, at least give a heads-up. This is a + # safety net for a case that shouldn't happen but isn't a fatal error. + self._disabled_tests = 'some' + return + + # Is it a line reporting flaky tests? + results = self._flaky.match(line) + if results: + try: + flaky = int(results.group(1)) + except ValueError: + flaky = 0 + if flaky > 0 and isinstance(self._flaky_tests, int): + self._flaky_tests = flaky + else: + # If we can't parse the line, at least give a heads-up. This is a + # safety net for a case that shouldn't happen but isn't a fatal error. + self._flaky_tests = 'some' + return + + # Is it the start of a test? + results = self._test_start.match(line) + if results: + if self._current_test: + if self._test_status[self._current_test][0] == 'started': + self._test_status[self._current_test] = ( + 'timeout', self._failure_description) + test_name = results.group(1) + self._test_status[test_name] = ('started', ['Did not complete.']) + self._current_test = test_name + if self.retrying_failed: + self._failure_description = self._test_status[test_name][1] + self._failure_description.extend(['', 'RETRY OUTPUT:', '']) + else: + self._failure_description = [] + return + + # Is it a test success line? + results = self._test_ok.match(line) + if results: + test_name = results.group(1) + status = self._StatusOfTest(test_name) + if status != 'started': + self._RecordError(line, 'success while in status %s' % status) + if self.retrying_failed: + self._test_status[test_name] = ('warning', self._failure_description) + else: + self._test_status[test_name] = ('OK', []) + self._failure_description = [] + self._current_test = '' + return + + # Is it a test failure line? + results = self._test_fail.match(line) + if results: + test_name = results.group(1) + status = self._StatusOfTest(test_name) + if status not in ('started', 'failed', 'timeout'): + self._RecordError(line, 'failure while in status %s' % status) + # Don't overwrite the failure description when a failing test is listed a + # second time in the summary, or if it was already recorded as timing + # out. + if status not in ('failed', 'timeout'): + self._test_status[test_name] = ('failed', self._failure_description) + self._failure_description = [] + self._current_test = '' + return + + # Is it a test timeout line? + results = self._test_timeout.search(line) + if results: + test_name = results.group(1) + status = self._StatusOfTest(test_name) + if status not in ('started', 'failed'): + self._RecordError(line, 'timeout while in status %s' % status) + self._test_status[test_name] = ( + 'timeout', self._failure_description + ['Killed (timed out).']) + self._failure_description = [] + self._current_test = '' + return + + # Is it the start of the retry tests? + results = self._retry_message.match(line) + if results: + self.retrying_failed = True + return + + # Random line: if we're in a test, collect it for the failure description. + # Tests may run simultaneously, so this might be off, but it's worth a try. + # This also won't work if a test times out before it begins running. + if self._current_test: + self._failure_description.append(line) + + # Parse the "Failing tests:" list at the end of the output, and add any + # additional failed tests to the list. For example, this includes tests + # that crash after the OK line. + if self._parsing_failures: + results = self._test_name.match(line) + if results: + test_name = results.group(1) + status = self._StatusOfTest(test_name) + if status in ('not known', 'OK'): + self._test_status[test_name] = ( + 'failed', ['Unknown error, see stdio log.']) + else: + self._parsing_failures = False + elif line.startswith('Failing tests:'): + self._parsing_failures = True diff --git a/ios/build/bots/scripts/run.py b/ios/build/bots/scripts/run.py new file mode 100755 index 0000000000..01ac2e0a07 --- /dev/null +++ b/ios/build/bots/scripts/run.py @@ -0,0 +1,152 @@ +#!/usr/bin/python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Run a test. + +Sample usage: + ./run.py \ + -a src/xcodebuild/Release-iphoneos/base_unittests.app \ + -o /tmp/out \ + -p iPhone 5s \ + -v 9.3 + + Installs base_unittests.app in an iPhone 5s simulator running iOS 9.3, + runs it, and captures all test data in /tmp/out. +""" + +import argparse +import json +import os +import sys +import traceback + +import test_runner + + +def main(args, test_args): + summary = {} + tr = None + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + try: + if args.iossim and args.platform and args.version: + tr = test_runner.SimulatorTestRunner( + args.app, + args.iossim, + args.platform, + args.version, + args.xcode_version, + args.out_dir, + env_vars=args.env_var, + test_args=test_args, + xctest=args.xctest, + ) + else: + tr = test_runner.DeviceTestRunner( + args.app, + args.xcode_version, + args.out_dir, + env_vars=args.env_var, + test_args=test_args, + xctest=args.xctest, + ) + + return 0 if tr.launch() else 1 + except test_runner.TestRunnerError as e: + sys.stderr.write(traceback.format_exc()) + summary['step_text'] = '%s%s' % ( + e.__class__.__name__, ': %s' % e.args[0] if e.args else '') + + # test_runner.Launch returns 0 on success, 1 on failure, so return 2 + # on exception to distinguish between a test failure, and a failure + # to launch the test at all. + return 2 + finally: + if tr: + summary['logs'] = tr.logs + + with open(os.path.join(args.out_dir, 'summary.json'), 'w') as f: + json.dump(summary, f) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument( + '-a', + '--app', + help='Compiled .app to run.', + metavar='app', + required=True, + ) + parser.add_argument( + '-e', + '--env-var', + action='append', + help='Environment variable to pass to the test itself.', + metavar='ENV=val', + ) + parser.add_argument( + '-i', + '--iossim', + help='Compiled iossim to run the app on.', + metavar='iossim', + ) + parser.add_argument( + '-j', + '--args-json', + default='{}', + help='Specify "env_var": [...] and "test_args": [...] using a JSON dict.', + metavar='{}', + ) + parser.add_argument( + '-o', + '--out-dir', + help='Directory to store all test data in.', + metavar='dir', + required=True, + ) + parser.add_argument( + '-p', + '--platform', + help='Platform to simulate.', + metavar='sim', + ) + parser.add_argument( + '-v', + '--version', + help='Version of iOS the simulator should run.', + metavar='ver', + ) + parser.add_argument( + '-x', + '--xcode-version', + help='Version of Xcode to use.', + metavar='ver', + required=True, + ) + parser.add_argument( + '--xctest', + action='store_true', + help='Whether or not the given app should be run as an XCTest.', + ) + + args, test_args = parser.parse_known_args() + if args.iossim or args.platform or args.version: + # If any of --iossim, --platform, or --version + # are specified then they must all be specified. + if not (args.iossim and args.platform and args.version): + parser.error( + 'must specify all or none of -i/--iossim, -p/--platform, -v/--version') + + args_json = json.loads(args.args_json) + args.env_var = args.env_var or [] + args.env_var.extend(args_json.get('env_var', [])) + args.xctest = args_json.get('xctest', args.xctest) + test_args.extend(args_json.get('test_args', [])) + + sys.exit(main(args, test_args)) diff --git a/ios/build/bots/scripts/test_runner.py b/ios/build/bots/scripts/test_runner.py new file mode 100644 index 0000000000..8027d65a9e --- /dev/null +++ b/ios/build/bots/scripts/test_runner.py @@ -0,0 +1,688 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Test runners for iOS.""" + +import argparse +import collections +import errno +import os +import shutil +import subprocess +import sys +import tempfile +import time + +import find_xcode +import gtest_utils +import xctest_utils + + +XCTEST_PROJECT = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'TestProject', + 'TestProject.xcodeproj', +)) + +XCTEST_SCHEME = 'TestProject' + + +class Error(Exception): + """Base class for errors.""" + pass + + +class TestRunnerError(Error): + """Base class for TestRunner-related errors.""" + pass + + +class AppLaunchError(TestRunnerError): + """The app failed to launch.""" + pass + + +class AppNotFoundError(TestRunnerError): + """The requested app was not found.""" + def __init__(self, app_path): + super(AppNotFoundError, self).__init__( + 'App does not exist: %s' % app_path) + + +class DeviceDetectionError(TestRunnerError): + """Unexpected number of devices detected.""" + def __init__(self, udids): + super(DeviceDetectionError, self).__init__( + 'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids))) + + +class PlugInsNotFoundError(TestRunnerError): + """The PlugIns directory was not found.""" + def __init__(self, plugins_dir): + super(PlugInsNotFoundError, self).__init__( + 'PlugIns directory does not exist: %s' % plugins_dir) + + +class SimulatorNotFoundError(TestRunnerError): + """The given simulator binary was not found.""" + def __init__(self, iossim_path): + super(SimulatorNotFoundError, self).__init__( + 'Simulator does not exist: %s' % iossim_path) + + +class XcodeVersionNotFoundError(TestRunnerError): + """The requested version of Xcode was not found.""" + def __init__(self, xcode_version): + super(XcodeVersionNotFoundError, self).__init__( + 'Xcode version not found: %s', xcode_version) + + +class XCTestPlugInNotFoundError(TestRunnerError): + """The .xctest PlugIn was not found.""" + def __init__(self, xctest_path): + super(XCTestPlugInNotFoundError, self).__init__( + 'XCTest not found: %s', xctest_path) + + +def get_kif_test_filter(tests, invert=False): + """Returns the KIF test filter to filter the given test cases. + + Args: + tests: List of test cases to filter. + invert: Whether to invert the filter or not. Inverted, the filter will match + everything except the given test cases. + + Returns: + A string which can be supplied to GKIF_SCENARIO_FILTER. + """ + # A pipe-separated list of test cases with the "KIF." prefix omitted. + # e.g. NAME:a|b|c matches KIF.a, KIF.b, KIF.c. + # e.g. -NAME:a|b|c matches everything except KIF.a, KIF.b, KIF.c. + test_filter = '|'.join(test.split('KIF.', 1)[-1] for test in tests) + if invert: + return '-NAME:%s' % test_filter + return 'NAME:%s' % test_filter + + +def get_gtest_filter(tests, invert=False): + """Returns the GTest filter to filter the given test cases. + + Args: + tests: List of test cases to filter. + invert: Whether to invert the filter or not. Inverted, the filter will match + everything except the given test cases. + + Returns: + A string which can be supplied to --gtest_filter. + """ + # A colon-separated list of tests cases. + # e.g. a:b:c matches a, b, c. + # e.g. -a:b:c matches everything except a, b, c. + test_filter = ':'.join(test for test in tests) + if invert: + return '-%s' % test_filter + return test_filter + + +class TestRunner(object): + """Base class containing common functionality.""" + + def __init__( + self, + app_path, + xcode_version, + out_dir, + env_vars=None, + test_args=None, + xctest=False, + ): + """Initializes a new instance of this class. + + Args: + app_path: Path to the compiled .app to run. + xcode_version: Version of Xcode to use when running the test. + out_dir: Directory to emit test data into. + env_vars: List of environment variables to pass to the test itself. + test_args: List of strings to pass as arguments to the test when + launching. + xctest: Whether or not this is an XCTest. + + Raises: + AppNotFoundError: If the given app does not exist. + PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests. + XcodeVersionNotFoundError: If the given Xcode version does not exist. + XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist. + """ + app_path = os.path.abspath(app_path) + if not os.path.exists(app_path): + raise AppNotFoundError(app_path) + + if not find_xcode.find_xcode(xcode_version)['found']: + raise XcodeVersionNotFoundError(xcode_version) + + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + self.app_name = os.path.splitext(os.path.split(app_path)[-1])[0] + self.app_path = app_path + self.cfbundleid = subprocess.check_output([ + '/usr/libexec/PlistBuddy', + '-c', 'Print:CFBundleIdentifier', + os.path.join(app_path, 'Info.plist'), + ]).rstrip() + self.env_vars = env_vars or [] + self.logs = collections.OrderedDict() + self.out_dir = out_dir + self.test_args = test_args or [] + self.xcode_version = xcode_version + self.xctest_path = '' + + if xctest: + plugins_dir = os.path.join(self.app_path, 'PlugIns') + if not os.path.exists(plugins_dir): + raise PlugInsNotFoundError(plugins_dir) + for plugin in os.listdir(plugins_dir): + if plugin.endswith('.xctest'): + self.xctest_path = os.path.join(plugins_dir, plugin) + if not os.path.exists(self.xctest_path): + raise XCTestPlugInNotFoundError(self.xctest_path) + + def get_launch_command(self, test_filter=None, invert=False): + """Returns the command that can be used to launch the test app. + + Args: + test_filter: List of test cases to filter. + invert: Whether to invert the filter or not. Inverted, the filter will + match everything except the given test cases. + + Returns: + A list of strings forming the command to launch the test. + """ + raise NotImplementedError + + def get_launch_env(self): + """Returns a dict of environment variables to use to launch the test app. + + Returns: + A dict of environment variables. + """ + return os.environ.copy() + + def set_up(self): + """Performs setup actions which must occur prior to every test launch.""" + raise NotImplementedError + + def tear_down(self): + """Performs cleanup actions which must occur after every test launch.""" + raise NotImplementedError + + def screenshot_desktop(self): + """Saves a screenshot of the desktop in the output directory.""" + subprocess.check_call([ + 'screencapture', + os.path.join(self.out_dir, 'desktop_%s.png' % time.time()), + ]) + + def _run(self, cmd): + """Runs the specified command, parsing GTest output. + + Args: + cmd: List of strings forming the command to run. + + Returns: + GTestResult instance. + """ + print ' '.join(cmd) + print + + result = gtest_utils.GTestResult(cmd) + if self.xctest_path: + parser = xctest_utils.XCTestLogParser() + else: + parser = gtest_utils.GTestLogParser() + + proc = subprocess.Popen( + cmd, + env=self.get_launch_env(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + while True: + line = proc.stdout.readline() + if not line: + break + line = line.rstrip() + parser.ProcessLine(line) + print line + sys.stdout.flush() + + proc.wait() + sys.stdout.flush() + + for test in parser.FailedTests(include_flaky=True): + # Test cases are named as .. If the test case + # is prefixed with "FLAKY_", it should be reported as flaked not failed. + if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): + result.flaked_tests[test] = parser.FailureDescription(test) + else: + result.failed_tests[test] = parser.FailureDescription(test) + + result.passed_tests.extend(parser.PassedTests(include_flaky=True)) + + print '%s returned %s' % (cmd[0], proc.returncode) + print + + # iossim can return 5 if it exits noncleanly even if all tests passed. + # Therefore we cannot rely on process exit code to determine success. + result.finalize(proc.returncode, parser.CompletedWithoutFailure()) + return result + + def launch(self): + """Launches the test app.""" + self.set_up() + cmd = self.get_launch_command() + try: + result = self._run(cmd) + if result.crashed and not result.crashed_test: + # If the app crashed but not during any particular test case, assume + # it crashed on startup. Try one more time. + print 'Crashed on startup, retrying...' + print + result = self._run(cmd) + + if result.crashed and not result.crashed_test: + raise AppLaunchError + + passed = result.passed_tests + failed = result.failed_tests + flaked = result.flaked_tests + + try: + # XCTests cannot currently be resumed at the next test case. + while not self.xctest_path and result.crashed and result.crashed_test: + # If the app crashes during a specific test case, then resume at the + # next test case. This is achieved by filtering out every test case + # which has already run. + print 'Crashed during %s, resuming...' % result.crashed_test + print + result = self._run(self.get_launch_command( + test_filter=passed + failed.keys() + flaked.keys(), invert=True, + )) + passed.extend(result.passed_tests) + failed.update(result.failed_tests) + flaked.update(result.flaked_tests) + except OSError as e: + if e.errno == errno.E2BIG: + print 'Too many test cases to resume.' + print + else: + raise + + self.logs['passed tests'] = passed + for test, log_lines in failed.iteritems(): + self.logs[test] = log_lines + for test, log_lines in flaked.iteritems(): + self.logs[test] = log_lines + + return not failed + finally: + self.tear_down() + + +class SimulatorTestRunner(TestRunner): + """Class for running tests on iossim.""" + + def __init__( + self, + app_path, + iossim_path, + platform, + version, + xcode_version, + out_dir, + env_vars=None, + test_args=None, + xctest=False, + ): + """Initializes a new instance of this class. + + Args: + app_path: Path to the compiled .app or .ipa to run. + iossim_path: Path to the compiled iossim binary to use. + platform: Name of the platform to simulate. Supported values can be found + by running "iossim -l". e.g. "iPhone 5s", "iPad Retina". + version: Version of iOS the platform should be running. Supported values + can be found by running "iossim -l". e.g. "9.3", "8.2", "7.1". + xcode_version: Version of Xcode to use when running the test. + out_dir: Directory to emit test data into. + env_vars: List of environment variables to pass to the test itself. + test_args: List of strings to pass as arguments to the test when + launching. + xctest: Whether or not this is an XCTest. + + Raises: + AppNotFoundError: If the given app does not exist. + PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests. + XcodeVersionNotFoundError: If the given Xcode version does not exist. + XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist. + """ + super(SimulatorTestRunner, self).__init__( + app_path, + xcode_version, + out_dir, + env_vars=env_vars, + test_args=test_args, + xctest=xctest, + ) + + iossim_path = os.path.abspath(iossim_path) + if not os.path.exists(iossim_path): + raise SimulatorNotFoundError(iossim_path) + + self.homedir = '' + self.iossim_path = iossim_path + self.platform = platform + self.start_time = None + self.version = version + + @staticmethod + def kill_simulators(): + """Kills all running simulators.""" + try: + subprocess.check_call([ + 'pkill', + '-9', + '-x', + # The simulator's name varies by Xcode version. + 'iPhone Simulator', # Xcode 5 + 'iOS Simulator', # Xcode 6 + 'Simulator', # Xcode 7+ + 'simctl', # https://crbug.com/637429 + ]) + # If a signal was sent, wait for the simulators to actually be killed. + time.sleep(5) + except subprocess.CalledProcessError as e: + if e.returncode != 1: + # Ignore a 1 exit code (which means there were no simulators to kill). + raise + + def wipe_simulator(self): + """Wipes the simulator.""" + subprocess.check_call([ + self.iossim_path, + '-d', self.platform, + '-s', self.version, + '-w', + ]) + + def get_home_directory(self): + """Returns the simulator's home directory.""" + return subprocess.check_output([ + self.iossim_path, + '-d', self.platform, + '-p', + '-s', self.version, + ]).rstrip() + + def set_up(self): + """Performs setup actions which must occur prior to every test launch.""" + self.kill_simulators() + self.wipe_simulator() + self.homedir = self.get_home_directory() + # Crash reports have a timestamp in their file name, formatted as + # YYYY-MM-DD-HHMMSS. Save the current time in the same format so + # we can compare and fetch crash reports from this run later on. + self.start_time = time.strftime('%Y-%m-%d-%H%M%S', time.localtime()) + + def extract_test_data(self): + """Extracts data emitted by the test.""" + # Find the Documents directory of the test app. The app directory names + # don't correspond with any known information, so we have to examine them + # all until we find one with a matching CFBundleIdentifier. + apps_dir = os.path.join( + self.homedir, 'Containers', 'Data', 'Application') + if os.path.exists(apps_dir): + for appid_dir in os.listdir(apps_dir): + docs_dir = os.path.join(apps_dir, appid_dir, 'Documents') + metadata_plist = os.path.join( + apps_dir, + appid_dir, + '.com.apple.mobile_container_manager.metadata.plist', + ) + if os.path.exists(docs_dir) and os.path.exists(metadata_plist): + cfbundleid = subprocess.check_output([ + '/usr/libexec/PlistBuddy', + '-c', 'Print:MCMMetadataIdentifier', + metadata_plist, + ]).rstrip() + if cfbundleid == self.cfbundleid: + shutil.copytree(docs_dir, os.path.join(self.out_dir, 'Documents')) + return + + def retrieve_crash_reports(self): + """Retrieves crash reports produced by the test.""" + # A crash report's naming scheme is [app]_[timestamp]_[hostname].crash. + # e.g. net_unittests_2014-05-13-15-0900_vm1-a1.crash. + crash_reports_dir = os.path.expanduser(os.path.join( + '~', 'Library', 'Logs', 'DiagnosticReports')) + + if not os.path.exists(crash_reports_dir): + return + + for crash_report in os.listdir(crash_reports_dir): + report_name, ext = os.path.splitext(crash_report) + if report_name.startswith(self.app_name) and ext == '.crash': + report_time = report_name[len(self.app_name) + 1:].split('_')[0] + + # The timestamp format in a crash report is big-endian and therefore + # a staight string comparison works. + if report_time > self.start_time: + with open(os.path.join(crash_reports_dir, crash_report)) as f: + self.logs['crash report (%s)' % report_time] = ( + f.read().splitlines()) + + def tear_down(self): + """Performs cleanup actions which must occur after every test launch.""" + self.extract_test_data() + self.retrieve_crash_reports() + self.screenshot_desktop() + self.kill_simulators() + self.wipe_simulator() + if os.path.exists(self.homedir): + shutil.rmtree(self.homedir, ignore_errors=True) + self.homedir = '' + + def get_launch_command(self, test_filter=None, invert=False): + """Returns the command that can be used to launch the test app. + + Args: + test_filter: List of test cases to filter. + invert: Whether to invert the filter or not. Inverted, the filter will + match everything except the given test cases. + + Returns: + A list of strings forming the command to launch the test. + """ + cmd = [ + self.iossim_path, + '-d', self.platform, + '-s', self.version, + ] + + if test_filter: + kif_filter = get_kif_test_filter(test_filter, invert=invert) + gtest_filter = get_gtest_filter(test_filter, invert=invert) + cmd.extend(['-e', 'GKIF_SCENARIO_FILTER=%s' % kif_filter]) + cmd.extend(['-c', '--gtest_filter=%s' % gtest_filter]) + + for env_var in self.env_vars: + cmd.extend(['-e', env_var]) + + for test_arg in self.test_args: + cmd.extend(['-c', test_arg]) + + cmd.append(self.app_path) + if self.xctest_path: + cmd.append(self.xctest_path) + return cmd + + def get_launch_env(self): + """Returns a dict of environment variables to use to launch the test app. + + Returns: + A dict of environment variables. + """ + env = super(SimulatorTestRunner, self).get_launch_env() + if self.xctest_path: + env['NSUnbufferedIO'] = 'YES' + return env + + +class DeviceTestRunner(TestRunner): + """Class for running tests on devices.""" + + def __init__( + self, + app_path, + xcode_version, + out_dir, + env_vars=None, + test_args=None, + xctest=False, + ): + """Initializes a new instance of this class. + + Args: + app_path: Path to the compiled .app to run. + xcode_version: Version of Xcode to use when running the test. + out_dir: Directory to emit test data into. + env_vars: List of environment variables to pass to the test itself. + test_args: List of strings to pass as arguments to the test when + launching. + xctest: Whether or not this is an XCTest. + + Raises: + AppNotFoundError: If the given app does not exist. + PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests. + XcodeVersionNotFoundError: If the given Xcode version does not exist. + XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist. + """ + super(DeviceTestRunner, self).__init__( + app_path, + xcode_version, + out_dir, + env_vars=env_vars, + test_args=test_args, + xctest=xctest, + ) + + self.udid = subprocess.check_output(['idevice_id', '--list']).rstrip() + if len(self.udid.splitlines()) != 1: + raise DeviceDetectionError(self.udid) + + def uninstall_apps(self): + """Uninstalls all apps found on the device.""" + for app in subprocess.check_output( + ['idevicefs', '--udid', self.udid, 'ls', '@']).splitlines(): + subprocess.check_call( + ['ideviceinstaller', '--udid', self.udid, '--uninstall', app]) + + def install_app(self): + """Installs the app.""" + subprocess.check_call( + ['ideviceinstaller', '--udid', self.udid, '--install', self.app_path]) + + def set_up(self): + """Performs setup actions which must occur prior to every test launch.""" + self.uninstall_apps() + self.install_app() + + def extract_test_data(self): + """Extracts data emitted by the test.""" + subprocess.check_call([ + 'idevicefs', + '--udid', self.udid, + 'pull', + '@%s/Documents' % self.cfbundleid, + os.path.join(self.out_dir, 'Documents'), + ]) + + def retrieve_crash_reports(self): + """Retrieves crash reports produced by the test.""" + logs_dir = os.path.join(self.out_dir, 'Logs') + os.mkdir(logs_dir) + subprocess.check_call([ + 'idevicecrashreport', + '--extract', + '--udid', self.udid, + logs_dir, + ]) + + def tear_down(self): + """Performs cleanup actions which must occur after every test launch.""" + self.extract_test_data() + self.retrieve_crash_reports() + self.screenshot_desktop() + self.uninstall_apps() + + def get_launch_command(self, test_filter=None, invert=False): + """Returns the command that can be used to launch the test app. + + Args: + test_filter: List of test cases to filter. + invert: Whether to invert the filter or not. Inverted, the filter will + match everything except the given test cases. + + Returns: + A list of strings forming the command to launch the test. + """ + if self.xctest_path: + return [ + 'xcodebuild', + 'test-without-building', + 'BUILT_PRODUCTS_DIR=%s' % os.path.dirname(self.app_path), + '-destination', 'id=%s' % self.udid, + '-project', XCTEST_PROJECT, + '-scheme', XCTEST_SCHEME, + ] + + cmd = [ + 'idevice-app-runner', + '--udid', self.udid, + '--start', self.cfbundleid, + ] + args = [] + + if test_filter: + kif_filter = get_kif_test_filter(test_filter, invert=invert) + gtest_filter = get_gtest_filter(test_filter, invert=invert) + cmd.extend(['-D', 'GKIF_SCENARIO_FILTER=%s' % kif_filter]) + args.append('--gtest-filter=%s' % gtest_filter) + + for env_var in self.env_vars: + cmd.extend(['-D', env_var]) + + if args or self.test_args: + cmd.append('--args') + cmd.extend(self.test_args) + cmd.extend(args) + + return cmd + + def get_launch_env(self): + """Returns a dict of environment variables to use to launch the test app. + + Returns: + A dict of environment variables. + """ + env = super(DeviceTestRunner, self).get_launch_env() + if self.xctest_path: + env['NSUnbufferedIO'] = 'YES' + # e.g. ios_web_shell_egtests + env['APP_TARGET_NAME'] = os.path.splitext( + os.path.basename(self.app_path))[0] + # e.g. ios_web_shell_egtests_module + env['TEST_TARGET_NAME'] = env['APP_TARGET_NAME'] + '_module' + return env diff --git a/ios/build/bots/scripts/test_runner_test.py b/ios/build/bots/scripts/test_runner_test.py new file mode 100755 index 0000000000..a9eef71775 --- /dev/null +++ b/ios/build/bots/scripts/test_runner_test.py @@ -0,0 +1,289 @@ +#!/usr/bin/python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unittests for test_runner.py.""" + +import collections +import json +import os +import sys +import unittest + +import test_runner + + +class TestCase(unittest.TestCase): + """Test case which supports installing mocks. Uninstalls on tear down.""" + + def __init__(self, *args, **kwargs): + """Initializes a new instance of this class.""" + super(TestCase, self).__init__(*args, **kwargs) + + # Maps object to a dict which maps names of mocked members to their + # original values. + self._mocks = collections.OrderedDict() + + def mock(self, obj, member, mock): + """Installs mock in place of the named member of the given obj. + + Args: + obj: Any object. + member: String naming the attribute of the object to mock. + mock: The mock to install. + """ + self._mocks.setdefault(obj, collections.OrderedDict()).setdefault( + member, getattr(obj, member)) + setattr(obj, member, mock) + + def tearDown(self, *args, **kwargs): + """Uninstalls mocks.""" + super(TestCase, self).tearDown(*args, **kwargs) + + for obj in self._mocks: + for member, original_value in self._mocks[obj].iteritems(): + setattr(obj, member, original_value) + + +class GetKIFTestFilterTest(TestCase): + """Tests for test_runner.get_kif_test_filter.""" + + def test_correct(self): + """Ensures correctness of filter.""" + tests = [ + 'KIF.test1', + 'KIF.test2', + ] + expected = 'NAME:test1|test2' + + self.assertEqual(test_runner.get_kif_test_filter(tests), expected) + + def test_correct_inverted(self): + """Ensures correctness of inverted filter.""" + tests = [ + 'KIF.test1', + 'KIF.test2', + ] + expected = '-NAME:test1|test2' + + self.assertEqual( + test_runner.get_kif_test_filter(tests, invert=True), expected) + + +class GetGTestFilterTest(TestCase): + """Tests for test_runner.get_gtest_filter.""" + + def test_correct(self): + """Ensures correctness of filter.""" + tests = [ + 'test.1', + 'test.2', + ] + expected = 'test.1:test.2' + + self.assertEqual(test_runner.get_gtest_filter(tests), expected) + + def test_correct_inverted(self): + """Ensures correctness of inverted filter.""" + tests = [ + 'test.1', + 'test.2', + ] + expected = '-test.1:test.2' + + self.assertEqual( + test_runner.get_gtest_filter(tests, invert=True), expected) + + +class SimulatorTestRunnerTest(TestCase): + """Tests for test_runner.SimulatorTestRunner.""" + + def test_app_not_found(self): + """Ensures AppNotFoundError is raised.""" + def exists(path): + if path == 'fake-app': + return False + return True + + def find_xcode(version): + return {'found': True} + + def check_output(command): + return 'fake-bundle-id' + + self.mock(test_runner.os.path, 'exists', exists) + self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) + self.mock(test_runner.subprocess, 'check_output', check_output) + + self.assertRaises( + test_runner.AppNotFoundError, + test_runner.SimulatorTestRunner, + 'fake-app', + 'fake-iossim', + 'platform', + 'os', + 'xcode-version', + 'out-dir', + ) + + def test_iossim_not_found(self): + """Ensures SimulatorNotFoundError is raised.""" + def exists(path): + if path == 'fake-iossim': + return False + return True + + def find_xcode(version): + return {'found': True} + + def check_output(command): + return 'fake-bundle-id' + + self.mock(test_runner.os.path, 'exists', exists) + self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) + self.mock(test_runner.subprocess, 'check_output', check_output) + + self.assertRaises( + test_runner.SimulatorNotFoundError, + test_runner.SimulatorTestRunner, + 'fake-app', + 'fake-iossim', + 'platform', + 'os', + 'xcode-version', + 'out-dir', + ) + + def test_init(self): + """Ensures instance is created.""" + def exists(path): + return True + + def find_xcode(version): + return {'found': True} + + def check_output(command): + return 'fake-bundle-id' + + self.mock(test_runner.os.path, 'exists', exists) + self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) + self.mock(test_runner.subprocess, 'check_output', check_output) + + tr = test_runner.SimulatorTestRunner( + 'fake-app', + 'fake-iossim', + 'platform', + 'os', + 'xcode-version', + 'out-dir', + ) + + self.failUnless(tr) + + def test_startup_crash(self): + """Ensures test is relaunched once on startup crash.""" + def exists(path): + return True + + def find_xcode(version): + return {'found': True} + + def check_output(command): + return 'fake-bundle-id' + + def set_up(self): + return + + @staticmethod + def _run(command): + return collections.namedtuple('result', ['crashed', 'crashed_test'])( + crashed=True, crashed_test=None) + + def tear_down(self): + return + + self.mock(test_runner.os.path, 'exists', exists) + self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) + self.mock(test_runner.subprocess, 'check_output', check_output) + self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) + self.mock(test_runner.TestRunner, '_run', _run) + self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) + + tr = test_runner.SimulatorTestRunner( + 'fake-app', + 'fake-iossim', + 'platform', + 'os', + 'xcode-version', + 'out-dir', + ) + self.assertRaises(test_runner.AppLaunchError, tr.launch) + + def test_relaunch(self): + """Ensures test is relaunched on test crash until tests complete.""" + def exists(path): + return True + + def find_xcode(version): + return {'found': True} + + def check_output(command): + return 'fake-bundle-id' + + def set_up(self): + return + + @staticmethod + def _run(command): + result = collections.namedtuple( + 'result', [ + 'crashed', + 'crashed_test', + 'failed_tests', + 'flaked_tests', + 'passed_tests', + ], + ) + if '-e' not in command: + # First run, has no test filter supplied. Mock a crash. + return result( + crashed=True, + crashed_test='c', + failed_tests={'b': ['b-out'], 'c': ['Did not complete.']}, + flaked_tests={'d': ['d-out']}, + passed_tests=['a'], + ) + else: + return result( + crashed=False, + crashed_test=None, + failed_tests={}, + flaked_tests={}, + passed_tests=[], + ) + + def tear_down(self): + return + + self.mock(test_runner.os.path, 'exists', exists) + self.mock(test_runner.find_xcode, 'find_xcode', find_xcode) + self.mock(test_runner.subprocess, 'check_output', check_output) + self.mock(test_runner.SimulatorTestRunner, 'set_up', set_up) + self.mock(test_runner.TestRunner, '_run', _run) + self.mock(test_runner.SimulatorTestRunner, 'tear_down', tear_down) + + tr = test_runner.SimulatorTestRunner( + 'fake-app', + 'fake-iossim', + 'platform', + 'os', + 'xcode-version', + 'out-dir', + ) + tr.launch() + self.failUnless(tr.logs) + + +if __name__ == '__main__': + unittest.main() diff --git a/ios/build/bots/scripts/xctest_utils.py b/ios/build/bots/scripts/xctest_utils.py new file mode 100644 index 0000000000..1d9d35ffe6 --- /dev/null +++ b/ios/build/bots/scripts/xctest_utils.py @@ -0,0 +1,266 @@ +# Copyright (c) 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import os +import re +import tempfile + + +# These labels should match the ones output by gtest's JSON. +TEST_UNKNOWN_LABEL = 'UNKNOWN' +TEST_SUCCESS_LABEL = 'SUCCESS' +TEST_FAILURE_LABEL = 'FAILURE' +TEST_CRASH_LABEL = 'CRASH' +TEST_TIMEOUT_LABEL = 'TIMEOUT' +TEST_WARNING_LABEL = 'WARNING' + + +class XCTestLogParser(object): + """This helper class process XCTest test output.""" + + def __init__(self): + # State tracking for log parsing + self.completed = False + self._current_test = '' + self._failure_description = [] + self._current_report_hash = '' + self._current_report = [] + self._parsing_failures = False + + # Line number currently being processed. + self._line_number = 0 + + # List of parsing errors, as human-readable strings. + self._internal_error_lines = [] + + # Tests are stored here as 'test.name': (status, [description]). + # The status should be one of ('started', 'OK', 'failed', 'timeout', + # 'warning'). Warning indicates that a test did not pass when run in + # parallel with other tests but passed when run alone. The description is + # a list of lines detailing the test's error, as reported in the log. + self._test_status = {} + + # This may be either text or a number. It will be used in the phrase + # '%s disabled' or '%s flaky' on the waterfall display. + self._disabled_tests = 0 + self._flaky_tests = 0 + + test_name_regexp = r'\-\[(\w+)\s(\w+)\]' + self._test_name = re.compile(test_name_regexp) + self._test_start = re.compile( + r'Test Case \'' + test_name_regexp + '\' started\.') + self._test_ok = re.compile( + r'Test Case \'' + test_name_regexp + + '\' passed\s+\(\d+\.\d+\s+seconds\)?.') + self._test_fail = re.compile( + r'Test Case \'' + test_name_regexp + + '\' failed\s+\(\d+\.\d+\s+seconds\)?.') + self._test_passed = re.compile(r'\*\*\s+TEST\s+EXECUTE\s+SUCCEEDED\s+\*\*') + self._retry_message = re.compile('RETRYING FAILED TESTS:') + self.retrying_failed = False + + self.TEST_STATUS_MAP = { + 'OK': TEST_SUCCESS_LABEL, + 'failed': TEST_FAILURE_LABEL, + 'timeout': TEST_TIMEOUT_LABEL, + 'warning': TEST_WARNING_LABEL + } + + def GetCurrentTest(self): + return self._current_test + + def _StatusOfTest(self, test): + """Returns the status code for the given test, or 'not known'.""" + test_status = self._test_status.get(test, ('not known', [])) + return test_status[0] + + def _TestsByStatus(self, status, include_fails, include_flaky): + """Returns list of tests with the given status. + + Args: + include_fails: If False, tests containing 'FAILS_' anywhere in their + names will be excluded from the list. + include_flaky: If False, tests containing 'FLAKY_' anywhere in their + names will be excluded from the list. + """ + test_list = [x[0] for x in self._test_status.items() + if self._StatusOfTest(x[0]) == status] + + if not include_fails: + test_list = [x for x in test_list if x.find('FAILS_') == -1] + if not include_flaky: + test_list = [x for x in test_list if x.find('FLAKY_') == -1] + + return test_list + + def _RecordError(self, line, reason): + """Record a log line that produced a parsing error. + + Args: + line: text of the line at which the error occurred + reason: a string describing the error + """ + self._internal_error_lines.append('%s: %s [%s]' % + (self._line_number, line.strip(), reason)) + + def RunningTests(self): + """Returns list of tests that appear to be currently running.""" + return self._TestsByStatus('started', True, True) + + def ParsingErrors(self): + """Returns a list of lines that have caused parsing errors.""" + return self._internal_error_lines + + def ClearParsingErrors(self): + """Clears the currently stored parsing errors.""" + self._internal_error_lines = ['Cleared.'] + + def PassedTests(self, include_fails=False, include_flaky=False): + """Returns list of tests that passed.""" + return self._TestsByStatus('OK', include_fails, include_flaky) + + def FailedTests(self, include_fails=False, include_flaky=False): + """Returns list of tests that failed, timed out, or didn't finish + (crashed). + + This list will be incorrect until the complete log has been processed, + because it will show currently running tests as having failed. + + Args: + include_fails: If true, all failing tests with FAILS_ in their names will + be included. Otherwise, they will only be included if they crashed or + timed out. + include_flaky: If true, all failing tests with FLAKY_ in their names will + be included. Otherwise, they will only be included if they crashed or + timed out. + + """ + return (self._TestsByStatus('failed', include_fails, include_flaky) + + self._TestsByStatus('timeout', True, True) + + self._TestsByStatus('warning', include_fails, include_flaky) + + self.RunningTests()) + + def TriesForTest(self, test): + """Returns a list containing the state for all tries of the given test. + This parser doesn't support retries so a single result is returned.""" + return [self.TEST_STATUS_MAP.get(self._StatusOfTest(test), + TEST_UNKNOWN_LABEL)] + + def FailureDescription(self, test): + """Returns a list containing the failure description for the given test. + + If the test didn't fail or timeout, returns []. + """ + test_status = self._test_status.get(test, ('', [])) + return ['%s: ' % test] + test_status[1] + + def CompletedWithoutFailure(self): + """Returns True if all tests completed and no tests failed unexpectedly.""" + return self.completed + + def ProcessLine(self, line): + """This is called once with each line of the test log.""" + + # Track line number for error messages. + self._line_number += 1 + + # Some tests (net_unittests in particular) run subprocesses which can write + # stuff to shared stdout buffer. Sometimes such output appears between new + # line and gtest directives ('[ RUN ]', etc) which breaks the parser. + # Code below tries to detect such cases and recognize a mixed line as two + # separate lines. + + # List of regexps that parses expects to find at the start of a line but + # which can be somewhere in the middle. + gtest_regexps = [ + self._test_start, + self._test_ok, + self._test_fail, + self._test_passed, + ] + + for regexp in gtest_regexps: + match = regexp.search(line) + if match: + break + + if not match or match.start() == 0: + self._ProcessLine(line) + else: + self._ProcessLine(line[:match.start()]) + self._ProcessLine(line[match.start():]) + + def _ProcessLine(self, line): + """Parses the line and changes the state of parsed tests accordingly. + + Will recognize newly started tests, OK or FAILED statuses, timeouts, etc. + """ + + # Is it a line declaring all tests passed? + results = self._test_passed.match(line) + if results: + self.completed = True + self._current_test = '' + return + + # Is it the start of a test? + results = self._test_start.match(line) + if results: + if self._current_test: + if self._test_status[self._current_test][0] == 'started': + self._test_status[self._current_test] = ( + 'timeout', self._failure_description) + test_name = '%s.%s' % (results.group(1), results.group(2)) + self._test_status[test_name] = ('started', ['Did not complete.']) + self._current_test = test_name + if self.retrying_failed: + self._failure_description = self._test_status[test_name][1] + self._failure_description.extend(['', 'RETRY OUTPUT:', '']) + else: + self._failure_description = [] + return + + # Is it a test success line? + results = self._test_ok.match(line) + if results: + test_name = '%s.%s' % (results.group(1), results.group(2)) + status = self._StatusOfTest(test_name) + if status != 'started': + self._RecordError(line, 'success while in status %s' % status) + if self.retrying_failed: + self._test_status[test_name] = ('warning', self._failure_description) + else: + self._test_status[test_name] = ('OK', []) + self._failure_description = [] + self._current_test = '' + return + + # Is it a test failure line? + results = self._test_fail.match(line) + if results: + test_name = '%s.%s' % (results.group(1), results.group(2)) + status = self._StatusOfTest(test_name) + if status not in ('started', 'failed', 'timeout'): + self._RecordError(line, 'failure while in status %s' % status) + # Don't overwrite the failure description when a failing test is listed a + # second time in the summary, or if it was already recorded as timing + # out. + if status not in ('failed', 'timeout'): + self._test_status[test_name] = ('failed', self._failure_description) + self._failure_description = [] + self._current_test = '' + return + + # Is it the start of the retry tests? + results = self._retry_message.match(line) + if results: + self.retrying_failed = True + return + + # Random line: if we're in a test, collect it for the failure description. + # Tests may run simultaneously, so this might be off, but it's worth a try. + # This also won't work if a test times out before it begins running. + if self._current_test: + self._failure_description.append(line)