3879e9e065
The `vendor` directory is a remnant of times long since gone when we had brightray and libchromiumcontent (never forget). It is currently the hope to a single header file that afaics we do not need anymore and randomly pyyaml gets cloned there but that is easily movable. This commit removes all references to the vendor directory from scripts, docs, helpers, etc. and removes the directory itself.
305 lines
9.1 KiB
Python
305 lines
9.1 KiB
Python
from __future__ import print_function
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from util import SRC_DIR
|
|
|
|
PYYAML_LIB_DIR = os.path.join(SRC_DIR, 'third_party', 'pyyaml', 'lib')
|
|
sys.path.append(PYYAML_LIB_DIR)
|
|
import yaml #pylint: disable=wrong-import-position,wrong-import-order
|
|
|
|
try:
|
|
basestring # Python 2
|
|
except NameError: # Python 3
|
|
basestring = str # pylint: disable=redefined-builtin
|
|
|
|
|
|
class Verbosity:
|
|
CHATTY = 'chatty' # stdout and stderr
|
|
ERRORS = 'errors' # stderr only
|
|
SILENT = 'silent' # no output
|
|
|
|
@staticmethod
|
|
def get_all():
|
|
return Verbosity.__get_all_in_order()
|
|
|
|
@staticmethod
|
|
def __get_all_in_order():
|
|
return [Verbosity.SILENT, Verbosity.ERRORS, Verbosity.CHATTY]
|
|
|
|
@staticmethod
|
|
def __get_indices(*values):
|
|
ordered = Verbosity.__get_all_in_order()
|
|
indices = map(ordered.index, values)
|
|
return indices
|
|
|
|
@staticmethod
|
|
def ge(a, b):
|
|
"""Greater or equal"""
|
|
a_index, b_index = Verbosity.__get_indices(a, b)
|
|
return a_index >= b_index
|
|
|
|
@staticmethod
|
|
def le(a, b):
|
|
"""Less or equal"""
|
|
a_index, b_index = Verbosity.__get_indices(a, b)
|
|
return a_index <= b_index
|
|
|
|
|
|
class DisabledTestsPolicy:
|
|
DISABLE = 'disable' # Disabled tests are disabled. Wow. Much sense.
|
|
ONLY = 'only' # Only disabled tests should be run.
|
|
INCLUDE = 'include' # Do not disable any tests.
|
|
|
|
|
|
class Platform:
|
|
LINUX = 'linux'
|
|
MAC = 'mac'
|
|
WINDOWS = 'windows'
|
|
|
|
@staticmethod
|
|
def get_current():
|
|
platform = sys.platform
|
|
if platform in ('linux', 'linux2'):
|
|
return Platform.LINUX
|
|
if platform == 'darwin':
|
|
return Platform.MAC
|
|
if platform in ('cygwin', 'win32'):
|
|
return Platform.WINDOWS
|
|
|
|
assert False, "unexpected current platform '{}'".format(platform)
|
|
|
|
@staticmethod
|
|
def get_all():
|
|
return [Platform.LINUX, Platform.MAC, Platform.WINDOWS]
|
|
|
|
@staticmethod
|
|
def is_valid(platform):
|
|
return platform in Platform.get_all()
|
|
|
|
|
|
class TestsList():
|
|
def __init__(self, config_path, tests_dir):
|
|
self.config_path = config_path
|
|
self.tests_dir = tests_dir
|
|
|
|
# A dict with binary names (e.g. 'base_unittests') as keys
|
|
# and various test data as values of dict type.
|
|
self.tests = TestsList.__get_tests_list(config_path)
|
|
|
|
def __len__(self):
|
|
return len(self.tests)
|
|
|
|
def get_for_current_platform(self):
|
|
all_binaries = self.tests.keys()
|
|
supported_binaries = filter(self.__platform_supports, all_binaries)
|
|
return supported_binaries
|
|
|
|
def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
|
|
disabled_tests_policy=DisabledTestsPolicy.DISABLE):
|
|
# Don't run anything twice.
|
|
binaries = set(binaries)
|
|
|
|
# First check that all names are present in the config.
|
|
for binary_name in binaries:
|
|
if binary_name not in self.tests:
|
|
raise Exception("binary {0} not found in config '{1}'".format(
|
|
binary_name, self.config_path))
|
|
|
|
# Respect the "platform" setting.
|
|
for binary_name in binaries:
|
|
if not self.__platform_supports(binary_name):
|
|
raise Exception(
|
|
"binary {0} cannot be run on {1}, check the config".format(
|
|
binary_name, Platform.get_current()))
|
|
|
|
suite_returncode = sum(
|
|
[self.__run(binary, output_dir, verbosity, disabled_tests_policy)
|
|
for binary in binaries])
|
|
return suite_returncode
|
|
|
|
def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY,
|
|
disabled_tests_policy=DisabledTestsPolicy.DISABLE):
|
|
return self.run([binary_name], output_dir, verbosity,
|
|
disabled_tests_policy)
|
|
|
|
def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY,
|
|
disabled_tests_policy=DisabledTestsPolicy.DISABLE):
|
|
return self.run(self.get_for_current_platform(), output_dir, verbosity,
|
|
disabled_tests_policy)
|
|
|
|
@staticmethod
|
|
def __get_tests_list(config_path):
|
|
tests_list = {}
|
|
config_data = TestsList.__get_config_data(config_path)
|
|
|
|
for data_item in config_data['tests']:
|
|
(binary_name, test_data) = TestsList.__get_test_data(data_item)
|
|
tests_list[binary_name] = test_data
|
|
|
|
return tests_list
|
|
|
|
@staticmethod
|
|
def __get_config_data(config_path):
|
|
with open(config_path, 'r') as stream:
|
|
return yaml.load(stream)
|
|
|
|
@staticmethod
|
|
def __expand_shorthand(value):
|
|
""" Treat a string as {'string_value': None}."""
|
|
if isinstance(value, dict):
|
|
return value
|
|
|
|
if isinstance(value, basestring):
|
|
return {value: None}
|
|
|
|
assert False, "unexpected shorthand type: {}".format(type(value))
|
|
|
|
@staticmethod
|
|
def __make_a_list(value):
|
|
"""Make a list if not already a list."""
|
|
if isinstance(value, list):
|
|
return value
|
|
return [value]
|
|
|
|
@staticmethod
|
|
def __merge_nested_lists(value):
|
|
"""Converts a dict of lists to a list."""
|
|
if isinstance(value, list):
|
|
return value
|
|
|
|
if isinstance(value, dict):
|
|
# It looks ugly as hell, but it does the job.
|
|
return [list_item for key in value for list_item in value[key]]
|
|
|
|
assert False, "unexpected type for list merging: {}".format(type(value))
|
|
|
|
def __platform_supports(self, binary_name):
|
|
return Platform.get_current() in self.tests[binary_name]['platforms']
|
|
|
|
@staticmethod
|
|
def __get_test_data(data_item):
|
|
data_item = TestsList.__expand_shorthand(data_item)
|
|
|
|
binary_name = data_item.keys()[0]
|
|
test_data = {
|
|
'excluded_tests': [],
|
|
'platforms': Platform.get_all()
|
|
}
|
|
|
|
configs = data_item[binary_name]
|
|
if configs is not None:
|
|
# List of excluded tests.
|
|
if 'disabled' in configs:
|
|
excluded_tests = TestsList.__merge_nested_lists(configs['disabled'])
|
|
test_data['excluded_tests'] = excluded_tests
|
|
|
|
# List of platforms to run the tests on.
|
|
if 'platform' in configs:
|
|
platforms = TestsList.__make_a_list(configs['platform'])
|
|
|
|
for platform in platforms:
|
|
assert Platform.is_valid(platform), \
|
|
"platform '{0}' is not supported, check {1} config" \
|
|
.format(platform, binary_name)
|
|
|
|
test_data['platforms'] = platforms
|
|
|
|
return (binary_name, test_data)
|
|
|
|
def __run(self, binary_name, output_dir, verbosity,
|
|
disabled_tests_policy):
|
|
binary_path = os.path.join(self.tests_dir, binary_name)
|
|
test_binary = TestBinary(binary_path)
|
|
|
|
test_data = self.tests[binary_name]
|
|
included_tests = []
|
|
excluded_tests = test_data['excluded_tests']
|
|
|
|
if disabled_tests_policy == DisabledTestsPolicy.ONLY:
|
|
if len(excluded_tests) == 0:
|
|
# There is nothing to run.
|
|
return 0
|
|
included_tests, excluded_tests = excluded_tests, included_tests
|
|
|
|
if disabled_tests_policy == DisabledTestsPolicy.INCLUDE:
|
|
excluded_tests = []
|
|
|
|
output_file_path = TestsList.__get_output_path(binary_name, output_dir)
|
|
|
|
return test_binary.run(included_tests=included_tests,
|
|
excluded_tests=excluded_tests,
|
|
output_file_path=output_file_path,
|
|
verbosity=verbosity)
|
|
|
|
@staticmethod
|
|
def __get_output_path(binary_name, output_dir=None):
|
|
if output_dir is None:
|
|
return None
|
|
|
|
return os.path.join(output_dir, "results_{}.xml".format(binary_name))
|
|
|
|
|
|
class TestBinary():
|
|
# Is only used when writing to a file.
|
|
output_format = 'xml'
|
|
|
|
def __init__(self, binary_path):
|
|
self.binary_path = binary_path
|
|
|
|
def run(self, included_tests=None, excluded_tests=None,
|
|
output_file_path=None, verbosity=Verbosity.CHATTY):
|
|
gtest_filter = TestBinary.__get_gtest_filter(included_tests,
|
|
excluded_tests)
|
|
gtest_output = TestBinary.__get_gtest_output(output_file_path)
|
|
|
|
args = [self.binary_path, gtest_filter, gtest_output]
|
|
stdout, stderr = TestBinary.__get_stdout_and_stderr(verbosity)
|
|
|
|
returncode = 0
|
|
try:
|
|
returncode = subprocess.call(args, stdout=stdout, stderr=stderr)
|
|
except Exception as exception:
|
|
if Verbosity.ge(verbosity, Verbosity.ERRORS):
|
|
print("An error occurred while running '{}':".format(self.binary_path),
|
|
'\n', exception, file=sys.stderr)
|
|
returncode = 1
|
|
|
|
return returncode
|
|
|
|
@staticmethod
|
|
def __get_gtest_filter(included_tests, excluded_tests):
|
|
included_tests_string = TestBinary.__list_tests(included_tests)
|
|
excluded_tests_string = TestBinary.__list_tests(excluded_tests)
|
|
|
|
gtest_filter = "--gtest_filter={}-{}".format(included_tests_string,
|
|
excluded_tests_string)
|
|
return gtest_filter
|
|
|
|
@staticmethod
|
|
def __get_gtest_output(output_file_path):
|
|
gtest_output = ""
|
|
if output_file_path is not None:
|
|
gtest_output = "--gtest_output={0}:{1}".format(TestBinary.output_format,
|
|
output_file_path)
|
|
return gtest_output
|
|
|
|
@staticmethod
|
|
def __list_tests(tests):
|
|
if tests is None:
|
|
return ''
|
|
return ':'.join(tests)
|
|
|
|
@staticmethod
|
|
def __get_stdout_and_stderr(verbosity):
|
|
stdout = stderr = None
|
|
|
|
if Verbosity.le(verbosity, Verbosity.ERRORS):
|
|
devnull = open(os.devnull, 'w')
|
|
stdout = devnull
|
|
if verbosity == Verbosity.SILENT:
|
|
stderr = devnull
|
|
|
|
return (stdout, stderr)
|