#!/usr/bin/env python import argparse import os import subprocess import sys SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) VENDOR_DIR = os.path.join(SOURCE_ROOT, 'vendor') PYYAML_LIB_DIR = os.path.join(VENDOR_DIR, 'pyyaml', 'lib') sys.path.append(PYYAML_LIB_DIR) import yaml #pylint: disable=wrong-import-position,wrong-import-order class Command: LIST = 'list' RUN = 'run' class Verbosity: ALL = 'all' # stdout and stderr ERRORS = 'errors' # stderr only SILENT = 'silent' # no output def parse_args(): parser = argparse.ArgumentParser(description='Run Google Test binaries') parser.add_argument('command', choices=[Command.LIST, Command.RUN], help='command to execute') parser.add_argument('-b', '--binary', nargs='+', required=False, help='binaries to run') parser.add_argument('-c', '--config', required=True, help='path to a tests config') parser.add_argument('-t', '--tests-dir', required=False, help='path to a directory with test binaries') parser.add_argument('-o', '--output-dir', required=False, help='path to a folder to save tests results') verbosity = parser.add_mutually_exclusive_group() verbosity.add_argument('-v', '--verbosity', required=False, default=Verbosity.ALL, choices=[ Verbosity.ALL, Verbosity.ERRORS, Verbosity.SILENT], help='set verbosity level') verbosity.add_argument('-q', '--quiet', required=False, action='store_const', const=Verbosity.ERRORS, dest='verbosity', help='suppress stdout from test binaries') verbosity.add_argument('-qq', '--quiet-quiet', # https://youtu.be/o0u4M6vppCI?t=1m18s required=False, action='store_const', const=Verbosity.SILENT, dest='verbosity', help='suppress stdout and stderr from test binaries') args = parser.parse_args() # Additional checks. if args.command == Command.RUN and args.tests_dir is None: parser.error("specify a path to a dir with test binaries via --tests-dir") # Absolutize and check paths. # 'config' must exist and be a file. args.config = os.path.abspath(args.config) if not os.path.isfile(args.config): parser.error("file '{}' doesn't exist".format(args.config)) # 'tests_dir' must exist and be a directory. if args.tests_dir is not None: args.tests_dir = os.path.abspath(args.tests_dir) if not os.path.isdir(args.tests_dir): parser.error("directory '{}' doesn't exist".format(args.tests_dir)) # 'output_dir' must exist and be a directory. if args.output_dir is not None: args.output_dir = os.path.abspath(args.output_dir) if not os.path.isdir(args.output_dir): parser.error("directory '{}' doesn't exist".format(args.output_dir)) return args def main(): args = parse_args() tests_list = TestsList(args.config, args.tests_dir) if args.command == Command.LIST: all_binaries_names = tests_list.get_names() print '\n'.join(all_binaries_names) return 0 if args.command == Command.RUN: if args.binary is not None: return tests_list.run(args.binary, args.output_dir, args.verbosity) else: return tests_list.run_all(args.output_dir, args.verbosity) raise Exception("unexpected command '{}'".format(args.command)) class TestsList(): def __init__(self, config_path, tests_dir): self.config_path = config_path self.tests_dir = tests_dir # A dict with binary names (e.g. 'base_unittests') as keys # and various test data as values of dict type. self.tests = TestsList.__get_tests_list(config_path) def __len__(self): return len(self.tests) def get_names(self): return self.tests.keys() def run(self, binaries, output_dir=None, verbosity=Verbosity.ALL): # Don't run anything twice. binaries = set(binaries) # First check that all names are present in the config. if any([binary_name not in self.tests for binary_name in binaries]): raise Exception("binary '{0}' not found in config '{1}'".format( binary_name, self.config_path)) # TODO(alexeykuzmin): Respect the "platform" setting. suite_returncode = sum( [self.__run(binary, output_dir, verbosity) for binary in binaries]) return suite_returncode def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.ALL): return self.run([binary_name], output_dir, verbosity) def run_all(self, output_dir=None, verbosity=Verbosity.ALL): return self.run(self.get_names(), output_dir, verbosity) @staticmethod def __get_tests_list(config_path): tests_list = {} config_data = TestsList.__get_config_data(config_path) for data_item in config_data['tests']: (binary_name, test_data) = TestsList.__get_test_data(data_item) tests_list[binary_name] = test_data return tests_list @staticmethod def __get_config_data(config_path): with open(config_path, 'r') as stream: return yaml.load(stream) @staticmethod def __expand_shorthand(value): """ Treat a string as {'string_value': None}.""" if isinstance(value, dict): return value if isinstance(value, basestring): return {value: None} assert False, "unexpected shorthand type: {}".format(type(value)) @staticmethod def __make_a_list(value): """Make a list if not already a list.""" if isinstance(value, list): return value return [value] @staticmethod def __get_test_data(data_item): data_item = TestsList.__expand_shorthand(data_item) binary_name = data_item.keys()[0] test_data = { 'excluded_tests': None, 'platforms': None # None means all? Weird. } configs = data_item[binary_name] if configs is not None: # List of excluded tests. if 'to_fix' in configs: test_data['excluded_tests'] = configs['to_fix'] # TODO(alexeykuzmin): Also add to 'excluded_tests' # those that should be permanently disabled. # List of platforms to run the tests on. if 'platform' in configs: test_data['platforms'] = TestsList.__make_a_list(configs['platform']) return (binary_name, test_data) def __run(self, binary_name, output_dir, verbosity): binary_path = os.path.join(self.tests_dir, binary_name) test_binary = TestBinary(binary_path) test_data = self.tests[binary_name] excluded_tests = test_data['excluded_tests'] output_file_path = TestsList.__get_output_path(binary_name, output_dir) return test_binary.run(excluded_tests=excluded_tests, output_file_path=output_file_path, verbosity=verbosity) @staticmethod def __get_output_path(binary_name, output_dir=None): if output_dir is None: return None return os.path.join(output_dir, "results_{}.xml".format(binary_name)) class TestBinary(): def __init__(self, binary_path): self.binary_path = binary_path # Is only used when writing to a file. self.output_format = 'xml' def run(self, excluded_tests=None, output_file_path=None, verbosity=Verbosity.ALL): gtest_filter = "" if excluded_tests is not None and len(excluded_tests) > 0: excluded_tests_string = TestBinary.__format_excluded_tests( excluded_tests) gtest_filter = "--gtest_filter={}".format(excluded_tests_string) gtest_output = "" if output_file_path is not None: gtest_output = "--gtest_output={0}:{1}".format(self.output_format, output_file_path) args = [self.binary_path, gtest_filter, gtest_output] stdout, stderr = TestBinary.__get_stdout_and_stderr(verbosity) returncode = subprocess.call(args, stdout=stdout, stderr=stderr) return returncode @staticmethod def __format_excluded_tests(excluded_tests): return "-" + ":".join(excluded_tests) @staticmethod def __get_stdout_and_stderr(verbosity): stdout = stderr = None if verbosity in (Verbosity.ERRORS, Verbosity.SILENT): devnull = open(os.devnull, 'w') stdout = devnull if verbosity == Verbosity.SILENT: stderr = devnull return (stdout, stderr) if __name__ == '__main__': sys.exit(main())