From ec4a0e096fee665991faad517214414177381538 Mon Sep 17 00:00:00 2001 From: Alexey Kuzmin Date: Thu, 25 Oct 2018 20:28:50 +0200 Subject: [PATCH] ci: run more Chromium unit tests (#15363) * test: add an option to native tests runner to run also disabled tests * ci: run all native tests on clean Chromium * ci: update the disabled tests list for native unittests --- .circleci/config.yml | 3 +- script/lib/native_tests.py | 32 ++++++----- script/native-tests.py | 22 +++++--- spec/configs/unittests.yml | 108 +++++++++++++++++++------------------ 4 files changed, 95 insertions(+), 70 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4e92466ee61..3312372dee81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -974,7 +974,7 @@ jobs: <<: *env-unittests <<: *env-enable-sccache <<: *env-headless-testing - TESTS_ARGS: '--run-only-disabled-tests' + TESTS_ARGS: '--only-disabled-tests' <<: *steps-native-tests linux-x64-chromium-unittests: @@ -983,6 +983,7 @@ jobs: <<: *env-unittests <<: *env-enable-sccache <<: *env-headless-testing + TESTS_ARGS: '--include-disabled-tests' <<: *steps-native-tests linux-x64-browsertests: diff --git a/script/lib/native_tests.py b/script/lib/native_tests.py index a313f021078b..d6fecd2d3d20 100644 --- a/script/lib/native_tests.py +++ b/script/lib/native_tests.py @@ -44,6 +44,12 @@ class Verbosity: return a_index <= b_index +class DisabledTestsPolicy: + DISABLE = 'disable' # Disabled tests are disabled. Wow. Much sense. + ONLY = 'only' # Only disabled tests should be run. + INCLUDE = 'include' # Do not disable any tests. + + class Platform: LINUX = 'linux' MAC = 'mac' @@ -88,7 +94,7 @@ class TestsList(): return supported_binaries def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY, - run_only_disabled_tests=False): + disabled_tests_policy=DisabledTestsPolicy.DISABLE): # Don't run anything twice. binaries = set(binaries) @@ -106,19 +112,19 @@ class TestsList(): binary_name, Platform.get_current())) suite_returncode = sum( - [self.__run(binary, output_dir, verbosity, run_only_disabled_tests) + [self.__run(binary, output_dir, verbosity, disabled_tests_policy) for binary in binaries]) return suite_returncode def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY, - run_only_disabled_tests=False): + disabled_tests_policy=DisabledTestsPolicy.DISABLE): return self.run([binary_name], output_dir, verbosity, - run_only_disabled_tests) + disabled_tests_policy) def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY, - run_only_disabled_tests=False): + disabled_tests_policy=DisabledTestsPolicy.DISABLE): return self.run(self.get_for_current_platform(), output_dir, verbosity, - run_only_disabled_tests) + disabled_tests_policy) @staticmethod def __get_tests_list(config_path): @@ -200,7 +206,7 @@ class TestsList(): return (binary_name, test_data) def __run(self, binary_name, output_dir, verbosity, - run_only_disabled_tests): + disabled_tests_policy): binary_path = os.path.join(self.tests_dir, binary_name) test_binary = TestBinary(binary_path) @@ -208,13 +214,15 @@ class TestsList(): included_tests = [] excluded_tests = test_data['excluded_tests'] - if run_only_disabled_tests and len(excluded_tests) == 0: - # There is nothing to run. - return 0 - - if run_only_disabled_tests: + if disabled_tests_policy == DisabledTestsPolicy.ONLY: + if len(excluded_tests) == 0: + # There is nothing to run. + return 0 included_tests, excluded_tests = excluded_tests, included_tests + if disabled_tests_policy == DisabledTestsPolicy.INCLUDE: + excluded_tests = [] + output_file_path = TestsList.__get_output_path(binary_name, output_dir) return test_binary.run(included_tests=included_tests, diff --git a/script/native-tests.py b/script/native-tests.py index f38a2394c4a6..4657b9233867 100755 --- a/script/native-tests.py +++ b/script/native-tests.py @@ -6,7 +6,7 @@ import argparse import os import sys -from lib.native_tests import TestsList, Verbosity +from lib.native_tests import TestsList, Verbosity, DisabledTestsPolicy class Command: @@ -25,14 +25,24 @@ def parse_args(): help='binaries to run') parser.add_argument('-c', '--config', required=True, help='path to a tests config') - parser.add_argument('--run-only-disabled-tests', - action='store_true', default=False, - help='if disabled tests should be run') parser.add_argument('-t', '--tests-dir', required=False, help='path to a directory with test binaries') parser.add_argument('-o', '--output-dir', required=False, help='path to a folder to save tests results') + disabled_tests = parser.add_mutually_exclusive_group() + disabled_tests.add_argument('--only-disabled-tests', + dest='disabled_tests_policy', + action='store_const', + const=DisabledTestsPolicy.ONLY, + help='run disabled tests only') + disabled_tests.add_argument('--include-disabled-tests', + dest='disabled_tests_policy', + action='store_const', + const=DisabledTestsPolicy.INCLUDE, + help='if disabled tests should be run as well') + parser.set_defaults(disabled_tests_policy=DisabledTestsPolicy.DISABLE) + verbosity = parser.add_mutually_exclusive_group() verbosity.add_argument('-v', '--verbosity', required=False, default=Verbosity.CHATTY, @@ -86,10 +96,10 @@ def main(): if args.command == Command.RUN: if args.binary is not None: return tests_list.run(args.binary, args.output_dir, args.verbosity, - args.run_only_disabled_tests) + args.disabled_tests_policy) else: return tests_list.run_all(args.output_dir, args.verbosity, - args.run_only_disabled_tests) + args.disabled_tests_policy) assert False, "unexpected command '{}'".format(args.command) diff --git a/spec/configs/unittests.yml b/spec/configs/unittests.yml index cd6c085b9d16..d4cbbb8de3b2 100644 --- a/spec/configs/unittests.yml +++ b/spec/configs/unittests.yml @@ -1,23 +1,34 @@ +# Note: a "disabled" dict can have arbitrary named properties +# with lists of disabled tests as a value. Those properties' names +# are for humans only and are there only to explain +# why a group of tests is disabled. + tests: - base_unittests: - # "disabled" dict can have arbitrary named properties - # with lists of disabled tests. Those properties' names - # are used only to explain why a group of tests is disabled. disabled: - undecided: - - FilePathTest* + upstream_failures: + - FilePathTest.FromUTF8Unsafe_And_AsUTF8Unsafe + - RTLTest.WrapPathWithLTRFormatting + - SysStrings.SysNativeMBAndWide + - SysStrings.SysNativeMBToWide + - SysStrings.SysWideToNativeMB + to_fix: - PartitionReallocReturnNullTest.RepeatedReturnNullDirect - - RTLTest* - - SysStrings* - - UTFOffsetStringConversionsTest* - WeakPtrDeathTest* - cc_unittests - cc_blink_unittests - content_unittests: disabled: - # TODO(alexeykuzmin): Should those be fixed? - - _/DOMStorageMapParamTest.EnforcesQuota/0 - - _/DOMStorageMapParamTest.EnforcesQuota/1 + upstream_failures: + - BackgroundSyncManagerTest.* + - FrameServiceBaseTest.ConnectionError + - MediaSessionControllerTest.* + - MediaStreamUIProxyFeaturePolicyTest.FeaturePolicy + - PresentationServiceImplTest.* + to_fix: + - _/DOMStorageMapParamTest.EnforcesQuota/0 + - _/DOMStorageMapParamTest.EnforcesQuota/1 + - RenderFrameAudioInputStreamFactoryTest.ConstructDestruct - crypto_unittests - device_unittests - gin_unittests @@ -31,39 +42,19 @@ tests: - mojo_common_unittests - net_unittests: disabled: - # TODO(alexeykuzmin): Should those be fixed? - - EndToEndTestsWithTls* - - FilenameUtilTest* - - HTTPSAIATest.AIAFetching - - HTTPSCRLSetTest* - - HTTPSEVCRLSetTest* - - HTTPSFallbackTest* - - HTTPSHardFailTest.FailsOnOCSPInvalid - - HTTPSOCSPTest* - - HTTPSRequestTest* - - HTTPSSessionTest* - - OCSPVerify* - - ProxyConfigServiceLinuxTest* - - SSLClientSocketCertRequestInfoTest* - - SSLClientSocketChannelIDTest* - - SSLClientSocketFalseStartTest* - - SSLClientSocketReadTest* - - SSLClientSocketTest* - - TokenBindingURLRequestTest* - - UDPSocketTest* - - URLRequestTestFTP* - - URLRequestTestHTTP* - - WebSocketEndToEndTest* + upstream_failures: + - EndToEndTestsWithTls/EndToEndTestWithTls.SimpleRequestResponsev6/* + - FilenameUtilTest.GenerateFileName + - SSLClientSocketTest.* + - UDPSocketTest.ClientGetLocalPeerAddresses + - UDPSocketTest.ClientSetDoNotFragment - ppapi_unittests - printing_unittests - skia_unittests - sql_unittests - storage_unittests - gn_unittests - - ui_base_unittests: - disabled: - # TODO(alexeykuzmin): Should it be fixed? - - AcceleratorTest.GetShortcutText + - ui_base_unittests - compositor_unittests - display_unittests - events_unittests @@ -73,31 +64,43 @@ tests: # V8 - unittests: disabled: - # TODO(alexeykuzmin): Should it be fixed? - - LiveRangeUnitTest* + upstream_failures: + - LiveRangeUnitTest.InvalidConstruction + - LiveRangeUnitTest.SplinterMultipleIntervalsInside + - LiveRangeUnitTest.SplitInvalidStart # Blink - blink_common_unittests - blink_heap_unittests - blink_platform_unittests: disabled: # TODO(alexeykuzmin): Should those be fixed? - - AnimatedPNGTests* - - AnimatedWebPTests* + - AnimatedPNGTests.* + - AnimatedWebPTests.* - AnimatedWEBPTests.clearCacheExceptFrameWithAncestors - - BitmapImageTest* + - BitmapImageTest.* - BMPImageDecoderTest* - DecodedImageOrientationHistogramTest* - DecodedImageTypeHistogramTest* + - DeferredImageDecoderTest.decodeOnOtherThread - DeferredImageDecoderTestWoPlatform* - GIFImageDecoderTest* - HighContrastImageClassifierTest* - ICOImageDecoderTests* - JPEGImageDecoderTest* - FilePathConversionTest* + - PaintPropertyNodeTest.InitialStateAndReset - PNGTests* - StaticPNGTests* - StaticWebPTests* - - webkit_unit_tests + - webkit_unit_tests: + disabled: + upstream_failures: + - Document/ThreadableLoaderTest.CancelAfterStart/0 + - ModernMediaControlsImplTest.ControlsShouldUseSafeAreaInsets + - ScriptRunnerTest.QueueReentrantScript_InOrder + - ScrollbarsTestWithVirtualTimer.PressScrollbarButtonOnInfiniteScrolling + - VirtualTimeTest.AllowVirtualTimeToAdvance + - VirtualTimeTest.VirtualTimeNotAllowedToAdvanceWhileResourcesLoading - wtf_unittests # Third party - angle_unittests @@ -108,13 +111,16 @@ tests: - sandbox_linux_unittests: platform: linux disabled: - # TODO(alexeykuzmin): Should these be fixed? - - BrokerProcess* - - SandboxBPF.SigBus + upstream_failures: + - BrokerProcess.OpenFilePermsWithClientCheck + - BrokerProcess.OpenFilePermsWithClientCheckNoEnt + - BrokerProcess.OpenOpenFilePermsNoClientCheck + - BrokerProcess.OpenOpenFilePermsNoClientCheckNoEnt + - SandboxBPF.SigBus - dbus_unittests: platform: linux disabled: - # TODO(alexeykuzmin): Should these be fixed? - - EndToEndAsyncTest.InvalidServiceName - - EndToEndSyncTest.InvalidServiceName - - MessageTest.SetInvalidHeaders + upstream_failures: + - EndToEndAsyncTest.InvalidServiceName + - EndToEndSyncTest.InvalidServiceName + - MessageTest.SetInvalidHeaders