ci: run more Chromium unit tests (#15363)

* test: add an option to native tests runner to run also disabled tests * ci: run all native tests on clean Chromium * ci: update the disabled tests list for native unittests
2018-10-25 20:28:50 +02:00 · 2018-10-25 20:28:50 +02:00 · ec4a0e096f
commit ec4a0e096f
parent d21d83cfc8
4 changed files with 95 additions and 70 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -974,7 +974,7 @@ jobs:
      <<: *env-unittests
      <<: *env-enable-sccache
      <<: *env-headless-testing
-      TESTS_ARGS: '--run-only-disabled-tests'
+      TESTS_ARGS: '--only-disabled-tests'
    <<: *steps-native-tests

  linux-x64-chromium-unittests:
@ -983,6 +983,7 @@ jobs:
      <<: *env-unittests
      <<: *env-enable-sccache
      <<: *env-headless-testing
+      TESTS_ARGS: '--include-disabled-tests'
    <<: *steps-native-tests

  linux-x64-browsertests:
--- a/script/lib/native_tests.py
+++ b/script/lib/native_tests.py
@ -44,6 +44,12 @@ class Verbosity:
    return a_index <= b_index


+class DisabledTestsPolicy:
+  DISABLE = 'disable'  # Disabled tests are disabled. Wow. Much sense.
+  ONLY = 'only'  # Only disabled tests should be run.
+  INCLUDE = 'include'  # Do not disable any tests.
+
+
 class Platform:
  LINUX = 'linux'
  MAC = 'mac'
@ -88,7 +94,7 @@ class TestsList():
    return supported_binaries

  def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
    # Don't run anything twice.
    binaries = set(binaries)

@ -106,19 +112,19 @@ class TestsList():
                binary_name, Platform.get_current()))

    suite_returncode = sum(
-        [self.__run(binary, output_dir, verbosity, run_only_disabled_tests)
+        [self.__run(binary, output_dir, verbosity, disabled_tests_policy)
        for binary in binaries])
    return suite_returncode

  def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
    return self.run([binary_name], output_dir, verbosity,
-                    run_only_disabled_tests)
+                    disabled_tests_policy)

  def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
    return self.run(self.get_for_current_platform(), output_dir, verbosity,
-                    run_only_disabled_tests)
+                    disabled_tests_policy)

  @staticmethod
  def __get_tests_list(config_path):
@ -200,7 +206,7 @@ class TestsList():
    return (binary_name, test_data)

  def __run(self, binary_name, output_dir, verbosity,
-      run_only_disabled_tests):
+      disabled_tests_policy):
    binary_path = os.path.join(self.tests_dir, binary_name)
    test_binary = TestBinary(binary_path)

@ -208,13 +214,15 @@ class TestsList():
    included_tests = []
    excluded_tests = test_data['excluded_tests']

-    if run_only_disabled_tests and len(excluded_tests) == 0:
-      # There is nothing to run.
-      return 0
-
-    if run_only_disabled_tests:
+    if disabled_tests_policy == DisabledTestsPolicy.ONLY:
+      if len(excluded_tests) == 0:
+        # There is nothing to run.
+        return 0
      included_tests, excluded_tests = excluded_tests, included_tests

+    if disabled_tests_policy == DisabledTestsPolicy.INCLUDE:
+      excluded_tests = []
+
    output_file_path = TestsList.__get_output_path(binary_name, output_dir)

    return test_binary.run(included_tests=included_tests,
--- a/script/native-tests.py
+++ b/script/native-tests.py
@ -6,7 +6,7 @@ import argparse
 import os
 import sys

-from lib.native_tests import TestsList, Verbosity
+from lib.native_tests import TestsList, Verbosity, DisabledTestsPolicy


 class Command:
@ -25,14 +25,24 @@ def parse_args():
                      help='binaries to run')
  parser.add_argument('-c', '--config', required=True,
                      help='path to a tests config')
-  parser.add_argument('--run-only-disabled-tests',
-                      action='store_true', default=False,
-                      help='if disabled tests should be run')
  parser.add_argument('-t', '--tests-dir', required=False,
                      help='path to a directory with test binaries')
  parser.add_argument('-o', '--output-dir', required=False,
                      help='path to a folder to save tests results')

+  disabled_tests = parser.add_mutually_exclusive_group()
+  disabled_tests.add_argument('--only-disabled-tests',
+                              dest='disabled_tests_policy',
+                              action='store_const',
+                              const=DisabledTestsPolicy.ONLY,
+                              help='run disabled tests only')
+  disabled_tests.add_argument('--include-disabled-tests',
+                              dest='disabled_tests_policy',
+                              action='store_const',
+                              const=DisabledTestsPolicy.INCLUDE,
+                              help='if disabled tests should be run as well')
+  parser.set_defaults(disabled_tests_policy=DisabledTestsPolicy.DISABLE)
+
  verbosity = parser.add_mutually_exclusive_group()
  verbosity.add_argument('-v', '--verbosity', required=False,
                         default=Verbosity.CHATTY,
@ -86,10 +96,10 @@ def main():
  if args.command == Command.RUN:
    if args.binary is not None:
      return tests_list.run(args.binary, args.output_dir, args.verbosity,
-                            args.run_only_disabled_tests)
+                            args.disabled_tests_policy)
    else:
      return tests_list.run_all(args.output_dir, args.verbosity,
-                                args.run_only_disabled_tests)
+                                args.disabled_tests_policy)

  assert False, "unexpected command '{}'".format(args.command)

--- a/spec/configs/unittests.yml
+++ b/spec/configs/unittests.yml
@ -1,23 +1,34 @@
+# Note: a "disabled" dict can have arbitrary named properties
+# with lists of disabled tests as a value. Those properties' names
+# are for humans only and are there only to explain
+# why a group of tests is disabled.
+
 tests:
  - base_unittests:
-      # "disabled" dict can have arbitrary named properties
-      # with lists of disabled tests. Those properties' names
-      # are used only to explain why a group of tests is disabled.
      disabled:
-        undecided:
-          - FilePathTest*
+        upstream_failures:
+          - FilePathTest.FromUTF8Unsafe_And_AsUTF8Unsafe
+          - RTLTest.WrapPathWithLTRFormatting
+          - SysStrings.SysNativeMBAndWide
+          - SysStrings.SysNativeMBToWide
+          - SysStrings.SysWideToNativeMB
+        to_fix:
          - PartitionReallocReturnNullTest.RepeatedReturnNullDirect
-          - RTLTest*
-          - SysStrings*
-          - UTFOffsetStringConversionsTest*
          - WeakPtrDeathTest*
  - cc_unittests
  - cc_blink_unittests
  - content_unittests:
      disabled:
-        # TODO(alexeykuzmin): Should those be fixed?
-        - _/DOMStorageMapParamTest.EnforcesQuota/0
-        - _/DOMStorageMapParamTest.EnforcesQuota/1
+        upstream_failures:
+          - BackgroundSyncManagerTest.*
+          - FrameServiceBaseTest.ConnectionError
+          - MediaSessionControllerTest.*
+          - MediaStreamUIProxyFeaturePolicyTest.FeaturePolicy
+          - PresentationServiceImplTest.*
+        to_fix:
+          - _/DOMStorageMapParamTest.EnforcesQuota/0
+          - _/DOMStorageMapParamTest.EnforcesQuota/1
+          - RenderFrameAudioInputStreamFactoryTest.ConstructDestruct
  - crypto_unittests
  - device_unittests
  - gin_unittests
@ -31,39 +42,19 @@ tests:
  - mojo_common_unittests
  - net_unittests:
      disabled:
-        # TODO(alexeykuzmin): Should those be fixed?
-        - EndToEndTestsWithTls*
-        - FilenameUtilTest*
-        - HTTPSAIATest.AIAFetching
-        - HTTPSCRLSetTest*
-        - HTTPSEVCRLSetTest*
-        - HTTPSFallbackTest*
-        - HTTPSHardFailTest.FailsOnOCSPInvalid
-        - HTTPSOCSPTest*
-        - HTTPSRequestTest*
-        - HTTPSSessionTest*
-        - OCSPVerify*
-        - ProxyConfigServiceLinuxTest*
-        - SSLClientSocketCertRequestInfoTest*
-        - SSLClientSocketChannelIDTest*
-        - SSLClientSocketFalseStartTest*
-        - SSLClientSocketReadTest*
-        - SSLClientSocketTest*
-        - TokenBindingURLRequestTest*
-        - UDPSocketTest*
-        - URLRequestTestFTP*
-        - URLRequestTestHTTP*
-        - WebSocketEndToEndTest*
+        upstream_failures:
+          - EndToEndTestsWithTls/EndToEndTestWithTls.SimpleRequestResponsev6/*
+          - FilenameUtilTest.GenerateFileName
+          - SSLClientSocketTest.*
+          - UDPSocketTest.ClientGetLocalPeerAddresses
+          - UDPSocketTest.ClientSetDoNotFragment
  - ppapi_unittests
  - printing_unittests
  - skia_unittests
  - sql_unittests
  - storage_unittests
  - gn_unittests
-  - ui_base_unittests:
-      disabled:
-        # TODO(alexeykuzmin): Should it be fixed?
-        - AcceleratorTest.GetShortcutText
+  - ui_base_unittests
  - compositor_unittests
  - display_unittests
  - events_unittests
@ -73,31 +64,43 @@ tests:
  # V8
  - unittests:
      disabled:
-        # TODO(alexeykuzmin): Should it be fixed?
-        - LiveRangeUnitTest*
+        upstream_failures:
+          - LiveRangeUnitTest.InvalidConstruction
+          - LiveRangeUnitTest.SplinterMultipleIntervalsInside
+          - LiveRangeUnitTest.SplitInvalidStart
  # Blink
  - blink_common_unittests
  - blink_heap_unittests
  - blink_platform_unittests:
      disabled:
        # TODO(alexeykuzmin): Should those be fixed?
-        - AnimatedPNGTests*
-        - AnimatedWebPTests*
+        - AnimatedPNGTests.*
+        - AnimatedWebPTests.*
        - AnimatedWEBPTests.clearCacheExceptFrameWithAncestors
-        - BitmapImageTest*
+        - BitmapImageTest.*
        - BMPImageDecoderTest*
        - DecodedImageOrientationHistogramTest*
        - DecodedImageTypeHistogramTest*
+        - DeferredImageDecoderTest.decodeOnOtherThread
        - DeferredImageDecoderTestWoPlatform*
        - GIFImageDecoderTest*
        - HighContrastImageClassifierTest*
        - ICOImageDecoderTests*
        - JPEGImageDecoderTest*
        - FilePathConversionTest*
+        - PaintPropertyNodeTest.InitialStateAndReset
        - PNGTests*
        - StaticPNGTests*
        - StaticWebPTests*
-  - webkit_unit_tests
+  - webkit_unit_tests:
+      disabled:
+        upstream_failures:
+          - Document/ThreadableLoaderTest.CancelAfterStart/0
+          - ModernMediaControlsImplTest.ControlsShouldUseSafeAreaInsets
+          - ScriptRunnerTest.QueueReentrantScript_InOrder
+          - ScrollbarsTestWithVirtualTimer.PressScrollbarButtonOnInfiniteScrolling
+          - VirtualTimeTest.AllowVirtualTimeToAdvance
+          - VirtualTimeTest.VirtualTimeNotAllowedToAdvanceWhileResourcesLoading
  - wtf_unittests
  # Third party
  - angle_unittests
@ -108,13 +111,16 @@ tests:
  - sandbox_linux_unittests:
      platform: linux
      disabled:
-        # TODO(alexeykuzmin): Should these be fixed?
-        - BrokerProcess*
-        - SandboxBPF.SigBus
+        upstream_failures:
+          - BrokerProcess.OpenFilePermsWithClientCheck
+          - BrokerProcess.OpenFilePermsWithClientCheckNoEnt
+          - BrokerProcess.OpenOpenFilePermsNoClientCheck
+          - BrokerProcess.OpenOpenFilePermsNoClientCheckNoEnt
+          - SandboxBPF.SigBus
  - dbus_unittests:
      platform: linux
      disabled:
-        # TODO(alexeykuzmin): Should these be fixed?
-        - EndToEndAsyncTest.InvalidServiceName
-        - EndToEndSyncTest.InvalidServiceName
-        - MessageTest.SetInvalidHeaders
+        upstream_failures:
+          - EndToEndAsyncTest.InvalidServiceName
+          - EndToEndSyncTest.InvalidServiceName
+          - MessageTest.SetInvalidHeaders