ci: run more Chromium unit tests (#15363)

* test: add an option to native tests runner to run also disabled tests

* ci: run all native tests on clean Chromium

* ci: update the disabled tests list for native unittests

script/lib/native_tests.py

@@ -44,6 +44,12 @@ class Verbosity:
     return a_index <= b_index
 
 
+class DisabledTestsPolicy:
+  DISABLE = 'disable'  # Disabled tests are disabled. Wow. Much sense.
+  ONLY = 'only'  # Only disabled tests should be run.
+  INCLUDE = 'include'  # Do not disable any tests.
+
+
 class Platform:
   LINUX = 'linux'
   MAC = 'mac'
@@ -88,7 +94,7 @@ class TestsList():
     return supported_binaries
 
   def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
     # Don't run anything twice.
     binaries = set(binaries)
 
@@ -106,19 +112,19 @@ class TestsList():
                 binary_name, Platform.get_current()))
 
     suite_returncode = sum(
-        [self.__run(binary, output_dir, verbosity, run_only_disabled_tests)
+        [self.__run(binary, output_dir, verbosity, disabled_tests_policy)
         for binary in binaries])
     return suite_returncode
 
   def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
     return self.run([binary_name], output_dir, verbosity,
-                    run_only_disabled_tests)
+                    disabled_tests_policy)
 
   def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY,
-      run_only_disabled_tests=False):
+      disabled_tests_policy=DisabledTestsPolicy.DISABLE):
     return self.run(self.get_for_current_platform(), output_dir, verbosity,
-                    run_only_disabled_tests)
+                    disabled_tests_policy)
 
   @staticmethod
   def __get_tests_list(config_path):
@@ -200,7 +206,7 @@ class TestsList():
     return (binary_name, test_data)
 
   def __run(self, binary_name, output_dir, verbosity,
-      run_only_disabled_tests):
+      disabled_tests_policy):
     binary_path = os.path.join(self.tests_dir, binary_name)
     test_binary = TestBinary(binary_path)
 
@@ -208,13 +214,15 @@ class TestsList():
     included_tests = []
     excluded_tests = test_data['excluded_tests']
 
-    if run_only_disabled_tests and len(excluded_tests) == 0:
-      # There is nothing to run.
-      return 0
-
-    if run_only_disabled_tests:
+    if disabled_tests_policy == DisabledTestsPolicy.ONLY:
+      if len(excluded_tests) == 0:
+        # There is nothing to run.
+        return 0
       included_tests, excluded_tests = excluded_tests, included_tests
 
+    if disabled_tests_policy == DisabledTestsPolicy.INCLUDE:
+      excluded_tests = []
+
     output_file_path = TestsList.__get_output_path(binary_name, output_dir)
 
     return test_binary.run(included_tests=included_tests,

script/native-tests.py

@@ -6,7 +6,7 @@ import argparse
 import os
 import sys
 
-from lib.native_tests import TestsList, Verbosity
+from lib.native_tests import TestsList, Verbosity, DisabledTestsPolicy
 
 
 class Command:
@@ -25,14 +25,24 @@ def parse_args():
                       help='binaries to run')
   parser.add_argument('-c', '--config', required=True,
                       help='path to a tests config')
-  parser.add_argument('--run-only-disabled-tests',
-                      action='store_true', default=False,
-                      help='if disabled tests should be run')
   parser.add_argument('-t', '--tests-dir', required=False,
                       help='path to a directory with test binaries')
   parser.add_argument('-o', '--output-dir', required=False,
                       help='path to a folder to save tests results')
 
+  disabled_tests = parser.add_mutually_exclusive_group()
+  disabled_tests.add_argument('--only-disabled-tests',
+                              dest='disabled_tests_policy',
+                              action='store_const',
+                              const=DisabledTestsPolicy.ONLY,
+                              help='run disabled tests only')
+  disabled_tests.add_argument('--include-disabled-tests',
+                              dest='disabled_tests_policy',
+                              action='store_const',
+                              const=DisabledTestsPolicy.INCLUDE,
+                              help='if disabled tests should be run as well')
+  parser.set_defaults(disabled_tests_policy=DisabledTestsPolicy.DISABLE)
+
   verbosity = parser.add_mutually_exclusive_group()
   verbosity.add_argument('-v', '--verbosity', required=False,
                          default=Verbosity.CHATTY,
@@ -86,10 +96,10 @@ def main():
   if args.command == Command.RUN:
     if args.binary is not None:
       return tests_list.run(args.binary, args.output_dir, args.verbosity,
-                            args.run_only_disabled_tests)
+                            args.disabled_tests_policy)
     else:
       return tests_list.run_all(args.output_dir, args.verbosity,
-                                args.run_only_disabled_tests)
+                                args.disabled_tests_policy)
 
   assert False, "unexpected command '{}'".format(args.command)