testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# Note that if you are excluding with variants, you will need to exclude
# per variant, with the identifier appended to the test suite name.
# For example, if you have ios_chrome_ui_eg2tests_module with variant
# SIM_IPHONE_6S_14_0 (identifier = "iPhone 6s 14.0"), you
# must exclude with the test name as:
# "ios_chrome_ui_eg2tests_module_iPhone 6s 14.0"

# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'accessibility_unittests': {
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'args': [
            # TODO(crbug.com/1442303): Re-enable.
          '--gtest_filter=-AXPlatformNodeAuraLinuxTest.AtkComponentScrollTo:'
          'AtkUtilAuraLinuxTest.*',
        ],
      },
    },
  },
  'android_browsertests': {
    'modifications': {
      'android-11-x86-rel': {
        'args': [
          # https://crbug.com/1289764
          '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
        ],
      },
      'android-12-x64-rel': {
        # TODO(crbug.com/1225851): Remove experiment and ci_only
        # once the test suite is stable.
        'ci_only': True,
        'experiment_percentage': 100,
        'args': [
          # https://crbug.com/1289764
          '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'android-12l-x64-dbg-tests': {
        'args': [
          # https://crbug.com/1289764
          '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*:'
          # https://crbug.com/1468262
          'All/PaymentHandlerEnforceFullDelegationTest.WhenEnabled_ShowPaymentSheet_WhenDisabled_Reject/1',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'android-12l-x64-fyi-dbg': {
        'args': [
          # https://crbug.com/1289764
          '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'android-13-x64-fyi-rel': {
        'args': [
          # https://crbug.com/1414886
          '--gtest_filter=-OfferNotificationControllerAndroidBrowserTestForMessagesUi.MessageShown',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'android-13-x64-rel': {
        'args': [
          # https://crbug.com/1414886
          '--gtest_filter=-OfferNotificationControllerAndroidBrowserTestForMessagesUi.MessageShown',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'android-code-coverage-native': {
        'swarming': {
          'shards': 3,
        },
      },
      # If you change this, make similar changes in android-x86-code-coverage below
      'android-oreo-x86-rel': {
        'ci_only': True,
        'swarming': {
          'shards': 9,
        },
      },
      'android-pie-arm64-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
        'swarming': {
          'dimensions': {
            'device_type': 'crosshatch',
          },
          # TODO(crbug/1456515): Temporarily increase the shard by 1 for the surging number of tests.
          # Recover to original number of shards(4) after the bug is fixed.
          'shards': 5,
        },
      },
      'android-pie-x86-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
        'swarming': {
          'dimensions': {
            'cores': '8', # crbug.com/1292221
          },
          'shards': 3,
        },
      },
      # Keep this same as android-oreo-x86-rel above
      'android-x86-code-coverage': {
        'ci_only': True,
        'swarming': {
          'shards': 9,
        },
      },
    },
  },
  'android_browsertests_no_fieldtrial': {
    'modifications': {
      'android-12-x64-rel': {
        # TODO(crbug.com/1225851): Remove experiment and ci_only
        # once the test suite is stable.
        'experiment_percentage': 100,
        'args': [
          # https://crbug.com/1289764
          '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
        ],
      }
    },
    'remove_from': [
      'android-12l-x64-fyi-dbg',
    ],
  },
  'android_sync_integration_tests': {
   'modifications': {
      'android-12-x64-rel': {
        'swarming': {
          'shards': 3,
        },
      },
      'android-code-coverage-native': {
        'swarming': {
          'shards': 2,
        },
      },
      # If you change this, make similar changes in android-x86-code-coverage below
      'android-oreo-x86-rel': {
        'ci_only': True,
        'swarming': {
          'shards': 2,
        },
      },
      # Keep this same as android-oreo-x86-rel above
      'android-x86-code-coverage': {
        'ci_only': True,
        'swarming': {
          'shards': 2,
        },
      },
   },
  },
  'android_webview_unittests': {
  },
  'angle_unittests': {
    'remove_from': [
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
      # Times out listing tests crbug.com/1167314
      'android-asan',
      # Does not currently work on Lacros configurations.
      'linux-lacros-asan-lsan-rel',
      'linux-lacros-code-coverage',
      'linux-lacros-tester-rel',
      'linux-lacros-tester-fyi-rel',
      'linux-lacros-dbg-tests-fyi',
    ],
    'modifications': {
      # anglebug.com/6894
      'fuchsia-fyi-x64-asan': {
        'args': [
          # crbug.com/
          '--gtest_filter=-ConstructCompilerTest.DefaultParameters',
        ],
      },
      'linux-chromeos-rel': {
        # b/41493162: angle_unittests has a high failure rate.
        # Re-enable cq when the issue is fixed.
        'ci_only': True,
      },
    },
  },
  'ash_crosapi_tests': {
    'remove_from': [
      # Disabling MSan tests since linux-chromeos chrome crashes on MSan build.
      # TODO(crbug.com/1376064): Re-enable.
      'Linux ChromiumOS MSan Tests',
    ]
  },
  'base_unittests': {
    'modifications': {
      'ToTAndroid': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.pie_tot.base_unittests.filter',
        ],
      },
      'android-12-x64-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
        ],
      },
      'android-12l-x64-dbg-tests': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
        ],
      },
      'android-12l-x64-fyi-dbg': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
        ],
      },
      'android-13-x64-fyi-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
        ],
      },
      'android-13-x64-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
        ],
      },
      'fuchsia-code-coverage': {
        'swarming': {
          'shards': 5,
        },
      },
      'fuchsia-fyi-x64-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/fuchsia.lsan.base_unittests.filter',
        ],
      },
    },
  },
  'bf_cache_content_browsertests': {
    'modifications': {
      'android-bfcache-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_o.content_browsertests.filter',
        ],
        'swarming': {
          'shards': 15,
        },
      }
    }
  },
  'blink_platform_unittests': {
    'modifications': {
      'android-pie-arm64-rel': {
        # TODO(crbug.com/1111436): Move this back to walleye if/when additional
        # capacity has been deployed.
        'swarming': {
          'dimensions': {
            'device_type': 'crosshatch',
          },
        },
      },
      'fuchsia-code-coverage': {
        'swarming': {
          'shards': 5,
        },
      },
    },
  },
  'blink_pytype': {
    'remove_from': [
      'linux-code-coverage',
    ],
  },
  'blink_unittests': {
    'modifications': {
      'fuchsia-code-coverage': {
        'swarming': {
          'shards': 30,
        },
      },
    },