testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'android_webview_unittests': {
    'remove_from': [
      # On chromium.android, these do not need to run prior to M.
      'android-kitkat-arm-rel',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
    ],
  },
  'angle_end2end_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # TODO(ynovikov) Investigate why the test breaks on older devices.
      'Android FYI Release (Nexus 5)',
      'Android FYI Release (Nexus 6)',
      'Android FYI Release (Nexus 9)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
      # Disabled due to numerous ES2_VULKAN failures crbug.com/940750
      'Linux FYI Experimental Release (Intel HD 630)',
      # Temporarily disabled due to bad NVIDIA driver upgrade crbug.com/950542
      'Linux FYI Debug (NVIDIA)',
      'Linux FYI GPU TSAN Release',
    ],
  },
  'angle_perftests': {
    'remove_from': [
      'Android FYI Release (Nexus 5)',  # crbug.com/915429
      'Android FYI Release (Nexus 6)',  # anglebug.com/2433
      'Linux FYI GPU TSAN Release',  # crbug.com/950542
    ],
  },
  'angle_unittests': {
    'remove_from': [
      # chromium.gpu.fyi
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'angle_white_box_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # Vulkan tests fail consistently due to missing extensions.
      # See crbug.com/940750
      'Linux FYI Experimental Release (Intel HD 630)',
    ],
  },
  'browser_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      # chromium.memory
      'Linux TSan Tests',  # https://crbug.com/368525
      # chromium.win
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      # chromium.fyi
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 21,
        },
      },
      'linux-chromeos-google-rel': {
        # TODO(https://crbug.com/932269): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
        'experiment_percentage': 100,
      },
      'linux-chromeos-oobe-code-coverage': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.clang
      'ToTMac': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'ToTMacASan': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac-dummy-rel': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',