testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'android_webview_unittests': {
    'remove_from': [
      # On chromium.android, these do not need to run prior to M.
      'android-kitkat-arm-rel',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
    ],
  },
  'angle_end2end_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # TODO(ynovikov) Investigate why the test breaks on older devices.
      'Android FYI Release (Nexus 5)',
      'Android FYI Release (Nexus 6)',
      'Android FYI Release (Nexus 9)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
      # Temporarily disabled due to bad NVIDIA driver upgrade crbug.com/950542
      'Linux FYI Debug (NVIDIA)',
      'Linux FYI GPU TSAN Release',
    ],
  },
  'angle_perftests': {
    'remove_from': [
      'Android FYI Release (Nexus 5)',  # crbug.com/915429
      'Android FYI Release (Nexus 6)',  # anglebug.com/2433
      'Linux FYI GPU TSAN Release',  # crbug.com/950542
    ],
  },
  'angle_unittests': {
    'remove_from': [
      # chromium.gpu.fyi
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'breakpad_unittests': {
    'modifications': {
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.breakpad_unittests.filter',
        ],
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      # chromium.memory
      'Linux TSan Tests',  # https://crbug.com/368525
      # chromium.win
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      # chromium.fyi
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 21,
        },
      },
      'linux-chromeos-google-rel': {
        # TODO(https://crbug.com/932269): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
      },
      'linux-chromeos-oobe-code-coverage': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.clang
      'ToTMac': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'ToTMacASan': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac-dummy-rel': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'mac-osxbeta-rel': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      # chromium.mac
      'Mac10.10 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.11 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.12 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.13 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.13 Tests (dbg)': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 30,
        },
      },
      'Mac ASan 64 Tests (1)': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 25,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 31,
        },
      },
      # chromium.win
      'Win7 Tests (1)': {
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 15,
        },
      },
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
        }
      },
      # client.v8.chromium
      'Linux - Future (dbg)': {
        'swarming': {
          'shards':  10,
        },
      },
    },
  },
  'cast_shell_browsertests': {
    'modifications': {
      'Cast Audio Linux': {
        'args': [
          '--enable-local-file-accesses',
          '--ozone-platform=headless',
          '--no-sandbox',
          '--test-launcher-jobs=1',
        ],
        'swarming': {
          'can_use_on_swarming_builders': False,  # https://crbug.com/861753
        },
      },
      'Cast Linux': {
        'args': [
          '--enable-local-file-accesses',
          '--ozone-platform=headless',
          '--no-sandbox',
          '--test-launcher-jobs=1',
        ],
        'swarming': {
          'can_use_on_swarming_builders': False,  # https://crbug.com/861753
        },
      },
    },
  },
  'checkbins': {
    'remove_from': [
      'linux-archive-dbg',
      'linux-archive-rel',
      'mac-archive-dbg',
      'mac-archive-rel',
    ],
  },
  'chrome_public_test_apk': {
    'remove_from': [
      'android-asan',  # https://crbug.com/964562
    ],
    'modifications': {
      # chromium.android
      'android-kitkat-arm-rel': {
        'swarming': {
          'shards': 12,
        },
      },
      'android-marshmallow-arm64-rel': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 0,
      },
      'KitKat Phone Tester (dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Marshmallow 64 bit Tester': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
        'swarming': {
          'shards': 14,
        },
      },
      'Oreo Phone Tester': {
        # TODO(https://crbug.com/884413): Re-enable this once the tests are
        # either passing or there is more capacity.
        'experiment_percentage': 0,
      },
      # chromium.swarm
      'Android N5X Swarm': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
      },
    },
  },
  'chrome_public_test_vr_apk': {
    'remove_from': [
      # chromium.android
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      # chromium.clang
      'ToTAndroid',
      'ToTAndroidCFI',
      # chromium.memory
      'android-asan',  # https://crbug.com/964562
    ],
  },
  'chrome_public_test_vr_apk-vega': {
    'modifications': {
      # chromium.android
      'Oreo Phone Tester': {
        'swarming': {
          'dimension_sets': [
            {
              'device_os': 'OVR1.180808.003',
              'device_type': 'vega',
              'os': 'Android',
            },
          ],
        },
      }
    }
  },
  'chromedriver_py_tests': {
    'remove_from': [
      # Timeout happens sometimes (crbug.com/951799).
      'Win10 Tests x64',
      'Win10 Tests x64 (dbg)',
    ],
  },
  'components_unittests': {
    'modifications': {
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'linux-chromeos-google-rel': {
        # TODO(crbug.com/980748): Remove this filter.
        'args': [
          '--gtest_filter=-CrashAnalyzerTest.StackTraceCollection',
        ],
      },
    },
  },
  'content_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
    'modifications': {
      # chromium.android
      'Lollipop Phone Tester': {
        'swarming': {
          'shards': 13,
        },
      },
      # chromium.linux
      'Cast Audio Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      'Cast Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      # chromium.memory