testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'android_webview_unittests': {
    'remove_from': [
      # On chromium.android, these do not need to run prior to M.
      'android-kitkat-arm-rel',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
      # chromium.android.fyi
      'x64 Device Tester',
    ],
  },
  'angle_end2end_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # TODO(ynovikov) Investigate why the test breaks on older devices.
      'Android FYI Release (Nexus 5)',
      'Android FYI Release (Nexus 6)',
      'Android FYI Release (Nexus 9)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
    ],
  },
  'angle_perftests': {
    'remove_from': [
      'Android FYI Release (Nexus 6)',  # anglebug.com/2433
    ],
  },
  'angle_unittests': {
    'remove_from': [
      # chromium.gpu.fyi
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsanCov',
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      'ToTMac', # https://crbug.com/828031
      'ToTMacASan', # https://crbug.com/828031
      # chromium.fyi
      'Chromium Mac 10.13', # https://crbug.com/828031
      'mac-osxbeta-rel', # https://crbug.com/828031
      'mac-views-rel', # https://crbug.com/828031
      # chromium.mac
      'Mac10.10 Tests', # https://crbug.com/828031
      # chromium.memory
      'Linux TSan Tests',  # https://crbug.com/368525
      'Mac ASan 64 Tests (1)', # https://crbug.com/828031
      # chromium.win
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.clang
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      # chromium.mac
      'Mac10.11 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.12 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.13 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.13 Tests (dbg)': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 30,
        },
      },
      'Mac ASan 64 Tests (1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 25,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 31,
        },
      },
      # client.v8.chromium
      'Linux - Future (dbg)': {
        'swarming': {
          'shards':  10,
        },
      },
      # chromium.win
      'Win7 Tests (1)': {
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 15,
        },
      },
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
    },
  },
  'cast_shell_browsertests': {
    'modifications': {
      'Cast Audio Linux': {
        'args': [
          '--enable-local-file-accesses',
          '--ozone-platform=headless',
          '--no-sandbox',
          '--test-launcher-jobs=1',
        ],
        'swarming': {
          'can_use_on_swarming_builders': False,  # https://crbug.com/861753
        },
      },
      'Cast Linux': {
        'args': [
          '--enable-local-file-accesses',
          '--ozone-platform=headless',
          '--no-sandbox',
          '--test-launcher-jobs=1',
        ],
        'swarming': {
          'can_use_on_swarming_builders': False,  # https://crbug.com/861753
        },
      },
    },
  },
  'checkbins': {
    'remove_from': [
      'linux-dbg',
      'linux-rel',
      'mac-dbg',
      'mac-rel',
    ],
  },
  # Note: keep experiment percentages the same as
  # network_service_chrome_public_test_apk.
  'chrome_public_test_apk': {
    'modifications': {
      # chromium.android
      'android-kitkat-arm-rel': {
        'swarming': {
          'shards': 12,
        },
      },
      'android-marshmallow-arm64-rel': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 0,
      },
      'KitKat Phone Tester (dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Marshmallow 64 bit Tester': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
        'swarming': {
          'shards': 14,
        },
      },
      'Oreo Phone Tester': {
        # TODO(https://crbug.com/884413): Re-enable this once the tests are
        # either passing or there is more capacity.
        'experiment_percentage': 0,
      },
      # chromium.swarm
      'Android N5X Swarm': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
      },
    },
  },
  'chrome_public_test_vr_apk': {
    'remove_from': [
      # chromium.android
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      # chromium.android.fyi
      'x64 Device Tester',
      'x86 Cloud Tester',
      # chromium.clang
      'ToTAndroid',
      'ToTAndroidCFI',
    ],
  },
  'chrome_public_test_vr_apk-vega': {
    'modifications': {
      # chromium.android
      'Oreo Phone Tester': {
        'swarming': {
          'dimension_sets': [
            {
              'device_os': 'OVR1.180808.003',
              'device_type': 'vega',
              'os': 'Android',
            },
          ],
        },
      }
    }
  },
  'components_unittests': {
    'modifications': {
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
    },
  },
  'content_browsertests': {
    'modifications': {
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 2,
        },
      },
      'linux-chromeos-rel': {
        'swarming': {
          'shards': 2,
        },
      },
      # chromium.clang
      'ToTLinuxTSan': {
        'swarming': {
          'shards': 5,
        },
      },
      'ToTLinuxUBSanVptr': {
        'swarming': {
          'shards': 5,
        },
      },
      'linux-win_cross-rel': {
        # content_browsertests sometimes times out without this override on
        # the cross bot, probably because of https://crbug.com/803591,
        # see also https://crbug.com/838559
        # TODO(thakis): Remove this once 803591 is fixed.
        'swarming': {
          'shards': 5,
        },
      },
      # chromium.linux
      'Cast Audio Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      'Cast Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 8,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # content_browsertests is slow on ASAN try bot. crbug.com/822461.
        'swarming': {
          'shards': 4,
        }
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 5,
        },
      },
      'Mac ASan 64 Tests (1)': {
        'swarming': {
          'shards': 2,
        },
      },
      # chromium.win
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 2,
        },
      },
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
    },
  },
  'content_shell_crash_test': {
    'remove_from': [
      # chromium.linux
      'Linux Tests (dbg)(1)(32)',  # https://crbug.com/859264
    ],
    'modifications': {
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,  # https://crbug.com/861730
      },
    },
  },
  'content_unittests': {
    'modifications': {
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 2,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'context_lost_tests': {
    'modifications': {
      'Mac FYI GPU ASAN Release': {
        'args': [
          '--is-asan',
        ]
      },
    },
  },
  'crashpad_tests': {
    'remove_from': [
      # chromium.clang
      'linux-win_cross-rel', # https://crbug.com/762167
    ],
  },
  'extensions_browsertests': {
    'modifications': {
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100, # https://crbug.com/876615
      },
    },
  },
  'gin_unittests': {
    'remove_from': [
      # chromium.clang
      'ToTLinuxASan',  # https://crbug.com/831667
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # https://crbug.com/831667
      'Linux Chromium OS ASan LSan Tests (1)',  # https://crbug.com/831667
    ],
  },
  'gl_tests': {
    'remove_from': [
      # chromium.clang
      'CFI Linux ToT',
      # chromium.gpu.fyi
      'Android FYI Release (Nexus 5X)',
      # chromium.memory
      # Can't run on MSAN because gl_tests uses the hardware driver,
      # which isn't instrumented.
      'Linux MSan Tests',
    ],
    'modifications': {
      # chromium.memory
      # TODO(kbr): figure out a better way to specify blocks of
      # arguments like this for tests on multiple machines.
      'Linux ASan LSan Tests (1)': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      'Linux ASan LSan Tests (1)': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      'Linux CFI': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      'Linux TSan Tests': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
    },
  },
  'gl_unittests': {
    'remove_from': [
      # chromium.gpu.fyi
      # On Android, these are already run on the main waterfall.
      # Run them on the one-off Android FYI bots, though.
      'Android FYI Release (Nexus 5X)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
    ],
    'modifications': {
      # chromium.clang
      'CFI Linux ToT': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      # chromium.memory
      'Linux CFI': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
    },
  },
  'gl_unittests_ozone': {
    'remove_from': [
      # Can't run on MSAN because gl_unittests_ozone uses the hardware driver,
      # which isn't instrumented.
      'Linux ChromiumOS MSan Tests',
    ],
  },
  'headless_browsertests': {
    'remove_from': [
      # chromium.clang
      'ToTLinuxASan',  # https://crbug.com/843732
      'ToTLinuxTSan',  # https://crbug.com/843734
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # https://crbug.com/843732
      'Linux TSan Tests',  # https://crbug.com/843734
    ],
  },
  'headless_unittests': {
    'remove_from': [
      # chromium.clang
      'ToTLinuxASan',  # https://crbug.com/843732
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # https://crbug.com/843732
    ],
  },
  'info_collection_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # The Mac ASAN swarming bot runs tests on two different GPU
      # types, so we can't have one expected vendor ID / device ID.
      'Mac FYI GPU ASAN Release',
      # client.v8.fyi
      'Android V8 FYI Release (Nexus 5X)',
      'Linux V8 FYI Release (NVIDIA)',
      'Linux V8 FYI Release - pointer compression (NVIDIA)',
      'Mac V8 FYI Release (Intel)',
      'Win V8 FYI Release (NVIDIA)',
    ],
  },
  'interactive_ui_tests': {
    'modifications': {
      # chromium.clang
      'ToTLinuxTSan': {
        # These are slow on the TSan bots for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      # chromium.win
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 4,
        },
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASan trybot for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      'Linux TSan Tests': {
        # These are slow on the TSan bots for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 5,
        },
      },
      # client.v8.chromium
      'Linux - Future (dbg)': {
        'swarming': {
          'shards': 3,
        },
      },
      # chromium.win; temporary, https://crbug.com/818832
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
      },
    },
  },
  'nacl_helper_nonsfi_unittests': {
    'remove_from': [
      # chromium.memory
      'Linux TSan Tests',  # Sets enable_nacl=false.
    ],
  },
  'nacl_loader_unittests': {
    'remove_from': [
      # chromium.clang
      'ToTMacASan',  # The mac asan tot bot sets enable_nacl=false
      'linux-win_cross-rel',  # Sets enable_nacl=false, https://crbug.com/774186
      # chromium.memory
      'Linux TSan Tests',  # The TSan bot sets enable_nacl=false
      'Mac ASan 64 Tests (1)',  # The mac asan bot sets enable_nacl=false
    ],
  },
  'net_unittests': {
    'remove_from': [
      # chromium.fyi
      'fuchsia-fyi-arm64-rel',  # https://crbug.com/847915
      'fuchsia-fyi-x64-dbg',  # https://crbug.com/844416 (and 847915)
      # chromium.linux
      'Fuchsia x64',  # https://crbug.com/847915
    ],
    'modifications': {
      # chromium.clang
      'ToTLinuxASan': {
        # TODO(crbug.com/794372): net_unittests is slow under ASan.
        'swarming': {
          'shards': 16,
        },
      },
      'ToTLinuxTSan': {
        'swarming': {
          'shards': 4,
        }
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 16,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 8,
        }
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 4,
        }
      },
      # The suite runs signficantly slower on linux dbg, so increase shards.
      'Linux Tests (dbg)(1)': {
        'swarming': {
          'shards': 2,
        },
      },
      'Linux Tests (dbg)(1)(32)': {
        'swarming': {
          'shards': 2,
        },
      },
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'network_service_browser_tests': {
    'remove_from': [
      # chromium.clang
      'ToTLinuxTSan',  # https://crbug.com/368525
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches browser_tests.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Chromium Mac 10.13',
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'mac-views-rel',
      'ToTMacASan',
    ],
    'modifications': {
      # chromium.win
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
    },
  },
  # Note: keep experiment percentages the same as chrome_public_test_apk.
  'network_service_chrome_public_test_apk': {
    # Only run these on the phone testers for now due to capacity.
    'remove_from': [
      # chromium.android
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      'KitKat Phone Tester (dbg)',
      # chromium.android.fyi
      'x86 Cloud Tester',
      # chromium.memory
      'Android CFI',
      # chromium.clang
      'ToTAndroid',
      'ToTAndroidCFI',
    ],
    'modifications': {
      # chromium.android
      'Marshmallow 64 bit Tester': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
      },
      'Oreo Phone Tester': {
        'experiment_percentage': 100,
      },
    },
  },
  'network_service_components_browsertests': {
    'remove_from': [
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches components_browsertests.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Chromium Mac 10.13',
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'mac-views-rel',
      'ToTMacASan',
    ],
  },
  'network_service_content_browsertests': {
    'remove_from': [
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # flaky: https://crbug.com/852786
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Chromium Mac 10.13',
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'mac-views-rel',
      'ToTMacASan',
    ],
  },
  'network_service_content_shell_test_apk': {
    # Only run these on the phone testers for now due to capacity.
    'remove_from': [
      # chromium.android
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      'KitKat Phone Tester (dbg)',
      # chromium.android.fyi
      'x86 Cloud Tester',
      # chromium.memory
      'Android CFI',
      # chromium.clang
      'ToTAndroid',
      'ToTAndroidCFI',
    ],
  },
  'network_service_extensions_browsertests': {
    'remove_from': [
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches extensions_browsertests.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Chromium Mac 10.13',
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'mac-views-rel',
      'ToTMacASan',
    ],
  },
  'network_service_interactive_ui_tests': {
    'remove_from': [
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Chromium Mac 10.13',
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'mac-views-rel',
      'ToTMacASan',
    ],
    'modifications': {
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,  # https://crbug.com/852786
      },
    },
  },
  'not_site_per_process_webkit_layout_tests': {
    'remove_from': [
      # chromium.linux
      # TODO(dpranke): Should we be running this step on Linux Tests (dbg)(1)?
      'Linux Tests (dbg)(1)',
      'Linux Tests (dbg)(1)(32)',
    ],
  },
  'notification_helper_unittests': {
    'remove_from': [
      # The test uses WinRT, which only exists in Windows 8 or above.
      # (https://crbug.com/813553)
      # chromium.win
      'Win 7 Tests x64 (1)',
      'Win7 Tests (1)',
      'Win7 Tests (dbg)(1)',
      # chromium.clang; all these bots run Windows 7.
      'ToTWin',
      'ToTWin(dbg)',
      'ToTWin(dll)',
      'ToTWin64',
      'ToTWin64(dbg)',
      'ToTWin64(dll)',
      'ToTWinCFI',
      'ToTWinCFI64',
      'ToTWinLibcxx64',
      'ToTWinThinLTO64',
    ],
  },
  'performance_test_suite': {
    'modifications': {
      'Mac10.13 Tests (dbg)': {
        'args': [
          '--browser=debug',
        ],
      },
      'Win10 Tests x64 (dbg)': {
        'args': [
          '--browser=debug_x64',
        ],
        'experiment_percentage': 100,
      },
      'Win7 Tests (dbg)(1)': {
        'args': [
          '--browser=debug',
        ],
      },
    },
  },
 'remoting_unittests': {
    'remove_from': [
      # chromium.linux
      'Linux Tests (dbg)(1)(32)',  # https://crbug.com/852895
    ],
  },
  'services_unittests': {
    'remove_from': [
      # chromium.clang
      'ToTLinuxMSan',  # https://crbug.com/831676
      # chromium.gpu.fyi
      # The face and barcode detection tests fail on the Mac Pros.
      'Mac Pro FYI Release (AMD)',
      # chromium.memory
      'Linux ChromiumOS MSan Tests',  # https://crbug.com/831676
      'Linux MSan Tests',  # https://crbug.com/831676
    ],
  },
  'single_process_mash_browser_tests': {
    'modifications': {
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 20,
        },
      },
      # chromium.memory
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 30,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        'swarming': {
          'shards': 20,
        },
      },
    },
  },
  'sizes': {
    'remove_from': [
      'win32-dbg',
      'win32-rel',
      'win-dbg',
      'win-rel',
    ],
    'modifications': {
      # chromium.android.fyi
      'Android Cronet KitKat Builder': {
        'args': [
          '--platform',
          'android-cronet',
          '--perf-id',
          'android_cronet_builder',
          'cronet-arm/sizes',
        ],
        'override_compile_targets': [
          'cronet',
        ],
      },
    },
  },
  'swiftshader_unittests': {
    'remove_from': [
      # Save capacity on the hardware where we have only a few machines.
      'Mac FYI Experimental Release (Intel)',
      'Mac FYI Experimental Retina Release (AMD)',
      'Mac FYI Experimental Retina Release (NVIDIA)',
      'Mac Pro FYI Release (AMD)',
    ],
  },
  'sync_integration_tests': {
    'modifications': {
      # chromium.clang
      'ToTLinuxTSan': {
        'swarming': {
          'shards': 6,
        },
      },
      # chromium.memory
      'Linux TSan Tests': {
        'swarming': {
          'shards': 6,
        },
      },
      # chromium.win
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 2,
        },
      },
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100, # https://crbug.com/840369
      },
    },
  },
  'tab_capture_end2end_tests': {
    # Run these only on Release bots.
    'remove_from': [
      # chromium.gpu
      'Linux Debug (NVIDIA)',
      'Mac Debug (Intel)',
      'Mac Retina Debug (AMD)',
      'Win10 Debug (NVIDIA)',
      # chromium.gpu.fyi
      'Win10 FYI Debug (NVIDIA)',
      'Win7 FYI Debug (AMD)',
    ],
  },
  'telemetry_perf_unittests': {
    'modifications': {
      # chromium.android
      'android-kitkat-arm-rel': {
        'args': [
          '--browser=android-chromium',
          '--device=android',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'temp_band': '<30',
            },
          ],
          'shards': 15,
        },
      },
      'android-marshmallow-arm64-rel': {
        'args': [
          '--browser=android-chromium',
          '--device=android',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'temp_band': '<30',
            },
          ],
          'shards': 15,
        },
      },
      # chromium.android.fyi
      'x86 Cloud Tester': {
        'args': [
          '--browser=android-chromium',
          '--device=android',
        ],
        'swarming': {
          'shards': 1,
        },
      },
      # chromium.fyi
      'linux-chromium-tests-staging-tests': {
        'args': [
          '--xvfb',
          '--jobs=1',
        ],
      },
      # chromium.linux
      'Linux Tests': {
        'args': [
          '--xvfb',
          '--jobs=1',
        ],
      },
      'Linux Tests (dbg)(1)': {
        'args': [
          '--xvfb',
          '--jobs=1',
        ],
      },
      'linux-xenial-rel': {
        'args': [
          '--xvfb',
          '--jobs=1',
        ],
      },
      # chromium.mac
      # Disable parallelism on all Mac bots.
      'Mac10.10 Tests': {
        'args': [
          '--jobs=1',
        ],
      },
      'Mac10.11 Tests': {
        'args': [
          '--jobs=1',
        ],
      },
      'Mac10.12 Tests': {
        'args': [
          '--jobs=1',
        ],
      },
      'Win7 Tests (dbg)(1)': {
        'args': [
          '--jobs=1',
        ],
        'swarming': {
          'shards': 20,
        },
        'experiment_percentage': 100,
      },
      # luci.chromium.ci
      'linux-tcmalloc-rel': {
        'args': [
          '--xvfb',
          '--jobs=1',
        ],
      },
    },
    'remove_from': [
      'Linux Tests (dbg)(1)(32)',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      # client.v8.chromium
      'Linux - Future (dbg)',
      # chromium.win
      'Win10 Tests x64 (dbg)',
    ],
  },
  'telemetry_unittests': {
    'modifications': {
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,  # crbug.com/870673
      }
    }
  },
  'unit_tests': {
    'modifications': {
      # chromium.clang
      'ToTLinuxASan': {
        'swarming': {
          'shards': 2,
        },
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 4,
        },
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 2,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 2,
        },
      },
      # chromium.swarm
      'Android N5X Swarm': {
        'swarming': {
          'shards': 8,
        },
      },
    },
  },
  'video_decode_accelerator_gl_unittest': {
    'remove_from': [
      # chromium.gpu.fyi
      # Windows Intel doesn't have the GL extensions to support this test.
      'Optional Win10 Release (Intel HD 630)',
      'Win10 FYI Release (Intel HD 630)',
      'Win10 FYI Exp Release (Intel HD 630)',
    ],
  },
  'viz_browser_tests': {
    'modifications': {
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 20,
        },
      },
      # chromium.memory
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 20,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 30,
        },
      },
    },
  },
  'viz_content_browsertests': {
    'modifications': {
      # chromium.memory
      'Linux Chromium OS ASan LSan Tests (1)': {
        # content_browsertests is slow on ASAN try bot. crbug.com/822461.
        'swarming': {
          'shards': 4,
        }
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'viz_unittests': {
    'remove_from': [
      # chromium.win
      'Win10 Tests x64',
    ],
  },
  'vr_android_unittests': {
    'remove_from': [
      # chromium.android.fyi
      'x64 Device Tester',
      'x86 Cloud Tester',
    ],
  },
  'vr_pixeltests': {
    'remove_from': [
      # chromium.fyi
      'VR Linux',
    ],
  },
  'wayland_client_perftests': {
    'remove_from': [
      'linux-chromeos-dbg',  # https://crbug.com/859307
    ],
  },
  'webgl2_conformance_tests': {
    'remove_from': [
      # The Mac NVIDIA Retina bots don't have the capacity to run
      # this test suite on mac_optional_gpu_tests_rel.
      'Optional Mac Retina Release (NVIDIA)',
    ],
    'modifications': {
      'Mac FYI GPU ASAN Release': {
        'args': [
          '--is-asan',
        ]
      },
    },
  },
  'webgl_conformance_gl_passthrough_tests': {
    'remove_from': [
      # crbug.com/555545 and crbug.com/649824:
      # Disable webgl_conformance_gl_tests on some Win/AMD cards.
      # Always fails on older cards, flaky on newer cards.
      'Win7 FYI Debug (AMD)',
      'Win7 FYI Release (AMD)',
    ],
  },
  'webgl_conformance_tests': {
    'modifications': {
      # On desktop platforms these don't take very long (~7 minutes),
      # but on Android they take ~30 minutes and we want to use more
      # shards than on desktop.
      # chromium.gpu
      'Android Release (Nexus 5X)': {
        'swarming': {
          'shards': 6,
        },
      },
      'Mac FYI GPU ASAN Release': {
        'args': [
          '--is-asan',
        ]
      },
    },
  },
  'webgl_conformance_vulkan_passthrough_tests': {
    'remove_from': [
      'Win7 FYI Debug (AMD)',
    ],
  },
  'webkit_layout_tests': {
    'remove_from': [
      # chromium.fyi
      'Chromium Mac 10.13',
      # chromium.linux
      'Linux Tests (dbg)(1)(32)', # 32-bit linux is unsupported
      # chromium.win
      'Win 7 Tests x64 (1)',  # 64-bit win is unsupported by the layout tests.
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      # chromium.android
      'KitKat Phone Tester (dbg)': {
        'args': [
          '--debug',
          '--android',
          # TODO(crbug.com/912120): Breakpad was entirely disabled on
          # Android. Remove this when it no longer affects Android crash
          # log handling.
          '--disable-breakpad',
          # TODO(crbug.com/875172): run on the real GPU since the
          # software fallback, SwiftShader, isn't yet for 32-bit ARM.
          '--additional-driver-flag=--use-gpu-in-tests',
        ],
      },
      # chromium.fyi
      'linux-blink-animation-use-time-delta': {
        'args': [
          '--debug',
        ],
      },
      'linux-blink-gen-property-trees': {
        'args': [
          '--debug',
          '--additional-driver-flag=--enable-blink-features=BlinkGenPropertyTrees',
        ],
      },
      'linux-blink-heap-incremental-marking': {
        'args': [
          '--debug',
          '--additional-driver-flag=--enable-blink-features=HeapIncrementalMarkingStress',
        ],
      },
      'linux-blink-heap-unified-gc': {
        'args': [
          '--debug',
          '--additional-driver-flag=--enable-blink-features=HeapUnifiedGarbageCollection',
        ],
      },
      'mac10.10-blink-rel-dummy': {
        'swarming': {
          'dimension_sets': [
            { # This should match Mac10.10 Tests on chromium.mac.
              'gpu': 'none',
              'os': 'Mac-10.10',
            }
          ]
        },
      },
      'mac10.11-blink-rel-dummy': {
        'swarming': {
          'dimension_sets': [
            { # This should match Mac10.11 Tests on chromium.mac.
              'gpu': 'none',
              'os': 'Mac-10.11',
            }
          ]
        },
      },
      'mac10.12-blink-rel-dummy': {
        'swarming': {
          'dimension_sets': [
            { # This should match Mac10.12 Tests on chromium.mac.
              'gpu': '8086:0a2e',
              'hidpi': '0',
              'os': 'Mac-10.12',
            },
          ],
        },
      },
      'mac10.13-blink-rel-dummy': {
        'swarming': {
          'dimension_sets': [
            { # This should match Chromium Mac 10.13 on chromium.fyi.
              'gpu': '8086:0a2e',
              'hidpi': '0',
              'os': 'Mac-10.13',
            },
          ],
          'shards': 12,
        },
      },
      'WebKit Linux composite_after_paint Dummy Builder': {
        'args': [
          '--additional-driver-flag=--enable-blink-features=CompositeAfterPaint',
        ],
      },
      'WebKit Linux layout_ng Dummy Builder': {
        'args': [
          '--additional-driver-flag=--enable-blink-features=LayoutNG',
        ],
      },
      'WebKit Linux root_layer_scrolls Dummy Builder': {
        'args': [
          '--additional-driver-flag=--root-layer-scrolls',
        ],
      },
      # chromium.linux
      'Linux Tests': {
        'swarming': {
          'dimension_sets': [
            {
              # TODO(kbr): this is essentially redundant.
              'os': 'Ubuntu-14.04',
            },
          ],
          'shards': 6,
        },
      },
      'Linux Tests (dbg)(1)': {
        'args': [
          '--debug',
        ],
        'swarming': {
          'dimension_sets': [
            {
              # TODO(kbr): this is essentially redundant.
              'os': 'Ubuntu-14.04',
            },
          ],
          'shards': 20,
        },
      },
      # chromium.mac
      'Mac10.10 Tests': {
        # TODO(kbr): this is essentially redundant.
        'swarming': {
          'dimension_sets': [
            {
              'gpu': 'none',
              'os': 'Mac-10.10.5',
            },
          ],
        },
      },
      'Mac10.11 Tests': {
        # TODO(kbr): this is essentially redundant.
        'swarming': {
          'dimension_sets': [
            {
              'gpu': 'none',
              'os': 'Mac-10.11.6',
            },
          ],
        },
      },
      'Mac10.12 Tests': {
        # TODO(kbr): if the Swarming dimensions were explicitly specified for
        # all jobs then this wouldn't be needed.
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '8086:0a2e',
              'os': 'Mac-10.12.6',
            },
          ],
          'shards': 12,
        },
      },
      'Mac10.13 Tests': {
        # TODO(kbr): if the Swarming dimensions were explicitly specified for
        # all jobs then this wouldn't be needed.
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '8086:0a2e',
              'os': 'Mac-10.12.6',
            },
          ],
          'shards': 12,
          # TODO(crbug.com/853356): Switch this fully to 10.13.
          # Starting with a 20 minute expiration for now, we will reduce this
          # later.
          'optional_dimensions': {
            1200: [
              {
                'os': 'Mac-10.13',
              }
            ],
          },
        },
      },
      'Mac10.13 Tests (dbg)': {
        'args': [
          '--debug',
        ],
      },
      # chromium.webkit.
      'WebKit Linux Trusty ASAN': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/ASANExpectations',
          '--time-out-ms',
          '48000',
          '--enable-sanitizer',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Ubuntu-14.04',
            },
          ],
          'shards': 20,
        },
      },
      'WebKit Linux Trusty Leak': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/LeakExpectations',
          '--time-out-ms',
          '48000',
          '--enable-leak-detection',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Ubuntu-14.04',
            },
          ],
          'shards': 10,
        },
      },
      'WebKit Linux Trusty MSAN': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/MSANExpectations',
          '--time-out-ms',
          '66000',
          '--enable-sanitizer',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Ubuntu-14.04',
            },
          ],
          'expiration': 36000,
          'hard_timeout': 10800,
          'io_timeout': 3600,
          'shards': 20,
        },
      },
      'WebKit Win10': {
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Windows-10-15063',
            },
          ],
        },
      },
      # chromium.win
      'Win7 Tests (dbg)(1)': {
        'args': [
          '--debug',
          '--time-out-ms',
          '60000'
        ],
        'experiment_percentage': 100,
        # TODO(kbr): the "Windows-7-SP1" Swarming dimension set is implicit,
        # from prefered_os_dimension in
        # tools/build/scripts/slave/recipe_modules/swarming/api.py. Ideally,
        # Swarming dimensions would be either specified for all testers on the
        # waterfall (and therefore, explicitly specified for all tests in the
        # generated JSON), or this would be removed, and the implicit one used.
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Windows-7-SP1',
            }
          ],
        },
      },
      'Win10 Tests x64 (dbg)': {
        'args': [
          '--debug',
          '--target',
          'Debug_x64',
        ],
      },
      # tryserver.chromium.android
      'android_blink_rel': {
        'args': [
          '--release',
          '--android',
          # TODO(crbug.com/912120): Breakpad was entirely disabled on
          # Android. Remove this when it no longer affects Android crash
          # log handling.
          '--disable-breakpad',
          # TODO(crbug.com/875172): run on the real GPU since the
          # software fallback, SwiftShader, isn't yet for 32-bit ARM.
          '--additional-driver-flag=--use-gpu-in-tests',
        ],
      },
    },
  },
  'webkit_unit_tests': {
    'modifications': {
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'Marshmallow 64 bit Tester': {
        # Increase timeout. See https://crbug.com/847229.
        'swarming': {
          'hard_timeout': 1200,
        },
      },
    },
  },
  'webui_polymer2_browser_tests': {
    'remove_from': [
      # chromium.win
      'Win10 Tests x64 (dbg)',  # Matches browser_tests.
    ],
  },
  'webui_polymer2_interactive_ui_tests': {
    'modifications': {
      # Match interactive_ui_tests, see https://crbug.com/840369
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
      },
    },
  },
  'webview_instrumentation_test_apk': {
    'remove_from': [
      # This test frequently fails on Android, https://crbug.com/824959
      # chromium.android
      'android-kitkat-arm-rel',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
      # chromium.android.fyi
      # chromium.clang; matching chromium.android, the test fails on these.
      'ToTAndroid',
      # On chromium.swarm, this should possibly be being run (or removed
      # from Android N5X Swarm).
      'Android N5 Swarm',
    ],
    'modifications': {
      'Oreo Phone Tester': {
        # TODO(crbug.com/863089): Enable this once it's passing.
        # TODO(https://crbug.com/884413): Re-enable this once the tests are
        # either passing or there is more capacity.
        'experiment_percentage': 0,
      },
    },
  },
  'xr_browser_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # This exception probably needs to stay due to lack of capacity
      # on the Win AMD bots.
      'Win7 FYI Debug (AMD)',
      # Fails on the bot, http://crbug.com/868143
      'Win7 FYI Release (AMD)',
    ],
  },
}