testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'android_browsertests': {
    'remove_from': [
      # TODO(crbug.com/961849): Remove these exceptions as the suite proves stable.
      # chromium.android.json
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      # chromium.fyi.json
      'android-code-coverage',
    ],
  },
  'android_webview_unittests': {
    'remove_from': [
      # On chromium.android, these do not need to run prior to M.
      'android-kitkat-arm-rel',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
    ],
  },
  'angle_end2end_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # TODO(ynovikov) Investigate why the test breaks on older devices.
      'Android FYI Release (Nexus 5)',
      'Android FYI Release (Nexus 6)',
      'Android FYI Release (Nexus 9)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
      # Temporarily disabled due to bad NVIDIA driver upgrade crbug.com/950542
      'Linux FYI Debug (NVIDIA)',
      'Linux FYI GPU TSAN Release',
    ],
  },
  'angle_perftests': {
    'remove_from': [
      'Android FYI Release (Nexus 5)',  # crbug.com/915429
      'Android FYI Release (Nexus 6)',  # anglebug.com/2433
      'Linux FYI GPU TSAN Release',  # crbug.com/950542
    ],
  },
  'angle_unittests': {
    'remove_from': [
      # chromium.gpu.fyi
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'breakpad_unittests': {
    'modifications': {
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.breakpad_unittests.filter',
        ],
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      # chromium.memory
      'Linux TSan Tests',  # https://crbug.com/368525
      # chromium.win
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      # chromium.fyi
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 21,
        },
      },
      'linux-chromeos-google-rel': {
        # TODO(https://crbug.com/932269): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
      },
      'linux-chromeos-oobe-code-coverage': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.chromiumos
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 21,
        },
      },
      # chromium.clang
      'ToTMac': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'ToTMacASan': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac-dummy-rel': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'mac-osxbeta-rel': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      # chromium.mac
      'Mac10.10 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.11 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.12 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.13 Tests': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'Mac10.13 Tests (dbg)': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 30,
        },
      },
      'Mac ASan 64 Tests (1)': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 25,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 31,
        },
      },
      # chromium.win
      'Win7 Tests (1)': {
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 15,
        },
      },
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
        }
      },
      # client.v8.chromium
      'Linux - Future (dbg)': {
        'swarming': {
          'shards':  10,
        },
      },
    },
  },
  'cast_shell_browsertests': {
    'modifications': {
      'Cast Audio Linux': {
        'args': [
          '--enable-local-file-accesses',
          '--ozone-platform=headless',
          '--no-sandbox',
          '--test-launcher-jobs=1',
        ],
        'swarming': {
          'can_use_on_swarming_builders': False,  # https://crbug.com/861753
        },
      },
      'Cast Linux': {
        'args': [
          '--enable-local-file-accesses',
          '--ozone-platform=headless',
          '--no-sandbox',
          '--test-launcher-jobs=1',
        ],
        'swarming': {
          'can_use_on_swarming_builders': False,  # https://crbug.com/861753
        },
      },
    },
  },
  'checkbins': {
    'remove_from': [
      'linux-archive-dbg',
      'linux-archive-rel',
      'mac-archive-dbg',
      'mac-archive-rel',
    ],
  },
  'chrome_public_test_apk': {
    'remove_from': [
      'android-asan',  # https://crbug.com/964562
    ],
    'modifications': {
      # chromium.android
      'android-kitkat-arm-rel': {
        'swarming': {
          'shards': 12,
        },
      },
      'android-marshmallow-arm64-rel': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 0,
      },
      'KitKat Phone Tester (dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Marshmallow 64 bit Tester': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
        'swarming': {
          'shards': 14,
        },
      },
      'Oreo Phone Tester': {
        # TODO(https://crbug.com/884413): Re-enable this once the tests are
        # either passing or there is more capacity.
        'experiment_percentage': 0,
      },
      # chromium.swarm
      'Android N5X Swarm': {
        # TODO(crbug.com/731759): Enable this once we're confident that it
        # passes.
        'experiment_percentage': 100,
      },
    },
  },
  'chrome_public_test_vr_apk': {
    'remove_from': [
      # chromium.android
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      # chromium.clang
      'ToTAndroid',
      'ToTAndroidCFI',
      # chromium.memory
      'android-asan',  # https://crbug.com/964562
    ],
  },
  'chrome_public_test_vr_apk-vega': {
    'modifications': {
      # chromium.android
      'Oreo Phone Tester': {
        'swarming': {
          'dimension_sets': [
            {
              'device_os': 'OVR1.180808.003',
              'device_type': 'vega',
              'os': 'Android',
            },
          ],
        },
      }
    }
  },
  'chromedriver_py_tests': {
    'remove_from': [
      # Timeout happens sometimes (crbug.com/951799).
      'Win10 Tests x64',
      'Win10 Tests x64 (dbg)',
    ],
  },
  'components_unittests': {
    'modifications': {
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'linux-chromeos-google-rel': {
        # TODO(crbug.com/980748): Remove this filter.
        'args': [
          '--gtest_filter=-CrashAnalyzerTest.StackTraceCollection',
        ],
      },
    },
  },
  'content_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
    'modifications': {
      # chromium.android
      'Lollipop Phone Tester': {
        'swarming': {
          'shards': 13,
        },
      },
      # chromium.linux
      'Cast Audio Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      'Cast Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      # chromium.memory
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.content_browsertests.filter',
        ],
        'swarming': {
          'shards': 18,
        },
      },
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
      'Linux ASan Tests (sandboxed)': {
        'swarming': {
          'shards': 2, # https://crbug.com/955853
        },
      },
    },
  },
  'content_shell_crash_test': {
    'modifications': {
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,  # https://crbug.com/861730
      },
    },
  },
  'content_shell_test_apk': {
    'remove_from': [
      # chromium.memory
      'android-asan',  # https://crbug.com/964562
    ],
  },
  'content_unittests': {
    'modifications': {
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 2,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'context_lost_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      # chromium.gpu.fyi
      'Android FYI Release (Pixel 2)',
    ],
    'modifications': {
      'Mac FYI GPU ASAN Release': {
        'args': [
          '--is-asan',
        ]
      },
    },
  },
  'crashpad_tests': {
    'remove_from': [
      # chromium.clang
      'linux-win_cross-rel', # https://crbug.com/762167
    ],
  },
  'cronet_sizes': {
    'modifications': {
      'android-cronet-arm-dbg': {
        'swarming': {
          'dimension_sets': [
            {
              'device_os': None,
              'device_type': None,
              'os': 'Ubuntu-16.04',
            },
          ],
        },
      },
    },
  },
  'dawn_end2end_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      'Linux FYI Release (AMD R7 240)', # https://crbug.com/915430
    ],
  },
  'depth_capture_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      # chromium.gpu.fyi
      'Android FYI Release (Pixel 2)',
    ],
  },
  'extensions_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
    'modifications': {
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100, # https://crbug.com/876615
      },
    },
  },
  'gin_unittests': {
    'remove_from': [
      # chromium.clang
      'ToTLinuxASan',  # https://crbug.com/831667
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # https://crbug.com/831667
      'Linux Chromium OS ASan LSan Tests (1)',  # https://crbug.com/831667
    ],
  },
  'gl_tests': {
    'remove_from': [
      # chromium.clang
      'CFI Linux ToT',
      # chromium.gpu.fyi
      'Android FYI Release (Nexus 5X)',
      # chromium.memory
      # Can't run on MSAN because gl_tests uses the hardware driver,
      # which isn't instrumented.
      'Linux MSan Tests',
    ],
    'modifications': {
      # chromium.memory
      # TODO(kbr): figure out a better way to specify blocks of
      # arguments like this for tests on multiple machines.
      'Linux ASan LSan Tests (1)': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      'Linux ASan LSan Tests (1)': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      'Linux CFI': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      'Linux TSan Tests': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
    },
  },
  'gl_unittests': {
    'remove_from': [
      # chromium.gpu.fyi
      # On Android, these are already run on the main waterfall.
      # Run them on the one-off Android FYI bots, though.
      'Android FYI Release (Nexus 5X)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
    ],
    'modifications': {
      # chromium.clang
      'CFI Linux ToT': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
      # chromium.memory
      'Linux CFI': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'Chrome-GPU',
            },
          ],
        },
        'use_xvfb': False,
      },
    },
  },
  'gl_unittests_ozone': {
    'remove_from': [
      # Can't run on MSAN because gl_unittests_ozone uses the hardware driver,
      # which isn't instrumented.
      'Linux ChromiumOS MSan Tests',
    ],
  },
  'gpu_process_launch_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      # chromium.gpu.fyi
      'Android FYI Release (Pixel 2)',
    ],
  },
  'hardware_accelerated_feature_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      # chromium.gpu.fyi
      'Android FYI Release (Pixel 2)',
    ],
  },
  'headless_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
  },
  'info_collection_tests': {
    'remove_from': [
      # chromium.gpu.fyi
      # The Mac ASAN swarming bot runs tests on two different GPU
      # types, so we can't have one expected vendor ID / device ID.
      'Mac FYI GPU ASAN Release',
      # client.v8.fyi
      'Android V8 FYI Release (Nexus 5X)',
      'Linux V8 FYI Release (NVIDIA)',
      'Linux V8 FYI Release - pointer compression (NVIDIA)',
      'Mac V8 FYI Release (Intel)',
      'Win V8 FYI Release (NVIDIA)',
    ],
  },
  'interactive_ui_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      # chromium.fyi
      # Not eough disk space in the VMs, crbug.com/913153
      'chromeos-vm-code-coverage',
    ],
    'modifications': {
      # chromium.clang
      'ToTLinuxTSan': {
        # These are slow on the TSan bots for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      # chromium.win
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 4,
        },
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASan trybot for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      'Linux TSan Tests': {
        # These are slow on the TSan bots for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 5,
        },
      },
      # client.v8.chromium
      'Linux - Future (dbg)': {
        'swarming': {
          'shards': 3,
        },
      },
      # chromium.win; temporary, https://crbug.com/818832
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
      },
      'linux-chromeos-google-rel': {
        # TODO(crbug.com/970649): Remove this filter.
        'args': [
          '--gtest_filter=-SadTabViewInteractiveUITest.ReloadMultipleSadTabs',
        ],
      },
    },
  },
  'ipc_tests': {
    'modifications': {
      'android-asan': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'maps_pixel_test': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      # chromium.gpu.fyi
      'Android FYI Release (Pixel 2)',
    ],
  },
  'mojo_unittests': {
    'modifications': {
      'android-asan': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'monochrome_public_test_ar_apk': {
    'modifications': {
      # chromium.android
      # We need to match the Playstore version as well because AR tests fail on
      # old versions of the Playstore.
      'Oreo Phone Tester': {
        'args': [
          # ArCore is not installed as a system app on O, so we can install it
          # normally instead of replacing the system version.
          '--additional-apk=//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ],
        'swarming': {
          'dimension_sets': [
            {
              'device_os': 'OPR3.170623.008',
              'device_playstore_version': '12.8.23-all',
              'device_type': 'marlin',
              'os': 'Android',
            },
          ],
        },
      },
      'android-pie-arm64-dbg': {
        'args': [
          '--replace-system-package=com.google.ar.core,//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ]
      }
    },
  },
  'nacl_helper_nonsfi_unittests': {
    'remove_from': [
      # chromium.memory
      'Linux TSan Tests',  # Sets enable_nacl=false.
    ],
  },
  'nacl_loader_unittests': {
    'remove_from': [
      # chromium.clang
      'ToTMacASan',  # The mac asan tot bot sets enable_nacl=false
      'linux-win_cross-rel',  # Sets enable_nacl=false, https://crbug.com/774186
      # chromium.memory
      'Linux TSan Tests',  # The TSan bot sets enable_nacl=false
      'Mac ASan 64 Tests (1)',  # The mac asan bot sets enable_nacl=false
    ],
  },
  'net_unittests': {
    'remove_from': [
      # chromium.fyi
      'fuchsia-fyi-arm64-rel',  # https://crbug.com/877248
      'fuchsia-fyi-x64-dbg',  # https://crbug.com/844416
    ],
    'modifications': {
      # chromium.clang
      'ToTLinuxASan': {
        # TODO(crbug.com/794372): net_unittests is slow under ASan.
        'swarming': {
          'shards': 16,
        },
      },
      'ToTLinuxTSan': {
        'swarming': {
          'shards': 4,
        }
      },
      # chromium.memory
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 16,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 8,
        }
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 4,
        }
      },
      # The suite runs signficantly slower on linux dbg, so increase shards.
      'Linux Tests (dbg)(1)': {
        'swarming': {
          'shards': 2,
        },
      },
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'network_service_browser_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      'ToTLinuxTSan',  # https://crbug.com/368525
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches browser_tests.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'ToTMacASan',
    ],
    'modifications': {
      # chromium.win
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 20,
        },
      },
    },
  },
  'network_service_components_browsertests': {
    'remove_from': [
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches components_browsertests.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'ToTMacASan',
    ],
  },
  'network_service_content_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      'ToTLinuxTSan',  # Match 'Linux TSan Tests' below since this is slow.
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # flaky: https://crbug.com/852786
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'ToTMacASan',
    ],
  },
  'network_service_extensions_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches extensions_browsertests.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'ToTMacASan',
    ],
  },
  'network_service_headless_browsertests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
  },
  'network_service_interactive_ui_tests': {
    'remove_from': [
      # chromium.clang
      'CrWinAsan(dll)', # https://crbug.com/935598
      'ToTLinuxTSan',  # Match 'Linux TSan Tests' below since this is slow.
      # chromium.memory
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      # chromium.win
      'Win7 Tests (1)',  # For swarming capacity reasons.
      # Only run network service tests on Mac 10.12 for capacity reasons.
      'Mac10.10 Tests',
      'Mac10.11 Tests',
      'Mac10.13 Tests',
      'Mac10.13 Tests (dbg)',
      'Mac ASan 64 Tests (1)',
      'Mac ASan 64 Tests (1)',
      'ToTMacASan',
    ],
    'modifications': {
      # chromium.win
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,  # https://crbug.com/852786
      },
    },
  },
  'network_service_out_of_process_chrome_public_test_apk': {
    'remove_from': [
      # chromium.android
      'android-asan',
      # Don't run on trybots; waterfall is enough.
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      # No capacity for tablet testers.
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
    ],
  },
  'network_service_out_of_process_components_browsertests': {
    'remove_from': [
      # chromium.android
      'android-asan',
      # Don't run on trybots; waterfall is enough.
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      # No capacity for tablet testers.
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
    ],
  },
  'network_service_out_of_process_content_browsertests': {
    'remove_from': [
      # chromium.android
      'android-asan',
      # Don't run on trybots; waterfall is enough.
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      # No capacity for tablet testers.
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
    ],
  },
  'network_service_out_of_process_content_shell_test_apk': {
    'remove_from': [
      # chromium.android
      'android-asan',
      # Don't run on trybots; waterfall is enough.
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      # No capacity for tablet testers.
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
    ],
  },
  'network_service_webview_instrumentation_test_apk': {
    # Only run these on the phone testers for now due to capacity.
    'remove_from': [
      # chromium.android
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      'KitKat Phone Tester (dbg)',
      # chromium.memory
      'android-asan',  # https://crbug.com/964562
      'Android CFI',
      # chromium.clang
      'ToTAndroid',
      'ToTAndroidCFI',
    ],
  },
  'non_network_service_content_browsertests': {
    'remove_from': [
      # chromium.android
      'android-asan',
      # Don't run on trybots; waterfall is enough.
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      # No capacity for tablet testers.
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
    ],
  },
  'non_network_service_content_shell_test_apk': {
    'remove_from': [
      # chromium.android
      'android-asan',
      # Don't run on trybots; waterfall is enough.
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      # No capacity for tablet testers.
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
    ],
  },
  'non_network_service_webkit_layout_tests' : {
    'remove_from': [
      # chromium.mac. TODO(https://crbug.com/944117): Remove this.
      'mac-dummy-rel',
    ],
    'modifications': {
      'Linux Tests Code Coverage': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%8m.profraw",
        ],
      },
      'Linux Release Code Coverage (NVIDIA)': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%8m.profraw",
        ],
      },
      'Linux Tests (dbg)(1)': {
        'args': [
          '--debug',
        ],
        'swarming': {
          'shards': 20,
        },
      },
    },
  },
  'nonperfetto_content_browsertests': {
    'remove_from': [
      'android-asan',
    ],
  },
  'not_site_per_process_webkit_layout_tests': {
    'remove_from': [
      # chromium.linux
      # TODO(dpranke): Should we be running this step on Linux Tests (dbg)(1)?
      'Linux Tests (dbg)(1)',

      # chromium.mac. TODO(https://crbug.com/944117): Remove this.
      'mac-dummy-rel',
    ],
    'modifications': {
      'Linux Tests Code Coverage': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%8m.profraw",
        ],
      },
      'Linux Release Code Coverage (NVIDIA)': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%8m.profraw",
        ],
      },
    },
  },
  'notification_helper_unittests': {
    'remove_from': [
      # The test uses WinRT, which only exists in Windows 8 or above.
      # (https://crbug.com/813553)
      # chromium.win
      'Win 7 Tests x64 (1)',
      'Win7 Tests (1)',
      'Win7 Tests (dbg)(1)',
      # chromium.clang; all these bots run Windows 7.
      'ToTWin',
      'ToTWin(dbg)',
      'ToTWin(dll)',
      'ToTWinOfficial',
      'ToTWin64',
      'ToTWin64(dbg)',
      'ToTWin64(dll)',
      'ToTWinCFI',
      'ToTWinCFI64',
      'ToTWinLibcxx64',
      'ToTWinThinLTO64',
    ],
  },
  'ozone_unittests': {
    'modifications': {
      'chromeos-betty-google-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.ozone_unittests.filter',
        ],
      },
      'linux-chromeos-google-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.ozone_unittests.filter',
        ],
      },
    },
  },
  'perfetto_unittests': {
    'remove_from': [
      'android-asan',
      'android-code-coverage',
      'android-kitkat-arm-rel',
      'android-marshmallow-arm64-rel',
      'Android CFI',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
      'Lollipop Phone Tester',
      'Lollipop Tablet Tester',
      'Marshmallow 64 bit Tester',
      'Marshmallow Tablet Tester',
      'Nougat Phone Tester',
      'ToTAndroid',
      'ToTAndroidCFI'
    ]
  },
  'performance_test_suite': {
    'modifications': {
      'Mac10.13 Tests (dbg)': {
        'args': [
          '--browser=debug',
        ],
      },
      'Win10 Tests x64 (dbg)': {
        'args': [
          '--browser=debug_x64',
        ],
        'experiment_percentage': 100,
      },
      'Win7 Tests (dbg)(1)': {
        'args': [
          '--browser=debug',
        ],
      },
    },
  },
  'pixel_skia_gold_test': {
    'replacements': {
      # client.v8.fyi
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Android V8 FYI Release (Nexus 5X)': {
        'args': {
          '--build-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release (NVIDIA)': {
        'args': {
          '--build-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release - pointer compression (NVIDIA)': {
        'args': {
          '--build-revision': '${got_cr_revision}',
        },
      },
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--build-revision': '${got_cr_revision}',
        },
      },
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--build-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'pixel_test':{
    'modifications': {
      'Android Release (Nexus 5X)': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'sandbox_linux_unittests': {
    'modifications': {
      'Lollipop Phone Tester': {
        'args': [
          '--shard-timeout=300',
        ],
      },
    },
    'remove_from': [
      # chromium.memory
      'android-asan',  # https://crbug.com/962650
    ],
  },
  'services_unittests': {
    'modifications': {
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.services_unittests.filter',
        ],
      },
    },
    'remove_from': [
      # chromium.clang
      'ToTLinuxMSan',  # https://crbug.com/831676
      # chromium.gpu.fyi
      # The face and barcode detection tests fail on the Mac Pros.
      'Mac Pro FYI Release (AMD)',
      # chromium.linux
      'Fuchsia x64',
      # chromium.memory