testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'android_browsertests': {
    'modifications': {
      'android-pie-arm64-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
      },
      'android-pie-x86-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
      },
    },
  },
  'android_webview_unittests': {
    'remove_from': [
      # On chromium.android, these do not need to run prior to M.
      'android-lollipop-arm-rel',
      'android-code-coverage-native', # crbug/1018434
    ],
  },
  'android_wpr_record_replay_tests': {
    # crbug/1060399: only enable wpr tests on marshmallow CI bot
    # due to the restriction that render test only works on marshamllow.
    'modifications': {
      'Marshmallow 64 bit Tester': {
        'experiment_percentage': 100, # https://crbug.com/1060399
      },
    },
  },
  'angle_end2end_tests': {
    'remove_from': [
      # TODO(ynovikov) Investigate why the test breaks on older devices.
      'Android FYI Release (Nexus 5)',
      'Android FYI Release (Nexus 6)',
      'Android FYI Release (Nexus 9)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
      # Temporarily disabled due to bad NVIDIA driver upgrade crbug.com/950542
      'Linux FYI Debug (NVIDIA)',
      'Linux FYI GPU TSAN Release',
      # Disabling on RX 550 as we are currently qualifying the card and not
      # checking for regressions. crbug.com/1025745
      'Win10 FYI x64 Release (AMD RX 550)',
    ],
  },
  'angle_perftests': {
    'remove_from': [
      'Android FYI Release (Nexus 5)',  # crbug.com/915429
      'Android FYI Release (Nexus 6)',  # anglebug.com/2433
      'Linux FYI GPU TSAN Release',  # crbug.com/950542
    ],
  },
  'angle_unittests': {
    'remove_from': [
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'angle_white_box_tests': {
    'remove_from': [
      # https://crbug.com/1035178
      'Win10 FYI x64 Release (AMD RX 550)',
      # https://crbug.com/1035461
      'Linux FYI GPU TSAN Release',
    ],
  },
  'blink_platform_unittests': {
    'modifications': {
      # TODO(crbug.com/1108121): Remove this filter
      'android-nougat-arm64-rel': {
        'args': [
          '--gtest_filter=-HarfBuzzShaperTest.ShapeHorizontalWithoutSubpixelPositionWithKerningIsRounded',
        ],
      },
    },
  },
  'blink_web_tests': {
    'remove_from': [
      'Win 7 Tests x64 (1)',  # 64-bit win is unsupported by the layout tests.
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      'Fuchsia x64': {
        'args': [
          '--platform=fuchsia',
        ],
        'swarming': {
          'shards': 1,
        },
      },
      'Linux Tests': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw",
        ],
      },
      'Linux Tests (dbg)(1)': {
        'args': [
          '--debug',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'Mac10.10 Tests': {
        # TODO(kbr): this is essentially redundant.
        'swarming': {
          'dimension_sets': [
            {
              'gpu': 'none',
              'os': 'Mac-10.10.5',
            },
          ],
          'shards': 25,
        },
      },
      'Mac10.11 Tests': {
        # TODO(kbr): this is essentially redundant.
        'swarming': {
          'dimension_sets': [
            {
              'gpu': 'none',
              'os': 'Mac-10.11.6',
            },
          ],
          'shards': 25,
        },
      },
      'Mac10.12 Tests': {
        'swarming': {
          'shards': 25,
        },
      },
      'Mac10.13 Tests': {
        'swarming': {
          'dimension_sets': [
            {
              # Run the layout tests on a consistent set of hardware
              # (Mac Minis).
              'gpu': '8086:0a2e',
            },
          ],
          'shards': 25,
        },
      },
      'Mac10.13 Tests (dbg)': {
        'args': [
          '--debug',
        ],
        'swarming': {
          'shards': 25,
        },
      },
      'V8 Blink Linux Debug': {
        # This builder compiles Chromium in release mode, but V8 in debug.
        # TODO(machenbach): Change name and out directory of this builder to
        # something less confusing.
        'args': [
          '-t',
          'Debug',
          '--release',
        ],
      },
      'V8 Blink Linux Future': {
        'args': [
          '--additional-driver-flag=--js-flags=--future',
        ],
      },
      'WebKit Linux ASAN': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/ASANExpectations',
          '--time-out-ms',
          '48000',
          '--enable-sanitizer',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'WebKit Linux Leak': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/LeakExpectations',
          '--time-out-ms',
          '48000',
          '--enable-leak-detection',
        ],
        'swarming': {
          'shards': 10,
        },
      },
      'WebKit Linux MSAN': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/MSANExpectations',
          '--time-out-ms',
          '66000',
          '--enable-sanitizer',
        ],
        'swarming': {
          'expiration': 36000,
          'hard_timeout': 10800,
          'io_timeout': 3600,
          'shards': 20,
        },
      },
      'WebKit Linux composite_after_paint Dummy Builder': {
        'args': [
          '--additional-driver-flag=--enable-blink-features=CompositeAfterPaint',
        ],
      },
      'WebKit Linux layout_ng_disabled Builder': {
        'args': [
          '--additional-driver-flag=--disable-blink-features=LayoutNG',
        ],
      },
      'Win10 Tests x64': {
        'args': [
          '--target',
          'Release_x64',
        ],
        'swarming': {
          "shards": 28
        },
      },
      'Win10 Tests x64 (dbg)': {
        'args': [
          '--debug',
          '--target',
          'Debug_x64',
        ],
      },
      'Win10 Tests x64 Code Coverage': {
        'args': [
          '--target',
          'Release_x64',
        ],
        'swarming': {
          "shards": 28
        },
      },
      'Win7 Tests (dbg)(1)': {
        'args': [
          '--debug',
          '--time-out-ms',
          '60000'
        ],
        'experiment_percentage': 100,
        # TODO(kbr): the "Windows-7-SP1" Swarming dimension set is implicit,
        # from prefered_os_dimension in
        # tools/build/scripts/slave/recipe_modules/swarming/api.py. Ideally,
        # Swarming dimensions would be either specified for all testers on the
        # waterfall (and therefore, explicitly specified for all tests in the
        # generated JSON), or this would be removed, and the implicit one used.
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Windows-7-SP1',
            }
          ],
        },
      },
      # tryserver.chromium.android
      'android_blink_rel': {
        'args': [
          '--release',
          '--android',
          # TODO(crbug.com/912120): Breakpad was entirely disabled on
          # Android. Remove this when it no longer affects Android crash
          # log handling.
          '--disable-breakpad',
          # TODO(crbug.com/875172): run on the real GPU since the
          # software fallback, SwiftShader, isn't yet for 32-bit ARM.
          '--additional-driver-flag=--use-gpu-in-tests',
        ],
      },
      'devtools_frontend_linux_blink_light_rel': {
        'args': [
          'http/tests/devtools',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'fuchsia-fyi-x64-rel': {
        'args': [
          '--platform=fuchsia',
        ],
        'swarming': {
          'shards': 1,
        },
      },
      'linux-autofill-assistant': {
        'args': [
          '--release',
        ],
      },
      'linux-bfcache-rel': {
        'args': [
          '--additional-driver-flag=--enable-features=BackForwardCache',
          '--additional-expectations=../../third_party/blink/web_tests/FlagExpectations/enable-features=BackForwardCache',
        ],
      },
      'linux-blink-animation-use-time-delta': {
        'args': [
          '--debug',
        ],
      },
      'linux-blink-cors-rel': {
        'args': [
          '--additional-driver-flag=--disable-features=OutOfBlinkCors',
          '--additional-expectations=../../third_party/blink/web_tests/FlagExpectations/disable-features=OutOfBlinkCors',
        ],
      },
      'linux-blink-heap-concurrent-marking-tsan-rel': {
        'args': [
          '--release',
          '--additional-driver-flag=--enable-features=BlinkHeapConcurrentMarking',
        ],
      },
      'linux-code-coverage': {
        'args': [
          '--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'linux-layout-tests-edit-ng': {
        'args': [
          '--additional-driver-flag=--enable-blink-features=EditingNG',
        ],
      },
      'linux-layout-tests-fragment-item': {
        'args': [
          '--additional-driver-flag=--enable-blink-features=LayoutNGFragmentItem',
        ],
      },
      'mac10.10-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.11-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.12-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.13-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.13_retina-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.14-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.15-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
    },
  },
  'boringssl_crypto_tests': {
    'remove_from': [
      # https://crbug.com/1054545
      'Fuchsia ARM64',
    ],
  },
  'boringssl_ssl_tests': {
    'remove_from': [
      # https://crbug.com/1054545
      'Fuchsia ARM64',
    ],
  },
  'breakpad_unittests': {
    'modifications': {
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.breakpad_unittests.filter',
        ],
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      'Mac10.10 Tests',  # https://crbug.com/828031
      'Mac10.13 Tests',  # https://crbug.com/1042757
      'Linux TSan Tests',  # https://crbug.com/368525
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      'CrWinAsan': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 40,
        },
      },
      'CrWinAsan(dll)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 40,
        },
      },
      'Linux - Future (dbg)': {  # client.v8.chromium
        'swarming': {
          'shards': 10,
        },
      },
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 40,
        },
      },
      'Linux ASan Tests (sandboxed)': {
        'swarming': {
          'shards': 30, # https://crbug.com/1103330
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 31,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 32,
        },
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.msan.browser_tests.oobe_negative.filter',
        ],
      },
      'Linux Tests (dbg)(1)': {
        # crbug.com/1066161
        'swarming': {
          'shards': 20,
        },
      },
      'Mac ASan 64 Tests (1)': {
        'swarming': {
          'dimension_sets': [
            {
              # These shards are liable to time out when running on minis.
              'cores': '8',
            }
          ],
          'shards': 20,
        },
      },
      'Mac10.13 Tests (dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
        }
      },
      'Win10 Tests x64 Code Coverage': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
        }
      },
      'Win7 Tests (1)': {
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 15,
        },
      },
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'linux-chromeos-chrome': {
        # TODO(https://crbug.com/932269): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
      },
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 50,
        },
      },
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 30,
        },
      },
      'linux-chromeos-rel': {
        'swarming': {
          'shards': 20,
          'dimension_sets': [
            {
              'kvm': '1',
            },
          ],
        },
      },
      'linux-code-coverage': {
        'args': [
          '--no-sandbox',
        ],
        'swarming': {
          'shards': 50,
        },
      },
      'linux-trusty-rel': {
       'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/linux_trusty_rel.browser_tests.filter',
        ],
      },
      'mac-code-coverage': {
        'swarming': {
          'shards': 15,
        },
      },
      'win-asan': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 40,
        },
      },
    },
  },
  'browser_tests_wayland': {
    'modifications': {
      'Linux Ozone Tester (Wayland)': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/ozone-linux.wayland_browser_tests.filter',
        ],
      },
    },
  },
  'browser_tests_x11': {
    'modifications': {
      'Linux Ozone Tester (X11)': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/ozone-linux.x11_browser_tests.filter',
        ],
      },
    },
  },
  'cc_unittests': {
    'modifications': {
      'Linux TSan Tests': {
        'swarming': {
          'shards': 3,
        },
      },
      'android-marshmallow-x86-rel': {
        # https://crbug.com/1039860
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.cc_unittests.filter',
        ],
      },
      'android-pie-x86-rel': {
        # https://crbug.com/1039860
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.cc_unittests.filter',
        ],
      },
    },
  },
  'checkbins': {
    'remove_from': [
      'linux-archive-dbg',
      'linux-archive-rel',
      'mac-archive-dbg',
      'mac-archive-rel',
    ],
  },
  'chrome_java_test_pagecontroller_tests': {
    'modifications': {
      'android-pie-arm64-rel': {
        'experiment_percentage': 100, # https://crbug.com/924194
      },
    },
  },
  'chrome_public_smoke_test': {
    'remove_from': [
      'android-code-coverage-native', # https://crbug.com/1018780
    ],
  },
  'chrome_public_test_apk': {
    'remove_from': [
      'android-asan',  # https://crbug.com/964562
      'android-code-coverage-native', # https://crbug.com/1018780
      'android-lollipop-arm-rel', # https://crbug.com/1060245
      'android-marshmallow-x86-rel', # crbug.com/1098111
      'android-pie-arm64-rel', # https://crbug.com/1108071
    ],
    'modifications': {
      'Android CFI': {
        'args': [
          '--exclude-annotation',
          'Feature=RenderTest', # https://crbug.com/1068294
        ],
      },
      'Marshmallow 64 bit Tester': {
        'swarming': {
          'shards': 14,
        },
      },
      'Marshmallow Tablet Tester': {
        'args': [
          '--timeout-scale',
          '2.0',
        ],
      },
      'Oreo Phone Tester': {
        # TODO(https://crbug.com/884413): Re-enable this once the tests are
        # either passing or there is more capacity.
        'experiment_percentage': 0,
      },
      'android-arm64-proguard-rel': {
        'swarming': {
          'shards': 25,
        },
      },
      'android-lollipop-arm-rel-swarming': {
        'swarming': {
          'service_account': 'chrome-gold-dev@chops-service-accounts.iam.gserviceaccount.com'
        },
      },
      'android-marshmallow-arm64-rel': {
        'swarming': {
          'shards': 25,
        },
      },
      'android-marshmallow-arm64-rel-swarming': {
        'swarming': {
          'service_account': 'chrome-gold-dev@chops-service-accounts.iam.gserviceaccount.com'
        },
      },
      'android-marshmallow-x86-fyi-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_m.chrome_public_test_apk.filter',
          '-v', # TODO(crbug.com/1093602) Remove once the issue is resolved
        ],
      },
      'android-pie-arm64-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.pie_arm64_rel.chrome_public_test_apk.filter', # https://crbug.com/1010211
        ],
      },
      'android-pie-x86-rel': {
        'args': [
          '--avd-config=../../tools/android/avd/proto/generic_playstore_android28.textpb',
          # https://crbug.com/1046059
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_p.chrome_public_test_apk.filter',
        ],
        'swarming': {
          'named_caches': [
            {
              'name': 'avd_generic_playstore_android28',
              'path': '.android',
            },
            {
              'name': 'system_images_android_28_google_apis_playstore_x86',
              'path': '.emulator_sdk',
            },
          ]
        },
      },
    },
  },
  'chrome_public_test_vr_apk': {
    'remove_from': [
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      'android-marshmallow-x86-rel',
      'android-pie-x86-rel',
      'android-pie-arm64-rel', # https://crbug.com/1010211
      'ToTAndroid',
      'ToTAndroidCFI',
      'android-asan',  # https://crbug.com/964562
      'android-code-coverage-native', # https://crbug.com/1018780
    ],
    'modifications': {
      # Use "--replace-system-package" according to crbug.com/931947#c1
      'android-nougat-arm64-rel': {
        'args': [
          '--replace-system-package=com.google.vr.vrcore,//third_party/gvr-android-sdk/test-apks/vr_services/vr_services_current.apk'
        ]
      }
    },
  },
  'chrome_public_test_vr_apk-vega': {
    'modifications': {
      'Oreo Phone Tester': {
        'swarming': {
          'dimension_sets': [
            {
              'device_os': 'OVR1.180808.003',
              'device_type': 'vega',
              'os': 'Android',
            },
          ],
        },
      }
    }
  },
  'chromedriver_py_tests': {
    'remove_from': [
      # Timeout happens sometimes (crbug.com/951799).
      'Win10 Tests x64 (dbg)',
    ],
  },
  'components_browsertests': {
    'remove_from': [
      'android-code-coverage-native', # crbug/1018434
    ],
  },
  'components_unittests': {
    'remove_from': [
      'Fuchsia ARM64',  # https://crbug.com/961457
      'Fuchsia x64', # https://crbug.com/961457
      'android-code-coverage-native', # crbug/1018431
      'fuchsia-fyi-arm64-dbg',  # https://crbug.com/961457
      'fuchsia-fyi-arm64-rel',  # https://crbug.com/961457
      'fuchsia-fyi-x64-dbg',  # https://crbug.com/961457
      'fuchsia-fyi-x64-rel',  # https://crbug.com/961457
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 5,
        },
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 2,
        },
      },
      'linux-blink-heap-concurrent-marking-tsan-rel': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'content_browsertests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
      'android-code-coverage-native', # crbug/1018434
      # TODO(crbug.com/1102974) Re-enable on L once it is not overloaded.
      'android-lollipop-arm-rel',
      # http://crbug.com/1060245#c30: due to low utility and capacity.
      'android-marshmallow-arm64-rel',
      'android-marshmallow-x86-rel', # crbug.com/1098111
    ],
    'modifications': {
      'Cast Audio Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      'Cast Linux': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/cast-linux.content_browsertests.filter',
        ],
      },
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 12,
        },
      },
      'Linux ASan Tests (sandboxed)': {
        'swarming': {
          'shards': 10, # https://crbug.com/1103330
        },
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 15,
        },
      },
      'Lollipop Phone Tester': {
        'swarming': {
          'shards': 13,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
      'Win10 Tests x64 Code Coverage': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
      },
      'android-arm64-proguard-rel': {
        'swarming': {
          'shards': 16,
        },
      },
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.content_browsertests.filter',
        ],
        'swarming': {
          'shards': 18,
        },
      },
      'android-lollipop-arm-rel': {
        'swarming': {
          'shards': 15,
        },
      },
      'android-marshmallow-arm64-rel': {
        'swarming': {
          'shards': 16,
        },
      },
      'android-marshmallow-x86-fyi-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_m.content_browsertests.filter',
          '-v', # TODO(crbug.com/1093602) Remove once the issue is resolved
        ],
      },
      'android-nougat-arm64-rel': {
        'swarming': {
          'shards': 16,
        },
        # TODO(crbug.com/1108055): Remove this filter
        'args': [
          '--gtest_filter=-BackgroundTracingManagerBrowserTest.PerfettoSystemBackgroundScenario*',
        ],
      },
      'android-pie-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_p.content_browsertests.filter',
        ],
      },
      'linux-blink-heap-concurrent-marking-tsan-rel': {
        'swarming': {
          'shards': 4,
        },
      },
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 12,
        },
      },
      'linux-chromeos-rel': {
        'swarming': {
          'shards': 3,
        },
      },
      'linux-code-coverage': {
        'args': [
          '--no-sandbox',
        ],
        'swarming': {
          'shards': 12,
        },
      },
    },
  },
  'content_browsertests_wayland': {
    'modifications': {
      'Linux Ozone Tester (Wayland)': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/ozone-linux.wayland_content_browsertests.filter',
        ],
      },
    },
  },
  'content_browsertests_x11': {
    'modifications': {
      'Linux Ozone Tester (X11)': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/ozone-linux.x11_content_browsertests.filter',
        ],
      },
    },
  },
  'content_shell_crash_test': {
    'remove_from': [
      'android-marshmallow-x86-rel', # crbug.com/1084353
    ],
    'modifications': {
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,  # https://crbug.com/861730
      },
    },
  },
  'content_shell_test_apk': {
    'remove_from': [
      'android-asan',  # https://crbug.com/964562
      'android-code-coverage-native', # crbug/1018434
    ],
    'modifications': {
      'android-marshmallow-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_m.content_shell_test_apk.filter',
        ],
      },
      'android-pie-x86-rel': {
        'args': [
          '--gtest_filter=-ContentViewScrollingTest.testFling',
        ],
      },
    },
  },
  'content_unittests': {
    'remove_from': [
      'android-code-coverage-native', # crbug/1018434
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 2,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'context_lost_passthrough_tests': {
    'modifications': {
      'Mac FYI GPU ASAN Release': {
        'args': [
          '--is-asan',
        ]
      },
      'Win10 x64 Debug (NVIDIA)': {
        'args': [
          # TODO(https://crbug.com/1007013): Remove this flag when it's no
          # longer necessary to figure out why the renderer process is hanging.
          '--extra-browser-args=--no-sandbox',
        ],
      },
    },
  },
  'context_lost_validating_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      'Android FYI Release (Pixel 2)',
      'Fuchsia x64' # https://crbug.com/1058255
    ],
    'modifications': {
      'Mac FYI GPU ASAN Release': {
        'args': [
          '--is-asan',
        ]
      },
    },
  },
  'crashpad_tests': {
    'remove_from': [
      'linux-win_cross-rel', # https://crbug.com/762167
      # https://crbug.com/crashpad/304
      'Linux TSan Tests',
      'ToTLinuxTSan',
      # https://crbug.com/crashpad/306
      'Linux CFI',
      'CFI Linux ToT',
    ],
  },
  'cronet_sizes': {
    'modifications': {
      'android-cronet-arm-dbg': {
        'swarming': {
          'dimension_sets': [
            {
              'device_os': None,
              'device_type': None,
              'os': 'Ubuntu-16.04',
            },
          ],
        },
      },
    },
  },
  'dawn_end2end_tests': {
    'remove_from': [
      'Linux FYI Release (AMD R7 240)', # https://crbug.com/915430
    ],
  },
  'depth_capture_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      'Android FYI Release (Pixel 2)',
    ],
    'modifications': {
      'Win10 x64 Debug (NVIDIA)': {
        'args': [
          # TODO(https://crbug.com/1007013): Remove this flag when it's no
          # longer necessary to figure out why the renderer process is hanging.
          '--extra-browser-args=--no-sandbox',
        ],
      },
    },
  },
  'disk_usage_tast_test': {
    # TODO(crbug.com/1077659): Add the test back after it's fixed.
    'remove_from': [
      'chromeos-kevin-rel',
    ],
  },
  'extensions_browsertests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
    'modifications': {
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100, # https://crbug.com/876615
      },
    },
  },
  'gin_unittests': {
    'remove_from': [
      'ToTLinuxASan',  # https://crbug.com/831667
      'Linux ASan LSan Tests (1)',  # https://crbug.com/831667
      'Linux Chromium OS ASan LSan Tests (1)',  # https://crbug.com/831667
    ],
  },
  'gl_tests_passthrough': {
    'remove_from': [
      'CFI Linux ToT',
      # Can't run on MSAN because gl_tests uses the hardware driver,
      # which isn't instrumented.
      'Linux MSan Tests',
    ],
    'modifications': {
      # TODO(kbr): figure out a better way to specify blocks of
      # arguments like this for tests on multiple machines.
      'Linux ASan LSan Tests (1)': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'chromium.tests.gpu',
            },
          ],
        },
      },
      'Linux CFI': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'chromium.tests.gpu',
            },
          ],
        },
      },
      'Linux TSan Tests': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'chromium.tests.gpu',
            },
          ],
        },
      },
    },
  },
  'gl_tests_validating': {
    'remove_from': [
      'CFI Linux ToT',
      'Android FYI Release (Nexus 5X)',
      # Can't run on MSAN because gl_tests uses the hardware driver,
      # which isn't instrumented.
      'Linux MSan Tests',
    ],
    'modifications': {
      # TODO(kbr): figure out a better way to specify blocks of
      # arguments like this for tests on multiple machines.
      'Linux ASan LSan Tests (1)': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'chromium.tests.gpu',
            },
          ],
        },
      },
      'Linux CFI': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'chromium.tests.gpu',
            },
          ],
        },
      },
      'Linux TSan Tests': {
        'args': [
          '--use-gpu-in-tests',
          '--no-xvfb',
        ],
        'swarming': {
          'dimension_sets': [
            {
              'gpu': '10de:1cb3',
              'os': 'Ubuntu',
              'pool': 'chromium.tests.gpu',
            },
          ],
        },
      },
      'android-marshmallow-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.gl_tests.filter',
        ],
      },
      'android-pie-arm64-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.pie_arm64_rel.gl_tests.filter', # https://crbug.com/1034007
        ],
      },
      'android-pie-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.gl_tests.filter',
        ],
      },
    },
  },
  'gl_unittests': {
    'remove_from': [
      # On Android, these are already run on the main waterfall.
      # Run them on the one-off Android FYI bots, though.
      'Android FYI Release (Nexus 5X)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
      # Can't run on MSAN because gl_unittests uses the hardware driver,
      # which isn't instrumented.
      'Linux MSan Tests',
    ],
  },
  'gl_unittests_ozone': {
    'remove_from': [
      # Can't run on MSAN because gl_unittests_ozone uses the hardware driver,
      # which isn't instrumented.
      'Linux ChromiumOS MSan Tests',
    ],
  },
  'gpu_process_launch_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      'Android FYI Release (Pixel 2)',
    ],
    'modifications': {
      'Win10 x64 Debug (NVIDIA)': {
        'args': [
          # TODO(https://crbug.com/1007013): Remove this flag when it's no
          # longer necessary to figure out why the renderer process is hanging.
          '--extra-browser-args=--no-sandbox',
        ],
      },
    },
  },
  'hardware_accelerated_feature_tests': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      'Android FYI Release (Pixel 2)',
    ],
  },
  'headless_browsertests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
    ],
  },
  'info_collection_tests': {
    'remove_from': [
      # The Mac ASAN swarming bot runs tests on two different GPU
      # types, so we can't have one expected vendor ID / device ID.
      'Mac FYI GPU ASAN Release',
      'Android V8 FYI Release (Nexus 5X)',
      'Linux V8 FYI Release (NVIDIA)',
      'Linux V8 FYI Release - pointer compression (NVIDIA)',
      'Mac V8 FYI Release (Intel)',
      'Win V8 FYI Release (NVIDIA)',
    ],
    'modifications': {
      'ChromeOS FYI Release (amd64-generic)': {
        'args': [
          # Swarming does not report a GPU since tests are run in a VM, but
          # the VM does report that a GPU is present.
          '--expected-device-id', '1050',
        ],
      },
      'Win10 x64 Debug (NVIDIA)': {
        'args': [
          # TODO(https://crbug.com/1007013): Remove this flag when it's no
          # longer necessary to figure out why the renderer process is hanging.
          '--extra-browser-args=--no-sandbox',
        ],
      },
    },
    'replacements': {
      'ChromeOS FYI Release (amd64-generic)': {
        'args': {
          # Swarming does not report a GPU since tests are run in a VM, but
          # the VM does report that a GPU is present.
          '--expected-vendor-id': '1af4',
          # Magic substitution happens after regular replacement, so remove it
          # now since we are manually applying the expected device ID above.
          '$$MAGIC_SUBSTITUTION_GPUExpectedDeviceId': None,
        },
      }
    },
  },
  'interactive_ui_tests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-lacros-tester-rel', # https://crbug.com/1106980
    ],
    'modifications': {
      'Linux - Future (dbg)': {  # client.v8.chromium
        'swarming': {
          'shards': 3,
        },
      },
      'Linux ASan LSan Tests (1)': {
        # These are slow on the ASan trybot for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 5,
        },
      },
      'Linux TSan Tests': {
        # These are slow on the TSan bots for some reason, crbug.com/794372
        'swarming': {
          'shards': 20,
        },
      },
      'ToTLinuxTSan': {
        # These are slow on the TSan bots for some reason, crbug.com/794372
        'swarming': {
          'shards': 6,
        },
      },
      'Win 7 Tests x64 (1)': {
        # These tests are continually flaky on Win7 bots, crbug.com/967588
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/autofill.interactive_ui_tests.filter',
        ],
      },
      # temporary, https://crbug.com/818832
      'Win10 Tests x64 (dbg)': {
        'experiment_percentage': 100,
      },
      'Win7 Tests (1)': {
        # These tests are continually flaky on Win7 bots, crbug.com/967588
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/autofill.interactive_ui_tests.filter',
        ],
      },
      'Win7 Tests (dbg)(1)': {
        # These tests are continually flaky on Win7 bots, crbug.com/967588
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/autofill.interactive_ui_tests.filter',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'linux-chromeos-chrome': {
        # TODO(crbug.com/970649): Remove this filter.
        'args': [
          '--gtest_filter=-SadTabViewInteractiveUITest.ReloadMultipleSadTabs',
        ],
      },
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 10,
        },
      },
      'linux-code-coverage': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'ipc_tests': {
    'modifications': {
      'android-asan': {
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'maps_pixel_passthrough_test': {
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Linux V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release - pointer compression (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'maps_pixel_validating_test': {
    # TODO(https://crbug.com/850107): Remove the Android FYI Release (Pixel 2)
    # exception once there is enough capacity to run these tests.
    'remove_from': [
      'Android FYI Release (Pixel 2)',
    ],
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Android V8 FYI Release (Nexus 5X)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release - pointer compression (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'media_unittests': {
    'modifications': {
      'android-marshmallow-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_m.media_unittests.filter',
        ],
      },
      'android-pie-arm64-rel': {
        'args': [
          '--gtest_filter=-AAudio/AudioOutputTest.Play200HzTone/0', # https://crbug.com/1034009
        ],
      },
      'android-pie-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_p.media_unittests.filter',
        ],
      },
    },
  },
  'mojo_unittests': {
    'modifications': {
      'android-asan': {
        'swarming': {
          'shards': 5,
        },
      },
    },
  },
  'monochrome_apk_checker': {
    'modifications': {
      'ToTAndroid': {
        'swarming': {
          'dimension_sets': [
            {
              'cpu': 'x86-64',
              'device_os': None,
              'device_playstore_version': None,
              'device_type': None,
              'os': 'Ubuntu-16.04',
            },
          ],
        },
      },
      'android-arm64-proguard-rel': {
        'swarming': {
          'dimension_sets': [
            {
              'cpu': 'x86-64',
              'device_os': None,
              'device_playstore_version': None,
              'device_type': None,
              'os': 'Ubuntu-16.04',
            },
          ],
        },
      },
      'android-pie-arm64-rel': {
        'swarming': {
          'dimension_sets': [
            {
              'cpu': 'x86-64',
              'device_os': None,
              'device_os_flavor': None,
              'device_playstore_version': None,
              'device_type': None,
              'os': 'Ubuntu-16.04',
            },
          ],
        },
      },
    },
  },
  'monochrome_public_test_ar_apk': {
    'modifications': {
      'Nougat Phone Tester': {
        'args': [
          # ArCore is not installed as a system app on N, so we can install it
          # normally instead of replacing the system version.
          '--additional-apk=//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ],
      },
      # We need to match the Playstore version as well because AR tests fail on
      # old versions of the Playstore.
      'Oreo Phone Tester': {
        'args': [
          # ArCore is not installed as a system app on O, so we can install it
          # normally instead of replacing the system version.
          '--additional-apk=//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ],
        'swarming': {
          'dimension_sets': [
            {
              'device_os': 'OPR3.170623.008',
              'device_playstore_version': '12.8.23-all',
              'device_type': 'marlin',
              'os': 'Android',
            },
          ],
        },
      },
      'android-10-arm64-rel': {
        'args': [
          '--replace-system-package=com.google.ar.core,//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ]
      },
      'android-nougat-arm64-rel': {
        'args': [
          # ArCore is not installed as a system app on N, so we can install it
          # normally instead of replacing the system version.
          '--additional-apk=//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ],
      },
      'android-pie-arm64-dbg': {
        'args': [
          '--replace-system-package=com.google.ar.core,//third_party/arcore-android-sdk/test-apks/arcore/arcore_current.apk'
        ]
      }
    },
  },
  'nacl_helper_nonsfi_unittests': {
    'remove_from': [
      'Linux TSan Tests',  # Sets enable_nacl=false.
    ],
  },
  'nacl_loader_unittests': {
    'remove_from': [
      'ToTMacASan',  # The mac asan tot bot sets enable_nacl=false
      'linux-win_cross-rel',  # Sets enable_nacl=false, https://crbug.com/774186
      'Linux TSan Tests',  # The TSan bot sets enable_nacl=false
      'Mac ASan 64 Tests (1)',  # The mac asan bot sets enable_nacl=false
    ],
  },
  'net_unittests': {
    'remove_from': [
      'Fuchsia ARM64',  # https://crbug.com/877248
      'fuchsia-fyi-x64-dbg',  # https://crbug.com/844416
    ],
    'modifications': {
      'CrWinAsan': {
        # TODO(crbug.com/794372): net_unittests is slow under ASan.
        'swarming': {
          'shards': 16,
        },
      },
      'CrWinAsan(dll)': {
        # TODO(crbug.com/794372): net_unittests is slow under ASan.
        'swarming': {
          'shards': 16,
        },
      },
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 16,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 8,
        }
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 2,
        },
      },
      'Linux TSan Tests': {
        'swarming': {
          'shards': 4,
        }
      },
      # The suite runs signficantly slower on linux dbg, so increase shards.
      'Linux Tests (dbg)(1)': {
        'swarming': {
          'shards': 2,
        },
      },
      'ToTLinuxASan': {
        # TODO(crbug.com/794372): net_unittests is slow under ASan.
        'swarming': {
          'shards': 16,
        },
      },
      'ToTLinuxTSan': {
        'swarming': {
          'shards': 4,
        }
      },
      'android-marshmallow-x86-rel': {
        # crbug.com/1046060
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.net_unittests.filter',
        ],
      },
      'android-pie-x86-rel': {
        # crbug.com/1046060
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.net_unittests.filter',
        ],
      },
      'linux-blink-heap-concurrent-marking-tsan-rel': {
        'swarming': {
          'shards': 4,
        }
      },
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 2,
        },
      },
      'win-asan': {
        # TODO(crbug.com/794372): net_unittests is slow under ASan.
        'swarming': {
          'shards': 16,
        },
      },
    },
  },
  'network_service_browser_tests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
      'ToTLinuxTSan',  # https://crbug.com/368525
      'Linux ASan LSan Tests (1)',  # No capacity, https://crbug.com/852442
      'Linux TSan Tests',  # No capacity, https://crbug.com/852442
      'Win7 Tests (1)',  # For swarming capacity reasons.
      'Win10 Tests x64 (dbg)',  # Matches browser_tests.
      # Only run network service tests on Mac 10.12 for capacity reasons.