testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# Note that if you are excluding with variants, you will need to exclude
# per variant, with the identifier appended to the test suite name.
# For example, if you have ios_chrome_ui_eg2tests_module with variant
# SIM_IPHONE_6S_14_0 (identifier = "iPhone 6s 14.0"), you
# must exclude with the test name as:
# "ios_chrome_ui_eg2tests_module_iPhone 6s 14.0"

# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'accessibility_unittests':{
    'modifications': {
      'fuchsia-fyi-x64-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/fuchsia.lsan.accessibility_unittests.filter',
        ],
      },
    },
  },
  'android_browsertests': {
    'modifications': {
      'android-11-x86-rel': {
        'args': [
          # https://crbug.com/1289764
          '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
        ],
        'swarming': {
          'shards': 2,
        },
      },
      'android-marshmallow-arm64-rel': {
        'swarming': {
          'quickrun_shards': 3,
        },
      },
      'android-marshmallow-arm64-rel-rts': {
        'swarming': {
          'shards': 2,
        },
      },
      'android-marshmallow-x86-rel': {
        'swarming': {
          'shards': 2,
          'quickrun_shards': 6,
        },
      },
      'android-pie-arm64-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
        # TODO(crbug.com/1111436): Move this back to walleye if/when additional
        # capacity has been deployed.
        'swarming': {
          'dimension_sets': [
            {
              'device_type': 'sailfish',
            },
          ],
          'shards': 2,
          'quickrun_shards': 6,
        },
      },
      'android-pie-x86-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'android_sync_integration_tests': {
   'remove_from': [
      # Remove from CQ bots due to capacity.
      'android-marshmallow-arm64-rel',
      'android-marshmallow-arm64-rel-rts',
   ],
  },
  'android_webview_unittests': {
  },
  'angle_end2end_tests': {
    'modifications': {
      'win10-angle-x64-intel': {
        'args': [
          '--max-processes=1',
        ],
      },
    },
  },
  'angle_unittests': {
    'remove_from': [
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
      # Times out listing tests crbug.com/1167314
      'android-asan',
      # Does not currently work on Lacros configurations.
      'linux-lacros-code-coverage',
      'linux-lacros-tester-rel',
      'linux-lacros-tester-fyi-rel',
      'linux-lacros-dbg-tests-fyi',
    ],
    'modifications': {
      # anglebug.com/6894
      'fuchsia-fyi-x64-asan': {
        'args': [
          # crbug.com/
          '--gtest_filter=-ConstructCompilerTest.DefaultParameters',
        ],
      },
    },
  },
  'angle_white_box_tests': {
    'modifications': {
      'win10-angle-x64-intel': {
        'args': [
          '--max-processes=1',
        ],
      },
    },
  },
  'aura_unittests_amd64-generic': {
    'remove_from': [
      # TODO(crbug.com/1204231): Re-enable.
      'lacros-amd64-generic-rel',
    ]
  },
  'aura_unittests_eve': {
    'remove_from': [
      # TODO(crbug.com/1204231): Re-enable.
      'lacros-amd64-generic-rel',
    ]
  },
  'base_unittests': {
    'modifications': {
      'android-11-x86-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
        ],
      },
      'android-12-x64-fyi-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
          '--gtest_filter=-ModuleCacheTest.CheckAgainstProcMaps',  # crbug.com/1260521
        ],
      },
      'fuchsia-code-coverage': {
        'swarming': {
          'shards': 5,
        },
      },
      'fuchsia-fyi-x64-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/fuchsia.lsan.base_unittests.filter',
        ],
      },
    },
  },
  'blink_platform_unittests': {
    'modifications': {
      'android-pie-arm64-rel': {
        # TODO(crbug.com/1111436): Move this back to walleye if/when additional
        # capacity has been deployed.
        'swarming': {
          'dimension_sets': [
            {
              'device_type': 'sailfish',
            },
          ],
        },
      },
      'fuchsia-code-coverage': {
        'swarming': {
          'shards': 5,
        },
      },
    },
  },
  'blink_unittests': {
    'modifications': {
      'fuchsia-code-coverage': {
        'swarming': {
          'shards': 30,
        },
      },
    },
  },
  'blink_web_tests': {
    'remove_from': [
      'Mac10.11 Tests',
      'Win 7 Tests x64 (1)',  # 64-bit win is unsupported by the layout tests.
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      'Fuchsia x64': {
        'args': [
          '--platform=fuchsia',
          '--jobs=1',
        ],
        'swarming': {
          'shards': 1,
        },
      },
      'Linux Tests': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw",
        ],
        'swarming': {
          'quickrun_shards': 18,
        },
      },
      'Linux Tests (dbg)(1)': {
        'args': [
          '--debug',
        ],
        'swarming': {
          'shards': 30,
        },
      },
      'Mac10.12 Tests': {
        'swarming': {
          'shards': 25,
        },
      },
      'Mac10.13 Tests': {
        'swarming': {
          'dimension_sets': [
            {
              'gpu': None,
            },
          ],
          'shards': 25,
        },
      },
      'Mac10.15 Tests': {
        'swarming': {
          'dimension_sets': [
            {
              'gpu': None,
            },
          ],
          'shards': 25,
        },
      },
      'Mac11 Tests': {
        'swarming': {
          'dimension_sets': [
            {
              'gpu': None,
            },
          ],
          'shards': 25,
          'quickrun_shards': 26,
        },
      },
      'Mac11 Tests (dbg)': {
        'experiment_percentage': 100,
        'args': [
          '--debug',
        ],
        'swarming': {
          'shards': 40,
        },
      },
      'ToTFuchsia x64': {
        'args': [
          '--platform=fuchsia',
        ],
        'swarming': {
          'shards': 1,
        },
      },
      'V8 Blink Linux Debug': {
        # This builder compiles Chromium in release mode, but V8 in debug.
        # TODO(machenbach): Change name and out directory of this builder to
        # something less confusing.
        'args': [
          '-t',
          'Debug',
          '--release',
        ],
      },
      'V8 Blink Linux Future': {
        'args': [
          '--flag-specific=future-js',
        ],
      },
      'WebKit Linux ASAN': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/ASANExpectations',
          '--timeout-ms',
          '48000',
          '--enable-sanitizer',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'WebKit Linux Leak': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/LeakExpectations',
          '--timeout-ms',
          '48000',
          '--enable-leak-detection',
        ],
        'swarming': {
          'shards': 10,
        },
      },
      'WebKit Linux MSAN': {
        'args': [
          '--additional-expectations',
          '../../third_party/blink/web_tests/MSANExpectations',
          '--timeout-ms',
          '66000',
          '--enable-sanitizer',
        ],
        'swarming': {
          'expiration': 36000,
          'hard_timeout': 10800,
          'io_timeout': 3600,
          'shards': 20,
        },
      },
      'WebKit Linux layout_ng_disabled Builder': {
        'args': [
          '--flag-specific=disable-layout-ng',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        'args': [
          '--target',
          'Release_x64',
        ],
        'swarming': {
          "shards": 28
        },
      },
      'Win10 Tests x64 (dbg)': {
        'args': [
          '--debug',
          '--target',
          'Debug_x64',
        ],
      },
      'Win11 Tests x64': {
        'args': [
          '--target',
          'Release_x64',
        ],
        'swarming': {
          "shards": 28
        },
      },
      'Win7 Tests (dbg)(1)': {
        'args': [
          '--debug',
          '--timeout-ms',
          '60000'
        ],
        'experiment_percentage': 100,
        # TODO(kbr): the "Windows-7-SP1" Swarming dimension set is implicit,
        # from prefered_os_dimension in
        # tools/build/scripts/slave/recipe_modules/swarming/api.py. Ideally,
        # Swarming dimensions would be either specified for all testers on the
        # waterfall (and therefore, explicitly specified for all tests in the
        # generated JSON), or this would be removed, and the implicit one used.
        'swarming': {
          'dimension_sets': [
            {
              'os': 'Windows-7-SP1',
            }
          ],
        },
      },
      # tryserver.chromium.android
      'android_blink_rel': {
        'args': [
          '--release',
          '--android',
          # TODO(crbug.com/912120): Breakpad was entirely disabled on
          # Android. Remove this when it no longer affects Android crash
          # log handling.
          '--disable-breakpad',
          # TODO(crbug.com/875172): run on the real GPU since the
          # software fallback, SwiftShader, isn't yet for 32-bit ARM.
          '--additional-driver-flag=--use-gpu-in-tests',
        ],
      },
      'devtools_frontend_linux_blink_light_rel': {
        'args': [
          'http/tests/devtools',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'devtools_frontend_linux_blink_light_rel_fastbuild': {
        'args': [
          'http/tests/devtools',
        ],
        'swarming': {
          'shards': 4,
        },
      },
      'fuchsia-fyi-x64-rel': {
        'args': [
          '--platform=fuchsia',
        ],
        'swarming': {
          'shards': 1,
        },
      },
      'linux-autofill-assistant': {
        'args': [
          '--release',
        ],
      },
      'linux-bfcache-rel': {
        'args': [
          '--flag-specific=enable-back-forward-cache-same-site',
          # TODO(crbug.com/1255537): Re-enable the test.
          '--ignore-tests=external/wpt/html/browsers/browsing-the-web/back-forward-cache/events.html',
        ],
      },
      'linux-blink-animation-use-time-delta': {
        'args': [
          '--debug',
        ],
      },
      'linux-blink-heap-concurrent-marking-tsan-rel': {
        'args': [
          '--release',
          '--flag-specific=enable-blink-heap-concurrent-marking',
        ],
      },
      'linux-blink-web-tests-force-accessibility-rel': {
        'args': [
          '--flag-specific=force-renderer-accessibility',
        ],
      },
      'linux-code-coverage': {
        'args': [
          '--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw',
        ],
        'swarming': {
          'shards': 20,
        },
      },
      'linux-layout-tests-edit-ng': {
        'args': [
          '--flag-specific=enable-editing-ng',
        ],
      },
      'linux-xenial-rel': {
        "args": [
          "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw",
        ],
      },
      # TODO (crbug.com/1254971) Re-enable once fixed
      'mac-osxbeta-rel': {
        'experiment_percentage': 0,
      },
      'mac10.12-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.13-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac10.14-blink-rel-dummy': {
        'swarming': {
          # Increase timeout and reduce shards.
          # See https://crbug.com/1203565.
          'shards': 12,
          'hard_timeout': 1800,
        },
      },
      'mac10.15-blink-rel-dummy': {
        'swarming': {
          'shards': 20,
        },
      },
      'mac11.0-blink-rel-dummy': {
        'swarming': {
          'shards': 12,
          'hard_timeout': 1800,
        },
      },
      'win10.20h2-blink-rel-dummy': {
        'swarming': {
          'shards': 14,
          'hard_timeout': 1200,
        },
      },
    },
  },
  'breakpad_unittests': {
    'modifications': {
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.breakpad_unittests.filter',
        ],
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      'Mac10.11 Tests', # https://crbug.com/1263696
      # TODO(crbug.com/1171053): Enable the suite in 'Mac10.15 Tests' once it's
      # off CQ.
      'Mac10.13 Tests', # https://crbug.com/1042757
      'Mac10.15 Tests', # https://crbug.com/1042757
      'Mac11 Tests (dbg)', # https://crbug.com/1201386
      'mac-code-coverage', # https://crbug.com/1201386
      'Linux TSan Tests',  # https://crbug.com/368525
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      'CrWinAsan': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/1257927
        'swarming': {
          'shards': 40,
        },
      },
      'CrWinAsan(dll)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/1257927
        'swarming': {
          'shards': 40,
        },
      },
      'Linux - Future (dbg)': {  # client.v8.chromium
        'swarming': {
          'shards': 10,
        },
      },
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/1257927
        'swarming': {
          'shards': 40,
          'quickrun_shards': 80
        },
      },
      'Linux ASan Tests (sandboxed)': {
        'swarming': {
          'shards': 30, # https://crbug.com/1103330
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/1257927
        'swarming': {
          'shards': 50,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 32,
        },
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.msan.browser_tests.oobe_negative.filter',
        ],
      },
      'Linux Tests': {
        'swarming': {
          'quickrun_shards': 15,
        },
      },
      # https://crbug.com/1084469
      'Linux Tests (Wayland)': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/ozone-linux.wayland_browser_tests.filter',
        ],
        'swarming': {
          'quickrun_shards': 24,
        },
      },
      'Linux Tests (dbg)(1)': {
        # crbug.com/1066161
        'swarming': {
          'shards': 20,
        },
      },
      'Mac ASan 64 Tests (1)': {
        'swarming': {
          'shards': 30,
        },
        # crbug.com/1196416
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac.mac-rel.browser_tests.filter',
        ],
        'experiment_percentage': 100, # https://crbug.com/1251657
      },
      # TODO(crbug.com/1232535) Remove once the bug is resolved.
      'Mac10.11 Tests': {
        'args': [
          '--test-launcher-print-timestamps',
        ],
      },
      'Mac10.15 Tests': {
        # crbug.com/1042757
        'swarming': {
          'dimension_sets': [
            {
              # These shards are liable to time out when running on macmini7,1.
              'cores': '8|12',
            }
          ],
        },
        # crbug.com/1196416
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac.mac-rel.browser_tests.filter',
        ],
      },
      'Mac11 Tests': {
        'ci_only': True,
      },
      'Mac11 Tests (dbg)': {
        # crbug.com/1042757
        'swarming': {
          'dimension_sets': [
            {
              # These shards are liable to time out when running on macmini7,1.
              'cores': '8|12',
            }
          ],
        },
        # crbug.com/1196416
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac.mac-rel.browser_tests.filter',
        ],
      },
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
          'quickrun_shards': 40,
        }
      },
      'Win11 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
          'quickrun_shards': 30,
        }
      },
      'Win7 Tests (1)': {
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 15,
        },
      },
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'linux-chromeos-chrome': {
        # TODO(https://crbug.com/932269): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
      },
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 50,
        },
      },
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 40,
        },
      },
      'linux-chromeos-rel': {
        'swarming': {
          'shards': 20,