testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly.
{
  'android_browsertests': {
    'modifications': {
      'android-pie-arm64-rel': {
        'args': [
          '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
        ],
      },
    },
  },
  'android_webview_unittests': {
    'remove_from': [
      # On chromium.android, these do not need to run prior to M.
      'android-kitkat-arm-rel',
      'KitKat Phone Tester (dbg)',
      'KitKat Tablet Tester',
      'android-code-coverage-native', # crbug/1018434
    ],
  },
  'angle_end2end_tests': {
    'remove_from': [
      # TODO(ynovikov) Investigate why the test breaks on older devices.
      'Android FYI Release (Nexus 5)',
      'Android FYI Release (Nexus 6)',
      'Android FYI Release (Nexus 9)',
      # Temporarily disabled due to AMDGPU-PRO issues crbug.com/786219
      'Linux FYI Release (AMD R7 240)',
      # Temporarily disabled due to bad NVIDIA driver upgrade crbug.com/950542
      'Linux FYI Debug (NVIDIA)',
      'Linux FYI GPU TSAN Release',
      # Disabling on RX 550 as we are currently qualifying the card and not
      # checking for regressions. crbug.com/1025745
      'Win10 FYI x64 Release (AMD RX 550)',
    ],
  },
  'angle_perftests': {
    'remove_from': [
      'Android FYI Release (Nexus 5)',  # crbug.com/915429
      'Android FYI Release (Nexus 6)',  # anglebug.com/2433
      'Linux FYI GPU TSAN Release',  # crbug.com/950542
    ],
  },
  'angle_unittests': {
    'remove_from': [
      # On Android, these are already run on the main waterfall.
      'Android FYI Release (Nexus 5X)',
    ],
    'modifications': {
      'Linux ASan LSan Tests (1)': {
        'swarming': {
          'shards': 10,
        },
      },
    },
  },
  'angle_white_box_tests': {
    'remove_from': [
      # https://crbug.com/1035178
      'Win10 FYI x64 Release (AMD RX 550)',
      # https://crbug.com/1035461
      'Linux FYI GPU TSAN Release',
    ],
  },
  'breakpad_unittests': {
    'modifications': {
      'android-asan': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.asan.breakpad_unittests.filter',
        ],
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      'CrWinAsan(dll)', # https://crbug.com/935598
      'linux-win_cross-rel',
      'ToTLinuxTSan',  # https://crbug.com/368525
      'Mac10.10 Tests',  # https://crbug.com/828031
      'Mac10.13 Tests',  # https://crbug.com/1042757
      'Linux TSan Tests',  # https://crbug.com/368525
      'Win10 Tests x64 (dbg)',
    ],
    'modifications': {
      'Linux - Future (dbg)': {  # client.v8.chromium
        'swarming': {
          'shards': 10,
        },
      },
      'Linux ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 40,
        },
      },
      'Linux Chromium OS ASan LSan Tests (1)': {
        # These are very slow on the ASAN trybot for some reason.
        # crbug.com/794372
        'swarming': {
          'shards': 31,
        },
      },
      'Linux ChromiumOS MSan Tests': {
        # These are very slow on the Chrome OS MSAN trybot for some reason.
        # crbug.com/865455
        'swarming': {
          'shards': 30,
        },
      },
      'Mac ASan 64 Tests (1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'ToTWin64(dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Win10 Tests x64': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
        }
      },
      'Win10 Tests x64 Code Coverage': {
        # crbug.com/868082
        'args': [
          '--disable-features=WebRTC-H264WithOpenH264FFmpeg',
        ],
        'swarming': {
          # This is for slow test execution that often becomes a critical path of
          # swarming jobs. crbug.com/868114
          'shards': 15,
        }
      },
      'Win7 Tests (1)': {
        # This is for slow test execution that often becomes a critical path of
        # swarming jobs. crbug.com/868114
        'swarming': {
          'shards': 15,
        },
      },
      'Win7 Tests (dbg)(1)': {
        'swarming': {
          'shards': 20,
        },
      },
      'linux-chromeos-chrome': {
        # TODO(https://crbug.com/932269): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
        # TODO(https://crbug.com/1039746): Move this back to the main pool.
        'swarming': {
          'dimension_sets': [
            {
              'pool': 'chrome.tests.browser_tests_killer',
            },
          ],
        },
      },
      'linux-chromeos-code-coverage': {
        'swarming': {
          'shards': 50,
        },
      },
      'linux-chromeos-dbg': {
        'swarming': {
          'shards': 30,
        },
      },
      'linux-chromeos-rel': {
        'swarming': {
          'shards': 20,
        },
      },
      'linux-code-coverage': {
        'swarming': {
          'shards': 50,
        },
      },
      'linux-trusty-rel': {
       'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/linux_trusty_rel.browser_tests.filter',
        ],
      },
      'mac-code-coverage': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
        'swarming': {
          'shards': 15,
        },
      },
      'mac-osxbeta-rel': {
        # A subset of tests seem to cause WindowServer deaths on VMs.
        # crbug.com/828031 et al.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/mac_window_server_killers.browser_tests.filter',
          '--gtest_shuffle',
        ],
      },
      'win-asan': {
        # browser_tests shards were reaching a timeout limit of 60 minutes on win-asan
        # (https://crbug.com/1031096).
        'swarming': {
          'shards': 20,
        },
      },
    },
  },
  'cc_unittests': {
    'modifications': {
      'Linux TSan Tests': {
        'swarming': {
          'shards': 3,
        },
      },
      'android-marshmallow-x86-fyi-rel': {
        # https://crbug.com/1039860
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.cc_unittests.filter',
        ],
      },
      'android-pie-x86-fyi-rel': {
        # https://crbug.com/1039860
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.cc_unittests.filter',
        ],
      },
    },
  },
  'checkbins': {
    'remove_from': [
      'linux-archive-dbg',
      'linux-archive-rel',
      'mac-archive-dbg',
      'mac-archive-rel',
    ],
  },
  'chrome_java_test_pagecontroller_tests': {
    'modifications': {
      'android-pie-arm64-rel': {
        'experiment_percentage': 100, # https://crbug.com/924194
      },
    },
  },
  'chrome_public_smoke_test': {
    'remove_from': [
      'android-code-coverage-native', # https://crbug.com/1018780
    ],
  },
  'chrome_public_test_apk': {
    'remove_from': [
      'android-asan',  # https://crbug.com/964562
      'android-code-coverage-native', # https://crbug.com/1018780
    ],
    'modifications': {
      'KitKat Phone Tester (dbg)': {
        'swarming': {
          'shards': 20,
        },
      },
      'Marshmallow 64 bit Tester': {
        'swarming': {
          'shards': 14,
        },
      },
      'Oreo Phone Tester': {
        # TODO(https://crbug.com/884413): Re-enable this once the tests are
        # either passing or there is more capacity.
        'experiment_percentage': 0,
      },
      'android-arm64-proguard-rel': {
        'swarming': {
          'shards': 25,
        },
      },
      'android-marshmallow-arm64-rel': {
        'swarming': {
          'shards': 25,
        },
      },
      'android-pie-arm64-rel': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/android.pie_arm64_rel.chrome_public_test_apk.filter', # https://crbug.com/1010211
        ],

      },
    },
  },
  'chrome_public_test_vr_apk': {
    'remove_from': [
      'KitKat Tablet Tester',
      'Lollipop Tablet Tester',
      'Marshmallow Tablet Tester',
      'android-marshmallow-x86-fyi-rel',
      'android-pie-x86-fyi-rel',