testing/buildbot/test_suite_exceptions.pyl

# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a .pyl, or "Python Literal", file. You can treat it just like a
# .json file, with the following exceptions:
# * all keys must be quoted (use single quotes, please);
# * comments are allowed, using '#' syntax; and
# * trailing commas are allowed.

# Exceptions to the common test suites can only include removal from
# particular bots, and modifications on particular bots. By design,
# it's not possible to add one-off tests to bots. Instead they have to
# be added to one of the test suites in test_suites.pyl.
#
# Note that if you are excluding with variants, you will need to exclude
# per variant, with the identifier appended to the test suite name.
# For example, if you have ios_chrome_ui_eg2tests_module with variant
# SIM_IPHONE_6S_14_0 (identifier = "iPhone 6s 14.0"), you
# must exclude with the test name as:
# "ios_chrome_ui_eg2tests_module_iPhone 6s 14.0"

# The goal is to drive the number of exceptions to zero, to make all
# the bots behave similarly. whitespace
{
  'android_webview_unittests': {
  },
  'blink_web_tests': {
    'modifications': {
      'V8 Blink Linux Future': {
        'args': [
          '--flag-specific=future-js',
        ],
      },
      'devtools_frontend_linux_blink_light_rel': {
        'args': [
          'http/tests/devtools',
        ],
        'swarming': {
          'shards': 2,
        },
      },
      'devtools_frontend_linux_blink_light_rel_fastbuild': {
        'args': [
          'http/tests/devtools',
        ],
        'swarming': {
          'shards': 2,
        },
      },
    },
  },
  'blink_wpt_tests': {
    'remove_from': [
      'devtools_frontend_linux_blink_light_rel',
      'devtools_frontend_linux_blink_light_rel_fastbuild',
    ],
    'modifications': {
      'V8 Blink Linux Future': {
        'args': [
          '--flag-specific=future-js',
        ],
      },
    },
  },
  'browser_tests': {
    'remove_from': [
      'linux-chromeos-chrome-preuprev', # http://b/331706144
    ],
    'modifications': {
      'Linux - Future (dbg)': {  # client.v8.chromium
        'swarming': {
          'shards': 10,
        },
      },
      'linux-chromeos-chrome': {
        # TODO(crbug.com/40614042): Promote out of experiment when the
        # tests are green.
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.browser_tests.filter',
        ],
        'swarming': {
          'shards': 20,
        },
      },
    },
  },
  'chrome_all_tast_tests': {
    'modifications': {
      'chromeos-eve-chrome': {
        'swarming': {
          # Increased timeout to 4 hours due to shard timeout.
          'hard_timeout': 14400,
        }
      },
      # TODO(crbug.com/40727045): Move out of experimental if/when the bots are
      # stable.
      'chromeos-jacuzzi-chrome': {
        'experiment_percentage': 100,
      },
      'chromeos-reven-chrome': {
        'args': [
          '--tast-shard-method=hash',
        ],
      },
    },
  },
  'content_browsertests': {
    'remove_from': [
      'linux-chromeos-chrome-preuprev', # http://b/331706144
    ],
  },
  'expected_color_pixel_gl_passthrough_ganesh_test': {
    'replacements': {
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'expected_color_pixel_metal_passthrough_ganesh_test': {
    'replacements': {
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'expected_color_pixel_metal_passthrough_graphite_test': {
    'replacements': {
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'expected_color_pixel_passthrough_graphite_test': {
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'expected_color_pixel_passthrough_test': {
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Linux V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release - pointer compression (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'expected_color_pixel_passthrough_test JACUZZI_RELEASE_LKGM': {
    'modifications': {
      'ChromeOS FYI Release Skylab (jacuzzi)': {
        'args': [
          # Skylab currently doesn't support use of LUCI_CONTEXT, so Gold cannot
          # automatically get the service account to authenticate with.
          '--service-account=/creds/service_accounts/skylab-drone.json',
        ],
      },
    },
  },
  'expected_color_pixel_validating_test': {
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Android V8 FYI Release': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'info_collection_tests': {
    'remove_from': [
      'Android V8 FYI Release',
      'Linux V8 FYI Release (NVIDIA)',
      'Linux V8 FYI Release - pointer compression (NVIDIA)',
      'Mac V8 FYI Release (Intel)',
      'Win V8 FYI Release (NVIDIA)',
    ],
  },
  'interactive_ui_tests': {
    'modifications': {
      'Linux - Future (dbg)': {  # client.v8.chromium
        'swarming': {
          'shards': 3,
        },
      },
      # TODO(crbug.com/40630654): Remove these filters.
      'linux-chromeos-chrome': {
        'args': [
          '--gtest_filter=-SadTabViewInteractiveUITest.ReloadMultipleSadTabs',
        ],
      },
    },
  },
  'model_validation_tests MODEL_VALIDATION_BASE': {
    'modifications': {
      'optimization_guide-win32': {
        'args': [
          '--target-platform=win32',
        ],
      },
    },
  },
  'net_unittests RELEASE_LKGM': {
    # TODO(b/319364912): vpython cannot be delivered via CIPD on skylab builders
    'remove_from': [
      'chromeos-brya-chrome-dchecks-skylab',
      'chromeos-brya-chrome-skylab',
      'chromeos-brya-chrome-tests',
    ],
  },
  'notification_helper_unittests': {
    'remove_from': [
      # all these bots run Windows 7.
      'ToTWinOfficial',
      'ToTWinOfficial64',
    ],
  },
  'ondevice_stability_tests AMD Radeon RX 5500 XT': {
    'modifications': {
      'optimization_guide-win32': {
        'args': [
          '--target-platform=win32',
        ],
      },
    },
  },
  'ondevice_stability_tests Intel UHD 630': {
    'modifications': {
      'optimization_guide-win32': {
        'args': [
          '--target-platform=win32',
        ],
      },
    },
  },
  'ondevice_stability_tests NVIDIA GeForce GTX 1660': {
    'modifications': {
      'optimization_guide-win32': {
        'args': [
          '--target-platform=win32',
        ],
      },
    },
  },
  'ozone_unittests': {
    'modifications': {
      'linux-chromeos-chrome': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.ozone_unittests.filter',
        ],
      },
      'linux-chromeos-chrome-preuprev': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.ozone_unittests.filter',
        ],
      },
    },
  },
  'pixel_skia_gold_gl_passthrough_ganesh_test': {
    'replacements': {
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'pixel_skia_gold_metal_passthrough_ganesh_test': {
    'replacements': {
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'pixel_skia_gold_metal_passthrough_graphite_test': {
    'replacements': {
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'pixel_skia_gold_passthrough_graphite_test': {
    'modifications': {
      'Win V8 FYI Release (NVIDIA)': {
        'args': [
          # TODO(crbug.com/382422293): Remove when fixed
          '--jobs=1',
        ],
      },
    },
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
          # Magic substitution happens after regular replacement, so remove it
          # now since we are manually applying the number of jobs above.
          '$$MAGIC_SUBSTITUTION_GPUParallelJobs': None,
        },
      },
    },
  },
  'pixel_skia_gold_passthrough_test': {
    'modifications': {
      'Win V8 FYI Release (NVIDIA)': {
        'args': [
          # TODO(crbug.com/382422293): Remove when fixed
          '--jobs=1',
        ],
      },
    },
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Linux V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release - pointer compression (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
          # Magic substitution happens after regular replacement, so remove it
          # now since we are manually applying the number of jobs above.
          '$$MAGIC_SUBSTITUTION_GPUParallelJobs': None,
        },
      },
    },
  },
  'pixel_skia_gold_passthrough_test JACUZZI_RELEASE_LKGM': {
    'modifications': {
      'ChromeOS FYI Release Skylab (jacuzzi)': {
        'args': [
          # Skylab currently doesn't support use of LUCI_CONTEXT, so Gold cannot
          # automatically get the service account to authenticate with.
          '--service-account=/creds/service_accounts/skylab-drone.json',
        ],
      },
    },
  },
  'pixel_skia_gold_validating_test': {
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Android V8 FYI Release': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'telemetry_perf_unittests': {
    'modifications': {
      'chromeos-eve-chrome': {
        'args': [
          # TODO(crbug.com/40756418): Re-enable.
          '--skip=benchmarks.benchmark_smoke_unittest.BenchmarkSmokeTest.wasmpspdfkit',
        ],
      },
    },
    'remove_from': [
      'Linux - Future (dbg)',  # client.v8.chromium
    ],
  },
  'telemetry_unittests': {
    'remove_from': [
      # TODO(crbug.com/40727045): Restore when there's more DUTs
      'chromeos-eve-chrome',
    ],
  },
  'unit_tests': {
    'modifications': {
      'linux-chromeos-chrome': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.unit_tests.filter',
        ],
      },
      'linux-chromeos-chrome-preuprev': {
        'args': [
          '--test-launcher-filter-file=../../testing/buildbot/filters/chromeos.unit_tests.filter',
        ],
      },
    },
  },
  'variations_smoke_tests': {
    'modifications': {
      'linux-chrome': {
        'swarming': {
          'dimensions': {
            'pool': 'chrome.tests.finch',
          },
        },
      },
      'win-chrome': {
        'experiment_percentage': 100,
        'swarming': {
          'dimensions': {
            'pool': 'chrome.tests.finch',
          },
        },
      },
    },
  },
  'vulkan_pixel_skia_gold_test': {
    'replacements': {
      # The V8 builders pass the V8 revision for ${got_revision}, so instead
      # use ${got_cr_revision}, which is only set on the V8 bots.
      'Linux V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Linux V8 FYI Release - pointer compression (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Mac V8 FYI Release (Intel)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
      'Win V8 FYI Release (NVIDIA)': {
        'args': {
          '--git-revision': '${got_cr_revision}',
        },
      },
    },
  },
  'webgl2_conformance_d3d11_passthrough_tests': {
    'remove_from': [
      # Removed from V8 builder since WebGL 1 should provide reasonable
      # upstream test coverage without the capacity hit from WebGL 2.
      'Win V8 FYI Release (NVIDIA)',
    ],
  },
  'webgl2_conformance_gl_passthrough_ganesh_tests': {
    'remove_from': [
      'Mac V8 FYI Release (Intel)',
    ],
  },
  'webgl2_conformance_gl_passthrough_tests': {
    'remove_from': [
      # Removed from V8 builders since WebGL 1 should provide reasonable
      # upstream test coverage without the capacity hit from WebGL 2.
      'Linux V8 FYI Release - pointer compression (NVIDIA)',
    ],
  },
  'webgl2_conformance_metal_passthrough_graphite_tests': {
    'remove_from': [
      # Removed from V8 builders since WebGL 1 should provide reasonable
      # upstream test coverage without the capacity hit from WebGL 2.
      'Mac V8 FYI Release (Intel)',
    ],
  },
  'webgl_conformance_vulkan_passthrough_tests': {
    'remove_from': [
      # TODO(crbug.com/380431384): Re-enable when fixed
      'Win V8 FYI Release (NVIDIA)',
    ],
  },
}