blob: 3c06ad448b6a7a9bde376fd301f8a661516fe171 [file] [log] [blame]
Avi Drissmandfd880852022-09-15 20:11:091# Copyright 2017 The Chromium Authors
Kenneth Russelleb60cbd22017-12-05 07:54:282# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5# This is a .pyl, or "Python Literal", file. You can treat it just like a
6# .json file, with the following exceptions:
7# * all keys must be quoted (use single quotes, please);
8# * comments are allowed, using '#' syntax; and
9# * trailing commas are allowed.
10
11# Exceptions to the common test suites can only include removal from
12# particular bots, and modifications on particular bots. By design,
13# it's not possible to add one-off tests to bots. Instead they have to
14# be added to one of the test suites in test_suites.pyl.
15#
Jeff Yoone2188f62020-08-24 20:55:5716# Note that if you are excluding with variants, you will need to exclude
17# per variant, with the identifier appended to the test suite name.
18# For example, if you have ios_chrome_ui_eg2tests_module with variant
19# SIM_IPHONE_6S_14_0 (identifier = "iPhone 6s 14.0"), you
20# must exclude with the test name as:
21# "ios_chrome_ui_eg2tests_module_iPhone 6s 14.0"
22
Kenneth Russelleb60cbd22017-12-05 07:54:2823# The goal is to drive the number of exceptions to zero, to make all
24# the bots behave similarly.
25{
Ben Pastenef2ca8e22023-05-03 21:52:2526 'accessibility_unittests': {
27 'modifications': {
28 'Linux ASan LSan Tests (1)': {
29 'swarming': {
30 'dimension_sets': [
31 {
32 # TODO(crbug.com/1442303): Move to Jammy when it's compatible.
33 'os': 'Ubuntu-18.04',
34 },
35 ],
36 },
37 },
Ian Struiksma01223f42023-06-05 15:52:3938 'linux-exp-asan-lsan-fyi-rel': {
39 'swarming': {
40 'dimension_sets': [
41 {
42 # TODO(crbug.com/1442303): Move to Jammy when it's compatible.
43 'os': 'Ubuntu-18.04',
44 },
45 ],
46 },
47 },
Ben Pastenef2ca8e22023-05-03 21:52:2548 },
49 },
Andrew Luo38b311f2019-10-04 03:42:0950 'android_browsertests': {
51 'modifications': {
Haiyang Pan9d68ca02021-08-04 21:30:4752 'android-11-x86-rel': {
Haiyang Pan0ea9b3ef2022-01-21 21:58:1453 'args': [
54 # https://crbug.com/1289764
55 '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
56 ],
Haiyang Panca7fcec2021-03-19 02:11:1057 },
Haiyang Panc1192662022-03-09 00:09:0258 'android-12-x64-rel': {
Haiyang Pan18b448d72022-12-13 21:33:4459 # TODO(crbug.com/1225851): Remove experiment and ci_only
60 # once the test suite is stable.
61 'ci_only': True,
62 'experiment_percentage': 100,
Haiyang Panc1192662022-03-09 00:09:0263 'args': [
64 # https://crbug.com/1289764
65 '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
66 ],
Haiyang Pand5524452023-04-25 01:56:4067 'swarming': {
68 'shards': 4,
69 },
Haiyang Pan1b8faa20b2023-02-10 23:44:3670 },
71 'android-12l-x64-dbg-tests': {
72 'args': [
73 # https://crbug.com/1289764
74 '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
75 ],
Haiyang Pand5524452023-04-25 01:56:4076 'swarming': {
77 'shards': 4,
78 },
Haiyang Pan1b8faa20b2023-02-10 23:44:3679 },
Haiyang Pan42c77cd92023-05-18 01:13:0380 'android-12l-x64-fyi-dbg': {
Haiyang Panef317e4372023-05-19 01:30:0881 'args': [
82 # https://crbug.com/1289764
83 '--gtest_filter=-All/ChromeBrowsingDataLifetimeManagerScheduledRemovalTest.History/*',
84 ],
Haiyang Pan42c77cd92023-05-18 01:13:0385 'swarming': {
86 'shards': 4,
87 },
88 },
89 'android-13-x64-fyi-rel': {
Haiyang Panef317e4372023-05-19 01:30:0890 'args': [
91 # https://crbug.com/1414886
92 '--gtest_filter=-OfferNotificationControllerAndroidBrowserTestForMessagesUi.MessageShown',
93 ],
Haiyang Pan42c77cd92023-05-18 01:13:0394 'swarming': {
95 'shards': 4,
96 },
97 },
Haiyang Pan1b8faa20b2023-02-10 23:44:3698 'android-13-x64-rel': {
99 'args': [
100 # https://crbug.com/1414886
101 '--gtest_filter=-OfferNotificationControllerAndroidBrowserTestForMessagesUi.MessageShown',
102 ],
Haiyang Pand5524452023-04-25 01:56:40103 'swarming': {
104 'shards': 4,
105 },
Haiyang Panc1192662022-03-09 00:09:02106 },
Prakharc60ec802023-03-06 07:49:29107 # If you change this, make similar changes in android-x86-code-coverage below
Haiyang Pancf24ceb2022-03-17 02:31:30108 'android-nougat-x86-rel': {
James Shenfb767a12023-02-17 01:47:29109 'ci_only': True,
Haiyang Pancf24ceb2022-03-17 02:31:30110 'swarming': {
Stephanie Kim53cc82d2022-11-01 19:42:57111 'shards': 9,
Haiyang Pancf24ceb2022-03-17 02:31:30112 },
113 },
Andrew Luo38b311f2019-10-04 03:42:09114 'android-pie-arm64-rel': {
Andrew Luobe83fd82019-12-19 03:07:58115 'args': [
116 '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
117 ],
Haiyang Panfbd2b482020-09-23 23:41:25118 # TODO(crbug.com/1111436): Move this back to walleye if/when additional
119 # capacity has been deployed.
120 'swarming': {
121 'dimension_sets': [
122 {
123 'device_type': 'sailfish',
124 },
125 ],
James Shend6574e852023-02-24 03:13:16126 'shards': 4,
Haiyang Panfbd2b482020-09-23 23:41:25127 },
Andrew Luo38b311f2019-10-04 03:42:09128 },
Haiyang Pan08d09dc2020-04-01 16:11:21129 'android-pie-x86-rel': {
John Budorick191adfcd2020-01-29 18:19:04130 'args': [
131 '--gtest_filter=-ImportantSitesUtilBrowserTest.DSENotConsideredImportantInRegularMode', # https://crbug.com/1034001
132 ],
Haiyang Pan7cd609d2021-04-08 20:17:24133 'swarming': {
Ben Joyce5bb3fad2022-09-23 06:05:18134 'dimension_sets': [
135 {
Haiyang Pane35ae642022-11-18 00:34:15136 'cores': '8', # crbug.com/1292221
Ben Joyce5bb3fad2022-09-23 06:05:18137 },
138 ],
Haiyang Pan7cd609d2021-04-08 20:17:24139 },
John Budorick191adfcd2020-01-29 18:19:04140 },
Prakharc60ec802023-03-06 07:49:29141 # Keep this same as android-nougat-x86-rel above
142 'android-x86-code-coverage': {
143 'ci_only': True,
144 'swarming': {
145 'shards': 9,
146 },
147 },
Andrew Luo38b311f2019-10-04 03:42:09148 },
149 },
Sven Zhengc60debc2020-09-10 18:40:52150 'android_sync_integration_tests': {
James Shen9e4c19e2022-04-15 17:57:50151 'modifications': {
152 'android-12-x64-rel': {
153 'swarming': {
154 'shards': 3,
155 },
156 },
Prakharc60ec802023-03-06 07:49:29157 # If you change this, make similar changes in android-x86-code-coverage below
Stephanie Kim11192362022-11-04 18:17:44158 'android-nougat-x86-rel': {
James Shen74810512023-02-22 20:27:39159 'ci_only': True,
Stephanie Kim11192362022-11-04 18:17:44160 'swarming': {
161 'shards': 2,
162 },
163 },
Prakharc60ec802023-03-06 07:49:29164 # Keep this same as android-nougat-x86-rel above
165 'android-x86-code-coverage': {
166 'ci_only': True,
167 'swarming': {
168 'shards': 2,
169 },
170 },
James Shen9e4c19e2022-04-15 17:57:50171 },
Sven Zhengc60debc2020-09-10 18:40:52172 },
Kenneth Russelleb60cbd22017-12-05 07:54:28173 'android_webview_unittests': {
Kenneth Russelleb60cbd22017-12-05 07:54:28174 },
Jamie Madillc9ae75d2020-10-23 19:19:48175 'angle_unittests': {
176 'remove_from': [
177 # On Android, these are already run on the main waterfall.
178 'Android FYI Release (Nexus 5X)',
Yuly Novikovc1117ae82021-01-15 22:33:36179 # Times out listing tests crbug.com/1167314
180 'android-asan',
Jamie Madillc9ae75d2020-10-23 19:19:48181 # Does not currently work on Lacros configurations.
Sven Zheng93bfaa92022-06-09 17:06:03182 'linux-lacros-asan-lsan-rel',
Junhua Chen39dd2dba2021-05-05 17:51:54183 'linux-lacros-code-coverage',
Jamie Madillc9ae75d2020-10-23 19:19:48184 'linux-lacros-tester-rel',
185 'linux-lacros-tester-fyi-rel',
Yuke Liaof27ebe52021-08-05 22:12:40186 'linux-lacros-dbg-tests-fyi',
Jamie Madillc9ae75d2020-10-23 19:19:48187 ],
Chong Gua10cd302022-01-14 19:22:19188 'modifications': {
189 # anglebug.com/6894
190 'fuchsia-fyi-x64-asan': {
191 'args': [
192 # crbug.com/
193 '--gtest_filter=-ConstructCompilerTest.DefaultParameters',
194 ],
195 },
196 },
Jamie Madillc9ae75d2020-10-23 19:19:48197 },
Eriko Kurimoto61a85332022-10-19 05:19:45198 'ash_crosapi_tests': {
199 'remove_from': [
200 # Disabling MSan tests since linux-chromeos chrome crashes on MSan build.
201 # TODO(crbug.com/1376064): Re-enable.
202 'Linux ChromiumOS MSan Tests',
203 ]
204 },
Sven Zhengef0d0872022-04-04 22:13:29205 'aura_unittests amd64-generic': {
Yuke Liaob6beaeca2021-04-29 23:41:51206 'remove_from': [
207 # TODO(crbug.com/1204231): Re-enable.
208 'lacros-amd64-generic-rel',
209 ]
210 },
Sven Zhengef0d0872022-04-04 22:13:29211 'aura_unittests eve': {
Yuke Liao79a8cf52021-06-11 18:14:34212 'remove_from': [
213 # TODO(crbug.com/1204231): Re-enable.
214 'lacros-amd64-generic-rel',
215 ]
216 },
Chong Gudf7fbe92021-02-24 02:55:58217 'base_unittests': {
218 'modifications': {
Ben Joycea0319112022-11-15 23:53:12219 'ToTAndroid': {
220 'args': [
221 '--test-launcher-filter-file=../../testing/buildbot/filters/android.pie_tot.base_unittests.filter',
222 ],
223 },
Haiyang Pan2a55a452022-03-03 08:18:33224 'android-12-x64-rel': {
Haiyang Panc08d02212021-10-18 18:58:38225 'args': [
Haiyang Pan32338b72023-02-08 23:48:58226 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
227 ],
228 },
229 'android-12l-x64-dbg-tests': {
230 'args': [
231 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
232 ],
233 },
Haiyang Panef317e4372023-05-19 01:30:08234 'android-12l-x64-fyi-dbg': {
235 'args': [
236 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
237 ],
238 },
239 'android-13-x64-fyi-rel': {
240 'args': [
241 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
242 ],
243 },
Haiyang Pan32338b72023-02-08 23:48:58244 'android-13-x64-rel': {
245 'args': [
246 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator.base_unittests.filter',
Haiyang Panc7c009e2021-03-18 19:31:23247 ],
248 },
Chong Gudf7fbe92021-02-24 02:55:58249 'fuchsia-code-coverage': {
250 'swarming': {
251 'shards': 5,
252 },
253 },
Chong Gua10cd302022-01-14 19:22:19254 'fuchsia-fyi-x64-asan': {
255 'args': [
256 '--test-launcher-filter-file=../../testing/buildbot/filters/fuchsia.lsan.base_unittests.filter',
257 ],
258 },
Chong Gudf7fbe92021-02-24 02:55:58259 },
260 },
James Shen5df9af872022-10-28 01:32:15261 'bf_cache_content_browsertests': {
262 'modifications': {
263 'android-bfcache-rel': {
264 'args': [
James Shenea291852022-11-08 04:19:45265 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_n.content_browsertests.filter',
James Shen5df9af872022-10-28 01:32:15266 ],
James Shen2c574ce02022-11-03 22:59:23267 'swarming': {
268 'shards': 15,
269 },
James Shen5df9af872022-10-28 01:32:15270 }
271 }
272 },
Haiyang Pan363c59a2020-07-23 05:47:04273 'blink_platform_unittests': {
274 'modifications': {
James Shen088356672022-06-28 01:21:14275 # TODO(crbug.com/1303439): Remove this filter
Prakharc60ec802023-03-06 07:49:29276 # If you change this, make similar changes in android-x86-code-coverage below
James Shen088356672022-06-28 01:21:14277 'android-nougat-x86-rel': {
278 'args': [
279 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_n.blink_platform_unittests.filter',
280 ],
281 },
Haiyang Panfbd2b482020-09-23 23:41:25282 'android-pie-arm64-rel': {
283 # TODO(crbug.com/1111436): Move this back to walleye if/when additional
284 # capacity has been deployed.
285 'swarming': {
286 'dimension_sets': [
287 {
288 'device_type': 'sailfish',
289 },
290 ],
291 },
292 },
Prakharc60ec802023-03-06 07:49:29293 # Keep this same as android-nougat-x86-rel above
294 'android-x86-code-coverage': {
295 'args': [
296 '--test-launcher-filter-file=../../testing/buildbot/filters/android.emulator_n.blink_platform_unittests.filter',
297 ],
298 },
Chong Gu52481452021-04-02 17:52:41299 'fuchsia-code-coverage': {
300 'swarming': {
301 'shards': 5,
302 },
303 },
Haiyang Pan363c59a2020-07-23 05:47:04304 },
305 },
Brian Sheedy944d0622022-07-27 20:42:42306 'blink_pytype': {
307 'remove_from': [
308 'linux-code-coverage',
309 ],
310 },
Chong Gu76046412021-09-22 17:49:21311 'blink_unittests': {
Chong Guc2d145e2021-03-31 19:55:34312 'modifications': {
313 'fuchsia-code-coverage': {
314 'swarming': {
315 'shards': 30,
316 },
317 },
318 },
319 },
John Chenaab16fc02020-03-03 06:35:45320 'blink_web_tests': {
321 'remove_from': [
John Chenaab16fc02020-03-03 06:35:45322 'Win10 Tests x64 (dbg)',
323 ],
324 'modifications': {
John Chenaab16fc02020-03-03 06:35:45325 'Linux Tests': {
326 "args": [
327 "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw",
328 ],
Struan Shrimpton57c594e42022-01-25 17:59:13329 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51330 'shards': 8,
Struan Shrimpton57c594e42022-01-25 17:59:13331 },
John Chenaab16fc02020-03-03 06:35:45332 },
333 'Linux Tests (dbg)(1)': {
334 'args': [
335 '--debug',
336 ],
337 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51338 'shards': 12,
John Chenaab16fc02020-03-03 06:35:45339 },
340 },
John Chenaab16fc02020-03-03 06:35:45341 'Mac10.13 Tests': {
342 'swarming': {
343 'dimension_sets': [
344 {
Erik Staab59591f32020-08-25 23:07:07345 'gpu': None,
John Chenaab16fc02020-03-03 06:35:45346 },
347 ],
Weizhong Xia8b786422022-06-23 14:46:51348 'shards': 10,
John Chenaab16fc02020-03-03 06:35:45349 },
350 },
Zhaoyang Li69e2e002021-02-19 21:57:35351 'Mac10.15 Tests': {
352 'swarming': {
353 'dimension_sets': [
354 {
355 'gpu': None,
356 },
357 ],
Weizhong Xia8b786422022-06-23 14:46:51358 'shards': 10,
Zhaoyang Li69e2e002021-02-19 21:57:35359 },
360 },
Lindsay Pasricha8d9a0c32021-12-08 17:08:29361 'Mac11 Tests': {
362 'swarming': {
363 'dimension_sets': [
364 {
365 'gpu': None,
366 },
367 ],
Weizhong Xia8b786422022-06-23 14:46:51368 'shards': 12,
Lindsay Pasricha8d9a0c32021-12-08 17:08:29369 },
370 },
Yue She0cad2922022-06-24 15:06:07371 'Mac12 Tests': {
372 'swarming': {
373 'dimension_sets': [
374 {
375 'gpu': None,
376 },
377 ],
Lindsay Pasrichadf6edb62022-07-06 22:08:35378 'shards': 12,
Yue She0cad2922022-06-24 15:06:07379 },
380 },
Yue Shed8e45742023-06-01 15:18:03381 'Mac13 Tests': {
382 'swarming': {
383 'dimension_sets': [
384 {
385 'gpu': None,
386 },
387 ],
388 'shards': 12,
389 },
390 },
Cameron Higgins2e4b7f12023-05-31 18:29:32391 'Mac13 Tests (dbg)': {
Zhaoyang Li9357e1e12021-12-07 18:53:17392 'experiment_percentage': 100,
393 'args': [
394 '--debug',
395 ],
396 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51397 'shards': 16,
Zhaoyang Li9357e1e12021-12-07 18:53:17398 },
399 },
Chong Gu75af969a2021-03-22 14:46:14400 'ToTFuchsia x64': {
401 'args': [
402 '--platform=fuchsia',
403 ],
404 'swarming': {
405 'shards': 1,
406 },
407 },
John Chenaab16fc02020-03-03 06:35:45408 'V8 Blink Linux Debug': {
Michael Achenbach455d39242020-06-22 13:33:45409 # This builder compiles Chromium in release mode, but V8 in debug.
410 # TODO(machenbach): Change name and out directory of this builder to
411 # something less confusing.
John Chenaab16fc02020-03-03 06:35:45412 'args': [
Michael Achenbach455d39242020-06-22 13:33:45413 '-t',
414 'Debug',
415 '--release',
John Chenaab16fc02020-03-03 06:35:45416 ],
417 },
418 'V8 Blink Linux Future': {
419 'args': [
Brian Sheedy10ddf0a2021-08-16 23:19:07420 '--flag-specific=future-js',
John Chenaab16fc02020-03-03 06:35:45421 ],
422 },
423 'WebKit Linux ASAN': {
424 'args': [
425 '--additional-expectations',
426 '../../third_party/blink/web_tests/ASANExpectations',
Weizhong Xia91b53362022-01-05 17:13:35427 '--timeout-ms',
John Chenaab16fc02020-03-03 06:35:45428 '48000',
429 '--enable-sanitizer',
430 ],
431 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51432 'shards': 8,
John Chenaab16fc02020-03-03 06:35:45433 },
434 },
435 'WebKit Linux Leak': {
436 'args': [
437 '--additional-expectations',
438 '../../third_party/blink/web_tests/LeakExpectations',
Weizhong Xia91b53362022-01-05 17:13:35439 '--timeout-ms',
John Chenaab16fc02020-03-03 06:35:45440 '48000',
441 '--enable-leak-detection',
442 ],
443 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51444 'shards': 4,
John Chenaab16fc02020-03-03 06:35:45445 },
446 },
447 'WebKit Linux MSAN': {
448 'args': [
449 '--additional-expectations',
450 '../../third_party/blink/web_tests/MSANExpectations',
Weizhong Xia91b53362022-01-05 17:13:35451 '--timeout-ms',
John Chenaab16fc02020-03-03 06:35:45452 '66000',
453 '--enable-sanitizer',
454 ],
455 'swarming': {
456 'expiration': 36000,
457 'hard_timeout': 10800,
458 'io_timeout': 3600,
Weizhong Xia8b786422022-06-23 14:46:51459 'shards': 8,
John Chenaab16fc02020-03-03 06:35:45460 },
461 },
John Chenaab16fc02020-03-03 06:35:45462 'Win10 Tests x64': {
463 'args': [
464 '--target',
Takuto Ikutab28b582e2021-02-10 01:41:21465 'Release_x64',
John Chenaab16fc02020-03-03 06:35:45466 ],
467 'swarming': {
Struan Shrimpton5786bfd02023-01-11 00:47:54468 "shards": 12,
John Chenaab16fc02020-03-03 06:35:45469 },
470 },
471 'Win10 Tests x64 (dbg)': {
472 'args': [
473 '--debug',
474 '--target',
Takuto Ikutab28b582e2021-02-10 01:41:21475 'Debug_x64',
John Chenaab16fc02020-03-03 06:35:45476 ],
477 },
Kuan Huanga89737d2022-01-07 19:42:11478 'Win11 Tests x64': {
479 'args': [
480 '--target',
481 'Release_x64',
482 ],
483 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51484 "shards": 12
Kuan Huanga89737d2022-01-07 19:42:11485 },
486 },
John Chenaab16fc02020-03-03 06:35:45487 # tryserver.chromium.android
488 'android_blink_rel': {
489 'args': [
490 '--release',
491 '--android',
John Chenaab16fc02020-03-03 06:35:45492 # TODO(crbug.com/875172): run on the real GPU since the
493 # software fallback, SwiftShader, isn't yet for 32-bit ARM.
494 '--additional-driver-flag=--use-gpu-in-tests',
495 ],
496 },
497 'devtools_frontend_linux_blink_light_rel': {
498 'args': [
499 'http/tests/devtools',
500 ],
501 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51502 'shards': 2,
John Chenaab16fc02020-03-03 06:35:45503 },
504 },
Takuto Ikutabace8f82022-01-26 23:47:47505 'devtools_frontend_linux_blink_light_rel_fastbuild': {
506 'args': [
507 'http/tests/devtools',
508 ],
509 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51510 'shards': 2,
Takuto Ikutabace8f82022-01-26 23:47:47511 },
512 },
David Dorwinf060a72d2022-10-05 22:09:03513 'fuchsia-x64-cast-receiver-rel': {
John Chenaab16fc02020-03-03 06:35:45514 'args': [
515 '--platform=fuchsia',
David Dorwinf060a72d2022-10-05 22:09:03516 '--jobs=1',
John Chenaab16fc02020-03-03 06:35:45517 ],
518 'swarming': {
519 'shards': 1,
520 },
521 },
David Dorwin49b2ddb2022-09-08 20:49:43522 'fuchsia-x64-rel': {
523 'args': [
524 '--platform=fuchsia',
525 '--jobs=1',
526 ],
527 'swarming': {
528 'shards': 1,
529 },
530 },
John Chenaab16fc02020-03-03 06:35:45531 'linux-blink-animation-use-time-delta': {
532 'args': [
533 '--debug',
534 ],
535 },
Jonathan Leed3ba7bf2023-01-13 21:15:16536 'linux-blink-rel': {
537 'swarming': {
538 'hard_timeout': 2400,
539 },
540 },
Daniel Libby178c9832020-10-08 20:36:14541 'linux-blink-web-tests-force-accessibility-rel': {
542 'args': [
Brian Sheedy10ddf0a2021-08-16 23:19:07543 '--flag-specific=force-renderer-accessibility',
Daniel Libby178c9832020-10-08 20:36:14544 ],
545 },
W. James MacLeance9b3a62022-10-05 16:58:58546 'linux-blink-wpt-reset-rel': {
547 'args': [
548 '--additional-driver-flag=--force-browsing-instance-reset-between-tests',
549 ],
550 # The flag above will slow the tests down, and we don't want the bot to
551 # timeout, so set a higher timeout here.
552 # TODO(wjmaclean): It would be nice if we could somehow specify a delta
553 # to the default/inherited timeout instead of an absolute.
554 'swarming': {
555 'hard_timeout': 1500,
556 },
557 },
John Chenaab16fc02020-03-03 06:35:45558 'linux-code-coverage': {
559 'args': [
Prakharbe44338a2023-05-24 19:02:04560 '--additional-driver-flag=--no-sandbox',
561 '--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%9m.profraw',
John Chenaab16fc02020-03-03 06:35:45562 ],
563 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51564 'shards': 8,
John Chenaab16fc02020-03-03 06:35:45565 },
566 },
Aaron Gable40f79aa2020-03-21 01:21:15567 'linux-layout-tests-edit-ng': {
568 'args': [
Brian Sheedy10ddf0a2021-08-16 23:19:07569 '--flag-specific=enable-editing-ng',
Aaron Gable40f79aa2020-03-21 01:21:15570 ],
571 },
Thiago Perrotta61487462023-01-20 12:52:32572 'linux-rel-cft': {
573 "args": [
Michael Lippautz60aa8f32023-05-24 14:27:02574 '--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw',
575 '--test-launcher-filter-file=../../testing/buildbot/filters/cft.blink_web_tests.filter',
576 '--flag-specific=chrome-for-testing',
Thiago Perrotta61487462023-01-20 12:52:32577 ],
578 'swarming': {
579 'shards': 8,
580 },
581 },
SrinivasBathula90213d62023-02-27 06:12:18582 'mac-osxbeta-rel': {
583 'swarming': {
584 'dimension_sets': [
585 {
586 'gpu': None
587 }
588 ],
589 'shards': 10,
590 },
591 },
Thiago Perrottaa289bad2023-01-20 12:44:39592 'mac-rel-cft': {
Thiago Perrotta4085d56b2023-03-22 23:05:09593 "args": [
594 '--test-launcher-filter-file=../../testing/buildbot/filters/cft.blink_web_tests.filter',
595 ],
Thiago Perrottaa289bad2023-01-20 12:44:39596 'swarming': {
597 'dimension_sets': [
598 {
599 'gpu': None,
600 },
601 ],
602 'shards': 12,
Thiago Perrottaa289bad2023-01-20 12:44:39603 },
604 },
Riley Wong89d6dc72022-09-08 20:29:07605 'mac10.13-blink-rel': {
606 'swarming': {
607 'shards': 1,
608 },
609 },
Riley Wong89d6dc72022-09-08 20:29:07610 'mac10.14-blink-rel': {
611 'swarming': {
612 'shards': 1,
613 },
614 },
Riley Wong89d6dc72022-09-08 20:29:07615 'mac10.15-blink-rel': {
616 'swarming': {
617 'shards': 8,
Weizhong Xia21063c72022-11-09 23:48:30618 'hard_timeout': 2400,
Riley Wong89d6dc72022-09-08 20:29:07619 },
620 },
Riley Wongc20ba4a2022-09-12 17:06:31621 'mac11.0-blink-rel': {
622 'swarming': {
623 'shards': 5,
Weizhong Xia21063c72022-11-09 23:48:30624 'hard_timeout': 2400,
625 },
626 },
627 'mac11.0.arm64-blink-rel': {
628 'swarming': {
629 'hard_timeout': 2400,
630 },
631 },
632 'mac12.0-blink-rel': {
633 'swarming': {
634 'hard_timeout': 2400,
635 },
636 },
637 'mac12.0.arm64-blink-rel': {
638 'swarming': {
639 'hard_timeout': 2400,
Riley Wongc20ba4a2022-09-12 17:06:31640 },
641 },
Thiago Perrotta770925d42023-01-17 18:38:51642 'win-rel-cft': {
643 'args': [
644 '--target',
645 'Release_x64',
Michael Lippautz60aa8f32023-05-24 14:27:02646 '--test-launcher-filter-file=../../testing/buildbot/filters/cft.blink_web_tests.filter',
Thiago Perrotta770925d42023-01-17 18:38:51647 ],
Thiago Perrotta314f7b02023-01-19 21:30:53648 'swarming': {
649 "shards": 12,
Thiago Perrotta314f7b02023-01-19 21:30:53650 },
Thiago Perrotta770925d42023-01-17 18:38:51651 },
Ben Pasteneebad8fc2022-08-12 23:34:28652 'win10-rel-no-external-ip': {
653 'args': [
654 '--target',
655 'Release_x64',
656 ],
657 'swarming': {
658 "shards": 12
659 },
660 },
Riley Wong5feb0eb2022-08-05 15:49:26661 'win10.20h2-blink-rel': {
662 'swarming': {
663 'shards': 6,
Weizhong Xia21063c72022-11-09 23:48:30664 'hard_timeout': 2400,
Riley Wong5feb0eb2022-08-05 15:49:26665 },
666 },
Nihar Damarasinguf31a5cc2023-05-24 18:38:06667 'win11-arm64-blink-rel': {
668 'swarming': {
669 'hard_timeout': 2400,
670 },
671 'args': [
672 '--target',
673 'Release_x64',
674 ],
675 },
Weizhong Xia8b786422022-06-23 14:46:51676 'win11-blink-rel': {
677 'swarming': {
Weizhong Xia21063c72022-11-09 23:48:30678 'hard_timeout': 2400,
Weizhong Xia8b786422022-06-23 14:46:51679 },
680 'args': [
681 '--target',
682 'Release_x64',
683 ],
684 },
685 },
686 },
687 'blink_wpt_tests': {
688 'remove_from': [
Weizhong Xia8b786422022-06-23 14:46:51689 'Win10 Tests x64 (dbg)',
Weizhong Xia6d85a442022-06-23 17:17:06690 'devtools_frontend_linux_blink_light_rel',
691 'devtools_frontend_linux_blink_light_rel_fastbuild',
Thiago Perrotta4085d56b2023-03-22 23:05:09692 'linux-rel-cft',
693 'mac-rel-cft',
694 'win-rel-cft',
Weizhong Xia8b786422022-06-23 14:46:51695 ],
696 'modifications': {
Weizhong Xia8b786422022-06-23 14:46:51697 'Linux Tests': {
698 "args": [
699 "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw",
700 ],
701 'swarming': {
Weizhong Xia8b786422022-06-23 14:46:51702 'shards': 10,
703 },
704 },
705 'Linux Tests (dbg)(1)': {
706 'args': [
707 '--debug',
708 ],
709 'swarming': {
710 'shards': 18,
711 },
712 },
713 'Mac10.13 Tests': {
714 'swarming': {
715 'dimension_sets': [
716 {
717 'gpu': None,
718 },
719 ],
720 'shards': 15,
721 },
722 },
723 'Mac10.15 Tests': {
724 'swarming': {
725 'dimension_sets': [
726 {
727 'gpu': None,
728 },
729 ],
730 'shards': 15,
731 },
732 },
733 'Mac11 Tests': {
734 'swarming': {
735 'dimension_sets': [
736 {
737 'gpu': None,
738 },
739 ],
740 'shards': 18,
Weizhong Xia8b786422022-06-23 14:46:51741 },
742 },
Lindsay Pasrichadf6edb62022-07-06 22:08:35743 'Mac12 Tests': {
744 'swarming': {
745 'dimension_sets': [
746 {
747 'gpu': None,
748 },
749 ],
750 'shards': 18,
Lindsay Pasrichadf6edb62022-07-06 22:08:35751 },
752 },
Yue Shed8e45742023-06-01 15:18:03753 'Mac13 Tests': {
754 'swarming': {
755 'dimension_sets': [
756 {
757 'gpu': None,
758 },
759 ],
760 'shards': 18,
761 },
762 },
Cameron Higgins2e4b7f12023-05-31 18:29:32763 'Mac13 Tests (dbg)': {
Weizhong Xia8b786422022-06-23 14:46:51764 'experiment_percentage': 100,
765 'args': [
766 '--debug',
767 ],
768 'swarming': {
769 'shards': 24,
770 },
771 },
772 'ToTFuchsia x64': {
773 'args': [
774 '--platform=fuchsia',
775 ],
776 'swarming': {
777 'shards': 1,
778 },
779 },
780 'V8 Blink Linux Debug': {
781 # This builder compiles Chromium in release mode, but V8 in debug.
782 # TODO(machenbach): Change name and out directory of this builder to
783 # something less confusing.
784 'args': [
785 '-t',
786 'Debug',
787 '--release',
788 ],
789 },
790 'V8 Blink Linux Future': {
791 'args': [
792 '--flag-specific=future-js',
793 ],
794 },
795 'WebKit Linux ASAN': {
796 'args': [
797 '--additional-expectations',
798 '../../third_party/blink/web_tests/ASANExpectations',
799 '--timeout-ms',
800 '48000',
801 '--enable-sanitizer',
802 ],
803 'swarming': {
804 'shards': 12,
805 },
806 },
807 'WebKit Linux Leak': {
808 'args': [
809 '--additional-expectations',
810 '../../third_party/blink/web_tests/LeakExpectations',
811 '--timeout-ms',
812 '48000',
813 '--enable-leak-detection',
814 ],
815 'swarming': {
816 'shards': 6,
817 },
818 },
819 'WebKit Linux MSAN': {
820 'args': [
821 '--additional-expectations',
822 '../../third_party/blink/web_tests/MSANExpectations',
823 '--timeout-ms',
824 '66000',
825 '--enable-sanitizer',
826 ],
827 'swarming': {
828 'expiration': 36000,
829 'hard_timeout': 10800,
830 'io_timeout': 3600,
831 'shards': 12,
832 },
833 },
834 'Win10 Tests x64': {
835 'args': [
836 '--target',
837 'Release_x64',
838 ],
839 'swarming': {
Struan Shrimpton5786bfd02023-01-11 00:47:54840 "shards": 18,
Weizhong Xia8b786422022-06-23 14:46:51841 },
Weizhong Xia8b786422022-06-23 14:46:51842 },
843 'Win10 Tests x64 (dbg)': {
844 'args': [
845 '--debug',
846 '--target',
847 'Debug_x64',
848 ],
849 },
850 'Win11 Tests x64': {
851 'args': [
852 '--target',
853 'Release_x64',
854 ],
855 'swarming': {
856 "shards": 18
857 },
858 },
859 # tryserver.chromium.android
860 'android_blink_rel': {
861 'args': [
862 '--release',
863 '--android',
Weizhong Xia8b786422022-06-23 14:46:51864 # TODO(crbug.com/875172): run on the real GPU since the
865 # software fallback, SwiftShader, isn't yet for 32-bit ARM.
866 '--additional-driver-flag=--use-gpu-in-tests',
867 ],
868 },
David Dorwinf060a72d2022-10-05 22:09:03869 'fuchsia-x64-cast-receiver-rel': {
Weizhong Xia8b786422022-06-23 14:46:51870 'args': [
871 '--platform=fuchsia',
David Dorwinf060a72d2022-10-05 22:09:03872 '--jobs=1',
Weizhong Xia8b786422022-06-23 14:46:51873 ],
874 'swarming': {
875 'shards': 1,
876 },
877 },
David Dorwin49b2ddb2022-09-08 20:49:43878 'fuchsia-x64-rel': {
879 'args': [
880 '--platform=fuchsia',
881 '--jobs=1',
882 ],
883 'swarming': {
884 'shards': 1,
885 },
886 },
Weizhong Xia8b786422022-06-23 14:46:51887 'linux-bfcache-rel': {
888 'args': [
Weizhong Xia8b786422022-06-23 14:46:51889 # TODO(crbug.com/1255537): Re-enable the test.
890 '--ignore-tests=external/wpt/html/browsers/browsing-the-web/back-forward-cache/events.html',
891 ],
892 },
893 'linux-blink-animation-use-time-delta': {
894 'args': [
895 '--debug',
896 ],
897 },
Jonathan Leed3ba7bf2023-01-13 21:15:16898 'linux-blink-rel': {
899 'swarming': {
900 'hard_timeout': 2400,
901 },
902 },
Weizhong Xia8b786422022-06-23 14:46:51903 'linux-blink-web-tests-force-accessibility-rel': {
904 'args': [
905 '--flag-specific=force-renderer-accessibility',
906 ],
907 },
W. James MacLeance9b3a62022-10-05 16:58:58908 'linux-blink-wpt-reset-rel': {
909 'args': [
910 '--additional-driver-flag=--force-browsing-instance-reset-between-tests',
911 ],
912 },
Weizhong Xia8b786422022-06-23 14:46:51913 'linux-code-coverage': {
914 'args': [
Prakharbe44338a2023-05-24 19:02:04915 '--additional-driver-flag=--no-sandbox',
916 '--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%9m.profraw',
Weizhong Xia8b786422022-06-23 14:46:51917 ],
Weizhong Xia8b786422022-06-23 14:46:51918 },
919 'linux-layout-tests-edit-ng': {
920 'args': [
921 '--flag-specific=enable-editing-ng',
922 ],
923 },
Thiago Perrotta61487462023-01-20 12:52:32924 'linux-rel-cft': {
925 "args": [
926 "--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw",
Michael Lippautz8f4016b2023-06-13 16:18:33927 '--flag-specific=chrome-for-testing',
Thiago Perrotta61487462023-01-20 12:52:32928 ],
929 'swarming': {
930 'shards': 10,
931 },
932 },
SrinivasBathula90213d62023-02-27 06:12:18933 'mac-osxbeta-rel': {
934 'swarming': {
935 'dimension_sets': [
936 {
937 'gpu': None
938 }
939 ],
940 'shards': 7,
941 },
942 },
Thiago Perrottaa289bad2023-01-20 12:44:39943 'mac-rel-cft': {
944 'args': [
945 '--driver-logging',
946 ],
947 'swarming': {
948 'dimension_sets': [
949 {
950 'gpu': None,
951 },
952 ],
953 'shards': 18,
Thiago Perrottaa289bad2023-01-20 12:44:39954 },
955 },
Riley Wong89d6dc72022-09-08 20:29:07956 'mac10.13-blink-rel': {
957 'swarming': {
958 'shards': 1,
959 },
960 },
Riley Wong89d6dc72022-09-08 20:29:07961 'mac10.14-blink-rel': {
962 'swarming': {
963 'shards': 1,
964 },
965 },
Riley Wong89d6dc72022-09-08 20:29:07966 'mac10.15-blink-rel': {
967 'swarming': {
968 'shards': 12,
Weizhong Xia21063c72022-11-09 23:48:30969 'hard_timeout': 2400,
Riley Wong89d6dc72022-09-08 20:29:07970 },
971 },
Riley Wongc20ba4a2022-09-12 17:06:31972 'mac11.0-blink-rel': {
973 'swarming': {
974 'shards': 7,
Weizhong Xia21063c72022-11-09 23:48:30975 'hard_timeout': 2400,
976 },
977 },
978 'mac11.0.arm64-blink-rel': {
979 'swarming': {
980 'hard_timeout': 2400,
981 },
982 },
983 'mac12.0-blink-rel': {
984 'swarming': {
985 'hard_timeout': 2400,
986 },
987 },
988 'mac12.0.arm64-blink-rel': {
989 'swarming': {
990 'hard_timeout': 2400,
Riley Wongc20ba4a2022-09-12 17:06:31991 },
992 },
Thiago Perrotta770925d42023-01-17 18:38:51993 'win-rel-cft': {
994 'args': [
995 '--target',
996 'Release_x64',
997 ],
Thiago Perrotta314f7b02023-01-19 21:30:53998 'swarming': {
999 "shards": 18,
Thiago Perrotta314f7b02023-01-19 21:30:531000 },
Thiago Perrotta770925d42023-01-17 18:38:511001 },
Ben Pasteneebad8fc2022-08-12 23:34:281002 'win10-rel-no-external-ip': {
1003 'args': [
1004 '--target',
1005 'Release_x64',
1006 ],
1007 'swarming': {
1008 "shards": 18
1009 },
1010 },
Riley Wong5feb0eb2022-08-05 15:49:261011 'win10.20h2-blink-rel': {
1012 'swarming': {
1013 'shards': 9,
Weizhong Xia21063c72022-11-09 23:48:301014 'hard_timeout': 2400,
Riley Wong5feb0eb2022-08-05 15:49:261015 },
1016 },
Nihar Damarasinguf31a5cc2023-05-24 18:38:061017 'win11-arm64-blink-rel': {
1018 'swarming': {
1019 'hard_timeout': 2400,
1020 },
1021 'args': [
1022 '--target',
1023 'Release_x64',
1024 ],
1025 },
Riley Wong5fc22762022-06-15 20:25:071026 'win11-blink-rel': {
1027 'swarming': {
Weizhong Xia21063c72022-11-09 23:48:301028 'hard_timeout': 2400,
Riley Wong5fc22762022-06-15 20:25:071029 },
1030 'args': [
1031 '--target',
1032 'Release_x64',
1033 ],
Thiago Perrotta770925d42023-01-17 18:38:511034 },
John Chenaab16fc02020-03-03 06:35:451035 },
1036 },
Kenneth Russelleb60cbd22017-12-05 07:54:281037 'browser_tests': {
Kenneth Russellfbda3c532017-12-08 23:57:241038 'remove_from': [
Brad Halla36886c2019-02-26 16:52:291039 'CrWinAsan(dll)', # https://crbug.com/935598
Nico Weber3af4da72018-02-27 01:59:411040 'linux-win_cross-rel',
Nico Weberdae13cd92018-12-14 13:56:261041 'ToTLinuxTSan', # https://crbug.com/368525
Lindsay Pasrichab3f72db2021-09-22 22:05:461042 # TODO(crbug.com/1171053): Enable the suite in 'Mac10.15 Tests' once it's
Zhaoyang Li69e2e002021-02-19 21:57:351043 # off CQ.
Gary Tongd5f5a742021-01-29 00:01:131044 'Mac10.13 Tests', # https://crbug.com/1042757
Zhaoyang Li69e2e002021-02-19 21:57:351045 'Mac10.15 Tests', # https://crbug.com/1042757
Cameron Higgins2e4b7f12023-05-31 18:29:321046 'Mac13 Tests (dbg)', # https://crbug.com/1201386
Lindsay Pasricha8155d052021-04-22 00:06:431047 'mac-code-coverage', # https://crbug.com/1201386
Lindsay Pasricha23227912023-01-11 22:36:341048 'mac11-arm64-rel-tests', # https://crbug.com/1406364
1049 'mac12-arm64-rel-tests', # https://crbug.com/1406364
Will Yeagera090e6a2023-06-09 14:59:411050 'mac13-arm64-rel-tests', # https://crbug.com/1406364
Nico Weberd36959682018-04-12 03:33:461051 'Linux TSan Tests', # https://crbug.com/368525
Ian Struiksma2e23f962023-05-26 22:37:201052 'linux-exp-tsan-fyi-rel', # https://crbug.com/368525
Stephen Martinise701f5a2018-05-04 01:25:111053 'Win10 Tests x64 (dbg)',
Kenneth Russellfbda3c532017-12-08 23:57:241054 ],
Kenneth Russelleb60cbd22017-12-05 07:54:281055 'modifications': {
Nico Weber3f919792020-06-12 23:34:52