Generate FYI bot config for linux-blink-wpt-reset-rel.
Updates trybot configs to add a blink web/wpt test runner that resets
the BrowsingInstance in between every test. Flakiness data from this bot will be compared the wpt/web test flakiness on linux-rel to determine which tests may benefit from having the reset applied.
This bot is modeled on linux-blink-rel, but with a longer timeout since the BrowsingInstance reset slows down the tests.
Bug: 1227911, 1366972
Change-Id: Ib71396f7e0d273913eceba3ebc65fd7435a2a777
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3869445
Commit-Queue: James Maclean <[email protected]>
Reviewed-by: Ben Pastene <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1055354}
diff --git a/testing/buildbot/chromium.fyi.json b/testing/buildbot/chromium.fyi.json
index 931d9ceaa..edd6425c 100644
--- a/testing/buildbot/chromium.fyi.json
+++ b/testing/buildbot/chromium.fyi.json
@@ -94932,6 +94932,85 @@
}
]
},
+ "linux-blink-wpt-reset-rel": {
+ "isolated_scripts": [
+ {
+ "args": [
+ "--num-retries=3",
+ "--write-run-histories-to=${ISOLATED_OUTDIR}/run_histories.json",
+ "--git-revision=${got_revision}",
+ "--additional-driver-flag=--force-browsing-instance-reset-between-tests"
+ ],
+ "check_flakiness_for_new_tests": false,
+ "isolate_name": "blink_web_tests",
+ "merge": {
+ "args": [
+ "--verbose"
+ ],
+ "script": "//third_party/blink/tools/merge_web_test_results.py"
+ },
+ "name": "blink_web_tests",
+ "precommit_args": [
+ "--gerrit-issue=${patch_issue}",
+ "--gerrit-patchset=${patch_set}",
+ "--buildbucket-id=${buildbucket_build_id}"
+ ],
+ "resultdb": {
+ "enable": true
+ },
+ "results_handler": "layout tests",
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "os": "Ubuntu-18.04"
+ }
+ ],
+ "hard_timeout": 1500,
+ "service_account": "[email protected]",
+ "shards": 5
+ },
+ "test_id_prefix": "ninja://:blink_web_tests/"
+ },
+ {
+ "args": [
+ "--num-retries=3",
+ "--write-run-histories-to=${ISOLATED_OUTDIR}/run_histories.json",
+ "--git-revision=${got_revision}",
+ "--additional-driver-flag=--force-browsing-instance-reset-between-tests"
+ ],
+ "check_flakiness_for_new_tests": false,
+ "isolate_name": "blink_wpt_tests",
+ "merge": {
+ "args": [
+ "--verbose"
+ ],
+ "script": "//third_party/blink/tools/merge_web_test_results.py"
+ },
+ "name": "blink_wpt_tests",
+ "precommit_args": [
+ "--gerrit-issue=${patch_issue}",
+ "--gerrit-patchset=${patch_set}",
+ "--buildbucket-id=${buildbucket_build_id}"
+ ],
+ "resultdb": {
+ "enable": true
+ },
+ "results_handler": "layout tests",
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "os": "Ubuntu-18.04"
+ }
+ ],
+ "service_account": "[email protected]",
+ "shards": 7
+ },
+ "test_id_prefix": "ninja://:blink_wpt_tests/"
+ }
+ ]
+ },
"linux-chromeos-annotator-rel": {
"scripts": [
{
diff --git a/testing/buildbot/test_suite_exceptions.pyl b/testing/buildbot/test_suite_exceptions.pyl
index b5d4831..6059d34e 100644
--- a/testing/buildbot/test_suite_exceptions.pyl
+++ b/testing/buildbot/test_suite_exceptions.pyl
@@ -464,6 +464,18 @@
'--flag-specific=force-renderer-accessibility',
],
},
+ 'linux-blink-wpt-reset-rel': {
+ 'args': [
+ '--additional-driver-flag=--force-browsing-instance-reset-between-tests',
+ ],
+ # The flag above will slow the tests down, and we don't want the bot to
+ # timeout, so set a higher timeout here.
+ # TODO(wjmaclean): It would be nice if we could somehow specify a delta
+ # to the default/inherited timeout instead of an absolute.
+ 'swarming': {
+ 'hard_timeout': 1500,
+ },
+ },
'linux-code-coverage': {
'args': [
'--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw',
@@ -762,6 +774,11 @@
'--flag-specific=force-renderer-accessibility',
],
},
+ 'linux-blink-wpt-reset-rel': {
+ 'args': [
+ '--additional-driver-flag=--force-browsing-instance-reset-between-tests',
+ ],
+ },
'linux-code-coverage': {
'args': [
'--additional-env-var=LLVM_PROFILE_FILE=${ISOLATED_OUTDIR}/profraw/default-%2m.profraw',
diff --git a/testing/buildbot/waterfalls.pyl b/testing/buildbot/waterfalls.pyl
index d034218..42c6c4c 100644
--- a/testing/buildbot/waterfalls.pyl
+++ b/testing/buildbot/waterfalls.pyl
@@ -3548,6 +3548,14 @@
'isolated_scripts': 'chromium_webkit_isolated_scripts',
},
},
+ 'linux-blink-wpt-reset-rel': {
+ 'mixins': [
+ 'linux-bionic',
+ ],
+ 'test_suites': {
+ 'isolated_scripts': 'chromium_webkit_isolated_scripts',
+ },
+ },
'linux-chromeos-annotator-rel': {
'test_suites': {
'scripts': 'test_traffic_annotation_auditor_script'
