Add a new 'Repeat Pageset Runs' checkbox to the Chromium Tryserver. * Setting the default value of discard_outliers in all benchmarks to 0. * Minor documentation improvements. BUG=358628, skia:2396 Review URL: https://codereview.chromium.org/232173006
diff --git a/appengine_scripts/skia-tree-status/skia_telemetry.py b/appengine_scripts/skia-tree-status/skia_telemetry.py index a2d1ea6..fbf3df1 100644 --- a/appengine_scripts/skia-tree-status/skia_telemetry.py +++ b/appengine_scripts/skia-tree-status/skia_telemetry.py
@@ -336,6 +336,7 @@ skia_patch = db.BlobProperty() chromium_patch = db.BlobProperty() blink_patch = db.BlobProperty() + num_repeated_runs = db.IntegerProperty() variance_threshold = db.FloatProperty(required=True) discard_outliers = db.FloatProperty(required=True) requested_time = db.DateTimeProperty(required=True) @@ -361,6 +362,7 @@ 'skia_patch': self.skia_patch, 'chromium_patch': self.chromium_patch, 'blink_patch': self.blink_patch, + 'num_repeated_runs': self.num_repeated_runs, 'variance_threshold': self.variance_threshold, 'discard_outliers': self.discard_outliers, 'requested_time': str(self.requested_time) @@ -847,6 +849,7 @@ benchmark_name = self.request.get('benchmark_name') benchmark_arguments = self.request.get('benchmark_arguments') pageset_type = self.request.get('pageset_type') + num_repeated_runs = int(self.request.get('num_repeated_runs')) variance_threshold = float(self.request.get('variance_threshold')) discard_outliers = float(self.request.get('discard_outliers')) description = self.request.get('description') @@ -865,6 +868,7 @@ skia_patch=skia_patch, chromium_patch=chromium_patch, blink_patch=blink_patch, + num_repeated_runs=num_repeated_runs, variance_threshold=variance_threshold, discard_outliers=discard_outliers, requested_time=requested_time,
diff --git a/appengine_scripts/skia-tree-status/templates/all_tasks.html b/appengine_scripts/skia-tree-status/templates/all_tasks.html index 6118032..1ac8f2c 100755 --- a/appengine_scripts/skia-tree-status/templates/all_tasks.html +++ b/appengine_scripts/skia-tree-status/templates/all_tasks.html
@@ -15,6 +15,7 @@ <td><b>Description</b></td> <td><b>Benchmark Name</b></td> <td><b>Benchmark Argument</b></td> + <td><b>Repeat Runs</b></td> <td><b>Variance Threshold</b></td> <td><b>Discard Outliers</b></td> <td><b>Pagesets Type</b></td> @@ -34,6 +35,13 @@ <td>{{ task.description }}</td> <td>{{ task.benchmark_name }}</td> <td>{{ task.benchmark_arguments }}</td> + <td> + {% if task.num_repeated_runs %} + {{ task.num_repeated_runs }} + {% else %} + 1 + {% endif %} + </td> <td>{{ task.variance_threshold }}</td> <td>{{ task.discard_outliers }}</td> <td>
diff --git a/appengine_scripts/skia-tree-status/templates/chromium_try.html b/appengine_scripts/skia-tree-status/templates/chromium_try.html index 1d3c67e..9675541 100755 --- a/appengine_scripts/skia-tree-status/templates/chromium_try.html +++ b/appengine_scripts/skia-tree-status/templates/chromium_try.html
@@ -10,21 +10,12 @@ {{ pending_tasks_count }} tasks currently in the <a align="center" href='pending_tasks'>Queue</a> <br/> -Documentation for Chromium Tryserver is <a href='https://docs.google.com/a/chromium.org/document/d/1GhqosQcwsy6F-eBAmFn_ITDF7_Iv_rY9FhCKwAnk9qQ/edit#'>here</a> +Documentation for Chromium Tryserver and benchmarks is <a href='https://docs.google.com/a/chromium.org/document/d/1GhqosQcwsy6F-eBAmFn_ITDF7_Iv_rY9FhCKwAnk9qQ/edit#'>here</a> <br/></br> - -<b>Note:</b> -<br/> -* The perf measurements are run on <a href='https://cloud.google.com/products/compute-engine/'>GCE</a> instances. There could be variance in runs depending on if there are other VMs running<br/>on the same physical machine. Please account for upto 1% variance in your runs till the framework moves to bare metal machines. -<br/><br/> -* The 'pixeldiffs' benchmark runs the <a href='https://skia.googlesource.com/buildbot/+/master/compute_engine_scripts/telemetry/telemetry_slave_scripts/capture_and_compare_pixeldiffs.py'>capture_and_compare_pixeldiffs.py</a> script against the live top 10k webpages (not using their<br/>archives). Images from this benchmark are visible to only @google.com accounts. - -<br/> -<br/> <a href="javascript:toggleDetails();">Show/hide algorithm details</a> <script type="text/javascript"> function toggleDetails() { - var details = ["benchmark_details", "variance_details", "outliers_details"]; + var details = ["benchmark_details", "variance_details", "outliers_details", "repeated_runs"]; for (var i=0; i<details.length; i++) { if (document.getElementById(details[i]).style.display == 'none') { document.getElementById(details[i]).style.display = ''; @@ -60,6 +51,17 @@ The Benchmark must output in CSV to do meaningful comparisions. </td> </tr> + <tr id="repeated_runs" style="display:none"> + <td>Repeat Pageset Runs</td> + <td> + <select id="num_repeated_runs" name="num_repeated_runs"> + <option value="1">1</option> + <option value="3" selected>3</option> + <option value="5">5</option> + </select> + Repeats each pageset and takes median of results. More documentation is here. + </td> + </tr> <tr id="variance_details" style="display:none"> <td>Variance Threshold</td> <td> @@ -70,8 +72,7 @@ <tr id="outliers_details" style="display:none"> <td>Discard Outliers</td> <td> - <input type="text" name="discard_outliers" size"5" value="10"/> - 10% is the recommended percentage of outliers to discard from top and bottom. + <input type="text" name="discard_outliers" size"5" value="0"/> </td> </tr> <tr> @@ -130,7 +131,7 @@ form.variance_threshold.value = '0'; break; default: - form.variance_threshold.value = '5'; + form.variance_threshold.value = '0'; } // Update the variance threshold information message. @@ -153,6 +154,7 @@ <td><b>Description</b></td> <td><b>Benchmark Name</b></td> <td><b>Benchmark Arguments</b></td> + <td><b>Repeat Runs</b></td> <td><b>Variance Threshold</b></td> <td><b>Discard Outliers</b></td> <td><b>Pagesets Type</b></td> @@ -172,6 +174,13 @@ <td>{{ task.description }}</td> <td>{{ task.benchmark_name }}</td> <td>{{ task.benchmark_arguments }}</td> + <td> + {% if task.num_repeated_runs %} + {{ task.num_repeated_runs }} + {% else %} + 1 + {% endif %} + </td> <td>{{ task.variance_threshold }}</td> <td>{{ task.discard_outliers }}</td> <td>