| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "This Jupyter Notebook has code and examples of how to pull data out of https://perf.skia.org.\n", | |
| "\n", | |
| "For further information see:\n", | |
| "\n", | |
| " * Pandas: http://pandas.pydata.org/pandas-docs/stable/\n", | |
| " * NumPy: https://docs.scipy.org/doc/numpy-dev/user/quickstart.html\n", | |
| " * matplotlib: http://matplotlib.org/2.0.0/index.html\n", | |
| " " | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# This is where the two functions, perf_calc and perf_query are defined.\n", | |
| "#\n", | |
| "# See the cells below this one for example of how to use them.\n", | |
| "import httplib2\n", | |
| "import json\n", | |
| "import time\n", | |
| "\n", | |
| "import matplotlib.pyplot as plt\n", | |
| "import numpy as np\n", | |
| "import pandas\n", | |
| "\n", | |
| "\n", | |
| "# perf_calc evaluates the formula against the last 50 commits\n", | |
| "# and returns a pandas.DataFrame with the results of the calculations.\n", | |
| "#\n", | |
| "# Example: perf_calc('count(filter(\"\"))')\n", | |
| "#\n", | |
| "def perf_calc(formula):\n", | |
| " body = {\n", | |
| " 'formulas': [formula], \n", | |
| " 'tz': 'America/New_York',\n", | |
| " }\n", | |
| " return perf_impl(body)\n", | |
| "\n", | |
| "\n", | |
| "# perf_query evaluates the query against the last 50 commits\n", | |
| "# and returns a pandas.DataFrame with the results of the query.\n", | |
| "#\n", | |
| "# Example: perf_query('source_type=skp&sub_result=min_ms')\n", | |
| "#\n", | |
| "def perf_query(query):\n", | |
| " body = {\n", | |
| " 'queries': [query], \n", | |
| " 'tz': 'America/New_York',\n", | |
| " }\n", | |
| " return perf_impl(body)\n", | |
| "\n", | |
| "\n", | |
| "# utility function.\n", | |
| "def noe(x):\n", | |
| " if x == 1e32:\n", | |
| " return np.nan\n", | |
| " else:\n", | |
| " return x\n", | |
| "\n", | |
| "def paramset():\n", | |
| " h = httplib2.Http()\n", | |
| " url = 'https://perf.skia.org/_/initpage/?tz=America/New_York'\n", | |
| " resp, content = h.request(url)\n", | |
| " if resp.status != 200:\n", | |
| " raise \"Failed to get initial bounds.\"\n", | |
| " init = json.loads(content)\n", | |
| " return init['dataframe']['paramset']\n", | |
| " \n", | |
| " \n", | |
| "# utility function.\n", | |
| "def perf_impl(body):\n", | |
| " h = httplib2.Http()\n", | |
| " url = 'https://perf.skia.org/_/initpage/?tz=America/New_York'\n", | |
| " resp, content = h.request(url)\n", | |
| " if resp.status != 200:\n", | |
| " raise \"Failed to get initial bounds.\"\n", | |
| " init = json.loads(content)\n", | |
| " body['begin'] = init['dataframe']['header'][0]['timestamp']\n", | |
| " body['end'] = init['dataframe']['header'][-1]['timestamp']+1\n", | |
| " (resp, content) = h.request(\"https://perf.skia.org/_/frame/start\", \"POST\",\n", | |
| " body=json.dumps(body),\n", | |
| " headers={'content-type': 'application/json'})\n", | |
| " if resp.status != 200:\n", | |
| " raise \"Failed to start query: \" + content\n", | |
| " id = json.loads(content)['id']\n", | |
| " state = {'state': 'Starting'}\n", | |
| " url = 'https://perf.skia.org/_/frame/status/' + id\n", | |
| " i = 0\n", | |
| " while state['state'] != 'Success':\n", | |
| " print '\\r', '|/-\\\\'[i%4],\n", | |
| " i+=1\n", | |
| " time.sleep(0.5)\n", | |
| " resp, content = h.request(url)\n", | |
| " if resp.status != 200:\n", | |
| " raise \"Failed during query: \" + content\n", | |
| " state = json.loads(content)\n", | |
| " url = 'https://perf.skia.org/_/frame/results/' + id\n", | |
| " resp, content = h.request(url)\n", | |
| " if resp.status != 200:\n", | |
| " raise \"Failed to load results: \" + content\n", | |
| "\n", | |
| " df = json.loads(content)\n", | |
| " clean = {}\n", | |
| " for key, value in df['dataframe']['traceset'].iteritems():\n", | |
| " clean[key] = [noe(x) for x in value]\n", | |
| "\n", | |
| " print '\\r ',\n", | |
| " return pandas.DataFrame(data=clean)\n" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# The following line makes the plots interactive.\n", | |
| "%matplotlib notebook\n", | |
| "\n", | |
| "# Perform a calculation over Perf data.\n", | |
| "df = perf_calc('count(filter(\"\"))')\n", | |
| "\n", | |
| "# pandas.DataFrame's can plot themselves.\n", | |
| "df.plot()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# The following line makes the plots interactive.\n", | |
| "%matplotlib notebook\n", | |
| "\n", | |
| "df = perf_query('sub_result=min_ms&test=AndroidCodec_01_original.jpg_SampleSize2_640_480')\n", | |
| "\n", | |
| "# You can also use matplotlib to do the plotting.\n", | |
| "plt.plot(df, linestyle='-', marker='o')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "%matplotlib notebook\n", | |
| "\n", | |
| "# DataFrames allow operating on traces in bulk. For example, to \n", | |
| "# normalize each trace to a mean of 0.0 and a std deviation of 1.0:\n", | |
| "normed = (df - df.mean())/df.std()\n", | |
| "plt.plot(normed)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "%matplotlib notebook\n", | |
| "df = perf_query('source_type=skp&sub_result=min_ms')\n", | |
| "df.mean(axis=1)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Find the noisiest models, from lowest to highest.\n", | |
| "#\n", | |
| "# Takes a while to run.\n", | |
| "params = paramset()\n", | |
| "results = pandas.DataFrame()\n", | |
| "for model in params['model']:\n", | |
| " df = perf_calc('ave(trace_cov(fill(filter(\"source_type=svg&sub_result=min_ms&model=%s\"))))' % model)\n", | |
| " if df.size > 0:\n", | |
| " df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n", | |
| " results[model] = pandas.Series([df.mean()[0]])\n", | |
| "results.sort_values(by=0,axis=1).transpose()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# Find the differences between CPU and GPU for Chorizo.\n", | |
| "#\n", | |
| "# Takes a while to run.\n", | |
| "params = paramset()\n", | |
| "results = pandas.DataFrame()\n", | |
| "i = 0\n", | |
| "for model in params['name']:\n", | |
| " if model.endswith(\".skp\") and (model.startswith(\"top\") or model.startswith(\"key\") or model.startswith(\"desk\")):\n", | |
| " print model\n", | |
| " df = perf_calc(\"\"\"trace_ave(ratio(\n", | |
| " ave(filter(\"cpu_or_gpu=GPU&model=Chorizo&sub_result=min_ms&name=%s\")),\n", | |
| " ave(filter(\"cpu_or_gpu=CPU&model=Chorizo&sub_result=min_ms&name=%s\"))\n", | |
| "))\"\"\" % (model, model))\n", | |
| " if df.size > 0:\n", | |
| " i+=1\n", | |
| " print model\n", | |
| " if i > 50:\n", | |
| " break\n", | |
| " df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n", | |
| " results[model] = pandas.Series([df.mean()[0]])\n", | |
| "results.sort_values(by=0,axis=1).transpose()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# Find the noisiest models, from lowest to highest.\n", | |
| "#\n", | |
| "# Takes a while to run.\n", | |
| "params = paramset()\n", | |
| "results = pandas.DataFrame()\n", | |
| "for model in params['test']:\n", | |
| " if model.startswith(\"GM_\"):\n", | |
| " df = perf_calc(\"\"\"trace_ave(ratio(\n", | |
| " ave(filter(\"cpu_or_gpu=GPU&model=Chorizo&sub_result=min_ms&test=%s\")),\n", | |
| " ave(filter(\"cpu_or_gpu=CPU&model=Chorizo&sub_result=min_ms&test=%s\"))\n", | |
| "))\"\"\" % (model, model))\n", | |
| " if df.size > 0:\n", | |
| " df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n", | |
| " results[model] = pandas.Series([df.mean()[0]])\n", | |
| "results.sort_values(by=0,axis=1).transpose()" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 2", | |
| "language": "python", | |
| "name": "python2" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 2 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython2", | |
| "version": "2.7.6" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |