| { |
| "cells": [ |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "This Jupyter Notebook has code and examples of how to pull data out of https://perf.skia.org.\n", |
| "\n", |
| "For further information see:\n", |
| "\n", |
| " * Pandas: http://pandas.pydata.org/pandas-docs/stable/\n", |
| " * NumPy: https://docs.scipy.org/doc/numpy-dev/user/quickstart.html\n", |
| " * matplotlib: http://matplotlib.org/2.0.0/index.html\n", |
| " " |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "# This is where the two functions, perf_calc and perf_query are defined.\n", |
| "#\n", |
| "# See the cells below this one for example of how to use them.\n", |
| "import httplib2\n", |
| "import json\n", |
| "import time\n", |
| "\n", |
| "import matplotlib.pyplot as plt\n", |
| "import numpy as np\n", |
| "import pandas\n", |
| "\n", |
| "\n", |
| "# perf_calc evaluates the formula against the last 50 commits\n", |
| "# and returns a pandas.DataFrame with the results of the calculations.\n", |
| "#\n", |
| "# Example: perf_calc('count(filter(\"\"))')\n", |
| "#\n", |
| "def perf_calc(formula):\n", |
| " body = {\n", |
| " 'formulas': [formula], \n", |
| " 'tz': 'America/New_York',\n", |
| " }\n", |
| " return perf_impl(body)\n", |
| "\n", |
| "\n", |
| "# perf_query evaluates the query against the last 50 commits\n", |
| "# and returns a pandas.DataFrame with the results of the query.\n", |
| "#\n", |
| "# Example: perf_query('source_type=skp&sub_result=min_ms')\n", |
| "#\n", |
| "def perf_query(query):\n", |
| " body = {\n", |
| " 'queries': [query], \n", |
| " 'tz': 'America/New_York',\n", |
| " }\n", |
| " return perf_impl(body)\n", |
| "\n", |
| "\n", |
| "# utility function.\n", |
| "def noe(x):\n", |
| " if x == 1e32:\n", |
| " return np.nan\n", |
| " else:\n", |
| " return x\n", |
| "\n", |
| "def paramset():\n", |
| " h = httplib2.Http()\n", |
| " url = 'https://perf.skia.org/_/initpage/?tz=America/New_York'\n", |
| " resp, content = h.request(url)\n", |
| " if resp.status != 200:\n", |
| " raise \"Failed to get initial bounds.\"\n", |
| " init = json.loads(content)\n", |
| " return init['dataframe']['paramset']\n", |
| " \n", |
| " \n", |
| "# utility function.\n", |
| "def perf_impl(body):\n", |
| " h = httplib2.Http()\n", |
| " url = 'https://perf.skia.org/_/initpage/?tz=America/New_York'\n", |
| " resp, content = h.request(url)\n", |
| " if resp.status != 200:\n", |
| " raise \"Failed to get initial bounds.\"\n", |
| " init = json.loads(content)\n", |
| " body['begin'] = init['dataframe']['header'][0]['timestamp']\n", |
| " body['end'] = init['dataframe']['header'][-1]['timestamp']+1\n", |
| " (resp, content) = h.request(\"https://perf.skia.org/_/frame/start\", \"POST\",\n", |
| " body=json.dumps(body),\n", |
| " headers={'content-type': 'application/json'})\n", |
| " if resp.status != 200:\n", |
| " raise \"Failed to start query: \" + content\n", |
| " id = json.loads(content)['id']\n", |
| " state = {'state': 'Starting'}\n", |
| " url = 'https://perf.skia.org/_/frame/status/' + id\n", |
| " i = 0\n", |
| " while state['state'] != 'Success':\n", |
| " print '\\r', '|/-\\\\'[i%4],\n", |
| " i+=1\n", |
| " time.sleep(0.5)\n", |
| " resp, content = h.request(url)\n", |
| " if resp.status != 200:\n", |
| " raise \"Failed during query: \" + content\n", |
| " state = json.loads(content)\n", |
| " url = 'https://perf.skia.org/_/frame/results/' + id\n", |
| " resp, content = h.request(url)\n", |
| " if resp.status != 200:\n", |
| " raise \"Failed to load results: \" + content\n", |
| "\n", |
| " df = json.loads(content)\n", |
| " clean = {}\n", |
| " for key, value in df['dataframe']['traceset'].iteritems():\n", |
| " clean[key] = [noe(x) for x in value]\n", |
| "\n", |
| " print '\\r ',\n", |
| " return pandas.DataFrame(data=clean)\n" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "# The following line makes the plots interactive.\n", |
| "%matplotlib notebook\n", |
| "\n", |
| "# Perform a calculation over Perf data.\n", |
| "df = perf_calc('count(filter(\"\"))')\n", |
| "\n", |
| "# pandas.DataFrame's can plot themselves.\n", |
| "df.plot()" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "# The following line makes the plots interactive.\n", |
| "%matplotlib notebook\n", |
| "\n", |
| "df = perf_query('sub_result=min_ms&test=AndroidCodec_01_original.jpg_SampleSize2_640_480')\n", |
| "\n", |
| "# You can also use matplotlib to do the plotting.\n", |
| "plt.plot(df, linestyle='-', marker='o')" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "%matplotlib notebook\n", |
| "\n", |
| "# DataFrames allow operating on traces in bulk. For example, to \n", |
| "# normalize each trace to a mean of 0.0 and a std deviation of 1.0:\n", |
| "normed = (df - df.mean())/df.std()\n", |
| "plt.plot(normed)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "%matplotlib notebook\n", |
| "df = perf_query('source_type=skp&sub_result=min_ms')\n", |
| "df.mean(axis=1)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "# Find the noisiest models, from lowest to highest.\n", |
| "#\n", |
| "# Takes a while to run.\n", |
| "params = paramset()\n", |
| "results = pandas.DataFrame()\n", |
| "for model in params['model']:\n", |
| " df = perf_calc('ave(trace_cov(fill(filter(\"source_type=svg&sub_result=min_ms&model=%s\"))))' % model)\n", |
| " if df.size > 0:\n", |
| " df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n", |
| " results[model] = pandas.Series([df.mean()[0]])\n", |
| "results.sort_values(by=0,axis=1).transpose()" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "# Find the differences between CPU and GPU for Chorizo.\n", |
| "#\n", |
| "# Takes a while to run.\n", |
| "params = paramset()\n", |
| "results = pandas.DataFrame()\n", |
| "i = 0\n", |
| "for model in params['name']:\n", |
| " if model.endswith(\".skp\") and (model.startswith(\"top\") or model.startswith(\"key\") or model.startswith(\"desk\")):\n", |
| " print model\n", |
| " df = perf_calc(\"\"\"trace_ave(ratio(\n", |
| " ave(filter(\"cpu_or_gpu=GPU&model=Chorizo&sub_result=min_ms&name=%s\")),\n", |
| " ave(filter(\"cpu_or_gpu=CPU&model=Chorizo&sub_result=min_ms&name=%s\"))\n", |
| "))\"\"\" % (model, model))\n", |
| " if df.size > 0:\n", |
| " i+=1\n", |
| " print model\n", |
| " if i > 50:\n", |
| " break\n", |
| " df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n", |
| " results[model] = pandas.Series([df.mean()[0]])\n", |
| "results.sort_values(by=0,axis=1).transpose()" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "# Find the noisiest models, from lowest to highest.\n", |
| "#\n", |
| "# Takes a while to run.\n", |
| "params = paramset()\n", |
| "results = pandas.DataFrame()\n", |
| "for model in params['test']:\n", |
| " if model.startswith(\"GM_\"):\n", |
| " df = perf_calc(\"\"\"trace_ave(ratio(\n", |
| " ave(filter(\"cpu_or_gpu=GPU&model=Chorizo&sub_result=min_ms&test=%s\")),\n", |
| " ave(filter(\"cpu_or_gpu=CPU&model=Chorizo&sub_result=min_ms&test=%s\"))\n", |
| "))\"\"\" % (model, model))\n", |
| " if df.size > 0:\n", |
| " df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n", |
| " results[model] = pandas.Series([df.mean()[0]])\n", |
| "results.sort_values(by=0,axis=1).transpose()" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [] |
| } |
| ], |
| "metadata": { |
| "kernelspec": { |
| "display_name": "Python 2", |
| "language": "python", |
| "name": "python2" |
| }, |
| "language_info": { |
| "codemirror_mode": { |
| "name": "ipython", |
| "version": 2 |
| }, |
| "file_extension": ".py", |
| "mimetype": "text/x-python", |
| "name": "python", |
| "nbconvert_exporter": "python", |
| "pygments_lexer": "ipython2", |
| "version": "2.7.6" |
| } |
| }, |
| "nbformat": 4, |
| "nbformat_minor": 2 |
| } |
