{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This Jupyter Notebook has code and examples of how to pull data out of https://perf.skia.org.\n",
    "\n",
    "For further information see:\n",
    "\n",
    "  * Pandas: http://pandas.pydata.org/pandas-docs/stable/\n",
    "  * NumPy:  https://docs.scipy.org/doc/numpy-dev/user/quickstart.html\n",
    "  * matplotlib: http://matplotlib.org/2.0.0/index.html\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is where the two functions, perf_calc and perf_query are defined.\n",
    "#\n",
    "# See the cells below this one for example of how to use them.\n",
    "import httplib2\n",
    "import json\n",
    "import time\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas\n",
    "\n",
    "\n",
    "# perf_calc evaluates the formula against the last 50 commits\n",
    "# and returns a pandas.DataFrame with the results of the calculations.\n",
    "#\n",
    "# Example: perf_calc('count(filter(\"\"))')\n",
    "#\n",
    "def perf_calc(formula):\n",
    "    body = {\n",
    "        'formulas': [formula], \n",
    "        'tz': 'America/New_York',\n",
    "    }\n",
    "    return perf_impl(body)\n",
    "\n",
    "\n",
    "# perf_query evaluates the query against the last 50 commits\n",
    "# and returns a pandas.DataFrame with the results of the query.\n",
    "#\n",
    "# Example: perf_query('source_type=skp&sub_result=min_ms')\n",
    "#\n",
    "def perf_query(query):\n",
    "    body = {\n",
    "        'queries': [query], \n",
    "        'tz': 'America/New_York',\n",
    "    }\n",
    "    return perf_impl(body)\n",
    "\n",
    "\n",
    "# utility function.\n",
    "def noe(x):\n",
    "    if x == 1e32:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return x\n",
    "\n",
    "def paramset():\n",
    "    h = httplib2.Http()\n",
    "    url = 'https://perf.skia.org/_/initpage/?tz=America/New_York'\n",
    "    resp, content = h.request(url)\n",
    "    if resp.status != 200:\n",
    "        raise \"Failed to get initial bounds.\"\n",
    "    init = json.loads(content)\n",
    "    return init['dataframe']['paramset']\n",
    "    \n",
    "    \n",
    "# utility function.\n",
    "def perf_impl(body):\n",
    "    h = httplib2.Http()\n",
    "    url = 'https://perf.skia.org/_/initpage/?tz=America/New_York'\n",
    "    resp, content = h.request(url)\n",
    "    if resp.status != 200:\n",
    "        raise \"Failed to get initial bounds.\"\n",
    "    init = json.loads(content)\n",
    "    body['begin'] = init['dataframe']['header'][0]['timestamp']\n",
    "    body['end'] = init['dataframe']['header'][-1]['timestamp']+1\n",
    "    (resp, content) = h.request(\"https://perf.skia.org/_/frame/start\", \"POST\",\n",
    "                            body=json.dumps(body),\n",
    "                            headers={'content-type': 'application/json'})\n",
    "    if resp.status != 200:\n",
    "        raise \"Failed to start query: \" + content\n",
    "    id = json.loads(content)['id']\n",
    "    state = {'state': 'Starting'}\n",
    "    url = 'https://perf.skia.org/_/frame/status/' + id\n",
    "    i = 0\n",
    "    while state['state'] != 'Success':\n",
    "        print '\\r', '|/-\\\\'[i%4],\n",
    "        i+=1\n",
    "        time.sleep(0.5)\n",
    "        resp, content = h.request(url)\n",
    "        if resp.status != 200:\n",
    "            raise \"Failed during query: \" + content\n",
    "        state = json.loads(content)\n",
    "    url = 'https://perf.skia.org/_/frame/results/' + id\n",
    "    resp, content = h.request(url)\n",
    "    if resp.status != 200:\n",
    "        raise \"Failed to load results: \" + content\n",
    "\n",
    "    df = json.loads(content)\n",
    "    clean = {}\n",
    "    for key, value in df['dataframe']['traceset'].iteritems():\n",
    "        clean[key] = [noe(x) for x in value]\n",
    "\n",
    "    print '\\r ',\n",
    "    return pandas.DataFrame(data=clean)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following line makes the plots interactive.\n",
    "%matplotlib notebook\n",
    "\n",
    "# Perform a calculation over Perf data.\n",
    "df = perf_calc('count(filter(\"\"))')\n",
    "\n",
    "# pandas.DataFrame's can plot themselves.\n",
    "df.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following line makes the plots interactive.\n",
    "%matplotlib notebook\n",
    "\n",
    "df = perf_query('sub_result=min_ms&test=AndroidCodec_01_original.jpg_SampleSize2_640_480')\n",
    "\n",
    "# You can also use matplotlib to do the plotting.\n",
    "plt.plot(df, linestyle='-', marker='o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "\n",
    "# DataFrames allow operating on traces in bulk. For example, to \n",
    "# normalize each trace to a mean of 0.0 and a std deviation of 1.0:\n",
    "normed = (df - df.mean())/df.std()\n",
    "plt.plot(normed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "df = perf_query('source_type=skp&sub_result=min_ms')\n",
    "df.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Find the noisiest models, from lowest to highest.\n",
    "#\n",
    "# Takes a while to run.\n",
    "params = paramset()\n",
    "results = pandas.DataFrame()\n",
    "for model in params['model']:\n",
    "    df = perf_calc('ave(trace_cov(fill(filter(\"source_type=svg&sub_result=min_ms&model=%s\"))))' % model)\n",
    "    if df.size > 0:\n",
    "        df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n",
    "        results[model] = pandas.Series([df.mean()[0]])\n",
    "results.sort_values(by=0,axis=1).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the differences between CPU and GPU for Chorizo.\n",
    "#\n",
    "# Takes a while to run.\n",
    "params = paramset()\n",
    "results = pandas.DataFrame()\n",
    "i = 0\n",
    "for model in params['name']:\n",
    "    if model.endswith(\".skp\") and (model.startswith(\"top\") or model.startswith(\"key\") or model.startswith(\"desk\")):\n",
    "        print model\n",
    "        df = perf_calc(\"\"\"trace_ave(ratio(\n",
    "  ave(filter(\"cpu_or_gpu=GPU&model=Chorizo&sub_result=min_ms&name=%s\")),\n",
    "  ave(filter(\"cpu_or_gpu=CPU&model=Chorizo&sub_result=min_ms&name=%s\"))\n",
    "))\"\"\" % (model, model))\n",
    "        if df.size > 0:\n",
    "            i+=1\n",
    "            print model\n",
    "            if i > 50:\n",
    "                break\n",
    "            df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n",
    "            results[model] = pandas.Series([df.mean()[0]])\n",
    "results.sort_values(by=0,axis=1).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the noisiest models, from lowest to highest.\n",
    "#\n",
    "# Takes a while to run.\n",
    "params = paramset()\n",
    "results = pandas.DataFrame()\n",
    "for model in params['test']:\n",
    "    if model.startswith(\"GM_\"):\n",
    "        df = perf_calc(\"\"\"trace_ave(ratio(\n",
    "  ave(filter(\"cpu_or_gpu=GPU&model=Chorizo&sub_result=min_ms&test=%s\")),\n",
    "  ave(filter(\"cpu_or_gpu=CPU&model=Chorizo&sub_result=min_ms&test=%s\"))\n",
    "))\"\"\" % (model, model))\n",
    "        if df.size > 0:\n",
    "            df.rename_axis({df.columns[0]: model}, axis=\"columns\")\n",
    "            results[model] = pandas.Series([df.mean()[0]])\n",
    "results.sort_values(by=0,axis=1).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}