Add lots of utils, PRESUBMIT.py

BUG=skia:2682
R=jcgregorio@google.com, rmistry@google.com

Review URL: https://codereview.chromium.org/341193004
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2f78cf5
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+*.pyc
+
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
new file mode 100644
index 0000000..ae0a076
--- /dev/null
+++ b/PRESUBMIT.py
@@ -0,0 +1,140 @@
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Presubmit checks for common code."""
+
+import os
+import subprocess
+import sys
+
+
+SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
+SKIP_RUNS_KEYWORD = '(SkipBuildbotRuns)'
+
+
+def _RunPyUnitTests(input_api, output_api):
+  """ Run the Python unit tests and return a list of strings containing any
+  errors.
+  """
+  results = []
+  success = True
+  try:
+    proc = subprocess.Popen(['python', os.path.join('py', 'run_unittests')],
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    success = proc.wait() == 0
+    long_text = proc.communicate()[0]
+  except Exception:
+    success = False
+    long_text = 'Failed to run the common tests!'
+  if not success:
+    results.append(output_api.PresubmitPromptWarning(
+        message='One or more unit tests failed.',
+        long_text=long_text))
+  return results
+
+
+def CheckChange(input_api, output_api):
+  """Presubmit checks for the change on upload or commit.
+
+  The presubmit checks have been handpicked from the list of canned checks
+  here:
+  https://chromium.googlesource.com/chromium/tools/depot_tools/+/master/presubmit_canned_checks.py
+
+  The following are the presubmit checks:
+  * Pylint is run if the change contains any .py files.
+  * Enforces max length for all lines is 80.
+  * Checks that the user didn't add TODO(name) without an owner.
+  * Checks that there is no stray whitespace at source lines end.
+  * Checks that there are no tab characters in any of the text files.
+  """
+  results = []
+
+  pylint_disabled_warnings = (
+      'F0401',  # Unable to import.
+      'E0611',  # No name in module.
+      'W0232',  # Class has no __init__ method.
+      'E1002',  # Use of super on an old style class.
+      'W0403',  # Relative import used.
+      'R0201',  # Method could be a function.
+      'E1003',  # Using class name in super.
+      'W0613',  # Unused argument.
+  )
+  # Run Pylint on only the modified python files. Unfortunately it still runs
+  # Pylint on the whole file instead of just the modified lines.
+  affected_python_files = []
+  for affected_file in input_api.AffectedSourceFiles(None):
+    affected_file_path = affected_file.LocalPath()
+    if affected_file_path.endswith('.py'):
+      affected_python_files.append(affected_file_path)
+  results += input_api.canned_checks.RunPylint(
+      input_api, output_api,
+      disabled_warnings=pylint_disabled_warnings,
+      white_list=affected_python_files)
+
+  # Use 100 for max length for files other than python. Python length is
+  # already checked during the Pylint above.
+  results += input_api.canned_checks.CheckLongLines(input_api, output_api, 100)
+  results += input_api.canned_checks.CheckChangeTodoHasOwner(
+      input_api, output_api)
+  results += input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
+      input_api, output_api)
+  results += input_api.canned_checks.CheckChangeHasNoTabs(input_api, output_api)
+
+  results += _RunPyUnitTests(input_api, output_api)
+
+  return results
+
+
+def _CheckTreeStatus(input_api, output_api, json_url):
+  """Check whether to allow commit.
+
+  Args:
+    input_api: input related apis.
+    output_api: output related apis.
+    json_url: url to download json style status.
+  """
+  results = []
+  tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
+      input_api, output_api, json_url=json_url)
+  display_skip_keyword_prompt = False
+  tree_status = None
+  if not tree_status_results:
+    # Check for caution state only if tree is not closed.
+    connection = input_api.urllib2.urlopen(json_url)
+    status = input_api.json.loads(connection.read())
+    connection.close()
+    tree_status = status['message']
+    if 'caution' in tree_status.lower():
+      # Display a prompt only if we are in an interactive shell. Without this
+      # check the commit queue behaves incorrectly because it considers
+      # prompts to be failures.
+      display_skip_keyword_prompt = True
+  else:
+    tree_status = tree_status_results[0]._message  # pylint: disable=W0212
+    display_skip_keyword_prompt = True
+
+  if (display_skip_keyword_prompt
+      and os.isatty(sys.stdout.fileno())
+      and not SKIP_RUNS_KEYWORD in input_api.change.DescriptionText()):
+    long_text = (
+        '%s\nAre you sure the change should be submitted. If it should be '
+        'submitted but not run on the buildbots you can use the %s keyword.' % (
+            tree_status, SKIP_RUNS_KEYWORD))
+    results.append(output_api.PresubmitPromptWarning(
+        message=tree_status, long_text=long_text))
+  return results
+
+
+def CheckChangeOnUpload(input_api, output_api):
+  return CheckChange(input_api, output_api)
+
+
+def CheckChangeOnCommit(input_api, output_api):
+  results = CheckChange(input_api, output_api)
+  results.extend(
+      _CheckTreeStatus(input_api, output_api, json_url=(
+          SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
+  return results
+
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1 @@
+
diff --git a/py/__init__.py b/py/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/py/__init__.py
@@ -0,0 +1 @@
+
diff --git a/py/run_unittests b/py/run_unittests
new file mode 100755
index 0000000..30c203e
--- /dev/null
+++ b/py/run_unittests
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs all unit tests under this base directory."""
+
+import os
+import subprocess
+import sys
+import unittest
+
+
+NO_CRAWL_DIRS = [
+  '.git',
+  '.svn',
+]
+
+
+SEARCH_PATH = os.path.dirname(os.path.abspath(__file__))
+
+
+def FilterDirectory(dirpath, filenames):
+  """ Determine whether to look for tests in the given directory.
+
+  dirpath: string; path of the directory in question.
+  filenames: list of strings; the files in the directory.
+  """
+  if not dirpath or not filenames:
+    return False
+  for no_crawl_dir in NO_CRAWL_DIRS:
+    if no_crawl_dir in dirpath:
+      return False
+  return True
+
+
+if __name__ == '__main__':
+  print 'Searching for tests.'
+  tests_to_run = []
+
+  for (dirpath, dirnames, filenames) in os.walk(SEARCH_PATH, topdown=True):
+    dirnames[:] = [d for d in dirnames if not d in NO_CRAWL_DIRS]
+    test_modules = [os.path.join(dirpath, filename) for filename in filenames
+                    if filename.endswith('_test.py')]
+    if not test_modules:
+      continue
+    tests_to_run.extend(test_modules)
+
+  print 'Found %d tests.' % len(tests_to_run)
+  errors = []
+  for test in tests_to_run:
+    proc = subprocess.Popen(['python', test], stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    if proc.wait() != 0:
+      errors.append(proc.communicate()[0])
+  if errors:
+    for error in errors:
+      print error
+    print 'Failed %d of %d.' % (len(errors), len(test_modules))
+    sys.exit(1)
+  else:
+    print 'All tests succeeded.'
diff --git a/py/utils/__init__.py b/py/utils/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/py/utils/__init__.py
@@ -0,0 +1 @@
+
diff --git a/py/utils/android_utils.py b/py/utils/android_utils.py
new file mode 100644
index 0000000..7b1c1a1
--- /dev/null
+++ b/py/utils/android_utils.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains tools used by Android-specific buildbot scripts. """
+
+import os
+import re
+import shell_utils
+import shlex
+import sys
+import time
+
+
+CPU_SCALING_MODES = ['performance', 'interactive']
+DEVICE_LOOKUP = {'nexus_s': 'crespo',
+                 'xoom': 'stingray',
+                 'galaxy_nexus': 'toro',
+                 'nexus_4': 'mako',
+                 'nexus_7': 'grouper',
+                 'nexus_10': 'manta'}
+PROCESS_MONITOR_INTERVAL = 5.0 # Seconds
+SKIA_RUNNING = 'running'
+SKIA_RETURN_CODE_REPEATS = 10
+SUBPROCESS_TIMEOUT = 30.0
+
+
+def GotADB(adb):
+  """ Returns True iff ADB exists at the given location.
+
+  adb: string; possible path to the ADB executable.
+  """
+  try:
+    shell_utils.run([adb, 'version'], echo=False)
+    return True
+  except Exception:
+    return False
+
+
+def FindADB(hint=None):
+  """ Attempt to find the ADB program using the following sequence of steps.
+  Returns the path to ADB if it can be found, or None otherwise.
+  1. If a hint was provided, is it a valid path to ADB?
+  2. Is ADB in PATH?
+  3. Is there an environment variable for ADB?
+  4. If the ANDROID_SDK_ROOT variable is set, try to find ADB in the SDK
+     directory.
+  5. Try to find ADB in a list of common locations.
+
+  hint: string indicating a possible path to ADB.
+  """
+  # 1. If a hint was provided, does it point to ADB?
+  if hint:
+    if os.path.basename(hint) == 'adb':
+      adb = hint
+    else:
+      adb = os.path.join(hint, 'adb')
+    if GotADB(adb):
+      return adb
+
+  # 2. Is 'adb' in our PATH?
+  adb = 'adb'
+  if GotADB(adb):
+    return adb
+
+  # 3. Is there an environment variable for ADB?
+  adb = os.environ.get('ADB')
+  if GotADB(adb):
+    return adb
+
+  # 4. If ANDROID_SDK_ROOT is set, try to find ADB in the SDK directory.
+  sdk_dir = os.environ.get('ANDROID_SDK_ROOT', '')
+  adb = os.path.join(sdk_dir, 'platform-tools', 'adb')
+  if GotADB(adb):
+    return adb
+
+  # 4. Try to find ADB relative to this file.
+  common_locations = []
+  os_dir = None
+  if sys.platform.startswith('linux'):
+    os_dir = 'linux'
+  elif sys.platform.startswith('darwin'):
+    os_dir = 'mac'
+  else:
+    os_dir = 'win'
+  common_locations.append(os.path.join('platform_tools', 'android', 'bin',
+                                       os_dir, 'adb'))
+  common_locations.append(os.path.join(os.environ.get('HOME', ''),
+                          'android-sdk-%s' % os_dir))
+  for location in common_locations:
+    if GotADB(location):
+      return location
+
+  raise Exception('android_utils: Unable to find ADB!')
+
+
+PATH_TO_ADB = FindADB(hint=os.path.join('platform_tools', 'android', 'bin',
+                                        'linux', 'adb'))
+
+
+def RunADB(serial, cmd, echo=True, attempts=5, secs_between_attempts=10,
+           timeout=None):
+  """ Run 'cmd' on an Android device, using ADB.  No return value; throws an
+  exception if the command fails more than the allotted number of attempts.
+
+  serial: string indicating the serial number of the target device
+  cmd: string; the command to issue on the device
+  attempts: number of times to attempt the command
+  secs_between_attempts: number of seconds to wait between attempts
+  timeout: optional, integer indicating the maximum elapsed time in seconds
+  """
+  adb_cmd = [PATH_TO_ADB, '-s', serial]
+  adb_cmd += cmd
+  shell_utils.run_retry(adb_cmd, echo=echo, attempts=attempts,
+                        secs_between_attempts=secs_between_attempts)
+
+
+def ADBShell(serial, cmd, echo=True):
+  """ Runs 'cmd' in the ADB shell on an Android device and returns the exit
+  code.
+
+  serial: string indicating the serial number of the target device
+  cmd: string; the command to issue on the device
+  """
+  # ADB doesn't exit with the exit code of the command we ran. It only exits
+  # non-zero when ADB itself encountered a problem. Therefore, we have to use
+  # the shell to print the exit code for the command and parse that from stdout.
+  adb_cmd = '%s -s %s shell "%s; echo \$?"' % (PATH_TO_ADB, serial,
+                                               ' '.join(cmd))
+  output = shell_utils.run(adb_cmd, shell=True, echo=echo)
+  output_lines = output.splitlines()
+  try:
+    real_exitcode = int(output_lines[-1].rstrip())
+  except ValueError:
+    real_exitcode = -1
+  if real_exitcode != 0:
+    raise Exception('Command failed with code %s' % real_exitcode)
+  return '\n'.join(output_lines[:-1])
+
+
+def ADBKill(serial, process, kill_app=False):
+  """ Kill a process running on an Android device.
+
+  serial: string indicating the serial number of the target device
+  process: string indicating the name of the process to kill
+  kill_app: bool indicating whether the process is an Android app, as opposed
+      to a normal executable process.
+  """
+  if kill_app:
+    ADBShell(serial, ['am', 'kill', process])
+  else:
+    try:
+      stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB,
+                                                                serial,
+                                                                process),
+                               shell=True)
+    except Exception:
+      return
+    for line in stdout.split('\n'):
+      if line != '':
+        split = shlex.split(line)
+        if len(split) < 2:
+          continue
+        pid = split[1]
+        ADBShell(serial, ['kill', pid])
+    # Raise an exception if any Skia processes are still running.
+    try:
+      stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB,
+                                                                serial,
+                                                                process),
+                               shell=True)
+    except Exception:
+      return
+    if stdout:
+      raise Exception('There are still some skia processes running:\n%s\n'
+                      'Maybe the device should be rebooted?' % stdout)
+
+
+def GetSerial(device_type):
+  """ Determine the serial number of the *first* connected device with the
+  specified type.  The ordering of 'adb devices' is not documented, and the
+  connected devices do not appear to be ordered by serial number.  Therefore,
+  we have to assume that, in the case of multiple devices of the same type being
+  connected to one host, we cannot predict which device will be chosen.
+
+  device_type: string indicating the 'common name' for the target device
+  """
+  if not device_type in DEVICE_LOOKUP:
+    raise ValueError('Unknown device: %s!' % device_type)
+  device_name = DEVICE_LOOKUP[device_type]
+  output = shell_utils.run_retry('%s devices' % PATH_TO_ADB, shell=True,
+                                 attempts=5)
+  print output
+  lines = output.split('\n')
+  device_ids = []
+  for line in lines:
+    # Filter garbage lines
+    if line != '' and not ('List of devices attached' in line) and \
+        line[0] != '*':
+      device_ids.append(line.split('\t')[0])
+  for device_id in device_ids:
+    print 'Finding type for id %s' % device_id
+    # Get device name
+    name_line = shell_utils.run_retry(
+        '%s -s %s shell cat /system/build.prop | grep "ro.product.device="' % (
+            PATH_TO_ADB, device_id), shell=True, attempts=5)
+    print name_line
+    name = name_line.split('=')[-1].rstrip()
+    # Just return the first attached device of the requested model.
+    if device_name in name:
+      return device_id
+  raise Exception('No %s device attached!' % device_name)
+
+
+def SetCPUScalingMode(serial, mode):
+  """ Set the CPU scaling governor for the device with the given serial number
+  to the given mode.
+
+  serial: string indicating the serial number of the device whose scaling mode
+          is to be modified
+  mode:   string indicating the desired CPU scaling mode.  Acceptable values
+          are listed in CPU_SCALING_MODES.
+  """
+  if mode not in CPU_SCALING_MODES:
+    raise ValueError('mode must be one of: %s' % CPU_SCALING_MODES)
+  cpu_dirs = shell_utils.run('%s -s %s shell ls /sys/devices/system/cpu' % (
+      PATH_TO_ADB, serial), echo=False, shell=True)
+  cpu_dirs_list = cpu_dirs.split('\n')
+  regex = re.compile('cpu\d')
+  for cpu_dir_from_list in cpu_dirs_list:
+    cpu_dir = cpu_dir_from_list.rstrip()
+    if regex.match(cpu_dir):
+      path = '/sys/devices/system/cpu/%s/cpufreq/scaling_governor' % cpu_dir
+      path_found = shell_utils.run('%s -s %s shell ls %s' % (
+                                       PATH_TO_ADB, serial, path),
+                                   echo=False, shell=True).rstrip()
+      if path_found == path:
+        # Unfortunately, we can't directly change the scaling_governor file over
+        # ADB. Instead, we write a script to do so, push it to the device, and
+        # run it.
+        old_mode = shell_utils.run('%s -s %s shell cat %s' % (
+                                       PATH_TO_ADB, serial, path),
+                                   echo=False, shell=True).rstrip()
+        print 'Current scaling mode for %s is: %s' % (cpu_dir, old_mode)
+        filename = 'skia_cpuscale.sh'
+        with open(filename, 'w') as script_file:
+          script_file.write('echo %s > %s\n' % (mode, path))
+        os.chmod(filename, 0777)
+        RunADB(serial, ['push', filename, '/system/bin'], echo=False)
+        RunADB(serial, ['shell', filename], echo=True)
+        RunADB(serial, ['shell', 'rm', '/system/bin/%s' % filename], echo=False)
+        os.remove(filename)
+        new_mode = shell_utils.run('%s -s %s shell cat %s' % (
+                                       PATH_TO_ADB, serial, path),
+                                   echo=False, shell=True).rstrip()
+        print 'New scaling mode for %s is: %s' % (cpu_dir, new_mode)
+
+
+def IsAndroidShellRunning(serial):
+  """ Find the status of the Android shell for the device with the given serial
+  number. Returns True if the shell is running and False otherwise.
+
+  serial: string indicating the serial number of the target device.
+  """
+  if 'Error:' in ADBShell(serial, ['pm', 'path', 'android'], echo=False):
+    return False
+  return True
+
+
+def StopShell(serial, timeout=60):
+  """ Halt the Android runtime on the device with the given serial number.
+  Blocks until the shell reports that it has stopped.
+
+  serial: string indicating the serial number of the target device.
+  timeout: maximum allotted time, in seconds.
+  """
+  ADBShell(serial, ['stop'])
+  start_time = time.time()
+  while IsAndroidShellRunning(serial):
+    time.sleep(1)
+    if time.time() - start_time > timeout:
+      raise Exception('Timeout while attempting to stop the Android runtime.')
+
+
+def StartShell(serial, timeout=60):
+  """ Start the Android runtime on the device with the given serial number.
+  Blocks until the shell reports that it has started.
+
+  serial: string indicating the serial number of the target device.
+  timeout: maximum allotted time, in seconds.
+  """
+  ADBShell(serial, ['start'])
+  start_time = time.time()
+  while not IsAndroidShellRunning(serial):
+    time.sleep(1)
+    if time.time() - start_time > timeout:
+      raise Exception('Timeout while attempting to start the Android runtime.')
+
+
+def RunSkia(serial, cmd, release, device):
+  """ Run the given command through skia_launcher on a given device.
+
+  serial: string indicating the serial number of the target device.
+  cmd: list of strings; the command line to run.
+  release: bool; whether or not to run the app in Release mode.
+  device: string indicating the target device.
+  """
+  RunADB(serial, ['logcat', '-c'])
+  try:
+    os.environ['SKIA_ANDROID_VERBOSE_SETUP'] = '1'
+    cmd_to_run = [os.path.join('platform_tools', 'android', 'bin',
+                               'android_run_skia')]
+    if release:
+      cmd_to_run.extend(['--release'])
+    cmd_to_run.extend(['-d', device])
+    cmd_to_run.extend(['-s', serial])
+    cmd_to_run.extend(cmd)
+    shell_utils.run(cmd_to_run)
+  finally:
+    RunADB(serial, ['logcat', '-d', '-v', 'time'])
diff --git a/py/utils/git_utils.py b/py/utils/git_utils.py
new file mode 100644
index 0000000..da7f097
--- /dev/null
+++ b/py/utils/git_utils.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module contains functions for using git."""
+
+
+import os
+import shell_utils
+
+
+GIT = 'git.bat' if os.name == 'nt' else 'git'
+
+
+def Add(addition):
+  """Run 'git add <addition>'"""
+  shell_utils.run([GIT, 'add', addition])
+
+def AIsAncestorOfB(a, b):
+  """Return true if a is an ancestor of b."""
+  return shell_utils.run([GIT, 'merge-base', a, b]).rstrip() == FullHash(a)
+
+def FullHash(commit):
+  """Return full hash of specified commit."""
+  return shell_utils.run([GIT, 'rev-parse', '--verify', commit]).rstrip()
+
+def IsMerge(commit):
+  """Return True if the commit is a merge, False otherwise."""
+  rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1',
+                               '--no-merges'])
+  last_non_merge = rev_parse.split('\n')[0]
+  # Get full hash since that is what was returned by rev-parse.
+  return FullHash(commit) != last_non_merge
+
+def MergeAbort():
+  """Abort in process merge."""
+  shell_utils.run([GIT, 'merge', '--abort'])
+
+def ShortHash(commit):
+  """Return short hash of the specified commit."""
+  return shell_utils.run([GIT, 'show', commit, '--format=%h', '-s']).rstrip()
+
+def GetRemoteMasterHash(git_url):
+  return shell_utils.run([GIT, 'ls-remote', git_url, '--verify',
+                          'refs/heads/master'])
diff --git a/py/utils/misc.py b/py/utils/misc.py
new file mode 100644
index 0000000..a61527e
--- /dev/null
+++ b/py/utils/misc.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains miscellaneous tools used by the buildbot scripts. """
+
+import os
+
+from git_utils import GIT
+import shell_utils
+
+
+# Absolute path to the root of this Skia buildbot checkout.
+BUILDBOT_PATH = os.path.realpath(os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    os.pardir, os.pardir, os.pardir))
+
+
+def ArgsToDict(argv):
+  """ Collect command-line arguments of the form '--key value' into a
+  dictionary.  Fail if the arguments do not fit this format. """
+  dictionary = {}
+  PREFIX = '--'
+  # Expect the first arg to be the path to the script, which we don't want.
+  argv = argv[1:]
+  while argv:
+    if argv[0].startswith(PREFIX):
+      dictionary[argv[0][len(PREFIX):]] = argv[1]
+      argv = argv[2:]
+    else:
+      raise Exception('Malformed input: %s' % argv)
+  return dictionary
+
+
+def ConfirmOptionsSet(name_value_dict):
+  """Raise an exception if any of the given command-line options were not set.
+
+  name_value_dict: dictionary mapping option names to option values
+  """
+  for (name, value) in name_value_dict.iteritems():
+    if value is None:
+      raise Exception('missing command-line option %s; rerun with --help' %
+                      name)
+
+
+def GetAbsPath(relative_path):
+  """My own implementation of os.path.abspath() that better handles paths
+  which approach Window's 260-character limit.
+  See https://code.google.com/p/skia/issues/detail?id=674
+
+  This implementation adds path components one at a time, resolving the
+  absolute path each time, to take advantage of any chdirs into outer
+  directories that will shorten the total path length.
+
+  TODO(epoger): share a single implementation with bench_graph_svg.py, instead
+  of pasting this same code into both files."""
+  if os.path.isabs(relative_path):
+    return relative_path
+  path_parts = relative_path.split(os.sep)
+  abs_path = os.path.abspath('.')
+  for path_part in path_parts:
+    abs_path = os.path.abspath(os.path.join(abs_path, path_part))
+  return abs_path
+
+
+class ChDir(object):
+  """Enter and exit the given directory appropriately."""
+
+  def __init__(self, directory):
+    """Instantiate the ChDir.
+
+    Args:
+        directory: string; the directory to enter.
+    """
+    self._destination = directory
+    self._origin = None
+
+  def __enter__(self):
+    """Change to the destination directory.
+
+    Does not check whether the directory exists.
+    """
+    self._origin = os.getcwd()
+    print 'chdir %s' % self._destination
+    os.chdir(self._destination)
+
+  def __exit__(self, *args):
+    """Change back to the original directory."""
+    print 'chdir %s' % self._origin
+    os.chdir(self._origin)
+
+
+class GitBranch(object):
+  """Class to manage git branches.
+
+  This class allows one to create a new branch in a repository to make changes,
+  then it commits the changes, switches to master branch, and deletes the
+  created temporary branch upon exit.
+  """
+  def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False):
+    self._branch_name = branch_name
+    self._commit_msg = commit_msg
+    self._upload = upload
+    self._commit_queue = commit_queue
+    self._patch_set = 0
+
+  def __enter__(self):
+    shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
+    shell_utils.run([GIT, 'checkout', 'master'])
+    if self._branch_name in shell_utils.run([GIT, 'branch']):
+      shell_utils.run([GIT, 'branch', '-D', self._branch_name])
+    shell_utils.run([GIT, 'checkout', '-b', self._branch_name,
+                     '-t', 'origin/master'])
+    return self
+
+  def commit_and_upload(self, use_commit_queue=False):
+    shell_utils.run([GIT, 'commit', '-a', '-m',
+                     self._commit_msg])
+    upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks',
+                  '--bypass-watchlists']
+    self._patch_set += 1
+    if self._patch_set > 1:
+      upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set])
+    if use_commit_queue:
+      upload_cmd.append('--use-commit-queue')
+    shell_utils.run(upload_cmd)
+
+  def __exit__(self, exc_type, _value, _traceback):
+    if self._upload:
+      # Only upload if no error occurred.
+      try:
+        if exc_type is None:
+          self.commit_and_upload(use_commit_queue=self._commit_queue)
+      finally:
+        shell_utils.run([GIT, 'checkout', 'master'])
+        shell_utils.run([GIT, 'branch', '-D', self._branch_name])
+
diff --git a/py/utils/shell_utils.py b/py/utils/shell_utils.py
new file mode 100644
index 0000000..a6b54ef
--- /dev/null
+++ b/py/utils/shell_utils.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains tools for running commands in a shell. """
+
+import datetime
+import os
+import Queue
+import select
+import subprocess
+import sys
+import threading
+import time
+
+if 'nt' in os.name:
+  import ctypes
+
+
+DEFAULT_SECS_BETWEEN_ATTEMPTS = 10
+POLL_MILLIS = 250
+
+
+class CommandFailedException(Exception):
+  """Exception which gets raised when a command fails."""
+
+  def __init__(self, output, *args):
+    """Initialize the CommandFailedException.
+
+    Args:
+        output: string; output from the command.
+    """
+    Exception.__init__(self, *args)
+    self._output = output
+
+  @property
+  def output(self):
+    """Output from the command."""
+    return self._output
+
+
+class TimeoutException(CommandFailedException):
+  """CommandFailedException which gets raised when a subprocess exceeds its
+  timeout. """
+  pass
+
+
+def run_async(cmd, echo=True, shell=False):
+  """ Run 'cmd' in a subprocess, returning a Popen class instance referring to
+  that process.  (Non-blocking) """
+  if echo:
+    print cmd
+  if 'nt' in os.name:
+    # Windows has a bad habit of opening a dialog when a console program
+    # crashes, rather than just letting it crash.  Therefore, when a program
+    # crashes on Windows, we don't find out until the build step times out.
+    # This code prevents the dialog from appearing, so that we find out
+    # immediately and don't waste time waiting around.
+    SEM_NOGPFAULTERRORBOX = 0x0002
+    ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
+    flags = 0x8000000 # CREATE_NO_WINDOW
+  else:
+    flags = 0
+  return subprocess.Popen(cmd, shell=shell, stderr=subprocess.STDOUT,
+                          stdout=subprocess.PIPE, creationflags=flags,
+                          bufsize=1)
+
+
+class EnqueueThread(threading.Thread):
+  """ Reads and enqueues lines from a file. """
+  def __init__(self, file_obj, queue):
+    threading.Thread.__init__(self)
+    self._file = file_obj
+    self._queue = queue
+    self._stopped = False
+
+  def run(self):
+    if sys.platform.startswith('linux'):
+      # Use a polling object to avoid the blocking call to readline().
+      poll = select.poll()
+      poll.register(self._file, select.POLLIN)
+      while not self._stopped:
+        has_output = poll.poll(POLL_MILLIS)
+        if has_output:
+          line = self._file.readline()
+          if line == '':
+            self._stopped = True
+          self._queue.put(line)
+    else:
+      # Only Unix supports polling objects, so just read from the file,
+      # Python-style.
+      for line in iter(self._file.readline, ''):
+        self._queue.put(line)
+        if self._stopped:
+          break
+
+  def stop(self):
+    self._stopped = True
+
+
+def log_process_in_real_time(proc, echo=True, timeout=None, log_file=None,
+                             halt_on_output=None, print_timestamps=True):
+  """ Log the output of proc in real time until it completes. Return a tuple
+  containing the exit code of proc and the contents of stdout.
+
+  proc: an instance of Popen referring to a running subprocess.
+  echo: boolean indicating whether to print the output received from proc.stdout
+  timeout: number of seconds allotted for the process to run. Raises a
+      TimeoutException if the run time exceeds the timeout.
+  log_file: an open file for writing output
+  halt_on_output: string; kill the process and return if this string is found
+      in the output stream from the process.
+  print_timestamps: boolean indicating whether a formatted timestamp should be
+      prepended to each line of output.
+  """
+  stdout_queue = Queue.Queue()
+  log_thread = EnqueueThread(proc.stdout, stdout_queue)
+  log_thread.start()
+  try:
+    all_output = []
+    t_0 = time.time()
+    while True:
+      code = proc.poll()
+      try:
+        output = stdout_queue.get_nowait()
+        all_output.append(output)
+        if output and print_timestamps:
+          timestamp = datetime.datetime.now().strftime('%H:%M:%S.%f')
+          output = ''.join(['[%s] %s\n' % (timestamp, line)
+                            for line in output.splitlines()])
+        if echo:
+          sys.stdout.write(output)
+          sys.stdout.flush()
+        if log_file:
+          log_file.write(output)
+          log_file.flush()
+        if halt_on_output and halt_on_output in output:
+          proc.terminate()
+          break
+      except Queue.Empty:
+        if code != None: # proc has finished running
+          break
+        time.sleep(0.5)
+      if timeout and time.time() - t_0 > timeout:
+        proc.terminate()
+        raise TimeoutException(
+            ''.join(all_output),
+            'Subprocess exceeded timeout of %ds' % timeout)
+  finally:
+    log_thread.stop()
+    log_thread.join()
+  return (code, ''.join(all_output))
+
+
+def log_process_after_completion(proc, echo=True, timeout=None, log_file=None):
+  """ Wait for proc to complete and return a tuple containing the exit code of
+  proc and the contents of stdout. Unlike log_process_in_real_time, does not
+  attempt to read stdout from proc in real time.
+
+  proc: an instance of Popen referring to a running subprocess.
+  echo: boolean indicating whether to print the output received from proc.stdout
+  timeout: number of seconds allotted for the process to run. Raises a
+      TimeoutException if the run time exceeds the timeout.
+  log_file: an open file for writing outout
+  """
+  t_0 = time.time()
+  code = None
+  while code is None:
+    if timeout and time.time() - t_0 > timeout:
+      raise TimeoutException(
+          proc.communicate()[0],
+          'Subprocess exceeded timeout of %ds' % timeout)
+    time.sleep(0.5)
+    code = proc.poll()
+  output = proc.communicate()[0]
+  if echo:
+    print output
+  if log_file:
+    log_file.write(output)
+    log_file.flush()
+  return (code, output)
+
+
+def run(cmd, echo=True, shell=False, timeout=None, print_timestamps=True,
+        log_in_real_time=True):
+  """ Run 'cmd' in a shell and return the combined contents of stdout and
+  stderr (Blocking).  Throws an exception if the command exits non-zero.
+
+  cmd: list of strings (or single string, iff shell==True) indicating the
+      command to run
+  echo: boolean indicating whether we should print the command and log output
+  shell: boolean indicating whether we are using advanced shell features. Use
+      only when absolutely necessary, since this allows a lot more freedom which
+      could be exploited by malicious code. See the warning here:
+      http://docs.python.org/library/subprocess.html#popen-constructor
+  timeout: optional, integer indicating the maximum elapsed time in seconds
+  print_timestamps: boolean indicating whether a formatted timestamp should be
+      prepended to each line of output. Unused if echo or log_in_real_time is
+      False.
+  log_in_real_time: boolean indicating whether to read stdout from the
+      subprocess in real time instead of when the process finishes. If echo is
+      False, we never log in real time, even if log_in_real_time is True.
+  """
+  proc = run_async(cmd, echo=echo, shell=shell)
+  # If we're not printing the output, we don't care if the output shows up in
+  # real time, so don't bother.
+  if log_in_real_time and echo:
+    (returncode, output) = log_process_in_real_time(proc, echo=echo,
+        timeout=timeout, print_timestamps=print_timestamps)
+  else:
+    (returncode, output) = log_process_after_completion(proc, echo=echo,
+                                                        timeout=timeout)
+  if returncode != 0:
+    raise CommandFailedException(
+        output,
+        'Command failed with code %d: %s' % (returncode, cmd))
+  return output
+
+
+def run_retry(cmd, echo=True, shell=False, attempts=1,
+              secs_between_attempts=DEFAULT_SECS_BETWEEN_ATTEMPTS,
+              timeout=None, print_timestamps=True):
+  """ Wrapper for run() which makes multiple attempts until either the command
+  succeeds or the maximum number of attempts is reached. """
+  attempt = 1
+  while True:
+    try:
+      return run(cmd, echo=echo, shell=shell, timeout=timeout,
+                 print_timestamps=print_timestamps)
+    except CommandFailedException:
+      if attempt >= attempts:
+        raise
+    print 'Command failed. Retrying in %d seconds...' % secs_between_attempts
+    time.sleep(secs_between_attempts)
+    attempt += 1
diff --git a/py/utils/ssh_utils.py b/py/utils/ssh_utils.py
new file mode 100644
index 0000000..868fd62
--- /dev/null
+++ b/py/utils/ssh_utils.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains tools related to ssh used by the buildbot scripts. """
+
+import atexit
+import os
+import re
+import shell_utils
+import signal
+
+def PutSCP(local_path, remote_path, username, host, port, recurse=False,
+           options=None):
+  """ Send a file to the given host over SCP. Assumes that public key
+  authentication is set up between the client and server.
+
+  local_path: path to the file to send on the client
+  remote_path: destination path for the file on the server
+  username: ssh login name
+  host: hostname or ip address of the server
+  port: port on the server to use
+  recurse: boolean indicating whether to transmit everything in a folder
+  options: list of extra options to pass to scp
+  """
+  # TODO(borenet): This will hang for a while if the host does not recognize
+  # the client.
+  cmd = ['scp']
+  if options:
+    cmd.extend(options)
+  if recurse:
+    cmd.append('-r')
+  cmd.extend(
+    ['-P', port, local_path, '%s@%s:%s' % (username, host, remote_path)])
+  shell_utils.run(cmd)
+
+
+def MultiPutSCP(local_paths, remote_path, username, host, port, options=None):
+  """ Send files to the given host over SCP. Assumes that public key
+  authentication is set up between the client and server.
+
+  local_paths: list of paths of files and directories to send on the client
+  remote_path: destination directory path on the server
+  username: ssh login name
+  host: hostname or ip address of the server
+  port: port on the server to use
+  options: list of extra options to pass to scp
+  """
+  # TODO(borenet): This will hang for a while if the host does not recognize
+  # the client.
+  cmd = ['scp']
+  if options:
+    cmd.extend(options)
+  cmd.extend(['-r', '-P', port])
+  cmd.extend(local_paths)
+  cmd.append('%s@%s:%s' % (username, host, remote_path))
+  shell_utils.run(cmd)
+
+
+def GetSCP(local_path, remote_path, username, host, port, recurse=False,
+           options=None):
+  """ Retrieve a file from the given host over SCP. Assumes that public key
+  authentication is set up between the client and server.
+
+  local_path: destination path for the file on the client
+  remote_path: path to the file to retrieve on the server
+  username: ssh login name
+  host: hostname or ip address of the server
+  port: port on the server to use
+  recurse: boolean indicating whether to transmit everything in a folder
+  options: list of extra options to pass to scp
+  """
+  # TODO(borenet): This will hang for a while if the host does not recognize
+  # the client.
+  cmd = ['scp']
+  if options:
+    cmd.extend(options)
+  if recurse:
+    cmd.append('-r')
+  cmd.extend(
+    ['-P', port, '%s@%s:%s' % (username, host, remote_path), local_path])
+  shell_utils.run(cmd)
+
+
+def RunSSHCmd(username, host, port, command, echo=True, options=None):
+  """ Login to the given host and run the given command.
+
+  username: ssh login name
+  host: hostname or ip address of the server
+  port: port on the server to use
+  command: (string) command to run on the server
+  options: list of extra options to pass to ssh
+  """
+  # TODO(borenet): This will hang for a while if the host does not recognize
+  # the client.
+  cmd = ['ssh']
+  if options:
+    cmd.extend(options)
+  cmd.extend(['-p', port, '%s@%s' % (username, host), command])
+  return shell_utils.run(cmd, echo=echo)
+
+
+def ShellEscape(arg):
+  """ Escape a single argument for passing into a remote shell
+  """
+  arg = re.sub(r'(["\\])', r'\\\1', arg)
+  return '"%s"' % arg if re.search(r'[\' \t\r\n]', arg) else arg
+
+
+def RunSSH(username, host, port, command, echo=True, options=None):
+  """ Login to the given host and run the given command.
+
+  username: ssh login name
+  host: hostname or ip address of the server
+  port: port on the server to use
+  command: command to run on the server in list format
+  options: list of extra options to pass to ssh
+  """
+  cmd = ' '.join(ShellEscape(arg) for arg in command)
+  return RunSSHCmd(username, host, port, cmd, echo=echo, options=options)
+
+
+class SshDestination(object):
+  """ Convenience class to remember a host, port, and username.
+  Wraps the other functions in this module.
+  """
+  def __init__(self, host, port, username, options=None):
+    """
+    host - (string) hostname of the target
+    port - (string or int) sshd port on the target
+    username - (string) remote username
+    options - (list of strings) extra options to pass to ssh and scp.
+    """
+    self.host = host
+    self.port = str(port)
+    self.user = username
+    self.options = options
+
+  def Put(self, local_path, remote_path, recurse=False):
+    return PutSCP(local_path, remote_path, self.user, self.host,
+                  self.port, recurse=recurse, options=self.options)
+
+  def MultiPut(self, local_paths, remote_path):
+    return MultiPutSCP(local_paths, remote_path, self.user, self.host,
+                       self.port, options=self.options)
+
+  def Get(self, local_path, remote_path, recurse=False):
+    return GetSCP(local_path, remote_path, self.user,
+                  self.host, self.port, recurse=recurse, options=self.options)
+
+  def RunCmd(self, command, echo=True):
+    return RunSSHCmd(self.user, self.host, self.port, command,
+                     echo=echo, options=self.options)
+
+  def Run(self, command, echo=True):
+    return RunSSH(self.user, self.host, self.port, command,
+                  echo=echo, options=self.options)
+
+
+def search_within_string(input_string, pattern):
+  """Search for regular expression in a string.
+
+  input_string: (string) to be searched
+  pattern: (string) to be passed to re.compile, with a symbolic
+           group named 'return'.
+  default: what to return if no match
+
+  Returns a string or None
+  """
+  match = re.search(pattern, input_string)
+  return match.group('return') if match else None
+
+def SSHAdd(key_file):
+  """ Call ssh-add, and call ssh-agent if necessary.
+  """
+  assert os.path.isfile(key_file)
+  try:
+    shell_utils.run(['ssh-add', key_file],
+                    log_in_real_time=False)
+    return
+  except shell_utils.CommandFailedException:
+    ssh_agent_output = shell_utils.run(['ssh-agent', '-s'],
+                                       log_in_real_time=False)
+    if not ssh_agent_output:
+      raise Exception('ssh-agent did not print anything')
+    ssh_auth_sock = search_within_string(
+      ssh_agent_output, r'SSH_AUTH_SOCK=(?P<return>[^;]*);')
+    ssh_agent_pid = search_within_string(
+      ssh_agent_output, r'SSH_AGENT_PID=(?P<return>[^;]*);')
+    if not (ssh_auth_sock and ssh_agent_pid):
+      raise Exception('ssh-agent did not print meaningful data')
+    os.environ['SSH_AUTH_SOCK'] = ssh_auth_sock
+    os.environ['SSH_AGENT_PID'] = ssh_agent_pid
+    atexit.register(os.kill, int(ssh_agent_pid), signal.SIGTERM)
+    shell_utils.run(['ssh-add', key_file])