Add lots of utils, PRESUBMIT.py
BUG=skia:2682
R=jcgregorio@google.com, rmistry@google.com
Review URL: https://codereview.chromium.org/341193004
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2f78cf5
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+*.pyc
+
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
new file mode 100644
index 0000000..ae0a076
--- /dev/null
+++ b/PRESUBMIT.py
@@ -0,0 +1,140 @@
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Presubmit checks for common code."""
+
+import os
+import subprocess
+import sys
+
+
+SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
+SKIP_RUNS_KEYWORD = '(SkipBuildbotRuns)'
+
+
+def _RunPyUnitTests(input_api, output_api):
+ """ Run the Python unit tests and return a list of strings containing any
+ errors.
+ """
+ results = []
+ success = True
+ try:
+ proc = subprocess.Popen(['python', os.path.join('py', 'run_unittests')],
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ success = proc.wait() == 0
+ long_text = proc.communicate()[0]
+ except Exception:
+ success = False
+ long_text = 'Failed to run the common tests!'
+ if not success:
+ results.append(output_api.PresubmitPromptWarning(
+ message='One or more unit tests failed.',
+ long_text=long_text))
+ return results
+
+
+def CheckChange(input_api, output_api):
+ """Presubmit checks for the change on upload or commit.
+
+ The presubmit checks have been handpicked from the list of canned checks
+ here:
+ https://chromium.googlesource.com/chromium/tools/depot_tools/+/master/presubmit_canned_checks.py
+
+ The following are the presubmit checks:
+ * Pylint is run if the change contains any .py files.
+ * Enforces max length for all lines is 80.
+ * Checks that the user didn't add TODO(name) without an owner.
+ * Checks that there is no stray whitespace at source lines end.
+ * Checks that there are no tab characters in any of the text files.
+ """
+ results = []
+
+ pylint_disabled_warnings = (
+ 'F0401', # Unable to import.
+ 'E0611', # No name in module.
+ 'W0232', # Class has no __init__ method.
+ 'E1002', # Use of super on an old style class.
+ 'W0403', # Relative import used.
+ 'R0201', # Method could be a function.
+ 'E1003', # Using class name in super.
+ 'W0613', # Unused argument.
+ )
+ # Run Pylint on only the modified python files. Unfortunately it still runs
+ # Pylint on the whole file instead of just the modified lines.
+ affected_python_files = []
+ for affected_file in input_api.AffectedSourceFiles(None):
+ affected_file_path = affected_file.LocalPath()
+ if affected_file_path.endswith('.py'):
+ affected_python_files.append(affected_file_path)
+ results += input_api.canned_checks.RunPylint(
+ input_api, output_api,
+ disabled_warnings=pylint_disabled_warnings,
+ white_list=affected_python_files)
+
+ # Use 100 for max length for files other than python. Python length is
+ # already checked during the Pylint above.
+ results += input_api.canned_checks.CheckLongLines(input_api, output_api, 100)
+ results += input_api.canned_checks.CheckChangeTodoHasOwner(
+ input_api, output_api)
+ results += input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
+ input_api, output_api)
+ results += input_api.canned_checks.CheckChangeHasNoTabs(input_api, output_api)
+
+ results += _RunPyUnitTests(input_api, output_api)
+
+ return results
+
+
+def _CheckTreeStatus(input_api, output_api, json_url):
+ """Check whether to allow commit.
+
+ Args:
+ input_api: input related apis.
+ output_api: output related apis.
+ json_url: url to download json style status.
+ """
+ results = []
+ tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
+ input_api, output_api, json_url=json_url)
+ display_skip_keyword_prompt = False
+ tree_status = None
+ if not tree_status_results:
+ # Check for caution state only if tree is not closed.
+ connection = input_api.urllib2.urlopen(json_url)
+ status = input_api.json.loads(connection.read())
+ connection.close()
+ tree_status = status['message']
+ if 'caution' in tree_status.lower():
+ # Display a prompt only if we are in an interactive shell. Without this
+ # check the commit queue behaves incorrectly because it considers
+ # prompts to be failures.
+ display_skip_keyword_prompt = True
+ else:
+ tree_status = tree_status_results[0]._message # pylint: disable=W0212
+ display_skip_keyword_prompt = True
+
+ if (display_skip_keyword_prompt
+ and os.isatty(sys.stdout.fileno())
+ and not SKIP_RUNS_KEYWORD in input_api.change.DescriptionText()):
+ long_text = (
+ '%s\nAre you sure the change should be submitted. If it should be '
+ 'submitted but not run on the buildbots you can use the %s keyword.' % (
+ tree_status, SKIP_RUNS_KEYWORD))
+ results.append(output_api.PresubmitPromptWarning(
+ message=tree_status, long_text=long_text))
+ return results
+
+
+def CheckChangeOnUpload(input_api, output_api):
+ return CheckChange(input_api, output_api)
+
+
+def CheckChangeOnCommit(input_api, output_api):
+ results = CheckChange(input_api, output_api)
+ results.extend(
+ _CheckTreeStatus(input_api, output_api, json_url=(
+ SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
+ return results
+
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1 @@
+
diff --git a/py/__init__.py b/py/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/py/__init__.py
@@ -0,0 +1 @@
+
diff --git a/py/run_unittests b/py/run_unittests
new file mode 100755
index 0000000..30c203e
--- /dev/null
+++ b/py/run_unittests
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs all unit tests under this base directory."""
+
+import os
+import subprocess
+import sys
+import unittest
+
+
+NO_CRAWL_DIRS = [
+ '.git',
+ '.svn',
+]
+
+
+SEARCH_PATH = os.path.dirname(os.path.abspath(__file__))
+
+
+def FilterDirectory(dirpath, filenames):
+ """ Determine whether to look for tests in the given directory.
+
+ dirpath: string; path of the directory in question.
+ filenames: list of strings; the files in the directory.
+ """
+ if not dirpath or not filenames:
+ return False
+ for no_crawl_dir in NO_CRAWL_DIRS:
+ if no_crawl_dir in dirpath:
+ return False
+ return True
+
+
+if __name__ == '__main__':
+ print 'Searching for tests.'
+ tests_to_run = []
+
+ for (dirpath, dirnames, filenames) in os.walk(SEARCH_PATH, topdown=True):
+ dirnames[:] = [d for d in dirnames if not d in NO_CRAWL_DIRS]
+ test_modules = [os.path.join(dirpath, filename) for filename in filenames
+ if filename.endswith('_test.py')]
+ if not test_modules:
+ continue
+ tests_to_run.extend(test_modules)
+
+ print 'Found %d tests.' % len(tests_to_run)
+ errors = []
+ for test in tests_to_run:
+ proc = subprocess.Popen(['python', test], stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ if proc.wait() != 0:
+ errors.append(proc.communicate()[0])
+ if errors:
+ for error in errors:
+ print error
+ print 'Failed %d of %d.' % (len(errors), len(test_modules))
+ sys.exit(1)
+ else:
+ print 'All tests succeeded.'
diff --git a/py/utils/__init__.py b/py/utils/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/py/utils/__init__.py
@@ -0,0 +1 @@
+
diff --git a/py/utils/android_utils.py b/py/utils/android_utils.py
new file mode 100644
index 0000000..7b1c1a1
--- /dev/null
+++ b/py/utils/android_utils.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains tools used by Android-specific buildbot scripts. """
+
+import os
+import re
+import shell_utils
+import shlex
+import sys
+import time
+
+
+CPU_SCALING_MODES = ['performance', 'interactive']
+DEVICE_LOOKUP = {'nexus_s': 'crespo',
+ 'xoom': 'stingray',
+ 'galaxy_nexus': 'toro',
+ 'nexus_4': 'mako',
+ 'nexus_7': 'grouper',
+ 'nexus_10': 'manta'}
+PROCESS_MONITOR_INTERVAL = 5.0 # Seconds
+SKIA_RUNNING = 'running'
+SKIA_RETURN_CODE_REPEATS = 10
+SUBPROCESS_TIMEOUT = 30.0
+
+
+def GotADB(adb):
+ """ Returns True iff ADB exists at the given location.
+
+ adb: string; possible path to the ADB executable.
+ """
+ try:
+ shell_utils.run([adb, 'version'], echo=False)
+ return True
+ except Exception:
+ return False
+
+
+def FindADB(hint=None):
+ """ Attempt to find the ADB program using the following sequence of steps.
+ Returns the path to ADB if it can be found, or None otherwise.
+ 1. If a hint was provided, is it a valid path to ADB?
+ 2. Is ADB in PATH?
+ 3. Is there an environment variable for ADB?
+ 4. If the ANDROID_SDK_ROOT variable is set, try to find ADB in the SDK
+ directory.
+ 5. Try to find ADB in a list of common locations.
+
+ hint: string indicating a possible path to ADB.
+ """
+ # 1. If a hint was provided, does it point to ADB?
+ if hint:
+ if os.path.basename(hint) == 'adb':
+ adb = hint
+ else:
+ adb = os.path.join(hint, 'adb')
+ if GotADB(adb):
+ return adb
+
+ # 2. Is 'adb' in our PATH?
+ adb = 'adb'
+ if GotADB(adb):
+ return adb
+
+ # 3. Is there an environment variable for ADB?
+ adb = os.environ.get('ADB')
+ if GotADB(adb):
+ return adb
+
+ # 4. If ANDROID_SDK_ROOT is set, try to find ADB in the SDK directory.
+ sdk_dir = os.environ.get('ANDROID_SDK_ROOT', '')
+ adb = os.path.join(sdk_dir, 'platform-tools', 'adb')
+ if GotADB(adb):
+ return adb
+
+ # 4. Try to find ADB relative to this file.
+ common_locations = []
+ os_dir = None
+ if sys.platform.startswith('linux'):
+ os_dir = 'linux'
+ elif sys.platform.startswith('darwin'):
+ os_dir = 'mac'
+ else:
+ os_dir = 'win'
+ common_locations.append(os.path.join('platform_tools', 'android', 'bin',
+ os_dir, 'adb'))
+ common_locations.append(os.path.join(os.environ.get('HOME', ''),
+ 'android-sdk-%s' % os_dir))
+ for location in common_locations:
+ if GotADB(location):
+ return location
+
+ raise Exception('android_utils: Unable to find ADB!')
+
+
+PATH_TO_ADB = FindADB(hint=os.path.join('platform_tools', 'android', 'bin',
+ 'linux', 'adb'))
+
+
+def RunADB(serial, cmd, echo=True, attempts=5, secs_between_attempts=10,
+ timeout=None):
+ """ Run 'cmd' on an Android device, using ADB. No return value; throws an
+ exception if the command fails more than the allotted number of attempts.
+
+ serial: string indicating the serial number of the target device
+ cmd: string; the command to issue on the device
+ attempts: number of times to attempt the command
+ secs_between_attempts: number of seconds to wait between attempts
+ timeout: optional, integer indicating the maximum elapsed time in seconds
+ """
+ adb_cmd = [PATH_TO_ADB, '-s', serial]
+ adb_cmd += cmd
+ shell_utils.run_retry(adb_cmd, echo=echo, attempts=attempts,
+ secs_between_attempts=secs_between_attempts)
+
+
+def ADBShell(serial, cmd, echo=True):
+ """ Runs 'cmd' in the ADB shell on an Android device and returns the exit
+ code.
+
+ serial: string indicating the serial number of the target device
+ cmd: string; the command to issue on the device
+ """
+ # ADB doesn't exit with the exit code of the command we ran. It only exits
+ # non-zero when ADB itself encountered a problem. Therefore, we have to use
+ # the shell to print the exit code for the command and parse that from stdout.
+ adb_cmd = '%s -s %s shell "%s; echo \$?"' % (PATH_TO_ADB, serial,
+ ' '.join(cmd))
+ output = shell_utils.run(adb_cmd, shell=True, echo=echo)
+ output_lines = output.splitlines()
+ try:
+ real_exitcode = int(output_lines[-1].rstrip())
+ except ValueError:
+ real_exitcode = -1
+ if real_exitcode != 0:
+ raise Exception('Command failed with code %s' % real_exitcode)
+ return '\n'.join(output_lines[:-1])
+
+
+def ADBKill(serial, process, kill_app=False):
+ """ Kill a process running on an Android device.
+
+ serial: string indicating the serial number of the target device
+ process: string indicating the name of the process to kill
+ kill_app: bool indicating whether the process is an Android app, as opposed
+ to a normal executable process.
+ """
+ if kill_app:
+ ADBShell(serial, ['am', 'kill', process])
+ else:
+ try:
+ stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB,
+ serial,
+ process),
+ shell=True)
+ except Exception:
+ return
+ for line in stdout.split('\n'):
+ if line != '':
+ split = shlex.split(line)
+ if len(split) < 2:
+ continue
+ pid = split[1]
+ ADBShell(serial, ['kill', pid])
+ # Raise an exception if any Skia processes are still running.
+ try:
+ stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB,
+ serial,
+ process),
+ shell=True)
+ except Exception:
+ return
+ if stdout:
+ raise Exception('There are still some skia processes running:\n%s\n'
+ 'Maybe the device should be rebooted?' % stdout)
+
+
+def GetSerial(device_type):
+ """ Determine the serial number of the *first* connected device with the
+ specified type. The ordering of 'adb devices' is not documented, and the
+ connected devices do not appear to be ordered by serial number. Therefore,
+ we have to assume that, in the case of multiple devices of the same type being
+ connected to one host, we cannot predict which device will be chosen.
+
+ device_type: string indicating the 'common name' for the target device
+ """
+ if not device_type in DEVICE_LOOKUP:
+ raise ValueError('Unknown device: %s!' % device_type)
+ device_name = DEVICE_LOOKUP[device_type]
+ output = shell_utils.run_retry('%s devices' % PATH_TO_ADB, shell=True,
+ attempts=5)
+ print output
+ lines = output.split('\n')
+ device_ids = []
+ for line in lines:
+ # Filter garbage lines
+ if line != '' and not ('List of devices attached' in line) and \
+ line[0] != '*':
+ device_ids.append(line.split('\t')[0])
+ for device_id in device_ids:
+ print 'Finding type for id %s' % device_id
+ # Get device name
+ name_line = shell_utils.run_retry(
+ '%s -s %s shell cat /system/build.prop | grep "ro.product.device="' % (
+ PATH_TO_ADB, device_id), shell=True, attempts=5)
+ print name_line
+ name = name_line.split('=')[-1].rstrip()
+ # Just return the first attached device of the requested model.
+ if device_name in name:
+ return device_id
+ raise Exception('No %s device attached!' % device_name)
+
+
+def SetCPUScalingMode(serial, mode):
+ """ Set the CPU scaling governor for the device with the given serial number
+ to the given mode.
+
+ serial: string indicating the serial number of the device whose scaling mode
+ is to be modified
+ mode: string indicating the desired CPU scaling mode. Acceptable values
+ are listed in CPU_SCALING_MODES.
+ """
+ if mode not in CPU_SCALING_MODES:
+ raise ValueError('mode must be one of: %s' % CPU_SCALING_MODES)
+ cpu_dirs = shell_utils.run('%s -s %s shell ls /sys/devices/system/cpu' % (
+ PATH_TO_ADB, serial), echo=False, shell=True)
+ cpu_dirs_list = cpu_dirs.split('\n')
+ regex = re.compile('cpu\d')
+ for cpu_dir_from_list in cpu_dirs_list:
+ cpu_dir = cpu_dir_from_list.rstrip()
+ if regex.match(cpu_dir):
+ path = '/sys/devices/system/cpu/%s/cpufreq/scaling_governor' % cpu_dir
+ path_found = shell_utils.run('%s -s %s shell ls %s' % (
+ PATH_TO_ADB, serial, path),
+ echo=False, shell=True).rstrip()
+ if path_found == path:
+ # Unfortunately, we can't directly change the scaling_governor file over
+ # ADB. Instead, we write a script to do so, push it to the device, and
+ # run it.
+ old_mode = shell_utils.run('%s -s %s shell cat %s' % (
+ PATH_TO_ADB, serial, path),
+ echo=False, shell=True).rstrip()
+ print 'Current scaling mode for %s is: %s' % (cpu_dir, old_mode)
+ filename = 'skia_cpuscale.sh'
+ with open(filename, 'w') as script_file:
+ script_file.write('echo %s > %s\n' % (mode, path))
+ os.chmod(filename, 0777)
+ RunADB(serial, ['push', filename, '/system/bin'], echo=False)
+ RunADB(serial, ['shell', filename], echo=True)
+ RunADB(serial, ['shell', 'rm', '/system/bin/%s' % filename], echo=False)
+ os.remove(filename)
+ new_mode = shell_utils.run('%s -s %s shell cat %s' % (
+ PATH_TO_ADB, serial, path),
+ echo=False, shell=True).rstrip()
+ print 'New scaling mode for %s is: %s' % (cpu_dir, new_mode)
+
+
+def IsAndroidShellRunning(serial):
+ """ Find the status of the Android shell for the device with the given serial
+ number. Returns True if the shell is running and False otherwise.
+
+ serial: string indicating the serial number of the target device.
+ """
+ if 'Error:' in ADBShell(serial, ['pm', 'path', 'android'], echo=False):
+ return False
+ return True
+
+
+def StopShell(serial, timeout=60):
+ """ Halt the Android runtime on the device with the given serial number.
+ Blocks until the shell reports that it has stopped.
+
+ serial: string indicating the serial number of the target device.
+ timeout: maximum allotted time, in seconds.
+ """
+ ADBShell(serial, ['stop'])
+ start_time = time.time()
+ while IsAndroidShellRunning(serial):
+ time.sleep(1)
+ if time.time() - start_time > timeout:
+ raise Exception('Timeout while attempting to stop the Android runtime.')
+
+
+def StartShell(serial, timeout=60):
+ """ Start the Android runtime on the device with the given serial number.
+ Blocks until the shell reports that it has started.
+
+ serial: string indicating the serial number of the target device.
+ timeout: maximum allotted time, in seconds.
+ """
+ ADBShell(serial, ['start'])
+ start_time = time.time()
+ while not IsAndroidShellRunning(serial):
+ time.sleep(1)
+ if time.time() - start_time > timeout:
+ raise Exception('Timeout while attempting to start the Android runtime.')
+
+
+def RunSkia(serial, cmd, release, device):
+ """ Run the given command through skia_launcher on a given device.
+
+ serial: string indicating the serial number of the target device.
+ cmd: list of strings; the command line to run.
+ release: bool; whether or not to run the app in Release mode.
+ device: string indicating the target device.
+ """
+ RunADB(serial, ['logcat', '-c'])
+ try:
+ os.environ['SKIA_ANDROID_VERBOSE_SETUP'] = '1'
+ cmd_to_run = [os.path.join('platform_tools', 'android', 'bin',
+ 'android_run_skia')]
+ if release:
+ cmd_to_run.extend(['--release'])
+ cmd_to_run.extend(['-d', device])
+ cmd_to_run.extend(['-s', serial])
+ cmd_to_run.extend(cmd)
+ shell_utils.run(cmd_to_run)
+ finally:
+ RunADB(serial, ['logcat', '-d', '-v', 'time'])
diff --git a/py/utils/git_utils.py b/py/utils/git_utils.py
new file mode 100644
index 0000000..da7f097
--- /dev/null
+++ b/py/utils/git_utils.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module contains functions for using git."""
+
+
+import os
+import shell_utils
+
+
+GIT = 'git.bat' if os.name == 'nt' else 'git'
+
+
+def Add(addition):
+ """Run 'git add <addition>'"""
+ shell_utils.run([GIT, 'add', addition])
+
+def AIsAncestorOfB(a, b):
+ """Return true if a is an ancestor of b."""
+ return shell_utils.run([GIT, 'merge-base', a, b]).rstrip() == FullHash(a)
+
+def FullHash(commit):
+ """Return full hash of specified commit."""
+ return shell_utils.run([GIT, 'rev-parse', '--verify', commit]).rstrip()
+
+def IsMerge(commit):
+ """Return True if the commit is a merge, False otherwise."""
+ rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1',
+ '--no-merges'])
+ last_non_merge = rev_parse.split('\n')[0]
+ # Get full hash since that is what was returned by rev-parse.
+ return FullHash(commit) != last_non_merge
+
+def MergeAbort():
+ """Abort in process merge."""
+ shell_utils.run([GIT, 'merge', '--abort'])
+
+def ShortHash(commit):
+ """Return short hash of the specified commit."""
+ return shell_utils.run([GIT, 'show', commit, '--format=%h', '-s']).rstrip()
+
+def GetRemoteMasterHash(git_url):
+ return shell_utils.run([GIT, 'ls-remote', git_url, '--verify',
+ 'refs/heads/master'])
diff --git a/py/utils/misc.py b/py/utils/misc.py
new file mode 100644
index 0000000..a61527e
--- /dev/null
+++ b/py/utils/misc.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains miscellaneous tools used by the buildbot scripts. """
+
+import os
+
+from git_utils import GIT
+import shell_utils
+
+
+# Absolute path to the root of this Skia buildbot checkout.
+BUILDBOT_PATH = os.path.realpath(os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ os.pardir, os.pardir, os.pardir))
+
+
+def ArgsToDict(argv):
+ """ Collect command-line arguments of the form '--key value' into a
+ dictionary. Fail if the arguments do not fit this format. """
+ dictionary = {}
+ PREFIX = '--'
+ # Expect the first arg to be the path to the script, which we don't want.
+ argv = argv[1:]
+ while argv:
+ if argv[0].startswith(PREFIX):
+ dictionary[argv[0][len(PREFIX):]] = argv[1]
+ argv = argv[2:]
+ else:
+ raise Exception('Malformed input: %s' % argv)
+ return dictionary
+
+
+def ConfirmOptionsSet(name_value_dict):
+ """Raise an exception if any of the given command-line options were not set.
+
+ name_value_dict: dictionary mapping option names to option values
+ """
+ for (name, value) in name_value_dict.iteritems():
+ if value is None:
+ raise Exception('missing command-line option %s; rerun with --help' %
+ name)
+
+
+def GetAbsPath(relative_path):
+ """My own implementation of os.path.abspath() that better handles paths
+ which approach Window's 260-character limit.
+ See https://code.google.com/p/skia/issues/detail?id=674
+
+ This implementation adds path components one at a time, resolving the
+ absolute path each time, to take advantage of any chdirs into outer
+ directories that will shorten the total path length.
+
+ TODO(epoger): share a single implementation with bench_graph_svg.py, instead
+ of pasting this same code into both files."""
+ if os.path.isabs(relative_path):
+ return relative_path
+ path_parts = relative_path.split(os.sep)
+ abs_path = os.path.abspath('.')
+ for path_part in path_parts:
+ abs_path = os.path.abspath(os.path.join(abs_path, path_part))
+ return abs_path
+
+
+class ChDir(object):
+ """Enter and exit the given directory appropriately."""
+
+ def __init__(self, directory):
+ """Instantiate the ChDir.
+
+ Args:
+ directory: string; the directory to enter.
+ """
+ self._destination = directory
+ self._origin = None
+
+ def __enter__(self):
+ """Change to the destination directory.
+
+ Does not check whether the directory exists.
+ """
+ self._origin = os.getcwd()
+ print 'chdir %s' % self._destination
+ os.chdir(self._destination)
+
+ def __exit__(self, *args):
+ """Change back to the original directory."""
+ print 'chdir %s' % self._origin
+ os.chdir(self._origin)
+
+
+class GitBranch(object):
+ """Class to manage git branches.
+
+ This class allows one to create a new branch in a repository to make changes,
+ then it commits the changes, switches to master branch, and deletes the
+ created temporary branch upon exit.
+ """
+ def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False):
+ self._branch_name = branch_name
+ self._commit_msg = commit_msg
+ self._upload = upload
+ self._commit_queue = commit_queue
+ self._patch_set = 0
+
+ def __enter__(self):
+ shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
+ shell_utils.run([GIT, 'checkout', 'master'])
+ if self._branch_name in shell_utils.run([GIT, 'branch']):
+ shell_utils.run([GIT, 'branch', '-D', self._branch_name])
+ shell_utils.run([GIT, 'checkout', '-b', self._branch_name,
+ '-t', 'origin/master'])
+ return self
+
+ def commit_and_upload(self, use_commit_queue=False):
+ shell_utils.run([GIT, 'commit', '-a', '-m',
+ self._commit_msg])
+ upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks',
+ '--bypass-watchlists']
+ self._patch_set += 1
+ if self._patch_set > 1:
+ upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set])
+ if use_commit_queue:
+ upload_cmd.append('--use-commit-queue')
+ shell_utils.run(upload_cmd)
+
+ def __exit__(self, exc_type, _value, _traceback):
+ if self._upload:
+ # Only upload if no error occurred.
+ try:
+ if exc_type is None:
+ self.commit_and_upload(use_commit_queue=self._commit_queue)
+ finally:
+ shell_utils.run([GIT, 'checkout', 'master'])
+ shell_utils.run([GIT, 'branch', '-D', self._branch_name])
+
diff --git a/py/utils/shell_utils.py b/py/utils/shell_utils.py
new file mode 100644
index 0000000..a6b54ef
--- /dev/null
+++ b/py/utils/shell_utils.py
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains tools for running commands in a shell. """
+
+import datetime
+import os
+import Queue
+import select
+import subprocess
+import sys
+import threading
+import time
+
+if 'nt' in os.name:
+ import ctypes
+
+
+DEFAULT_SECS_BETWEEN_ATTEMPTS = 10
+POLL_MILLIS = 250
+
+
+class CommandFailedException(Exception):
+ """Exception which gets raised when a command fails."""
+
+ def __init__(self, output, *args):
+ """Initialize the CommandFailedException.
+
+ Args:
+ output: string; output from the command.
+ """
+ Exception.__init__(self, *args)
+ self._output = output
+
+ @property
+ def output(self):
+ """Output from the command."""
+ return self._output
+
+
+class TimeoutException(CommandFailedException):
+ """CommandFailedException which gets raised when a subprocess exceeds its
+ timeout. """
+ pass
+
+
+def run_async(cmd, echo=True, shell=False):
+ """ Run 'cmd' in a subprocess, returning a Popen class instance referring to
+ that process. (Non-blocking) """
+ if echo:
+ print cmd
+ if 'nt' in os.name:
+ # Windows has a bad habit of opening a dialog when a console program
+ # crashes, rather than just letting it crash. Therefore, when a program
+ # crashes on Windows, we don't find out until the build step times out.
+ # This code prevents the dialog from appearing, so that we find out
+ # immediately and don't waste time waiting around.
+ SEM_NOGPFAULTERRORBOX = 0x0002
+ ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
+ flags = 0x8000000 # CREATE_NO_WINDOW
+ else:
+ flags = 0
+ return subprocess.Popen(cmd, shell=shell, stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE, creationflags=flags,
+ bufsize=1)
+
+
+class EnqueueThread(threading.Thread):
+ """ Reads and enqueues lines from a file. """
+ def __init__(self, file_obj, queue):
+ threading.Thread.__init__(self)
+ self._file = file_obj
+ self._queue = queue
+ self._stopped = False
+
+ def run(self):
+ if sys.platform.startswith('linux'):
+ # Use a polling object to avoid the blocking call to readline().
+ poll = select.poll()
+ poll.register(self._file, select.POLLIN)
+ while not self._stopped:
+ has_output = poll.poll(POLL_MILLIS)
+ if has_output:
+ line = self._file.readline()
+ if line == '':
+ self._stopped = True
+ self._queue.put(line)
+ else:
+ # Only Unix supports polling objects, so just read from the file,
+ # Python-style.
+ for line in iter(self._file.readline, ''):
+ self._queue.put(line)
+ if self._stopped:
+ break
+
+ def stop(self):
+ self._stopped = True
+
+
+def log_process_in_real_time(proc, echo=True, timeout=None, log_file=None,
+ halt_on_output=None, print_timestamps=True):
+ """ Log the output of proc in real time until it completes. Return a tuple
+ containing the exit code of proc and the contents of stdout.
+
+ proc: an instance of Popen referring to a running subprocess.
+ echo: boolean indicating whether to print the output received from proc.stdout
+ timeout: number of seconds allotted for the process to run. Raises a
+ TimeoutException if the run time exceeds the timeout.
+ log_file: an open file for writing output
+ halt_on_output: string; kill the process and return if this string is found
+ in the output stream from the process.
+ print_timestamps: boolean indicating whether a formatted timestamp should be
+ prepended to each line of output.
+ """
+ stdout_queue = Queue.Queue()
+ log_thread = EnqueueThread(proc.stdout, stdout_queue)
+ log_thread.start()
+ try:
+ all_output = []
+ t_0 = time.time()
+ while True:
+ code = proc.poll()
+ try:
+ output = stdout_queue.get_nowait()
+ all_output.append(output)
+ if output and print_timestamps:
+ timestamp = datetime.datetime.now().strftime('%H:%M:%S.%f')
+ output = ''.join(['[%s] %s\n' % (timestamp, line)
+ for line in output.splitlines()])
+ if echo:
+ sys.stdout.write(output)
+ sys.stdout.flush()
+ if log_file:
+ log_file.write(output)
+ log_file.flush()
+ if halt_on_output and halt_on_output in output:
+ proc.terminate()
+ break
+ except Queue.Empty:
+ if code != None: # proc has finished running
+ break
+ time.sleep(0.5)
+ if timeout and time.time() - t_0 > timeout:
+ proc.terminate()
+ raise TimeoutException(
+ ''.join(all_output),
+ 'Subprocess exceeded timeout of %ds' % timeout)
+ finally:
+ log_thread.stop()
+ log_thread.join()
+ return (code, ''.join(all_output))
+
+
+def log_process_after_completion(proc, echo=True, timeout=None, log_file=None):
+ """ Wait for proc to complete and return a tuple containing the exit code of
+ proc and the contents of stdout. Unlike log_process_in_real_time, does not
+ attempt to read stdout from proc in real time.
+
+ proc: an instance of Popen referring to a running subprocess.
+ echo: boolean indicating whether to print the output received from proc.stdout
+ timeout: number of seconds allotted for the process to run. Raises a
+ TimeoutException if the run time exceeds the timeout.
+ log_file: an open file for writing outout
+ """
+ t_0 = time.time()
+ code = None
+ while code is None:
+ if timeout and time.time() - t_0 > timeout:
+ raise TimeoutException(
+ proc.communicate()[0],
+ 'Subprocess exceeded timeout of %ds' % timeout)
+ time.sleep(0.5)
+ code = proc.poll()
+ output = proc.communicate()[0]
+ if echo:
+ print output
+ if log_file:
+ log_file.write(output)
+ log_file.flush()
+ return (code, output)
+
+
+def run(cmd, echo=True, shell=False, timeout=None, print_timestamps=True,
+ log_in_real_time=True):
+ """ Run 'cmd' in a shell and return the combined contents of stdout and
+ stderr (Blocking). Throws an exception if the command exits non-zero.
+
+ cmd: list of strings (or single string, iff shell==True) indicating the
+ command to run
+ echo: boolean indicating whether we should print the command and log output
+ shell: boolean indicating whether we are using advanced shell features. Use
+ only when absolutely necessary, since this allows a lot more freedom which
+ could be exploited by malicious code. See the warning here:
+ http://docs.python.org/library/subprocess.html#popen-constructor
+ timeout: optional, integer indicating the maximum elapsed time in seconds
+ print_timestamps: boolean indicating whether a formatted timestamp should be
+ prepended to each line of output. Unused if echo or log_in_real_time is
+ False.
+ log_in_real_time: boolean indicating whether to read stdout from the
+ subprocess in real time instead of when the process finishes. If echo is
+ False, we never log in real time, even if log_in_real_time is True.
+ """
+ proc = run_async(cmd, echo=echo, shell=shell)
+ # If we're not printing the output, we don't care if the output shows up in
+ # real time, so don't bother.
+ if log_in_real_time and echo:
+ (returncode, output) = log_process_in_real_time(proc, echo=echo,
+ timeout=timeout, print_timestamps=print_timestamps)
+ else:
+ (returncode, output) = log_process_after_completion(proc, echo=echo,
+ timeout=timeout)
+ if returncode != 0:
+ raise CommandFailedException(
+ output,
+ 'Command failed with code %d: %s' % (returncode, cmd))
+ return output
+
+
+def run_retry(cmd, echo=True, shell=False, attempts=1,
+ secs_between_attempts=DEFAULT_SECS_BETWEEN_ATTEMPTS,
+ timeout=None, print_timestamps=True):
+ """ Wrapper for run() which makes multiple attempts until either the command
+ succeeds or the maximum number of attempts is reached. """
+ attempt = 1
+ while True:
+ try:
+ return run(cmd, echo=echo, shell=shell, timeout=timeout,
+ print_timestamps=print_timestamps)
+ except CommandFailedException:
+ if attempt >= attempts:
+ raise
+ print 'Command failed. Retrying in %d seconds...' % secs_between_attempts
+ time.sleep(secs_between_attempts)
+ attempt += 1
diff --git a/py/utils/ssh_utils.py b/py/utils/ssh_utils.py
new file mode 100644
index 0000000..868fd62
--- /dev/null
+++ b/py/utils/ssh_utils.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" This module contains tools related to ssh used by the buildbot scripts. """
+
+import atexit
+import os
+import re
+import shell_utils
+import signal
+
+def PutSCP(local_path, remote_path, username, host, port, recurse=False,
+ options=None):
+ """ Send a file to the given host over SCP. Assumes that public key
+ authentication is set up between the client and server.
+
+ local_path: path to the file to send on the client
+ remote_path: destination path for the file on the server
+ username: ssh login name
+ host: hostname or ip address of the server
+ port: port on the server to use
+ recurse: boolean indicating whether to transmit everything in a folder
+ options: list of extra options to pass to scp
+ """
+ # TODO(borenet): This will hang for a while if the host does not recognize
+ # the client.
+ cmd = ['scp']
+ if options:
+ cmd.extend(options)
+ if recurse:
+ cmd.append('-r')
+ cmd.extend(
+ ['-P', port, local_path, '%s@%s:%s' % (username, host, remote_path)])
+ shell_utils.run(cmd)
+
+
+def MultiPutSCP(local_paths, remote_path, username, host, port, options=None):
+ """ Send files to the given host over SCP. Assumes that public key
+ authentication is set up between the client and server.
+
+ local_paths: list of paths of files and directories to send on the client
+ remote_path: destination directory path on the server
+ username: ssh login name
+ host: hostname or ip address of the server
+ port: port on the server to use
+ options: list of extra options to pass to scp
+ """
+ # TODO(borenet): This will hang for a while if the host does not recognize
+ # the client.
+ cmd = ['scp']
+ if options:
+ cmd.extend(options)
+ cmd.extend(['-r', '-P', port])
+ cmd.extend(local_paths)
+ cmd.append('%s@%s:%s' % (username, host, remote_path))
+ shell_utils.run(cmd)
+
+
+def GetSCP(local_path, remote_path, username, host, port, recurse=False,
+ options=None):
+ """ Retrieve a file from the given host over SCP. Assumes that public key
+ authentication is set up between the client and server.
+
+ local_path: destination path for the file on the client
+ remote_path: path to the file to retrieve on the server
+ username: ssh login name
+ host: hostname or ip address of the server
+ port: port on the server to use
+ recurse: boolean indicating whether to transmit everything in a folder
+ options: list of extra options to pass to scp
+ """
+ # TODO(borenet): This will hang for a while if the host does not recognize
+ # the client.
+ cmd = ['scp']
+ if options:
+ cmd.extend(options)
+ if recurse:
+ cmd.append('-r')
+ cmd.extend(
+ ['-P', port, '%s@%s:%s' % (username, host, remote_path), local_path])
+ shell_utils.run(cmd)
+
+
+def RunSSHCmd(username, host, port, command, echo=True, options=None):
+ """ Login to the given host and run the given command.
+
+ username: ssh login name
+ host: hostname or ip address of the server
+ port: port on the server to use
+ command: (string) command to run on the server
+ options: list of extra options to pass to ssh
+ """
+ # TODO(borenet): This will hang for a while if the host does not recognize
+ # the client.
+ cmd = ['ssh']
+ if options:
+ cmd.extend(options)
+ cmd.extend(['-p', port, '%s@%s' % (username, host), command])
+ return shell_utils.run(cmd, echo=echo)
+
+
+def ShellEscape(arg):
+ """ Escape a single argument for passing into a remote shell
+ """
+ arg = re.sub(r'(["\\])', r'\\\1', arg)
+ return '"%s"' % arg if re.search(r'[\' \t\r\n]', arg) else arg
+
+
+def RunSSH(username, host, port, command, echo=True, options=None):
+ """ Login to the given host and run the given command.
+
+ username: ssh login name
+ host: hostname or ip address of the server
+ port: port on the server to use
+ command: command to run on the server in list format
+ options: list of extra options to pass to ssh
+ """
+ cmd = ' '.join(ShellEscape(arg) for arg in command)
+ return RunSSHCmd(username, host, port, cmd, echo=echo, options=options)
+
+
+class SshDestination(object):
+ """ Convenience class to remember a host, port, and username.
+ Wraps the other functions in this module.
+ """
+ def __init__(self, host, port, username, options=None):
+ """
+ host - (string) hostname of the target
+ port - (string or int) sshd port on the target
+ username - (string) remote username
+ options - (list of strings) extra options to pass to ssh and scp.
+ """
+ self.host = host
+ self.port = str(port)
+ self.user = username
+ self.options = options
+
+ def Put(self, local_path, remote_path, recurse=False):
+ return PutSCP(local_path, remote_path, self.user, self.host,
+ self.port, recurse=recurse, options=self.options)
+
+ def MultiPut(self, local_paths, remote_path):
+ return MultiPutSCP(local_paths, remote_path, self.user, self.host,
+ self.port, options=self.options)
+
+ def Get(self, local_path, remote_path, recurse=False):
+ return GetSCP(local_path, remote_path, self.user,
+ self.host, self.port, recurse=recurse, options=self.options)
+
+ def RunCmd(self, command, echo=True):
+ return RunSSHCmd(self.user, self.host, self.port, command,
+ echo=echo, options=self.options)
+
+ def Run(self, command, echo=True):
+ return RunSSH(self.user, self.host, self.port, command,
+ echo=echo, options=self.options)
+
+
+def search_within_string(input_string, pattern):
+ """Search for regular expression in a string.
+
+ input_string: (string) to be searched
+ pattern: (string) to be passed to re.compile, with a symbolic
+ group named 'return'.
+ default: what to return if no match
+
+ Returns a string or None
+ """
+ match = re.search(pattern, input_string)
+ return match.group('return') if match else None
+
+def SSHAdd(key_file):
+ """ Call ssh-add, and call ssh-agent if necessary.
+ """
+ assert os.path.isfile(key_file)
+ try:
+ shell_utils.run(['ssh-add', key_file],
+ log_in_real_time=False)
+ return
+ except shell_utils.CommandFailedException:
+ ssh_agent_output = shell_utils.run(['ssh-agent', '-s'],
+ log_in_real_time=False)
+ if not ssh_agent_output:
+ raise Exception('ssh-agent did not print anything')
+ ssh_auth_sock = search_within_string(
+ ssh_agent_output, r'SSH_AUTH_SOCK=(?P<return>[^;]*);')
+ ssh_agent_pid = search_within_string(
+ ssh_agent_output, r'SSH_AGENT_PID=(?P<return>[^;]*);')
+ if not (ssh_auth_sock and ssh_agent_pid):
+ raise Exception('ssh-agent did not print meaningful data')
+ os.environ['SSH_AUTH_SOCK'] = ssh_auth_sock
+ os.environ['SSH_AGENT_PID'] = ssh_agent_pid
+ atexit.register(os.kill, int(ssh_agent_pid), signal.SIGTERM)
+ shell_utils.run(['ssh-add', key_file])