Add lots of utils, PRESUBMIT.py BUG=skia:2682 R=jcgregorio@google.com, rmistry@google.com Review URL: https://codereview.chromium.org/341193004
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f78cf5 --- /dev/null +++ b/.gitignore
@@ -0,0 +1,2 @@ +*.pyc +
diff --git a/PRESUBMIT.py b/PRESUBMIT.py new file mode 100644 index 0000000..ae0a076 --- /dev/null +++ b/PRESUBMIT.py
@@ -0,0 +1,140 @@ +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Presubmit checks for common code.""" + +import os +import subprocess +import sys + + +SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com' +SKIP_RUNS_KEYWORD = '(SkipBuildbotRuns)' + + +def _RunPyUnitTests(input_api, output_api): + """ Run the Python unit tests and return a list of strings containing any + errors. + """ + results = [] + success = True + try: + proc = subprocess.Popen(['python', os.path.join('py', 'run_unittests')], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + success = proc.wait() == 0 + long_text = proc.communicate()[0] + except Exception: + success = False + long_text = 'Failed to run the common tests!' + if not success: + results.append(output_api.PresubmitPromptWarning( + message='One or more unit tests failed.', + long_text=long_text)) + return results + + +def CheckChange(input_api, output_api): + """Presubmit checks for the change on upload or commit. + + The presubmit checks have been handpicked from the list of canned checks + here: + https://chromium.googlesource.com/chromium/tools/depot_tools/+/master/presubmit_canned_checks.py + + The following are the presubmit checks: + * Pylint is run if the change contains any .py files. + * Enforces max length for all lines is 80. + * Checks that the user didn't add TODO(name) without an owner. + * Checks that there is no stray whitespace at source lines end. + * Checks that there are no tab characters in any of the text files. + """ + results = [] + + pylint_disabled_warnings = ( + 'F0401', # Unable to import. + 'E0611', # No name in module. + 'W0232', # Class has no __init__ method. + 'E1002', # Use of super on an old style class. + 'W0403', # Relative import used. + 'R0201', # Method could be a function. + 'E1003', # Using class name in super. + 'W0613', # Unused argument. + ) + # Run Pylint on only the modified python files. Unfortunately it still runs + # Pylint on the whole file instead of just the modified lines. + affected_python_files = [] + for affected_file in input_api.AffectedSourceFiles(None): + affected_file_path = affected_file.LocalPath() + if affected_file_path.endswith('.py'): + affected_python_files.append(affected_file_path) + results += input_api.canned_checks.RunPylint( + input_api, output_api, + disabled_warnings=pylint_disabled_warnings, + white_list=affected_python_files) + + # Use 100 for max length for files other than python. Python length is + # already checked during the Pylint above. + results += input_api.canned_checks.CheckLongLines(input_api, output_api, 100) + results += input_api.canned_checks.CheckChangeTodoHasOwner( + input_api, output_api) + results += input_api.canned_checks.CheckChangeHasNoStrayWhitespace( + input_api, output_api) + results += input_api.canned_checks.CheckChangeHasNoTabs(input_api, output_api) + + results += _RunPyUnitTests(input_api, output_api) + + return results + + +def _CheckTreeStatus(input_api, output_api, json_url): + """Check whether to allow commit. + + Args: + input_api: input related apis. + output_api: output related apis. + json_url: url to download json style status. + """ + results = [] + tree_status_results = input_api.canned_checks.CheckTreeIsOpen( + input_api, output_api, json_url=json_url) + display_skip_keyword_prompt = False + tree_status = None + if not tree_status_results: + # Check for caution state only if tree is not closed. + connection = input_api.urllib2.urlopen(json_url) + status = input_api.json.loads(connection.read()) + connection.close() + tree_status = status['message'] + if 'caution' in tree_status.lower(): + # Display a prompt only if we are in an interactive shell. Without this + # check the commit queue behaves incorrectly because it considers + # prompts to be failures. + display_skip_keyword_prompt = True + else: + tree_status = tree_status_results[0]._message # pylint: disable=W0212 + display_skip_keyword_prompt = True + + if (display_skip_keyword_prompt + and os.isatty(sys.stdout.fileno()) + and not SKIP_RUNS_KEYWORD in input_api.change.DescriptionText()): + long_text = ( + '%s\nAre you sure the change should be submitted. If it should be ' + 'submitted but not run on the buildbots you can use the %s keyword.' % ( + tree_status, SKIP_RUNS_KEYWORD)) + results.append(output_api.PresubmitPromptWarning( + message=tree_status, long_text=long_text)) + return results + + +def CheckChangeOnUpload(input_api, output_api): + return CheckChange(input_api, output_api) + + +def CheckChangeOnCommit(input_api, output_api): + results = CheckChange(input_api, output_api) + results.extend( + _CheckTreeStatus(input_api, output_api, json_url=( + SKIA_TREE_STATUS_URL + '/banner-status?format=json'))) + return results +
diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/__init__.py
@@ -0,0 +1 @@ +
diff --git a/py/__init__.py b/py/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/py/__init__.py
@@ -0,0 +1 @@ +
diff --git a/py/run_unittests b/py/run_unittests new file mode 100755 index 0000000..30c203e --- /dev/null +++ b/py/run_unittests
@@ -0,0 +1,62 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs all unit tests under this base directory.""" + +import os +import subprocess +import sys +import unittest + + +NO_CRAWL_DIRS = [ + '.git', + '.svn', +] + + +SEARCH_PATH = os.path.dirname(os.path.abspath(__file__)) + + +def FilterDirectory(dirpath, filenames): + """ Determine whether to look for tests in the given directory. + + dirpath: string; path of the directory in question. + filenames: list of strings; the files in the directory. + """ + if not dirpath or not filenames: + return False + for no_crawl_dir in NO_CRAWL_DIRS: + if no_crawl_dir in dirpath: + return False + return True + + +if __name__ == '__main__': + print 'Searching for tests.' + tests_to_run = [] + + for (dirpath, dirnames, filenames) in os.walk(SEARCH_PATH, topdown=True): + dirnames[:] = [d for d in dirnames if not d in NO_CRAWL_DIRS] + test_modules = [os.path.join(dirpath, filename) for filename in filenames + if filename.endswith('_test.py')] + if not test_modules: + continue + tests_to_run.extend(test_modules) + + print 'Found %d tests.' % len(tests_to_run) + errors = [] + for test in tests_to_run: + proc = subprocess.Popen(['python', test], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if proc.wait() != 0: + errors.append(proc.communicate()[0]) + if errors: + for error in errors: + print error + print 'Failed %d of %d.' % (len(errors), len(test_modules)) + sys.exit(1) + else: + print 'All tests succeeded.'
diff --git a/py/utils/__init__.py b/py/utils/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/py/utils/__init__.py
@@ -0,0 +1 @@ +
diff --git a/py/utils/android_utils.py b/py/utils/android_utils.py new file mode 100644 index 0000000..7b1c1a1 --- /dev/null +++ b/py/utils/android_utils.py
@@ -0,0 +1,321 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" This module contains tools used by Android-specific buildbot scripts. """ + +import os +import re +import shell_utils +import shlex +import sys +import time + + +CPU_SCALING_MODES = ['performance', 'interactive'] +DEVICE_LOOKUP = {'nexus_s': 'crespo', + 'xoom': 'stingray', + 'galaxy_nexus': 'toro', + 'nexus_4': 'mako', + 'nexus_7': 'grouper', + 'nexus_10': 'manta'} +PROCESS_MONITOR_INTERVAL = 5.0 # Seconds +SKIA_RUNNING = 'running' +SKIA_RETURN_CODE_REPEATS = 10 +SUBPROCESS_TIMEOUT = 30.0 + + +def GotADB(adb): + """ Returns True iff ADB exists at the given location. + + adb: string; possible path to the ADB executable. + """ + try: + shell_utils.run([adb, 'version'], echo=False) + return True + except Exception: + return False + + +def FindADB(hint=None): + """ Attempt to find the ADB program using the following sequence of steps. + Returns the path to ADB if it can be found, or None otherwise. + 1. If a hint was provided, is it a valid path to ADB? + 2. Is ADB in PATH? + 3. Is there an environment variable for ADB? + 4. If the ANDROID_SDK_ROOT variable is set, try to find ADB in the SDK + directory. + 5. Try to find ADB in a list of common locations. + + hint: string indicating a possible path to ADB. + """ + # 1. If a hint was provided, does it point to ADB? + if hint: + if os.path.basename(hint) == 'adb': + adb = hint + else: + adb = os.path.join(hint, 'adb') + if GotADB(adb): + return adb + + # 2. Is 'adb' in our PATH? + adb = 'adb' + if GotADB(adb): + return adb + + # 3. Is there an environment variable for ADB? + adb = os.environ.get('ADB') + if GotADB(adb): + return adb + + # 4. If ANDROID_SDK_ROOT is set, try to find ADB in the SDK directory. + sdk_dir = os.environ.get('ANDROID_SDK_ROOT', '') + adb = os.path.join(sdk_dir, 'platform-tools', 'adb') + if GotADB(adb): + return adb + + # 4. Try to find ADB relative to this file. + common_locations = [] + os_dir = None + if sys.platform.startswith('linux'): + os_dir = 'linux' + elif sys.platform.startswith('darwin'): + os_dir = 'mac' + else: + os_dir = 'win' + common_locations.append(os.path.join('platform_tools', 'android', 'bin', + os_dir, 'adb')) + common_locations.append(os.path.join(os.environ.get('HOME', ''), + 'android-sdk-%s' % os_dir)) + for location in common_locations: + if GotADB(location): + return location + + raise Exception('android_utils: Unable to find ADB!') + + +PATH_TO_ADB = FindADB(hint=os.path.join('platform_tools', 'android', 'bin', + 'linux', 'adb')) + + +def RunADB(serial, cmd, echo=True, attempts=5, secs_between_attempts=10, + timeout=None): + """ Run 'cmd' on an Android device, using ADB. No return value; throws an + exception if the command fails more than the allotted number of attempts. + + serial: string indicating the serial number of the target device + cmd: string; the command to issue on the device + attempts: number of times to attempt the command + secs_between_attempts: number of seconds to wait between attempts + timeout: optional, integer indicating the maximum elapsed time in seconds + """ + adb_cmd = [PATH_TO_ADB, '-s', serial] + adb_cmd += cmd + shell_utils.run_retry(adb_cmd, echo=echo, attempts=attempts, + secs_between_attempts=secs_between_attempts) + + +def ADBShell(serial, cmd, echo=True): + """ Runs 'cmd' in the ADB shell on an Android device and returns the exit + code. + + serial: string indicating the serial number of the target device + cmd: string; the command to issue on the device + """ + # ADB doesn't exit with the exit code of the command we ran. It only exits + # non-zero when ADB itself encountered a problem. Therefore, we have to use + # the shell to print the exit code for the command and parse that from stdout. + adb_cmd = '%s -s %s shell "%s; echo \$?"' % (PATH_TO_ADB, serial, + ' '.join(cmd)) + output = shell_utils.run(adb_cmd, shell=True, echo=echo) + output_lines = output.splitlines() + try: + real_exitcode = int(output_lines[-1].rstrip()) + except ValueError: + real_exitcode = -1 + if real_exitcode != 0: + raise Exception('Command failed with code %s' % real_exitcode) + return '\n'.join(output_lines[:-1]) + + +def ADBKill(serial, process, kill_app=False): + """ Kill a process running on an Android device. + + serial: string indicating the serial number of the target device + process: string indicating the name of the process to kill + kill_app: bool indicating whether the process is an Android app, as opposed + to a normal executable process. + """ + if kill_app: + ADBShell(serial, ['am', 'kill', process]) + else: + try: + stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB, + serial, + process), + shell=True) + except Exception: + return + for line in stdout.split('\n'): + if line != '': + split = shlex.split(line) + if len(split) < 2: + continue + pid = split[1] + ADBShell(serial, ['kill', pid]) + # Raise an exception if any Skia processes are still running. + try: + stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB, + serial, + process), + shell=True) + except Exception: + return + if stdout: + raise Exception('There are still some skia processes running:\n%s\n' + 'Maybe the device should be rebooted?' % stdout) + + +def GetSerial(device_type): + """ Determine the serial number of the *first* connected device with the + specified type. The ordering of 'adb devices' is not documented, and the + connected devices do not appear to be ordered by serial number. Therefore, + we have to assume that, in the case of multiple devices of the same type being + connected to one host, we cannot predict which device will be chosen. + + device_type: string indicating the 'common name' for the target device + """ + if not device_type in DEVICE_LOOKUP: + raise ValueError('Unknown device: %s!' % device_type) + device_name = DEVICE_LOOKUP[device_type] + output = shell_utils.run_retry('%s devices' % PATH_TO_ADB, shell=True, + attempts=5) + print output + lines = output.split('\n') + device_ids = [] + for line in lines: + # Filter garbage lines + if line != '' and not ('List of devices attached' in line) and \ + line[0] != '*': + device_ids.append(line.split('\t')[0]) + for device_id in device_ids: + print 'Finding type for id %s' % device_id + # Get device name + name_line = shell_utils.run_retry( + '%s -s %s shell cat /system/build.prop | grep "ro.product.device="' % ( + PATH_TO_ADB, device_id), shell=True, attempts=5) + print name_line + name = name_line.split('=')[-1].rstrip() + # Just return the first attached device of the requested model. + if device_name in name: + return device_id + raise Exception('No %s device attached!' % device_name) + + +def SetCPUScalingMode(serial, mode): + """ Set the CPU scaling governor for the device with the given serial number + to the given mode. + + serial: string indicating the serial number of the device whose scaling mode + is to be modified + mode: string indicating the desired CPU scaling mode. Acceptable values + are listed in CPU_SCALING_MODES. + """ + if mode not in CPU_SCALING_MODES: + raise ValueError('mode must be one of: %s' % CPU_SCALING_MODES) + cpu_dirs = shell_utils.run('%s -s %s shell ls /sys/devices/system/cpu' % ( + PATH_TO_ADB, serial), echo=False, shell=True) + cpu_dirs_list = cpu_dirs.split('\n') + regex = re.compile('cpu\d') + for cpu_dir_from_list in cpu_dirs_list: + cpu_dir = cpu_dir_from_list.rstrip() + if regex.match(cpu_dir): + path = '/sys/devices/system/cpu/%s/cpufreq/scaling_governor' % cpu_dir + path_found = shell_utils.run('%s -s %s shell ls %s' % ( + PATH_TO_ADB, serial, path), + echo=False, shell=True).rstrip() + if path_found == path: + # Unfortunately, we can't directly change the scaling_governor file over + # ADB. Instead, we write a script to do so, push it to the device, and + # run it. + old_mode = shell_utils.run('%s -s %s shell cat %s' % ( + PATH_TO_ADB, serial, path), + echo=False, shell=True).rstrip() + print 'Current scaling mode for %s is: %s' % (cpu_dir, old_mode) + filename = 'skia_cpuscale.sh' + with open(filename, 'w') as script_file: + script_file.write('echo %s > %s\n' % (mode, path)) + os.chmod(filename, 0777) + RunADB(serial, ['push', filename, '/system/bin'], echo=False) + RunADB(serial, ['shell', filename], echo=True) + RunADB(serial, ['shell', 'rm', '/system/bin/%s' % filename], echo=False) + os.remove(filename) + new_mode = shell_utils.run('%s -s %s shell cat %s' % ( + PATH_TO_ADB, serial, path), + echo=False, shell=True).rstrip() + print 'New scaling mode for %s is: %s' % (cpu_dir, new_mode) + + +def IsAndroidShellRunning(serial): + """ Find the status of the Android shell for the device with the given serial + number. Returns True if the shell is running and False otherwise. + + serial: string indicating the serial number of the target device. + """ + if 'Error:' in ADBShell(serial, ['pm', 'path', 'android'], echo=False): + return False + return True + + +def StopShell(serial, timeout=60): + """ Halt the Android runtime on the device with the given serial number. + Blocks until the shell reports that it has stopped. + + serial: string indicating the serial number of the target device. + timeout: maximum allotted time, in seconds. + """ + ADBShell(serial, ['stop']) + start_time = time.time() + while IsAndroidShellRunning(serial): + time.sleep(1) + if time.time() - start_time > timeout: + raise Exception('Timeout while attempting to stop the Android runtime.') + + +def StartShell(serial, timeout=60): + """ Start the Android runtime on the device with the given serial number. + Blocks until the shell reports that it has started. + + serial: string indicating the serial number of the target device. + timeout: maximum allotted time, in seconds. + """ + ADBShell(serial, ['start']) + start_time = time.time() + while not IsAndroidShellRunning(serial): + time.sleep(1) + if time.time() - start_time > timeout: + raise Exception('Timeout while attempting to start the Android runtime.') + + +def RunSkia(serial, cmd, release, device): + """ Run the given command through skia_launcher on a given device. + + serial: string indicating the serial number of the target device. + cmd: list of strings; the command line to run. + release: bool; whether or not to run the app in Release mode. + device: string indicating the target device. + """ + RunADB(serial, ['logcat', '-c']) + try: + os.environ['SKIA_ANDROID_VERBOSE_SETUP'] = '1' + cmd_to_run = [os.path.join('platform_tools', 'android', 'bin', + 'android_run_skia')] + if release: + cmd_to_run.extend(['--release']) + cmd_to_run.extend(['-d', device]) + cmd_to_run.extend(['-s', serial]) + cmd_to_run.extend(cmd) + shell_utils.run(cmd_to_run) + finally: + RunADB(serial, ['logcat', '-d', '-v', 'time'])
diff --git a/py/utils/git_utils.py b/py/utils/git_utils.py new file mode 100644 index 0000000..da7f097 --- /dev/null +++ b/py/utils/git_utils.py
@@ -0,0 +1,46 @@ +#!/usr/bin/env python +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module contains functions for using git.""" + + +import os +import shell_utils + + +GIT = 'git.bat' if os.name == 'nt' else 'git' + + +def Add(addition): + """Run 'git add <addition>'""" + shell_utils.run([GIT, 'add', addition]) + +def AIsAncestorOfB(a, b): + """Return true if a is an ancestor of b.""" + return shell_utils.run([GIT, 'merge-base', a, b]).rstrip() == FullHash(a) + +def FullHash(commit): + """Return full hash of specified commit.""" + return shell_utils.run([GIT, 'rev-parse', '--verify', commit]).rstrip() + +def IsMerge(commit): + """Return True if the commit is a merge, False otherwise.""" + rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1', + '--no-merges']) + last_non_merge = rev_parse.split('\n')[0] + # Get full hash since that is what was returned by rev-parse. + return FullHash(commit) != last_non_merge + +def MergeAbort(): + """Abort in process merge.""" + shell_utils.run([GIT, 'merge', '--abort']) + +def ShortHash(commit): + """Return short hash of the specified commit.""" + return shell_utils.run([GIT, 'show', commit, '--format=%h', '-s']).rstrip() + +def GetRemoteMasterHash(git_url): + return shell_utils.run([GIT, 'ls-remote', git_url, '--verify', + 'refs/heads/master'])
diff --git a/py/utils/misc.py b/py/utils/misc.py new file mode 100644 index 0000000..a61527e --- /dev/null +++ b/py/utils/misc.py
@@ -0,0 +1,138 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" This module contains miscellaneous tools used by the buildbot scripts. """ + +import os + +from git_utils import GIT +import shell_utils + + +# Absolute path to the root of this Skia buildbot checkout. +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + os.pardir, os.pardir, os.pardir)) + + +def ArgsToDict(argv): + """ Collect command-line arguments of the form '--key value' into a + dictionary. Fail if the arguments do not fit this format. """ + dictionary = {} + PREFIX = '--' + # Expect the first arg to be the path to the script, which we don't want. + argv = argv[1:] + while argv: + if argv[0].startswith(PREFIX): + dictionary[argv[0][len(PREFIX):]] = argv[1] + argv = argv[2:] + else: + raise Exception('Malformed input: %s' % argv) + return dictionary + + +def ConfirmOptionsSet(name_value_dict): + """Raise an exception if any of the given command-line options were not set. + + name_value_dict: dictionary mapping option names to option values + """ + for (name, value) in name_value_dict.iteritems(): + if value is None: + raise Exception('missing command-line option %s; rerun with --help' % + name) + + +def GetAbsPath(relative_path): + """My own implementation of os.path.abspath() that better handles paths + which approach Window's 260-character limit. + See https://code.google.com/p/skia/issues/detail?id=674 + + This implementation adds path components one at a time, resolving the + absolute path each time, to take advantage of any chdirs into outer + directories that will shorten the total path length. + + TODO(epoger): share a single implementation with bench_graph_svg.py, instead + of pasting this same code into both files.""" + if os.path.isabs(relative_path): + return relative_path + path_parts = relative_path.split(os.sep) + abs_path = os.path.abspath('.') + for path_part in path_parts: + abs_path = os.path.abspath(os.path.join(abs_path, path_part)) + return abs_path + + +class ChDir(object): + """Enter and exit the given directory appropriately.""" + + def __init__(self, directory): + """Instantiate the ChDir. + + Args: + directory: string; the directory to enter. + """ + self._destination = directory + self._origin = None + + def __enter__(self): + """Change to the destination directory. + + Does not check whether the directory exists. + """ + self._origin = os.getcwd() + print 'chdir %s' % self._destination + os.chdir(self._destination) + + def __exit__(self, *args): + """Change back to the original directory.""" + print 'chdir %s' % self._origin + os.chdir(self._origin) + + +class GitBranch(object): + """Class to manage git branches. + + This class allows one to create a new branch in a repository to make changes, + then it commits the changes, switches to master branch, and deletes the + created temporary branch upon exit. + """ + def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False): + self._branch_name = branch_name + self._commit_msg = commit_msg + self._upload = upload + self._commit_queue = commit_queue + self._patch_set = 0 + + def __enter__(self): + shell_utils.run([GIT, 'reset', '--hard', 'HEAD']) + shell_utils.run([GIT, 'checkout', 'master']) + if self._branch_name in shell_utils.run([GIT, 'branch']): + shell_utils.run([GIT, 'branch', '-D', self._branch_name]) + shell_utils.run([GIT, 'checkout', '-b', self._branch_name, + '-t', 'origin/master']) + return self + + def commit_and_upload(self, use_commit_queue=False): + shell_utils.run([GIT, 'commit', '-a', '-m', + self._commit_msg]) + upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks', + '--bypass-watchlists'] + self._patch_set += 1 + if self._patch_set > 1: + upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set]) + if use_commit_queue: + upload_cmd.append('--use-commit-queue') + shell_utils.run(upload_cmd) + + def __exit__(self, exc_type, _value, _traceback): + if self._upload: + # Only upload if no error occurred. + try: + if exc_type is None: + self.commit_and_upload(use_commit_queue=self._commit_queue) + finally: + shell_utils.run([GIT, 'checkout', 'master']) + shell_utils.run([GIT, 'branch', '-D', self._branch_name]) +
diff --git a/py/utils/shell_utils.py b/py/utils/shell_utils.py new file mode 100644 index 0000000..a6b54ef --- /dev/null +++ b/py/utils/shell_utils.py
@@ -0,0 +1,236 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" This module contains tools for running commands in a shell. """ + +import datetime +import os +import Queue +import select +import subprocess +import sys +import threading +import time + +if 'nt' in os.name: + import ctypes + + +DEFAULT_SECS_BETWEEN_ATTEMPTS = 10 +POLL_MILLIS = 250 + + +class CommandFailedException(Exception): + """Exception which gets raised when a command fails.""" + + def __init__(self, output, *args): + """Initialize the CommandFailedException. + + Args: + output: string; output from the command. + """ + Exception.__init__(self, *args) + self._output = output + + @property + def output(self): + """Output from the command.""" + return self._output + + +class TimeoutException(CommandFailedException): + """CommandFailedException which gets raised when a subprocess exceeds its + timeout. """ + pass + + +def run_async(cmd, echo=True, shell=False): + """ Run 'cmd' in a subprocess, returning a Popen class instance referring to + that process. (Non-blocking) """ + if echo: + print cmd + if 'nt' in os.name: + # Windows has a bad habit of opening a dialog when a console program + # crashes, rather than just letting it crash. Therefore, when a program + # crashes on Windows, we don't find out until the build step times out. + # This code prevents the dialog from appearing, so that we find out + # immediately and don't waste time waiting around. + SEM_NOGPFAULTERRORBOX = 0x0002 + ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX) + flags = 0x8000000 # CREATE_NO_WINDOW + else: + flags = 0 + return subprocess.Popen(cmd, shell=shell, stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, creationflags=flags, + bufsize=1) + + +class EnqueueThread(threading.Thread): + """ Reads and enqueues lines from a file. """ + def __init__(self, file_obj, queue): + threading.Thread.__init__(self) + self._file = file_obj + self._queue = queue + self._stopped = False + + def run(self): + if sys.platform.startswith('linux'): + # Use a polling object to avoid the blocking call to readline(). + poll = select.poll() + poll.register(self._file, select.POLLIN) + while not self._stopped: + has_output = poll.poll(POLL_MILLIS) + if has_output: + line = self._file.readline() + if line == '': + self._stopped = True + self._queue.put(line) + else: + # Only Unix supports polling objects, so just read from the file, + # Python-style. + for line in iter(self._file.readline, ''): + self._queue.put(line) + if self._stopped: + break + + def stop(self): + self._stopped = True + + +def log_process_in_real_time(proc, echo=True, timeout=None, log_file=None, + halt_on_output=None, print_timestamps=True): + """ Log the output of proc in real time until it completes. Return a tuple + containing the exit code of proc and the contents of stdout. + + proc: an instance of Popen referring to a running subprocess. + echo: boolean indicating whether to print the output received from proc.stdout + timeout: number of seconds allotted for the process to run. Raises a + TimeoutException if the run time exceeds the timeout. + log_file: an open file for writing output + halt_on_output: string; kill the process and return if this string is found + in the output stream from the process. + print_timestamps: boolean indicating whether a formatted timestamp should be + prepended to each line of output. + """ + stdout_queue = Queue.Queue() + log_thread = EnqueueThread(proc.stdout, stdout_queue) + log_thread.start() + try: + all_output = [] + t_0 = time.time() + while True: + code = proc.poll() + try: + output = stdout_queue.get_nowait() + all_output.append(output) + if output and print_timestamps: + timestamp = datetime.datetime.now().strftime('%H:%M:%S.%f') + output = ''.join(['[%s] %s\n' % (timestamp, line) + for line in output.splitlines()]) + if echo: + sys.stdout.write(output) + sys.stdout.flush() + if log_file: + log_file.write(output) + log_file.flush() + if halt_on_output and halt_on_output in output: + proc.terminate() + break + except Queue.Empty: + if code != None: # proc has finished running + break + time.sleep(0.5) + if timeout and time.time() - t_0 > timeout: + proc.terminate() + raise TimeoutException( + ''.join(all_output), + 'Subprocess exceeded timeout of %ds' % timeout) + finally: + log_thread.stop() + log_thread.join() + return (code, ''.join(all_output)) + + +def log_process_after_completion(proc, echo=True, timeout=None, log_file=None): + """ Wait for proc to complete and return a tuple containing the exit code of + proc and the contents of stdout. Unlike log_process_in_real_time, does not + attempt to read stdout from proc in real time. + + proc: an instance of Popen referring to a running subprocess. + echo: boolean indicating whether to print the output received from proc.stdout + timeout: number of seconds allotted for the process to run. Raises a + TimeoutException if the run time exceeds the timeout. + log_file: an open file for writing outout + """ + t_0 = time.time() + code = None + while code is None: + if timeout and time.time() - t_0 > timeout: + raise TimeoutException( + proc.communicate()[0], + 'Subprocess exceeded timeout of %ds' % timeout) + time.sleep(0.5) + code = proc.poll() + output = proc.communicate()[0] + if echo: + print output + if log_file: + log_file.write(output) + log_file.flush() + return (code, output) + + +def run(cmd, echo=True, shell=False, timeout=None, print_timestamps=True, + log_in_real_time=True): + """ Run 'cmd' in a shell and return the combined contents of stdout and + stderr (Blocking). Throws an exception if the command exits non-zero. + + cmd: list of strings (or single string, iff shell==True) indicating the + command to run + echo: boolean indicating whether we should print the command and log output + shell: boolean indicating whether we are using advanced shell features. Use + only when absolutely necessary, since this allows a lot more freedom which + could be exploited by malicious code. See the warning here: + http://docs.python.org/library/subprocess.html#popen-constructor + timeout: optional, integer indicating the maximum elapsed time in seconds + print_timestamps: boolean indicating whether a formatted timestamp should be + prepended to each line of output. Unused if echo or log_in_real_time is + False. + log_in_real_time: boolean indicating whether to read stdout from the + subprocess in real time instead of when the process finishes. If echo is + False, we never log in real time, even if log_in_real_time is True. + """ + proc = run_async(cmd, echo=echo, shell=shell) + # If we're not printing the output, we don't care if the output shows up in + # real time, so don't bother. + if log_in_real_time and echo: + (returncode, output) = log_process_in_real_time(proc, echo=echo, + timeout=timeout, print_timestamps=print_timestamps) + else: + (returncode, output) = log_process_after_completion(proc, echo=echo, + timeout=timeout) + if returncode != 0: + raise CommandFailedException( + output, + 'Command failed with code %d: %s' % (returncode, cmd)) + return output + + +def run_retry(cmd, echo=True, shell=False, attempts=1, + secs_between_attempts=DEFAULT_SECS_BETWEEN_ATTEMPTS, + timeout=None, print_timestamps=True): + """ Wrapper for run() which makes multiple attempts until either the command + succeeds or the maximum number of attempts is reached. """ + attempt = 1 + while True: + try: + return run(cmd, echo=echo, shell=shell, timeout=timeout, + print_timestamps=print_timestamps) + except CommandFailedException: + if attempt >= attempts: + raise + print 'Command failed. Retrying in %d seconds...' % secs_between_attempts + time.sleep(secs_between_attempts) + attempt += 1
diff --git a/py/utils/ssh_utils.py b/py/utils/ssh_utils.py new file mode 100644 index 0000000..868fd62 --- /dev/null +++ b/py/utils/ssh_utils.py
@@ -0,0 +1,196 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" This module contains tools related to ssh used by the buildbot scripts. """ + +import atexit +import os +import re +import shell_utils +import signal + +def PutSCP(local_path, remote_path, username, host, port, recurse=False, + options=None): + """ Send a file to the given host over SCP. Assumes that public key + authentication is set up between the client and server. + + local_path: path to the file to send on the client + remote_path: destination path for the file on the server + username: ssh login name + host: hostname or ip address of the server + port: port on the server to use + recurse: boolean indicating whether to transmit everything in a folder + options: list of extra options to pass to scp + """ + # TODO(borenet): This will hang for a while if the host does not recognize + # the client. + cmd = ['scp'] + if options: + cmd.extend(options) + if recurse: + cmd.append('-r') + cmd.extend( + ['-P', port, local_path, '%s@%s:%s' % (username, host, remote_path)]) + shell_utils.run(cmd) + + +def MultiPutSCP(local_paths, remote_path, username, host, port, options=None): + """ Send files to the given host over SCP. Assumes that public key + authentication is set up between the client and server. + + local_paths: list of paths of files and directories to send on the client + remote_path: destination directory path on the server + username: ssh login name + host: hostname or ip address of the server + port: port on the server to use + options: list of extra options to pass to scp + """ + # TODO(borenet): This will hang for a while if the host does not recognize + # the client. + cmd = ['scp'] + if options: + cmd.extend(options) + cmd.extend(['-r', '-P', port]) + cmd.extend(local_paths) + cmd.append('%s@%s:%s' % (username, host, remote_path)) + shell_utils.run(cmd) + + +def GetSCP(local_path, remote_path, username, host, port, recurse=False, + options=None): + """ Retrieve a file from the given host over SCP. Assumes that public key + authentication is set up between the client and server. + + local_path: destination path for the file on the client + remote_path: path to the file to retrieve on the server + username: ssh login name + host: hostname or ip address of the server + port: port on the server to use + recurse: boolean indicating whether to transmit everything in a folder + options: list of extra options to pass to scp + """ + # TODO(borenet): This will hang for a while if the host does not recognize + # the client. + cmd = ['scp'] + if options: + cmd.extend(options) + if recurse: + cmd.append('-r') + cmd.extend( + ['-P', port, '%s@%s:%s' % (username, host, remote_path), local_path]) + shell_utils.run(cmd) + + +def RunSSHCmd(username, host, port, command, echo=True, options=None): + """ Login to the given host and run the given command. + + username: ssh login name + host: hostname or ip address of the server + port: port on the server to use + command: (string) command to run on the server + options: list of extra options to pass to ssh + """ + # TODO(borenet): This will hang for a while if the host does not recognize + # the client. + cmd = ['ssh'] + if options: + cmd.extend(options) + cmd.extend(['-p', port, '%s@%s' % (username, host), command]) + return shell_utils.run(cmd, echo=echo) + + +def ShellEscape(arg): + """ Escape a single argument for passing into a remote shell + """ + arg = re.sub(r'(["\\])', r'\\\1', arg) + return '"%s"' % arg if re.search(r'[\' \t\r\n]', arg) else arg + + +def RunSSH(username, host, port, command, echo=True, options=None): + """ Login to the given host and run the given command. + + username: ssh login name + host: hostname or ip address of the server + port: port on the server to use + command: command to run on the server in list format + options: list of extra options to pass to ssh + """ + cmd = ' '.join(ShellEscape(arg) for arg in command) + return RunSSHCmd(username, host, port, cmd, echo=echo, options=options) + + +class SshDestination(object): + """ Convenience class to remember a host, port, and username. + Wraps the other functions in this module. + """ + def __init__(self, host, port, username, options=None): + """ + host - (string) hostname of the target + port - (string or int) sshd port on the target + username - (string) remote username + options - (list of strings) extra options to pass to ssh and scp. + """ + self.host = host + self.port = str(port) + self.user = username + self.options = options + + def Put(self, local_path, remote_path, recurse=False): + return PutSCP(local_path, remote_path, self.user, self.host, + self.port, recurse=recurse, options=self.options) + + def MultiPut(self, local_paths, remote_path): + return MultiPutSCP(local_paths, remote_path, self.user, self.host, + self.port, options=self.options) + + def Get(self, local_path, remote_path, recurse=False): + return GetSCP(local_path, remote_path, self.user, + self.host, self.port, recurse=recurse, options=self.options) + + def RunCmd(self, command, echo=True): + return RunSSHCmd(self.user, self.host, self.port, command, + echo=echo, options=self.options) + + def Run(self, command, echo=True): + return RunSSH(self.user, self.host, self.port, command, + echo=echo, options=self.options) + + +def search_within_string(input_string, pattern): + """Search for regular expression in a string. + + input_string: (string) to be searched + pattern: (string) to be passed to re.compile, with a symbolic + group named 'return'. + default: what to return if no match + + Returns a string or None + """ + match = re.search(pattern, input_string) + return match.group('return') if match else None + +def SSHAdd(key_file): + """ Call ssh-add, and call ssh-agent if necessary. + """ + assert os.path.isfile(key_file) + try: + shell_utils.run(['ssh-add', key_file], + log_in_real_time=False) + return + except shell_utils.CommandFailedException: + ssh_agent_output = shell_utils.run(['ssh-agent', '-s'], + log_in_real_time=False) + if not ssh_agent_output: + raise Exception('ssh-agent did not print anything') + ssh_auth_sock = search_within_string( + ssh_agent_output, r'SSH_AUTH_SOCK=(?P<return>[^;]*);') + ssh_agent_pid = search_within_string( + ssh_agent_output, r'SSH_AGENT_PID=(?P<return>[^;]*);') + if not (ssh_auth_sock and ssh_agent_pid): + raise Exception('ssh-agent did not print meaningful data') + os.environ['SSH_AUTH_SOCK'] = ssh_auth_sock + os.environ['SSH_AGENT_PID'] = ssh_agent_pid + atexit.register(os.kill, int(ssh_agent_pid), signal.SIGTERM) + shell_utils.run(['ssh-add', key_file])