Update common.py.utils to be used in Skia repo BUG=skia:2682 R=rmistry@google.com Review URL: https://codereview.chromium.org/346743008
diff --git a/py/utils/find_depot_tools.py b/py/utils/find_depot_tools.py new file mode 100644 index 0000000..3ea5dd7 --- /dev/null +++ b/py/utils/find_depot_tools.py
@@ -0,0 +1,51 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Small utility function to find depot_tools and add it to the python path. + +Will throw an ImportError exception if depot_tools can't be found since it +imports breakpad. + +This file is directly copied from: +https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/common/find_depot_tools.py +""" + +import os +import sys + + +def directory_really_is_depot_tools(directory): + return os.path.isfile(os.path.join(directory, 'breakpad.py')) + + +def add_depot_tools_to_path(): + """Search for depot_tools and add it to sys.path.""" + # First look if depot_tools is already in PYTHONPATH. + for i in sys.path: + if i.rstrip(os.sep).endswith('depot_tools'): + if directory_really_is_depot_tools(i): + return i + + # Then look if depot_tools is in PATH, common case. + for i in os.environ['PATH'].split(os.pathsep): + if i.rstrip(os.sep).endswith('depot_tools'): + if directory_really_is_depot_tools(i): + sys.path.insert(0, i.rstrip(os.sep)) + return i + # Rare case, it's not even in PATH, look upward up to root. + root_dir = os.path.dirname(os.path.abspath(__file__)) + previous_dir = os.path.abspath(__file__) + while root_dir and root_dir != previous_dir: + if directory_really_is_depot_tools(os.path.join(root_dir, 'depot_tools')): + i = os.path.join(root_dir, 'depot_tools') + sys.path.insert(0, i) + return i + previous_dir = root_dir + root_dir = os.path.dirname(root_dir) + print >> sys.stderr, 'Failed to find depot_tools' + return None + +add_depot_tools_to_path() + +# pylint: disable=W0611 +import breakpad
diff --git a/py/utils/git_utils.py b/py/utils/git_utils.py index da7f097..5ac63ec 100644 --- a/py/utils/git_utils.py +++ b/py/utils/git_utils.py
@@ -6,25 +6,54 @@ """This module contains functions for using git.""" -import os +import re import shell_utils -GIT = 'git.bat' if os.name == 'nt' else 'git' +def _FindGit(): + """Find the git executable. + + Returns: + A string suitable for passing to subprocess functions, or None. + """ + def test_git_executable(git): + """Test the git executable. + + Args: + git: git executable path. + Returns: + True if test is successful. + """ + try: + shell_utils.run([git, '--version'], echo=False) + return True + except (OSError,): + return False + + for git in ('git', 'git.exe', 'git.bat'): + if test_git_executable(git): + return git + return None + + +GIT = _FindGit() def Add(addition): """Run 'git add <addition>'""" shell_utils.run([GIT, 'add', addition]) + def AIsAncestorOfB(a, b): """Return true if a is an ancestor of b.""" return shell_utils.run([GIT, 'merge-base', a, b]).rstrip() == FullHash(a) + def FullHash(commit): """Return full hash of specified commit.""" return shell_utils.run([GIT, 'rev-parse', '--verify', commit]).rstrip() + def IsMerge(commit): """Return True if the commit is a merge, False otherwise.""" rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1', @@ -33,14 +62,85 @@ # Get full hash since that is what was returned by rev-parse. return FullHash(commit) != last_non_merge + def MergeAbort(): """Abort in process merge.""" shell_utils.run([GIT, 'merge', '--abort']) + def ShortHash(commit): """Return short hash of the specified commit.""" return shell_utils.run([GIT, 'show', commit, '--format=%h', '-s']).rstrip() + +def Fetch(remote=None): + """Run "git fetch". """ + cmd = [GIT, 'fetch'] + if remote: + cmd.append(remote) + shell_utils.run(cmd) + + def GetRemoteMasterHash(git_url): return shell_utils.run([GIT, 'ls-remote', git_url, '--verify', - 'refs/heads/master']) + 'refs/heads/master']).rstrip() + + +def GetCurrentBranch(): + return shell_utils.run([GIT, 'rev-parse', '--abbrev-ref', 'HEAD']).rstrip() + + +class GitBranch(object): + """Class to manage git branches. + + This class allows one to create a new branch in a repository to make changes, + then it commits the changes, switches to master branch, and deletes the + created temporary branch upon exit. + """ + def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False, + delete_when_finished=True): + self._branch_name = branch_name + self._commit_msg = commit_msg + self._upload = upload + self._commit_queue = commit_queue + self._patch_set = 0 + self._delete_when_finished = delete_when_finished + + def __enter__(self): + shell_utils.run([GIT, 'reset', '--hard', 'HEAD']) + shell_utils.run([GIT, 'checkout', 'master']) + if self._branch_name in shell_utils.run([GIT, 'branch']): + shell_utils.run([GIT, 'branch', '-D', self._branch_name]) + shell_utils.run([GIT, 'checkout', '-b', self._branch_name, + '-t', 'origin/master']) + return self + + def commit_and_upload(self, use_commit_queue=False): + """Commit all changes and upload a CL, returning the issue URL.""" + try: + shell_utils.run([GIT, 'commit', '-a', '-m', self._commit_msg]) + except shell_utils.CommandFailedException as e: + if not 'nothing to commit' in e.output: + raise + upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks', + '--bypass-watchlists'] + self._patch_set += 1 + if self._patch_set > 1: + upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set]) + if use_commit_queue: + upload_cmd.append('--use-commit-queue') + shell_utils.run(upload_cmd) + output = shell_utils.run([GIT, 'cl', 'issue']).rstrip() + return re.match('^Issue number: (?P<issue>\d+) \((?P<issue_url>.+)\)$', + output).group('issue_url') + + def __exit__(self, exc_type, _value, _traceback): + if self._upload: + # Only upload if no error occurred. + try: + if exc_type is None: + self.commit_and_upload(use_commit_queue=self._commit_queue) + finally: + shell_utils.run([GIT, 'checkout', 'master']) + if self._delete_when_finished: + shell_utils.run([GIT, 'branch', '-D', self._branch_name])
diff --git a/py/utils/misc.py b/py/utils/misc.py index a1bec6a..c3e4dc8 100644 --- a/py/utils/misc.py +++ b/py/utils/misc.py
@@ -3,13 +3,10 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" This module contains miscellaneous tools used by the buildbot scripts. """ +""" This module contains miscellaneous tools. """ import os -from git_utils import GIT -import shell_utils - # Absolute path to the root of this Skia buildbot checkout. BUILDBOT_PATH = os.path.realpath(os.path.join( @@ -93,50 +90,3 @@ if self._verbose: print 'chdir %s' % self._origin os.chdir(self._origin) - - -class GitBranch(object): - """Class to manage git branches. - - This class allows one to create a new branch in a repository to make changes, - then it commits the changes, switches to master branch, and deletes the - created temporary branch upon exit. - """ - def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False): - self._branch_name = branch_name - self._commit_msg = commit_msg - self._upload = upload - self._commit_queue = commit_queue - self._patch_set = 0 - - def __enter__(self): - shell_utils.run([GIT, 'reset', '--hard', 'HEAD']) - shell_utils.run([GIT, 'checkout', 'master']) - if self._branch_name in shell_utils.run([GIT, 'branch']): - shell_utils.run([GIT, 'branch', '-D', self._branch_name]) - shell_utils.run([GIT, 'checkout', '-b', self._branch_name, - '-t', 'origin/master']) - return self - - def commit_and_upload(self, use_commit_queue=False): - shell_utils.run([GIT, 'commit', '-a', '-m', - self._commit_msg]) - upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks', - '--bypass-watchlists'] - self._patch_set += 1 - if self._patch_set > 1: - upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set]) - if use_commit_queue: - upload_cmd.append('--use-commit-queue') - shell_utils.run(upload_cmd) - - def __exit__(self, exc_type, _value, _traceback): - if self._upload: - # Only upload if no error occurred. - try: - if exc_type is None: - self.commit_and_upload(use_commit_queue=self._commit_queue) - finally: - shell_utils.run([GIT, 'checkout', 'master']) - shell_utils.run([GIT, 'branch', '-D', self._branch_name]) -
diff --git a/py/utils/shell_utils.py b/py/utils/shell_utils.py index a6b54ef..37d124b 100644 --- a/py/utils/shell_utils.py +++ b/py/utils/shell_utils.py
@@ -20,6 +20,7 @@ DEFAULT_SECS_BETWEEN_ATTEMPTS = 10 POLL_MILLIS = 250 +VERBOSE = True class CommandFailedException(Exception): @@ -46,9 +47,11 @@ pass -def run_async(cmd, echo=True, shell=False): +def run_async(cmd, echo=None, shell=False): """ Run 'cmd' in a subprocess, returning a Popen class instance referring to that process. (Non-blocking) """ + if echo is None: + echo = VERBOSE if echo: print cmd if 'nt' in os.name: @@ -99,7 +102,7 @@ self._stopped = True -def log_process_in_real_time(proc, echo=True, timeout=None, log_file=None, +def log_process_in_real_time(proc, echo=None, timeout=None, log_file=None, halt_on_output=None, print_timestamps=True): """ Log the output of proc in real time until it completes. Return a tuple containing the exit code of proc and the contents of stdout. @@ -114,6 +117,8 @@ print_timestamps: boolean indicating whether a formatted timestamp should be prepended to each line of output. """ + if echo is None: + echo = VERBOSE stdout_queue = Queue.Queue() log_thread = EnqueueThread(proc.stdout, stdout_queue) log_thread.start() @@ -153,7 +158,8 @@ return (code, ''.join(all_output)) -def log_process_after_completion(proc, echo=True, timeout=None, log_file=None): +def log_process_after_completion(proc, echo=None, timeout=None, + log_file=None): """ Wait for proc to complete and return a tuple containing the exit code of proc and the contents of stdout. Unlike log_process_in_real_time, does not attempt to read stdout from proc in real time. @@ -164,6 +170,8 @@ TimeoutException if the run time exceeds the timeout. log_file: an open file for writing outout """ + if echo is None: + echo = VERBOSE t_0 = time.time() code = None while code is None: @@ -182,7 +190,7 @@ return (code, output) -def run(cmd, echo=True, shell=False, timeout=None, print_timestamps=True, +def run(cmd, echo=None, shell=False, timeout=None, print_timestamps=True, log_in_real_time=True): """ Run 'cmd' in a shell and return the combined contents of stdout and stderr (Blocking). Throws an exception if the command exits non-zero. @@ -202,6 +210,8 @@ subprocess in real time instead of when the process finishes. If echo is False, we never log in real time, even if log_in_real_time is True. """ + if echo is None: + echo = VERBOSE proc = run_async(cmd, echo=echo, shell=shell) # If we're not printing the output, we don't care if the output shows up in # real time, so don't bother. @@ -218,11 +228,13 @@ return output -def run_retry(cmd, echo=True, shell=False, attempts=1, +def run_retry(cmd, echo=None, shell=False, attempts=1, secs_between_attempts=DEFAULT_SECS_BETWEEN_ATTEMPTS, timeout=None, print_timestamps=True): """ Wrapper for run() which makes multiple attempts until either the command succeeds or the maximum number of attempts is reached. """ + if echo is None: + echo = VERBOSE attempt = 1 while True: try: