Update common.py.utils to be used in Skia repo
BUG=skia:2682
R=rmistry@google.com
Review URL: https://codereview.chromium.org/346743008
diff --git a/py/utils/find_depot_tools.py b/py/utils/find_depot_tools.py
new file mode 100644
index 0000000..3ea5dd7
--- /dev/null
+++ b/py/utils/find_depot_tools.py
@@ -0,0 +1,51 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Small utility function to find depot_tools and add it to the python path.
+
+Will throw an ImportError exception if depot_tools can't be found since it
+imports breakpad.
+
+This file is directly copied from:
+https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/common/find_depot_tools.py
+"""
+
+import os
+import sys
+
+
+def directory_really_is_depot_tools(directory):
+ return os.path.isfile(os.path.join(directory, 'breakpad.py'))
+
+
+def add_depot_tools_to_path():
+ """Search for depot_tools and add it to sys.path."""
+ # First look if depot_tools is already in PYTHONPATH.
+ for i in sys.path:
+ if i.rstrip(os.sep).endswith('depot_tools'):
+ if directory_really_is_depot_tools(i):
+ return i
+
+ # Then look if depot_tools is in PATH, common case.
+ for i in os.environ['PATH'].split(os.pathsep):
+ if i.rstrip(os.sep).endswith('depot_tools'):
+ if directory_really_is_depot_tools(i):
+ sys.path.insert(0, i.rstrip(os.sep))
+ return i
+ # Rare case, it's not even in PATH, look upward up to root.
+ root_dir = os.path.dirname(os.path.abspath(__file__))
+ previous_dir = os.path.abspath(__file__)
+ while root_dir and root_dir != previous_dir:
+ if directory_really_is_depot_tools(os.path.join(root_dir, 'depot_tools')):
+ i = os.path.join(root_dir, 'depot_tools')
+ sys.path.insert(0, i)
+ return i
+ previous_dir = root_dir
+ root_dir = os.path.dirname(root_dir)
+ print >> sys.stderr, 'Failed to find depot_tools'
+ return None
+
+add_depot_tools_to_path()
+
+# pylint: disable=W0611
+import breakpad
diff --git a/py/utils/git_utils.py b/py/utils/git_utils.py
index da7f097..5ac63ec 100644
--- a/py/utils/git_utils.py
+++ b/py/utils/git_utils.py
@@ -6,25 +6,54 @@
"""This module contains functions for using git."""
-import os
+import re
import shell_utils
-GIT = 'git.bat' if os.name == 'nt' else 'git'
+def _FindGit():
+ """Find the git executable.
+
+ Returns:
+ A string suitable for passing to subprocess functions, or None.
+ """
+ def test_git_executable(git):
+ """Test the git executable.
+
+ Args:
+ git: git executable path.
+ Returns:
+ True if test is successful.
+ """
+ try:
+ shell_utils.run([git, '--version'], echo=False)
+ return True
+ except (OSError,):
+ return False
+
+ for git in ('git', 'git.exe', 'git.bat'):
+ if test_git_executable(git):
+ return git
+ return None
+
+
+GIT = _FindGit()
def Add(addition):
"""Run 'git add <addition>'"""
shell_utils.run([GIT, 'add', addition])
+
def AIsAncestorOfB(a, b):
"""Return true if a is an ancestor of b."""
return shell_utils.run([GIT, 'merge-base', a, b]).rstrip() == FullHash(a)
+
def FullHash(commit):
"""Return full hash of specified commit."""
return shell_utils.run([GIT, 'rev-parse', '--verify', commit]).rstrip()
+
def IsMerge(commit):
"""Return True if the commit is a merge, False otherwise."""
rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1',
@@ -33,14 +62,85 @@
# Get full hash since that is what was returned by rev-parse.
return FullHash(commit) != last_non_merge
+
def MergeAbort():
"""Abort in process merge."""
shell_utils.run([GIT, 'merge', '--abort'])
+
def ShortHash(commit):
"""Return short hash of the specified commit."""
return shell_utils.run([GIT, 'show', commit, '--format=%h', '-s']).rstrip()
+
+def Fetch(remote=None):
+ """Run "git fetch". """
+ cmd = [GIT, 'fetch']
+ if remote:
+ cmd.append(remote)
+ shell_utils.run(cmd)
+
+
def GetRemoteMasterHash(git_url):
return shell_utils.run([GIT, 'ls-remote', git_url, '--verify',
- 'refs/heads/master'])
+ 'refs/heads/master']).rstrip()
+
+
+def GetCurrentBranch():
+ return shell_utils.run([GIT, 'rev-parse', '--abbrev-ref', 'HEAD']).rstrip()
+
+
+class GitBranch(object):
+ """Class to manage git branches.
+
+ This class allows one to create a new branch in a repository to make changes,
+ then it commits the changes, switches to master branch, and deletes the
+ created temporary branch upon exit.
+ """
+ def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False,
+ delete_when_finished=True):
+ self._branch_name = branch_name
+ self._commit_msg = commit_msg
+ self._upload = upload
+ self._commit_queue = commit_queue
+ self._patch_set = 0
+ self._delete_when_finished = delete_when_finished
+
+ def __enter__(self):
+ shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
+ shell_utils.run([GIT, 'checkout', 'master'])
+ if self._branch_name in shell_utils.run([GIT, 'branch']):
+ shell_utils.run([GIT, 'branch', '-D', self._branch_name])
+ shell_utils.run([GIT, 'checkout', '-b', self._branch_name,
+ '-t', 'origin/master'])
+ return self
+
+ def commit_and_upload(self, use_commit_queue=False):
+ """Commit all changes and upload a CL, returning the issue URL."""
+ try:
+ shell_utils.run([GIT, 'commit', '-a', '-m', self._commit_msg])
+ except shell_utils.CommandFailedException as e:
+ if not 'nothing to commit' in e.output:
+ raise
+ upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks',
+ '--bypass-watchlists']
+ self._patch_set += 1
+ if self._patch_set > 1:
+ upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set])
+ if use_commit_queue:
+ upload_cmd.append('--use-commit-queue')
+ shell_utils.run(upload_cmd)
+ output = shell_utils.run([GIT, 'cl', 'issue']).rstrip()
+ return re.match('^Issue number: (?P<issue>\d+) \((?P<issue_url>.+)\)$',
+ output).group('issue_url')
+
+ def __exit__(self, exc_type, _value, _traceback):
+ if self._upload:
+ # Only upload if no error occurred.
+ try:
+ if exc_type is None:
+ self.commit_and_upload(use_commit_queue=self._commit_queue)
+ finally:
+ shell_utils.run([GIT, 'checkout', 'master'])
+ if self._delete_when_finished:
+ shell_utils.run([GIT, 'branch', '-D', self._branch_name])
diff --git a/py/utils/misc.py b/py/utils/misc.py
index a1bec6a..c3e4dc8 100644
--- a/py/utils/misc.py
+++ b/py/utils/misc.py
@@ -3,13 +3,10 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-""" This module contains miscellaneous tools used by the buildbot scripts. """
+""" This module contains miscellaneous tools. """
import os
-from git_utils import GIT
-import shell_utils
-
# Absolute path to the root of this Skia buildbot checkout.
BUILDBOT_PATH = os.path.realpath(os.path.join(
@@ -93,50 +90,3 @@
if self._verbose:
print 'chdir %s' % self._origin
os.chdir(self._origin)
-
-
-class GitBranch(object):
- """Class to manage git branches.
-
- This class allows one to create a new branch in a repository to make changes,
- then it commits the changes, switches to master branch, and deletes the
- created temporary branch upon exit.
- """
- def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False):
- self._branch_name = branch_name
- self._commit_msg = commit_msg
- self._upload = upload
- self._commit_queue = commit_queue
- self._patch_set = 0
-
- def __enter__(self):
- shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
- shell_utils.run([GIT, 'checkout', 'master'])
- if self._branch_name in shell_utils.run([GIT, 'branch']):
- shell_utils.run([GIT, 'branch', '-D', self._branch_name])
- shell_utils.run([GIT, 'checkout', '-b', self._branch_name,
- '-t', 'origin/master'])
- return self
-
- def commit_and_upload(self, use_commit_queue=False):
- shell_utils.run([GIT, 'commit', '-a', '-m',
- self._commit_msg])
- upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks',
- '--bypass-watchlists']
- self._patch_set += 1
- if self._patch_set > 1:
- upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set])
- if use_commit_queue:
- upload_cmd.append('--use-commit-queue')
- shell_utils.run(upload_cmd)
-
- def __exit__(self, exc_type, _value, _traceback):
- if self._upload:
- # Only upload if no error occurred.
- try:
- if exc_type is None:
- self.commit_and_upload(use_commit_queue=self._commit_queue)
- finally:
- shell_utils.run([GIT, 'checkout', 'master'])
- shell_utils.run([GIT, 'branch', '-D', self._branch_name])
-
diff --git a/py/utils/shell_utils.py b/py/utils/shell_utils.py
index a6b54ef..37d124b 100644
--- a/py/utils/shell_utils.py
+++ b/py/utils/shell_utils.py
@@ -20,6 +20,7 @@
DEFAULT_SECS_BETWEEN_ATTEMPTS = 10
POLL_MILLIS = 250
+VERBOSE = True
class CommandFailedException(Exception):
@@ -46,9 +47,11 @@
pass
-def run_async(cmd, echo=True, shell=False):
+def run_async(cmd, echo=None, shell=False):
""" Run 'cmd' in a subprocess, returning a Popen class instance referring to
that process. (Non-blocking) """
+ if echo is None:
+ echo = VERBOSE
if echo:
print cmd
if 'nt' in os.name:
@@ -99,7 +102,7 @@
self._stopped = True
-def log_process_in_real_time(proc, echo=True, timeout=None, log_file=None,
+def log_process_in_real_time(proc, echo=None, timeout=None, log_file=None,
halt_on_output=None, print_timestamps=True):
""" Log the output of proc in real time until it completes. Return a tuple
containing the exit code of proc and the contents of stdout.
@@ -114,6 +117,8 @@
print_timestamps: boolean indicating whether a formatted timestamp should be
prepended to each line of output.
"""
+ if echo is None:
+ echo = VERBOSE
stdout_queue = Queue.Queue()
log_thread = EnqueueThread(proc.stdout, stdout_queue)
log_thread.start()
@@ -153,7 +158,8 @@
return (code, ''.join(all_output))
-def log_process_after_completion(proc, echo=True, timeout=None, log_file=None):
+def log_process_after_completion(proc, echo=None, timeout=None,
+ log_file=None):
""" Wait for proc to complete and return a tuple containing the exit code of
proc and the contents of stdout. Unlike log_process_in_real_time, does not
attempt to read stdout from proc in real time.
@@ -164,6 +170,8 @@
TimeoutException if the run time exceeds the timeout.
log_file: an open file for writing outout
"""
+ if echo is None:
+ echo = VERBOSE
t_0 = time.time()
code = None
while code is None:
@@ -182,7 +190,7 @@
return (code, output)
-def run(cmd, echo=True, shell=False, timeout=None, print_timestamps=True,
+def run(cmd, echo=None, shell=False, timeout=None, print_timestamps=True,
log_in_real_time=True):
""" Run 'cmd' in a shell and return the combined contents of stdout and
stderr (Blocking). Throws an exception if the command exits non-zero.
@@ -202,6 +210,8 @@
subprocess in real time instead of when the process finishes. If echo is
False, we never log in real time, even if log_in_real_time is True.
"""
+ if echo is None:
+ echo = VERBOSE
proc = run_async(cmd, echo=echo, shell=shell)
# If we're not printing the output, we don't care if the output shows up in
# real time, so don't bother.
@@ -218,11 +228,13 @@
return output
-def run_retry(cmd, echo=True, shell=False, attempts=1,
+def run_retry(cmd, echo=None, shell=False, attempts=1,
secs_between_attempts=DEFAULT_SECS_BETWEEN_ATTEMPTS,
timeout=None, print_timestamps=True):
""" Wrapper for run() which makes multiple attempts until either the command
succeeds or the maximum number of attempts is reached. """
+ if echo is None:
+ echo = VERBOSE
attempt = 1
while True:
try:
