Use new common utils where possible. BUG=skia:2682 R=rmistry@google.com Review URL: https://codereview.chromium.org/344183004
diff --git a/DEPS b/DEPS index a31c731..7e4692b 100644 --- a/DEPS +++ b/DEPS
@@ -26,7 +26,7 @@ deps = { # Utilities shared between the Skia and Skia-Buildbot repositories. "common": - Var("skia_git") + "/common.git@9bda9ca2f55b584189b83457d1cfea7805713f64", + Var("skia_git") + "/common.git@c92e6d8058240b0804b28fdc4f78261b7133431d", # Chromium trunk code for running telemetry binaries. "third_party/chromium_trunk/src/tools/perf":
diff --git a/slave/skia_slave_scripts/android_verify_device.py b/slave/skia_slave_scripts/android_verify_device.py index 86ff6d6..c645f9c 100644 --- a/slave/skia_slave_scripts/android_verify_device.py +++ b/slave/skia_slave_scripts/android_verify_device.py
@@ -16,8 +16,9 @@ CHROMIUM_BUILDBOT = os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot') sys.path.append(os.path.join(CHROMIUM_BUILDBOT, 'scripts')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) -from utils import android_utils, misc +from py.utils import android_utils, misc class AndroidVerifyDevice:
diff --git a/slave/skia_slave_scripts/apply_patch.py b/slave/skia_slave_scripts/apply_patch.py index 7b2b4cb..6d334aa 100644 --- a/slave/skia_slave_scripts/apply_patch.py +++ b/slave/skia_slave_scripts/apply_patch.py
@@ -9,8 +9,8 @@ from ast import literal_eval from build_step import BuildStep, BuildStepFailure -from utils import shell_utils -from utils.git_utils import GIT +from py.utils import shell_utils +from py.utils.git_utils import GIT import os import shutil import sys
diff --git a/slave/skia_slave_scripts/build_step.py b/slave/skia_slave_scripts/build_step.py index 81ba807..9a821de 100644 --- a/slave/skia_slave_scripts/build_step.py +++ b/slave/skia_slave_scripts/build_step.py
@@ -19,17 +19,23 @@ from playback_dirs import LocalSkpPlaybackDirs from playback_dirs import StorageSkpPlaybackDirs -from utils import misc +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) # Add important directories to the PYTHONPATH -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'site_config')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'master')) +sys.path.append(os.path.join(BUILDBOT_PATH)) +sys.path.append(os.path.join(BUILDBOT_PATH, 'site_config')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'master')) +sys.path.insert(0, os.path.join(BUILDBOT_PATH, 'common')) import builder_name_schema import slave_hosts_cfg import slaves_cfg +from py.utils import misc + + DEFAULT_TIMEOUT = 4800 DEFAULT_NO_OUTPUT_TIMEOUT = 3600 DEFAULT_NUM_CORES = 2
diff --git a/slave/skia_slave_scripts/check_buildslave_host_disk_usage.py b/slave/skia_slave_scripts/check_buildslave_host_disk_usage.py index 3dcf3cd..a93a3d1 100644 --- a/slave/skia_slave_scripts/check_buildslave_host_disk_usage.py +++ b/slave/skia_slave_scripts/check_buildslave_host_disk_usage.py
@@ -11,7 +11,7 @@ import sys from build_step import BuildStep, BuildStepWarning, BuildStepFailure -from utils import misc +from py.utils import misc sys.path.append(misc.BUILDBOT_PATH)
diff --git a/slave/skia_slave_scripts/check_for_regressions.py b/slave/skia_slave_scripts/check_for_regressions.py index 16056e4..c53c584 100644 --- a/slave/skia_slave_scripts/check_for_regressions.py +++ b/slave/skia_slave_scripts/check_for_regressions.py
@@ -6,7 +6,7 @@ """ Check for regressions in bench data. """ from build_step import BuildStep -from utils import shell_utils +from py.utils import shell_utils import builder_name_schema import os
diff --git a/slave/skia_slave_scripts/chrome_drt_canary_run_webkit_tests.py b/slave/skia_slave_scripts/chrome_drt_canary_run_webkit_tests.py index 3f5f444..bd7b1f0 100644 --- a/slave/skia_slave_scripts/chrome_drt_canary_run_webkit_tests.py +++ b/slave/skia_slave_scripts/chrome_drt_canary_run_webkit_tests.py
@@ -6,7 +6,7 @@ """ Run the WebKit tests. """ from build_step import BuildStep, BuildStepWarning -from utils import shell_utils +from py.utils import shell_utils import os import sys
diff --git a/slave/skia_slave_scripts/chrome_drt_canary_upload_results.py b/slave/skia_slave_scripts/chrome_drt_canary_upload_results.py index 5491391..3223fc4 100644 --- a/slave/skia_slave_scripts/chrome_drt_canary_upload_results.py +++ b/slave/skia_slave_scripts/chrome_drt_canary_upload_results.py
@@ -6,7 +6,7 @@ """ Run this before running any tests. """ from build_step import BuildStep -from utils import shell_utils +from py.utils import shell_utils from utils import upload_to_bucket import os import posixpath
diff --git a/slave/skia_slave_scripts/compare_gms.py b/slave/skia_slave_scripts/compare_gms.py index a889b4a..69fee02 100644 --- a/slave/skia_slave_scripts/compare_gms.py +++ b/slave/skia_slave_scripts/compare_gms.py
@@ -10,7 +10,7 @@ import sys from build_step import BuildStep, BuildStepWarning -from utils import misc +from py.utils import misc import run_gm import skia_vars
diff --git a/slave/skia_slave_scripts/detect_static_initializers.py b/slave/skia_slave_scripts/detect_static_initializers.py index f38c35d..576dd61 100644 --- a/slave/skia_slave_scripts/detect_static_initializers.py +++ b/slave/skia_slave_scripts/detect_static_initializers.py
@@ -5,8 +5,8 @@ """ Detect static initializers in compiled Skia code. """ -from utils import shell_utils from build_step import BuildStep, BuildStepWarning +from py.utils import shell_utils import os import re import sys
diff --git a/slave/skia_slave_scripts/do_auto_roll.py b/slave/skia_slave_scripts/do_auto_roll.py index 4a0f284..c3be94e 100644 --- a/slave/skia_slave_scripts/do_auto_roll.py +++ b/slave/skia_slave_scripts/do_auto_roll.py
@@ -13,8 +13,8 @@ from build_step import BuildStep from slave import slave_utils -from utils import misc -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils sys.path.append(misc.BUILDBOT_PATH)
diff --git a/slave/skia_slave_scripts/do_skps_capture.py b/slave/skia_slave_scripts/do_skps_capture.py index 8ed0f19..e9048ba 100644 --- a/slave/skia_slave_scripts/do_skps_capture.py +++ b/slave/skia_slave_scripts/do_skps_capture.py
@@ -13,7 +13,7 @@ from build_step import BuildStep from utils import gs_utils -from utils import shell_utils +from py.utils import shell_utils class SKPsCapture(BuildStep):
diff --git a/slave/skia_slave_scripts/flavor_utils/android_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/android_build_step_utils.py index f0edaa9..1be7069 100644 --- a/slave/skia_slave_scripts/flavor_utils/android_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/android_build_step_utils.py
@@ -5,9 +5,9 @@ """ Utilities for Android build steps. """ from default_build_step_utils import DefaultBuildStepUtils, DeviceDirs -from utils import android_utils +from py.utils import android_utils from utils import gs_utils -from utils import shell_utils +from py.utils import shell_utils import os import posixpath
diff --git a/slave/skia_slave_scripts/flavor_utils/arm64model_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/arm64model_build_step_utils.py index 46dd9da..110ad48 100644 --- a/slave/skia_slave_scripts/flavor_utils/arm64model_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/arm64model_build_step_utils.py
@@ -6,8 +6,8 @@ import os import sys -from utils import shell_utils -from utils import ssh_utils +from py.utils import shell_utils +from py.utils import ssh_utils from flavor_utils.ssh_build_step_utils import SshBuildStepUtils class Arm64ModelBuildStepUtils(SshBuildStepUtils):
diff --git a/slave/skia_slave_scripts/flavor_utils/chrome_canary_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/chrome_canary_build_step_utils.py index 1c5df6d..f0333b9 100644 --- a/slave/skia_slave_scripts/flavor_utils/chrome_canary_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/chrome_canary_build_step_utils.py
@@ -8,8 +8,8 @@ from default_build_step_utils import DefaultBuildStepUtils from common import chromium_utils from utils import gclient_utils -from utils import misc -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils import os
diff --git a/slave/skia_slave_scripts/flavor_utils/chromeos_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/chromeos_build_step_utils.py index cc557ca..949c91e 100644 --- a/slave/skia_slave_scripts/flavor_utils/chromeos_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/chromeos_build_step_utils.py
@@ -6,7 +6,7 @@ from flavor_utils.ssh_build_step_utils import SshBuildStepUtils from slave import slave_utils -from utils import shell_utils +from py.utils import shell_utils import os
diff --git a/slave/skia_slave_scripts/flavor_utils/default_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/default_build_step_utils.py index 5783595..5cdc465 100644 --- a/slave/skia_slave_scripts/flavor_utils/default_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/default_build_step_utils.py
@@ -9,7 +9,7 @@ import sys from utils import file_utils -from utils import shell_utils +from py.utils import shell_utils class DeviceDirs(object):
diff --git a/slave/skia_slave_scripts/flavor_utils/moz2d_canary_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/moz2d_canary_build_step_utils.py index f1cf431..99b9b7f 100644 --- a/slave/skia_slave_scripts/flavor_utils/moz2d_canary_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/moz2d_canary_build_step_utils.py
@@ -7,8 +7,8 @@ from common import chromium_utils from default_build_step_utils import DefaultBuildStepUtils -from utils import misc -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils import os
diff --git a/slave/skia_slave_scripts/flavor_utils/nacl_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/nacl_build_step_utils.py index e301322..0f072af 100644 --- a/slave/skia_slave_scripts/flavor_utils/nacl_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/nacl_build_step_utils.py
@@ -6,7 +6,7 @@ """ Utilities for NaCl build steps. """ from default_build_step_utils import DefaultBuildStepUtils -from utils import shell_utils +from py.utils import shell_utils import os
diff --git a/slave/skia_slave_scripts/flavor_utils/ssh_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/ssh_build_step_utils.py index 2a1bd6c..cb0fa25 100644 --- a/slave/skia_slave_scripts/flavor_utils/ssh_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/ssh_build_step_utils.py
@@ -8,8 +8,8 @@ from default_build_step_utils import DefaultBuildStepUtils, DeviceDirs from slave import slave_utils from utils import gs_utils -from utils import shell_utils -from utils import ssh_utils +from py.utils import shell_utils +from py.utils import ssh_utils import os import posixpath import shutil
diff --git a/slave/skia_slave_scripts/flavor_utils/valgrind_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/valgrind_build_step_utils.py index 85bb681..f961968 100644 --- a/slave/skia_slave_scripts/flavor_utils/valgrind_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/valgrind_build_step_utils.py
@@ -5,7 +5,7 @@ """ Utilities for Valgrind build steps. """ from default_build_step_utils import DefaultBuildStepUtils -from utils import shell_utils +from py.utils import shell_utils import os
diff --git a/slave/skia_slave_scripts/flavor_utils/xsan_build_step_utils.py b/slave/skia_slave_scripts/flavor_utils/xsan_build_step_utils.py index 77f4b7d..df464c3 100644 --- a/slave/skia_slave_scripts/flavor_utils/xsan_build_step_utils.py +++ b/slave/skia_slave_scripts/flavor_utils/xsan_build_step_utils.py
@@ -6,7 +6,7 @@ """ Utilities for ASAN,TSAN,etc. build steps. """ from default_build_step_utils import DefaultBuildStepUtils -from utils import shell_utils +from py.utils import shell_utils import os
diff --git a/slave/skia_slave_scripts/generate_doxygen.py b/slave/skia_slave_scripts/generate_doxygen.py index 768bb89..02dd65d 100755 --- a/slave/skia_slave_scripts/generate_doxygen.py +++ b/slave/skia_slave_scripts/generate_doxygen.py
@@ -11,7 +11,8 @@ import sys from build_step import BuildStep -from utils import file_utils, shell_utils +from utils import file_utils +from py.utils import shell_utils DOXYFILE_BASENAME = 'Doxyfile' # must match name of Doxyfile in skia root DOXYGEN_BINARY = 'doxygen'
diff --git a/slave/skia_slave_scripts/merge_into_android.py b/slave/skia_slave_scripts/merge_into_android.py index 27ebcd1..af872cf 100644 --- a/slave/skia_slave_scripts/merge_into_android.py +++ b/slave/skia_slave_scripts/merge_into_android.py
@@ -13,10 +13,10 @@ from build_step import BuildStep, BuildStepFailure, BuildStepWarning import skia_vars from sync_android import ANDROID_CHECKOUT_PATH, REPO, GitAuthenticate -from utils.git_utils import GIT -from utils import git_utils -from utils import misc -from utils import shell_utils +from py.utils.git_utils import GIT +from py.utils import git_utils +from py.utils import misc +from py.utils import shell_utils SKIA_REPO_URL = skia_vars.GetGlobalVariable('skia_git_url') SKIA_REV_URL = skia_vars.GetGlobalVariable('revlink_tmpl')
diff --git a/slave/skia_slave_scripts/moz2d_canary_update.py b/slave/skia_slave_scripts/moz2d_canary_update.py index deb888c..4d99432 100644 --- a/slave/skia_slave_scripts/moz2d_canary_update.py +++ b/slave/skia_slave_scripts/moz2d_canary_update.py
@@ -9,8 +9,8 @@ from build_step import BuildStep from flavor_utils import moz2d_canary_build_step_utils from update import Update -from utils import misc -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils import sys
diff --git a/slave/skia_slave_scripts/run_android_platform_self_tests.py b/slave/skia_slave_scripts/run_android_platform_self_tests.py index ba28b38..8da9942 100755 --- a/slave/skia_slave_scripts/run_android_platform_self_tests.py +++ b/slave/skia_slave_scripts/run_android_platform_self_tests.py
@@ -9,7 +9,7 @@ import sys from build_step import BuildStep -from utils import shell_utils +from py.utils import shell_utils class RunAndroidPlatformSelfTests(BuildStep):
diff --git a/slave/skia_slave_scripts/run_buildbot_self_tests.py b/slave/skia_slave_scripts/run_buildbot_self_tests.py index 16cc3b2..7a23719 100755 --- a/slave/skia_slave_scripts/run_buildbot_self_tests.py +++ b/slave/skia_slave_scripts/run_buildbot_self_tests.py
@@ -8,8 +8,8 @@ import sys from build_step import BuildStep -from utils import misc -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils class BuildbotSelfTests(BuildStep):
diff --git a/slave/skia_slave_scripts/run_gm_self_tests.py b/slave/skia_slave_scripts/run_gm_self_tests.py index a740d83..9f5147d25 100755 --- a/slave/skia_slave_scripts/run_gm_self_tests.py +++ b/slave/skia_slave_scripts/run_gm_self_tests.py
@@ -9,7 +9,7 @@ import sys from build_step import BuildStep -from utils import shell_utils +from py.utils import shell_utils class RunGmSelfTests(BuildStep):
diff --git a/slave/skia_slave_scripts/run_tool_self_tests.py b/slave/skia_slave_scripts/run_tool_self_tests.py index 8caf356..e0cb83b 100755 --- a/slave/skia_slave_scripts/run_tool_self_tests.py +++ b/slave/skia_slave_scripts/run_tool_self_tests.py
@@ -9,7 +9,7 @@ import sys from build_step import BuildStep -from utils import shell_utils +from py.utils import shell_utils class RunToolSelfTests(BuildStep):
diff --git a/slave/skia_slave_scripts/sync_android.py b/slave/skia_slave_scripts/sync_android.py index c328955..8a7712b 100644 --- a/slave/skia_slave_scripts/sync_android.py +++ b/slave/skia_slave_scripts/sync_android.py
@@ -12,8 +12,8 @@ import sys from build_step import BuildStep -from utils import misc -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils from utils.gclient_utils import GIT
diff --git a/slave/skia_slave_scripts/update.py b/slave/skia_slave_scripts/update.py index 34a3ba4..d248a5d 100644 --- a/slave/skia_slave_scripts/update.py +++ b/slave/skia_slave_scripts/update.py
@@ -6,13 +6,14 @@ """ Check out the Skia sources. """ +from build_step import BuildStep, BuildStepFailure + from common import chromium_utils -from utils.git_utils import GIT +from py.utils.git_utils import GIT from utils import file_utils from utils import gclient_utils -from utils import misc -from utils import shell_utils -from build_step import BuildStep, BuildStepFailure +from py.utils import misc +from py.utils import shell_utils import ast import config_private
diff --git a/slave/skia_slave_scripts/update_all_buildslave_checkouts.py b/slave/skia_slave_scripts/update_all_buildslave_checkouts.py index 10203b2..0cfecf8 100644 --- a/slave/skia_slave_scripts/update_all_buildslave_checkouts.py +++ b/slave/skia_slave_scripts/update_all_buildslave_checkouts.py
@@ -17,12 +17,8 @@ import sys from build_step import BuildStep, BuildStepWarning -from utils import force_update_checkout -from utils import misc - -sys.path.append(misc.BUILDBOT_PATH) - from scripts import run_cmd +from utils import force_update_checkout BUILDBOT_GIT_URL = skia_vars.GetGlobalVariable('buildbot_git_url')
diff --git a/slave/skia_slave_scripts/update_all_slave_hosts.py b/slave/skia_slave_scripts/update_all_slave_hosts.py index 6ce6946..a80af65 100644 --- a/slave/skia_slave_scripts/update_all_slave_hosts.py +++ b/slave/skia_slave_scripts/update_all_slave_hosts.py
@@ -18,7 +18,7 @@ from build_step import BuildStep, BuildStepWarning from utils import force_update_checkout -from utils import misc +from py.utils import misc sys.path.append(misc.BUILDBOT_PATH)
diff --git a/slave/skia_slave_scripts/update_skp_version.py b/slave/skia_slave_scripts/update_skp_version.py index a6e338a..01ed88a 100644 --- a/slave/skia_slave_scripts/update_skp_version.py +++ b/slave/skia_slave_scripts/update_skp_version.py
@@ -12,9 +12,9 @@ from build_step import BuildStep from config_private import SKIA_GIT_URL -from utils.git_utils import GIT -from utils import misc -from utils import shell_utils +from py.utils.git_utils import GIT +from py.utils import misc +from py.utils import shell_utils CHROMIUM_SKIA = 'https://chromium.googlesource.com/skia.git'
diff --git a/slave/skia_slave_scripts/upload_bench_results_appengine.py b/slave/skia_slave_scripts/upload_bench_results_appengine.py index 61d3200..cd34c88 100644 --- a/slave/skia_slave_scripts/upload_bench_results_appengine.py +++ b/slave/skia_slave_scripts/upload_bench_results_appengine.py
@@ -6,7 +6,7 @@ """ Upload benchmark results to AppEngine.. """ from build_step import BuildStep -from utils import shell_utils +from py.utils import shell_utils import os import skia_vars
diff --git a/slave/skia_slave_scripts/upload_bench_results_test.py b/slave/skia_slave_scripts/upload_bench_results_test.py index c902f40..4a10044 100644 --- a/slave/skia_slave_scripts/upload_bench_results_test.py +++ b/slave/skia_slave_scripts/upload_bench_results_test.py
@@ -9,14 +9,16 @@ import sys import unittest -from utils import misc +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) + # Appending to PYTHONPATH to find common and config. -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', 'scripts')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', 'site_config')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'third_party', 'twisted_8_1')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'third_party', 'twisted_8_1')) import upload_bench_results
diff --git a/slave/skia_slave_scripts/utils/android_utils.py b/slave/skia_slave_scripts/utils/android_utils.py deleted file mode 100644 index 7b1c1a1..0000000 --- a/slave/skia_slave_scripts/utils/android_utils.py +++ /dev/null
@@ -1,321 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" This module contains tools used by Android-specific buildbot scripts. """ - -import os -import re -import shell_utils -import shlex -import sys -import time - - -CPU_SCALING_MODES = ['performance', 'interactive'] -DEVICE_LOOKUP = {'nexus_s': 'crespo', - 'xoom': 'stingray', - 'galaxy_nexus': 'toro', - 'nexus_4': 'mako', - 'nexus_7': 'grouper', - 'nexus_10': 'manta'} -PROCESS_MONITOR_INTERVAL = 5.0 # Seconds -SKIA_RUNNING = 'running' -SKIA_RETURN_CODE_REPEATS = 10 -SUBPROCESS_TIMEOUT = 30.0 - - -def GotADB(adb): - """ Returns True iff ADB exists at the given location. - - adb: string; possible path to the ADB executable. - """ - try: - shell_utils.run([adb, 'version'], echo=False) - return True - except Exception: - return False - - -def FindADB(hint=None): - """ Attempt to find the ADB program using the following sequence of steps. - Returns the path to ADB if it can be found, or None otherwise. - 1. If a hint was provided, is it a valid path to ADB? - 2. Is ADB in PATH? - 3. Is there an environment variable for ADB? - 4. If the ANDROID_SDK_ROOT variable is set, try to find ADB in the SDK - directory. - 5. Try to find ADB in a list of common locations. - - hint: string indicating a possible path to ADB. - """ - # 1. If a hint was provided, does it point to ADB? - if hint: - if os.path.basename(hint) == 'adb': - adb = hint - else: - adb = os.path.join(hint, 'adb') - if GotADB(adb): - return adb - - # 2. Is 'adb' in our PATH? - adb = 'adb' - if GotADB(adb): - return adb - - # 3. Is there an environment variable for ADB? - adb = os.environ.get('ADB') - if GotADB(adb): - return adb - - # 4. If ANDROID_SDK_ROOT is set, try to find ADB in the SDK directory. - sdk_dir = os.environ.get('ANDROID_SDK_ROOT', '') - adb = os.path.join(sdk_dir, 'platform-tools', 'adb') - if GotADB(adb): - return adb - - # 4. Try to find ADB relative to this file. - common_locations = [] - os_dir = None - if sys.platform.startswith('linux'): - os_dir = 'linux' - elif sys.platform.startswith('darwin'): - os_dir = 'mac' - else: - os_dir = 'win' - common_locations.append(os.path.join('platform_tools', 'android', 'bin', - os_dir, 'adb')) - common_locations.append(os.path.join(os.environ.get('HOME', ''), - 'android-sdk-%s' % os_dir)) - for location in common_locations: - if GotADB(location): - return location - - raise Exception('android_utils: Unable to find ADB!') - - -PATH_TO_ADB = FindADB(hint=os.path.join('platform_tools', 'android', 'bin', - 'linux', 'adb')) - - -def RunADB(serial, cmd, echo=True, attempts=5, secs_between_attempts=10, - timeout=None): - """ Run 'cmd' on an Android device, using ADB. No return value; throws an - exception if the command fails more than the allotted number of attempts. - - serial: string indicating the serial number of the target device - cmd: string; the command to issue on the device - attempts: number of times to attempt the command - secs_between_attempts: number of seconds to wait between attempts - timeout: optional, integer indicating the maximum elapsed time in seconds - """ - adb_cmd = [PATH_TO_ADB, '-s', serial] - adb_cmd += cmd - shell_utils.run_retry(adb_cmd, echo=echo, attempts=attempts, - secs_between_attempts=secs_between_attempts) - - -def ADBShell(serial, cmd, echo=True): - """ Runs 'cmd' in the ADB shell on an Android device and returns the exit - code. - - serial: string indicating the serial number of the target device - cmd: string; the command to issue on the device - """ - # ADB doesn't exit with the exit code of the command we ran. It only exits - # non-zero when ADB itself encountered a problem. Therefore, we have to use - # the shell to print the exit code for the command and parse that from stdout. - adb_cmd = '%s -s %s shell "%s; echo \$?"' % (PATH_TO_ADB, serial, - ' '.join(cmd)) - output = shell_utils.run(adb_cmd, shell=True, echo=echo) - output_lines = output.splitlines() - try: - real_exitcode = int(output_lines[-1].rstrip()) - except ValueError: - real_exitcode = -1 - if real_exitcode != 0: - raise Exception('Command failed with code %s' % real_exitcode) - return '\n'.join(output_lines[:-1]) - - -def ADBKill(serial, process, kill_app=False): - """ Kill a process running on an Android device. - - serial: string indicating the serial number of the target device - process: string indicating the name of the process to kill - kill_app: bool indicating whether the process is an Android app, as opposed - to a normal executable process. - """ - if kill_app: - ADBShell(serial, ['am', 'kill', process]) - else: - try: - stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB, - serial, - process), - shell=True) - except Exception: - return - for line in stdout.split('\n'): - if line != '': - split = shlex.split(line) - if len(split) < 2: - continue - pid = split[1] - ADBShell(serial, ['kill', pid]) - # Raise an exception if any Skia processes are still running. - try: - stdout = shell_utils.run('%s -s %s shell ps | grep %s' % (PATH_TO_ADB, - serial, - process), - shell=True) - except Exception: - return - if stdout: - raise Exception('There are still some skia processes running:\n%s\n' - 'Maybe the device should be rebooted?' % stdout) - - -def GetSerial(device_type): - """ Determine the serial number of the *first* connected device with the - specified type. The ordering of 'adb devices' is not documented, and the - connected devices do not appear to be ordered by serial number. Therefore, - we have to assume that, in the case of multiple devices of the same type being - connected to one host, we cannot predict which device will be chosen. - - device_type: string indicating the 'common name' for the target device - """ - if not device_type in DEVICE_LOOKUP: - raise ValueError('Unknown device: %s!' % device_type) - device_name = DEVICE_LOOKUP[device_type] - output = shell_utils.run_retry('%s devices' % PATH_TO_ADB, shell=True, - attempts=5) - print output - lines = output.split('\n') - device_ids = [] - for line in lines: - # Filter garbage lines - if line != '' and not ('List of devices attached' in line) and \ - line[0] != '*': - device_ids.append(line.split('\t')[0]) - for device_id in device_ids: - print 'Finding type for id %s' % device_id - # Get device name - name_line = shell_utils.run_retry( - '%s -s %s shell cat /system/build.prop | grep "ro.product.device="' % ( - PATH_TO_ADB, device_id), shell=True, attempts=5) - print name_line - name = name_line.split('=')[-1].rstrip() - # Just return the first attached device of the requested model. - if device_name in name: - return device_id - raise Exception('No %s device attached!' % device_name) - - -def SetCPUScalingMode(serial, mode): - """ Set the CPU scaling governor for the device with the given serial number - to the given mode. - - serial: string indicating the serial number of the device whose scaling mode - is to be modified - mode: string indicating the desired CPU scaling mode. Acceptable values - are listed in CPU_SCALING_MODES. - """ - if mode not in CPU_SCALING_MODES: - raise ValueError('mode must be one of: %s' % CPU_SCALING_MODES) - cpu_dirs = shell_utils.run('%s -s %s shell ls /sys/devices/system/cpu' % ( - PATH_TO_ADB, serial), echo=False, shell=True) - cpu_dirs_list = cpu_dirs.split('\n') - regex = re.compile('cpu\d') - for cpu_dir_from_list in cpu_dirs_list: - cpu_dir = cpu_dir_from_list.rstrip() - if regex.match(cpu_dir): - path = '/sys/devices/system/cpu/%s/cpufreq/scaling_governor' % cpu_dir - path_found = shell_utils.run('%s -s %s shell ls %s' % ( - PATH_TO_ADB, serial, path), - echo=False, shell=True).rstrip() - if path_found == path: - # Unfortunately, we can't directly change the scaling_governor file over - # ADB. Instead, we write a script to do so, push it to the device, and - # run it. - old_mode = shell_utils.run('%s -s %s shell cat %s' % ( - PATH_TO_ADB, serial, path), - echo=False, shell=True).rstrip() - print 'Current scaling mode for %s is: %s' % (cpu_dir, old_mode) - filename = 'skia_cpuscale.sh' - with open(filename, 'w') as script_file: - script_file.write('echo %s > %s\n' % (mode, path)) - os.chmod(filename, 0777) - RunADB(serial, ['push', filename, '/system/bin'], echo=False) - RunADB(serial, ['shell', filename], echo=True) - RunADB(serial, ['shell', 'rm', '/system/bin/%s' % filename], echo=False) - os.remove(filename) - new_mode = shell_utils.run('%s -s %s shell cat %s' % ( - PATH_TO_ADB, serial, path), - echo=False, shell=True).rstrip() - print 'New scaling mode for %s is: %s' % (cpu_dir, new_mode) - - -def IsAndroidShellRunning(serial): - """ Find the status of the Android shell for the device with the given serial - number. Returns True if the shell is running and False otherwise. - - serial: string indicating the serial number of the target device. - """ - if 'Error:' in ADBShell(serial, ['pm', 'path', 'android'], echo=False): - return False - return True - - -def StopShell(serial, timeout=60): - """ Halt the Android runtime on the device with the given serial number. - Blocks until the shell reports that it has stopped. - - serial: string indicating the serial number of the target device. - timeout: maximum allotted time, in seconds. - """ - ADBShell(serial, ['stop']) - start_time = time.time() - while IsAndroidShellRunning(serial): - time.sleep(1) - if time.time() - start_time > timeout: - raise Exception('Timeout while attempting to stop the Android runtime.') - - -def StartShell(serial, timeout=60): - """ Start the Android runtime on the device with the given serial number. - Blocks until the shell reports that it has started. - - serial: string indicating the serial number of the target device. - timeout: maximum allotted time, in seconds. - """ - ADBShell(serial, ['start']) - start_time = time.time() - while not IsAndroidShellRunning(serial): - time.sleep(1) - if time.time() - start_time > timeout: - raise Exception('Timeout while attempting to start the Android runtime.') - - -def RunSkia(serial, cmd, release, device): - """ Run the given command through skia_launcher on a given device. - - serial: string indicating the serial number of the target device. - cmd: list of strings; the command line to run. - release: bool; whether or not to run the app in Release mode. - device: string indicating the target device. - """ - RunADB(serial, ['logcat', '-c']) - try: - os.environ['SKIA_ANDROID_VERBOSE_SETUP'] = '1' - cmd_to_run = [os.path.join('platform_tools', 'android', 'bin', - 'android_run_skia')] - if release: - cmd_to_run.extend(['--release']) - cmd_to_run.extend(['-d', device]) - cmd_to_run.extend(['-s', serial]) - cmd_to_run.extend(cmd) - shell_utils.run(cmd_to_run) - finally: - RunADB(serial, ['logcat', '-d', '-v', 'time'])
diff --git a/slave/skia_slave_scripts/utils/file_utils_test.py b/slave/skia_slave_scripts/utils/file_utils_test.py index b8b4d6b..8afa8efe 100644 --- a/slave/skia_slave_scripts/utils/file_utils_test.py +++ b/slave/skia_slave_scripts/utils/file_utils_test.py
@@ -9,10 +9,13 @@ import sys import unittest -import misc +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, os.pardir) +) + # Appending to PYTHONPATH to find common. -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'scripts')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'scripts')) import file_utils from common import chromium_utils
diff --git a/slave/skia_slave_scripts/utils/force_update_checkout.py b/slave/skia_slave_scripts/utils/force_update_checkout.py index 631d1c7..a68f34a 100644 --- a/slave/skia_slave_scripts/utils/force_update_checkout.py +++ b/slave/skia_slave_scripts/utils/force_update_checkout.py
@@ -12,15 +12,19 @@ import shlex import sys -import misc +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, os.pardir) +) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'site_config')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', +sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'site_config')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', 'scripts')) import gclient_utils -import git_utils -import shell_utils +from py.utils import git_utils +from py.utils import misc +from py.utils import shell_utils import skia_vars
diff --git a/slave/skia_slave_scripts/utils/gclient_utils.py b/slave/skia_slave_scripts/utils/gclient_utils.py index c0db120..a840c62 100644 --- a/slave/skia_slave_scripts/utils/gclient_utils.py +++ b/slave/skia_slave_scripts/utils/gclient_utils.py
@@ -8,10 +8,10 @@ from common import find_depot_tools -from git_utils import GIT -import misc +from py.utils.git_utils import GIT +from py.utils import shell_utils +from py.utils import misc import os -import shell_utils WHICH = 'where' if os.name == 'nt' else 'which'
diff --git a/slave/skia_slave_scripts/utils/git_utils.py b/slave/skia_slave_scripts/utils/git_utils.py deleted file mode 100644 index da7f097..0000000 --- a/slave/skia_slave_scripts/utils/git_utils.py +++ /dev/null
@@ -1,46 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""This module contains functions for using git.""" - - -import os -import shell_utils - - -GIT = 'git.bat' if os.name == 'nt' else 'git' - - -def Add(addition): - """Run 'git add <addition>'""" - shell_utils.run([GIT, 'add', addition]) - -def AIsAncestorOfB(a, b): - """Return true if a is an ancestor of b.""" - return shell_utils.run([GIT, 'merge-base', a, b]).rstrip() == FullHash(a) - -def FullHash(commit): - """Return full hash of specified commit.""" - return shell_utils.run([GIT, 'rev-parse', '--verify', commit]).rstrip() - -def IsMerge(commit): - """Return True if the commit is a merge, False otherwise.""" - rev_parse = shell_utils.run([GIT, 'rev-parse', commit, '--max-count=1', - '--no-merges']) - last_non_merge = rev_parse.split('\n')[0] - # Get full hash since that is what was returned by rev-parse. - return FullHash(commit) != last_non_merge - -def MergeAbort(): - """Abort in process merge.""" - shell_utils.run([GIT, 'merge', '--abort']) - -def ShortHash(commit): - """Return short hash of the specified commit.""" - return shell_utils.run([GIT, 'show', commit, '--format=%h', '-s']).rstrip() - -def GetRemoteMasterHash(git_url): - return shell_utils.run([GIT, 'ls-remote', git_url, '--verify', - 'refs/heads/master'])
diff --git a/slave/skia_slave_scripts/utils/gs_utils.py b/slave/skia_slave_scripts/utils/gs_utils.py index 48f7cbc..0cbe342 100644 --- a/slave/skia_slave_scripts/utils/gs_utils.py +++ b/slave/skia_slave_scripts/utils/gs_utils.py
@@ -17,10 +17,10 @@ import tempfile import time +from py.utils import shell_utils from slave import slave_utils import file_utils -import shell_utils DEFAULT_DEST_GSBASE = 'gs://chromium-skia-gm'
diff --git a/slave/skia_slave_scripts/utils/gs_utils_test.py b/slave/skia_slave_scripts/utils/gs_utils_test.py index 0a226e1..aa86c26 100644 --- a/slave/skia_slave_scripts/utils/gs_utils_test.py +++ b/slave/skia_slave_scripts/utils/gs_utils_test.py
@@ -6,27 +6,28 @@ """Tests for module gs_utils.""" import __builtin__ -import misc import os import posixpath -import shell_utils import shutil import sys import tempfile import time # Appending to PYTHONPATH to find common. -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'scripts')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'scripts', 'common')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'site_config')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'third_party', - 'twisted_10_2')) +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, os.pardir) +) +sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'scripts')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'scripts', 'common')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'site_config')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'third_party', 'twisted_10_2')) - +from py.utils import shell_utils from slave import slave_utils import gs_utils import unittest
diff --git a/slave/skia_slave_scripts/utils/misc.py b/slave/skia_slave_scripts/utils/misc.py deleted file mode 100644 index a61527e..0000000 --- a/slave/skia_slave_scripts/utils/misc.py +++ /dev/null
@@ -1,138 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" This module contains miscellaneous tools used by the buildbot scripts. """ - -import os - -from git_utils import GIT -import shell_utils - - -# Absolute path to the root of this Skia buildbot checkout. -BUILDBOT_PATH = os.path.realpath(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - os.pardir, os.pardir, os.pardir)) - - -def ArgsToDict(argv): - """ Collect command-line arguments of the form '--key value' into a - dictionary. Fail if the arguments do not fit this format. """ - dictionary = {} - PREFIX = '--' - # Expect the first arg to be the path to the script, which we don't want. - argv = argv[1:] - while argv: - if argv[0].startswith(PREFIX): - dictionary[argv[0][len(PREFIX):]] = argv[1] - argv = argv[2:] - else: - raise Exception('Malformed input: %s' % argv) - return dictionary - - -def ConfirmOptionsSet(name_value_dict): - """Raise an exception if any of the given command-line options were not set. - - name_value_dict: dictionary mapping option names to option values - """ - for (name, value) in name_value_dict.iteritems(): - if value is None: - raise Exception('missing command-line option %s; rerun with --help' % - name) - - -def GetAbsPath(relative_path): - """My own implementation of os.path.abspath() that better handles paths - which approach Window's 260-character limit. - See https://code.google.com/p/skia/issues/detail?id=674 - - This implementation adds path components one at a time, resolving the - absolute path each time, to take advantage of any chdirs into outer - directories that will shorten the total path length. - - TODO(epoger): share a single implementation with bench_graph_svg.py, instead - of pasting this same code into both files.""" - if os.path.isabs(relative_path): - return relative_path - path_parts = relative_path.split(os.sep) - abs_path = os.path.abspath('.') - for path_part in path_parts: - abs_path = os.path.abspath(os.path.join(abs_path, path_part)) - return abs_path - - -class ChDir(object): - """Enter and exit the given directory appropriately.""" - - def __init__(self, directory): - """Instantiate the ChDir. - - Args: - directory: string; the directory to enter. - """ - self._destination = directory - self._origin = None - - def __enter__(self): - """Change to the destination directory. - - Does not check whether the directory exists. - """ - self._origin = os.getcwd() - print 'chdir %s' % self._destination - os.chdir(self._destination) - - def __exit__(self, *args): - """Change back to the original directory.""" - print 'chdir %s' % self._origin - os.chdir(self._origin) - - -class GitBranch(object): - """Class to manage git branches. - - This class allows one to create a new branch in a repository to make changes, - then it commits the changes, switches to master branch, and deletes the - created temporary branch upon exit. - """ - def __init__(self, branch_name, commit_msg, upload=True, commit_queue=False): - self._branch_name = branch_name - self._commit_msg = commit_msg - self._upload = upload - self._commit_queue = commit_queue - self._patch_set = 0 - - def __enter__(self): - shell_utils.run([GIT, 'reset', '--hard', 'HEAD']) - shell_utils.run([GIT, 'checkout', 'master']) - if self._branch_name in shell_utils.run([GIT, 'branch']): - shell_utils.run([GIT, 'branch', '-D', self._branch_name]) - shell_utils.run([GIT, 'checkout', '-b', self._branch_name, - '-t', 'origin/master']) - return self - - def commit_and_upload(self, use_commit_queue=False): - shell_utils.run([GIT, 'commit', '-a', '-m', - self._commit_msg]) - upload_cmd = [GIT, 'cl', 'upload', '-f', '--bypass-hooks', - '--bypass-watchlists'] - self._patch_set += 1 - if self._patch_set > 1: - upload_cmd.extend(['-t', 'Patch set %d' % self._patch_set]) - if use_commit_queue: - upload_cmd.append('--use-commit-queue') - shell_utils.run(upload_cmd) - - def __exit__(self, exc_type, _value, _traceback): - if self._upload: - # Only upload if no error occurred. - try: - if exc_type is None: - self.commit_and_upload(use_commit_queue=self._commit_queue) - finally: - shell_utils.run([GIT, 'checkout', 'master']) - shell_utils.run([GIT, 'branch', '-D', self._branch_name]) -
diff --git a/slave/skia_slave_scripts/utils/shell_utils.py b/slave/skia_slave_scripts/utils/shell_utils.py deleted file mode 100644 index 7f66c73..0000000 --- a/slave/skia_slave_scripts/utils/shell_utils.py +++ /dev/null
@@ -1,236 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" This module contains tools for running commands in a shell. """ - -import datetime -import os -import Queue -import select -import subprocess -import sys -import threading -import time - -if 'nt' in os.name: - import ctypes - - -DEFAULT_SECS_BETWEEN_ATTEMPTS = 10 -POLL_MILLIS = 250 - - -class CommandFailedException(Exception): - """Exception which gets raised when a command fails.""" - - def __init__(self, output, *args): - """Initialize the CommandFailedException. - - Args: - output: string; output from the command. - """ - Exception.__init__(self, *args) - self._output = output - - @property - def output(self): - """Output from the command.""" - return self._output - - -class TimeoutException(CommandFailedException): - """CommandFailedException which gets raised when a subprocess exceeds its - timeout. """ - pass - - -def run_async(cmd, echo=True, shell=False): - """ Run 'cmd' in a subprocess, returning a Popen class instance referring to - that process. (Non-blocking) """ - if echo: - print cmd - if 'nt' in os.name: - # Windows has a bad habit of opening a dialog when a console program - # crashes, rather than just letting it crash. Therefore, when a program - # crashes on Windows, we don't find out until the build step times out. - # This code prevents the dialog from appearing, so that we find out - # immediately and don't waste time waiting around. - SEM_NOGPFAULTERRORBOX = 0x0002 - ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX) - flags = 0x8000000 # CREATE_NO_WINDOW - else: - flags = 0 - return subprocess.Popen(cmd, shell=shell, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, creationflags=flags, - bufsize=1) - - -class EnqueueThread(threading.Thread): - """ Reads and enqueues lines from a file. """ - def __init__(self, file_obj, queue): - threading.Thread.__init__(self) - self._file = file_obj - self._queue = queue - self._stopped = False - - def run(self): - if sys.platform.startswith('linux'): - # Use a polling object to avoid the blocking call to readline(). - poll = select.poll() - poll.register(self._file, select.POLLIN) - while not self._stopped: - has_output = poll.poll(POLL_MILLIS) - if has_output: - line = self._file.readline() - if line == '': - self._stopped = True - self._queue.put(line) - else: - # Only Unix supports polling objects, so just read from the file, - # Python-style. - for line in iter(self._file.readline, ''): - self._queue.put(line) - if self._stopped: - break - - def stop(self): - self._stopped = True - - -def log_process_in_real_time(proc, echo=True, timeout=None, log_file=None, - halt_on_output=None, print_timestamps=True): - """ Log the output of proc in real time until it completes. Return a tuple - containing the exit code of proc and the contents of stdout. - - proc: an instance of Popen referring to a running subprocess. - echo: boolean indicating whether to print the output received from proc.stdout - timeout: number of seconds allotted for the process to run. Raises a - TimeoutException if the run time exceeds the timeout. - log_file: an open file for writing output - halt_on_output: string; kill the process and return if this string is found - in the output stream from the process. - print_timestamps: boolean indicating whether a formatted timestamp should be - prepended to each line of output. - """ - stdout_queue = Queue.Queue() - log_thread = EnqueueThread(proc.stdout, stdout_queue) - log_thread.start() - try: - all_output = [] - t_0 = time.time() - while True: - code = proc.poll() - try: - output = stdout_queue.get_nowait() - all_output.append(output) - if output and print_timestamps: - timestamp = datetime.datetime.now().strftime('%H:%M:%S.%f') - output = ''.join(['[%s] %s\n' % (timestamp, line) - for line in output.splitlines()]) - if echo: - sys.stdout.write(output) - sys.stdout.flush() - if log_file: - log_file.write(output) - log_file.flush() - if halt_on_output and halt_on_output in output: - proc.terminate() - break - except Queue.Empty: - if code != None: # proc has finished running - break - time.sleep(0.5) - if timeout and time.time() - t_0 > timeout: - proc.terminate() - raise TimeoutException( - ''.join(all_output), - 'Subprocess exceeded timeout of %ds' % timeout) - finally: - log_thread.stop() - log_thread.join() - return (code, ''.join(all_output)) - - -def log_process_after_completion(proc, echo=True, timeout=None, log_file=None): - """ Wait for proc to complete and return a tuple containing the exit code of - proc and the contents of stdout. Unlike log_process_in_real_time, does not - attempt to read stdout from proc in real time. - - proc: an instance of Popen referring to a running subprocess. - echo: boolean indicating whether to print the output received from proc.stdout - timeout: number of seconds allotted for the process to run. Raises a - TimeoutException if the run time exceeds the timeout. - log_file: an open file for writing outout - """ - t_0 = time.time() - code = None - while code is None: - if timeout and time.time() - t_0 > timeout: - raise TimeoutException( - proc.communicate()[0], - 'Subprocess exceeded timeout of %ds' % timeout) - time.sleep(0.5) - code = proc.poll() - output = proc.communicate()[0] - if echo: - print output - if log_file: - log_file.write(output) - log_file.flush() - return (code, output) - - -def run(cmd, echo=True, shell=False, timeout=None, print_timestamps=True, - log_in_real_time=True): - """ Run 'cmd' in a shell and return the combined contents of stdout and - stderr (Blocking). Throws an exception if the command exits non-zero. - - cmd: list of strings (or single string, iff shell==True) indicating the - command to run - echo: boolean indicating whether we should print the command and log output - shell: boolean indicating whether we are using advanced shell features. Use - only when absolutely necessary, since this allows a lot more freedom which - could be exploited by malicious code. See the warning here: - http://docs.python.org/library/subprocess.html#popen-constructor - timeout: optional, integer indicating the maximum elapsed time in seconds - print_timestamps: boolean indicating whether a formatted timestamp should be - prepended to each line of output. Unused if echo or log_in_real_time is - False. - log_in_real_time: boolean indicating whether to read stdout from the - subprocess in real time instead of when the process finishes. If echo is - False, we never log in real time, even if log_in_real_time is True. - """ - proc = run_async(cmd, echo=echo, shell=shell) - # If we're not printing the output, we don't care if the output shows up in - # real time, so don't bother. - if log_in_real_time and echo: - (returncode, output) = log_process_in_real_time(proc, echo=echo, - timeout=timeout, print_timestamps=print_timestamps) - else: - (returncode, output) = log_process_after_completion(proc, echo=echo, - timeout=timeout) - if returncode != 0: - raise CommandFailedException( - output, - 'Command failed with code %d: %s' % (returncode, cmd)) - return output - - -def run_retry(cmd, echo=True, shell=False, attempts=1, - secs_between_attempts=DEFAULT_SECS_BETWEEN_ATTEMPTS, - timeout=None, print_timestamps=True): - """ Wrapper for run() which makes multiple attempts until either the command - succeeds or the maximum number of attempts is reached. """ - attempt = 1 - while True: - try: - return run(cmd, echo=echo, shell=shell, timeout=timeout, - print_timestamps=print_timestamps) - except CommandFailedException: - if attempt >= attempts: - raise - print 'Command failed. Retrying in %d seconds...' % secs_between_attempts - time.sleep(secs_between_attempts) - attempt += 1
diff --git a/slave/skia_slave_scripts/utils/ssh_utils.py b/slave/skia_slave_scripts/utils/ssh_utils.py deleted file mode 100644 index 4feb56b..0000000 --- a/slave/skia_slave_scripts/utils/ssh_utils.py +++ /dev/null
@@ -1,192 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" This module contains tools related to ssh used by the buildbot scripts. """ - -import atexit -import os -import re -import shell_utils -import signal - -def PutSCP(local_path, remote_path, username, host, port, recurse=False, - options=None): - """ Send a file to the given host over SCP. Assumes that public key - authentication is set up between the client and server. - - local_path: path to the file to send on the client - remote_path: destination path for the file on the server - username: ssh login name - host: hostname or ip address of the server - port: port on the server to use - recurse: boolean indicating whether to transmit everything in a folder - options: list of extra options to pass to scp - """ - # TODO: This will hang for a while if the host does not recognize the client - cmd = ['scp'] - if options: - cmd.extend(options) - if recurse: - cmd.append('-r') - cmd.extend( - ['-P', port, local_path, '%s@%s:%s' % (username, host, remote_path)]) - shell_utils.run(cmd) - - -def MultiPutSCP(local_paths, remote_path, username, host, port, options=None): - """ Send files to the given host over SCP. Assumes that public key - authentication is set up between the client and server. - - local_paths: list of paths of files and directories to send on the client - remote_path: destination directory path on the server - username: ssh login name - host: hostname or ip address of the server - port: port on the server to use - options: list of extra options to pass to scp - """ - # TODO: This will hang for a while if the host does not recognize the client - cmd = ['scp'] - if options: - cmd.extend(options) - cmd.extend(['-r', '-P', port]) - cmd.extend(local_paths) - cmd.append('%s@%s:%s' % (username, host, remote_path)) - shell_utils.run(cmd) - - -def GetSCP(local_path, remote_path, username, host, port, recurse=False, - options=None): - """ Retrieve a file from the given host over SCP. Assumes that public key - authentication is set up between the client and server. - - local_path: destination path for the file on the client - remote_path: path to the file to retrieve on the server - username: ssh login name - host: hostname or ip address of the server - port: port on the server to use - recurse: boolean indicating whether to transmit everything in a folder - options: list of extra options to pass to scp - """ - # TODO: This will hang for a while if the host does not recognize the client - cmd = ['scp'] - if options: - cmd.extend(options) - if recurse: - cmd.append('-r') - cmd.extend( - ['-P', port, '%s@%s:%s' % (username, host, remote_path), local_path]) - shell_utils.run(cmd) - - -def RunSSHCmd(username, host, port, command, echo=True, options=None): - """ Login to the given host and run the given command. - - username: ssh login name - host: hostname or ip address of the server - port: port on the server to use - command: (string) command to run on the server - options: list of extra options to pass to ssh - """ - # TODO: This will hang for a while if the host does not recognize the client - cmd = ['ssh'] - if options: - cmd.extend(options) - cmd.extend(['-p', port, '%s@%s' % (username, host), command]) - return shell_utils.run(cmd, echo=echo) - - -def ShellEscape(arg): - """ Escape a single argument for passing into a remote shell - """ - arg = re.sub(r'(["\\])', r'\\\1', arg) - return '"%s"' % arg if re.search(r'[\' \t\r\n]', arg) else arg - - -def RunSSH(username, host, port, command, echo=True, options=None): - """ Login to the given host and run the given command. - - username: ssh login name - host: hostname or ip address of the server - port: port on the server to use - command: command to run on the server in list format - options: list of extra options to pass to ssh - """ - cmd = ' '.join(ShellEscape(arg) for arg in command) - return RunSSHCmd(username, host, port, cmd, echo=echo, options=options) - - -class SshDestination(object): - """ Convenience class to remember a host, port, and username. - Wraps the other functions in this module. - """ - def __init__(self, host, port, username, options=None): - """ - host - (string) hostname of the target - port - (string or int) sshd port on the target - username - (string) remote username - options - (list of strings) extra options to pass to ssh and scp. - """ - self.host = host - self.port = str(port) - self.user = username - self.options = options - - def Put(self, local_path, remote_path, recurse=False): - return PutSCP(local_path, remote_path, self.user, self.host, - self.port, recurse=recurse, options=self.options) - - def MultiPut(self, local_paths, remote_path): - return MultiPutSCP(local_paths, remote_path, self.user, self.host, - self.port, options=self.options) - - def Get(self, local_path, remote_path, recurse=False): - return GetSCP(local_path, remote_path, self.user, - self.host, self.port, recurse=recurse, options=self.options) - - def RunCmd(self, command, echo=True): - return RunSSHCmd(self.user, self.host, self.port, command, - echo=echo, options=self.options) - - def Run(self, command, echo=True): - return RunSSH(self.user, self.host, self.port, command, - echo=echo, options=self.options) - - -def search_within_string(input_string, pattern): - """Search for regular expression in a string. - - input_string: (string) to be searched - pattern: (string) to be passed to re.compile, with a symbolic - group named 'return'. - default: what to return if no match - - Returns a string or None - """ - match = re.search(pattern, input_string) - return match.group('return') if match else None - -def SSHAdd(key_file): - """ Call ssh-add, and call ssh-agent if necessary. - """ - assert os.path.isfile(key_file) - try: - shell_utils.run(['ssh-add', key_file], - log_in_real_time=False) - return - except shell_utils.CommandFailedException: - ssh_agent_output = shell_utils.run(['ssh-agent', '-s'], - log_in_real_time=False) - if not ssh_agent_output: - raise Exception('ssh-agent did not print anything') - ssh_auth_sock = search_within_string( - ssh_agent_output, r'SSH_AUTH_SOCK=(?P<return>[^;]*);') - ssh_agent_pid = search_within_string( - ssh_agent_output, r'SSH_AGENT_PID=(?P<return>[^;]*);') - if not (ssh_auth_sock and ssh_agent_pid): - raise Exception('ssh-agent did not print meaningful data') - os.environ['SSH_AUTH_SOCK'] = ssh_auth_sock - os.environ['SSH_AGENT_PID'] = ssh_agent_pid - atexit.register(os.kill, int(ssh_agent_pid), signal.SIGTERM) - shell_utils.run(['ssh-add', key_file])
diff --git a/slave/skia_slave_scripts/utils/sync_skia_in_chrome.py b/slave/skia_slave_scripts/utils/sync_skia_in_chrome.py index 66c7c5b..77281ab 100644 --- a/slave/skia_slave_scripts/utils/sync_skia_in_chrome.py +++ b/slave/skia_slave_scripts/utils/sync_skia_in_chrome.py
@@ -7,14 +7,14 @@ from config_private import SKIA_GIT_URL from optparse import OptionParser +from py.utils.git_utils import GIT +from py.utils import git_utils +from py.utils import misc +from py.utils import shell_utils -from git_utils import GIT import gclient_utils -import git_utils -import misc import os import re -import shell_utils import shlex import sys import urllib2
diff --git a/slave/skia_slave_scripts/utils/upload_to_bucket.py b/slave/skia_slave_scripts/utils/upload_to_bucket.py index 23057fa..8bf496b 100755 --- a/slave/skia_slave_scripts/utils/upload_to_bucket.py +++ b/slave/skia_slave_scripts/utils/upload_to_bucket.py
@@ -13,7 +13,7 @@ --source_filepath=../../../DEPS --dest_gsbase=gs://chromium-skia-gm """ -from utils import misc +from py.utils import misc import optparse import os import sys
diff --git a/slave/skia_slave_scripts/webpages_playback.py b/slave/skia_slave_scripts/webpages_playback.py index c61b022..385fa14 100644 --- a/slave/skia_slave_scripts/webpages_playback.py +++ b/slave/skia_slave_scripts/webpages_playback.py
@@ -58,17 +58,20 @@ import traceback -from utils import misc # Set the PYTHONPATH for this script to include chromium_buildbot scripts, # and site_config. -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'scripts')) -sys.path.append(os.path.join(misc.BUILDBOT_PATH, 'third_party', - 'chromium_buildbot', 'site_config')) +BUILDBOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) +sys.path.append(os.path.join(BUILDBOT_PATH, 'common')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'scripts')) +sys.path.append(os.path.join(BUILDBOT_PATH, 'third_party', 'chromium_buildbot', + 'site_config')) from utils import file_utils from utils import gs_utils -from utils import shell_utils +from py.utils import misc +from py.utils import shell_utils from slave import slave_utils