infra/bots/recipe_modules/gsutil/api.py

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


from recipe_engine import recipe_api

UPLOAD_ATTEMPTS = 5

class GSUtilApi(recipe_api.RecipeApi):
  def __call__(self, step_name, *args):
    """Run gsutil with the given args.

       This assumes there exists an executable called gsutil on the PATH.
       This probably only works for Linux/Mac, but those are the only
       hosts that we try to upload to GCS from anyway.
    """
    return self.m.step(step_name, cmd=['gsutil'] + list(args))

  def cp(self, name, src, dst, extra_gsutil_args=None, extra_args=None,
         multithread=False):
    """Attempt to upload or download files to/from Google Cloud Storage (GCS).

    Args:
      name: string. Will be used to fill out the step name.
      src: string. Absolute path for a local file or gcs file (e.g. gs://...)
      dst: string. Same as src.
      extra_gsutil_args: optional list of args to be passed to gsutil before the
        cp command.
      extra_args: optional list of args to be passed to gsutil cp. e.g. [-Z]
        asks all files be compressed with gzip after upload and before download.
      multi_thread: if the -m argument should be used to copy multiple items
        at once (e.g. gsutil -m cp foo* gs://bar/dir)

    If the operation fails, it will be retried multiple times.
    """
    cmd = []
    if extra_gsutil_args:
      cmd.extend(extra_gsutil_args)
    if multithread:
      cmd.append('-m')
    cmd.append('cp')
    if extra_args:
      cmd.extend(extra_args)
    cmd.extend([src, dst])

    name = 'upload %s' % name
    for i in range(UPLOAD_ATTEMPTS):
      step_name = name
      if i > 0:
        step_name += ' (attempt %d)' % (i+1)
      try:
        self(step_name, *cmd)
        break
      except self.m.step.StepFailure:
        if i == UPLOAD_ATTEMPTS - 1:
          raise