gs_utils.py: use boto instead of google-api-python-client library so we can use .boto file credentials
Also adds these public functions:
- delete_file(bucket, path)
- upload_file(source_path, dest_bucket, dest_path)
and a self-test that exercises all functions.
BUG=skia:2732
R=jcgregorio@google.com
Review URL: https://codereview.chromium.org/387233003
diff --git a/DEPS b/DEPS
index 3580231..c866727 100644
--- a/DEPS
+++ b/DEPS
@@ -3,8 +3,5 @@
# Dependencies on outside packages.
#
deps = {
- "third_party/externals/google-api-python-client" : "https://github.com/google/google-api-python-client.git@ecc64a0a2baa4a77f35dec83ad05c6c9ba2d2841",
- "third_party/externals/httplib2" : "https://github.com/jcgregorio/httplib2.git@7d1b88a3cf34774242bf4c0578c09c0092bb05d8",
- "third_party/externals/oauth2client" : "https://github.com/google/oauth2client.git@d02b317af0313dcf66755844f5421651af5eb356",
- "third_party/externals/uritemplate-py" : "https://github.com/uri-templates/uritemplate-py.git@1e780a49412cdbb273e9421974cb91845c124f3f",
+ "third_party/externals/boto" : "https://github.com/boto/boto.git@410909e993746c1b7a2604721d82afc0d2778a8a",
}
diff --git a/py/utils/gs_utils.py b/py/utils/gs_utils.py
index efab8ad..c30295b 100755
--- a/py/utils/gs_utils.py
+++ b/py/utils/gs_utils.py
@@ -1,49 +1,79 @@
#!/usr/bin/python
+# pylint: disable=C0301
"""
Copyright 2014 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
-Utilities for accessing Google Cloud Storage.
+Utilities for accessing Google Cloud Storage, using the boto library.
+
+See http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html
+for implementation tips.
"""
+# pylint: enable=C0301
# System-level imports
+import errno
import os
import posixpath
+import random
+import re
+import shutil
import sys
+import tempfile
# Imports from third-party code
TRUNK_DIRECTORY = os.path.abspath(os.path.join(
os.path.dirname(__file__), os.pardir, os.pardir))
-for import_subdir in ['google-api-python-client', 'httplib2', 'oauth2client',
- 'uritemplate-py']:
+for import_subdir in ['boto']:
import_dirpath = os.path.join(
TRUNK_DIRECTORY, 'third_party', 'externals', import_subdir)
if import_dirpath not in sys.path:
# We need to insert at the beginning of the path, to make sure that our
# imported versions are favored over others that might be in the path.
- # Also, the google-api-python-client checkout contains an empty
- # oauth2client directory, which will confuse things unless we insert
- # our checked-out oauth2client in front of it in the path.
sys.path.insert(0, import_dirpath)
-try:
- from googleapiclient.discovery import build as build_service
-except ImportError:
- # We should not require any googleapiclient dependencies to be
- # installed at a system level, but in the meanwhile, if developers run into
- # trouble they can install those system-level dependencies to get unblocked.
- print ('We should not require any googleapiclient dependencies to be '
- 'installed at a system level, but it seems like some are missing. '
- 'Please install google-api-python-client to get those dependencies; '
- 'directions can be found at https://developers.google.com/'
- 'api-client-library/python/start/installation . '
- 'More details in http://skbug.com/2641 ')
- raise
+from boto.gs.connection import GSConnection
+from boto.gs.key import Key
+from boto.s3.bucketlistresultset import BucketListResultSet
+from boto.s3.prefix import Prefix
-# Local imports
-import url_utils
+
+def delete_file(bucket, path):
+ """Delete a single file within a GS bucket.
+
+ TODO(epoger): what if bucket or path does not exist? Should probably raise
+ an exception. Implement, and add a test to exercise this.
+
+ Params:
+ bucket: GS bucket to delete a file from
+ path: full path (Posix-style) of the file within the bucket to delete
+ """
+ conn = _create_connection()
+ b = conn.get_bucket(bucket_name=bucket)
+ item = Key(b)
+ item.key = path
+ item.delete()
+
+
+def upload_file(source_path, dest_bucket, dest_path):
+ """Upload contents of a local file to Google Storage.
+
+ TODO(epoger): Add the extra parameters provided by upload_file() within
+ https://github.com/google/skia-buildbot/blob/master/slave/skia_slave_scripts/utils/old_gs_utils.py ,
+ so we can replace that function with this one.
+
+ params:
+ source_path: full path (local-OS-style) on local disk to read from
+ dest_bucket: GCS bucket to copy the file to
+ dest_path: full path (Posix-style) within that bucket
+ """
+ conn = _create_connection()
+ b = conn.get_bucket(bucket_name=dest_bucket)
+ item = Key(b)
+ item.key = dest_path
+ item.set_contents_from_filename(filename=source_path)
def download_file(source_bucket, source_path, dest_path,
@@ -57,44 +87,162 @@
create_subdirs_if_needed: boolean; whether to create subdirectories as
needed to create dest_path
"""
- source_http_url = posixpath.join(
- 'http://storage.googleapis.com', source_bucket, source_path)
- url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path,
- create_subdirs_if_needed=create_subdirs_if_needed)
+ conn = _create_connection()
+ b = conn.get_bucket(bucket_name=source_bucket)
+ item = Key(b)
+ item.key = source_path
+ if create_subdirs_if_needed:
+ _makedirs_if_needed(os.path.dirname(dest_path))
+ with open(dest_path, 'w') as f:
+ item.get_contents_to_file(fp=f)
def list_bucket_contents(bucket, subdir=None):
""" Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple.
- Uses the API documented at
- https://developers.google.com/storage/docs/json_api/v1/objects/list
-
Args:
bucket: name of the Google Storage bucket
subdir: directory within the bucket to list, or None for root directory
"""
- # The GCS command relies on the subdir name (if any) ending with a slash.
- if subdir and not subdir.endswith('/'):
- subdir += '/'
- subdir_length = len(subdir) if subdir else 0
+ # The GS command relies on the prefix (if any) ending with a slash.
+ prefix = subdir or ''
+ if prefix and not prefix.endswith('/'):
+ prefix += '/'
+ prefix_length = len(prefix) if prefix else 0
- storage = build_service('storage', 'v1')
- command = storage.objects().list(
- bucket=bucket, delimiter='/', fields='items(name),prefixes',
- prefix=subdir)
- results = command.execute()
-
- # The GCS command returned two subdicts:
- # prefixes: the full path of every directory within subdir, with trailing '/'
- # items: property dict for each file object within subdir
- # (including 'name', which is full path of the object)
+ conn = _create_connection()
+ b = conn.get_bucket(bucket_name=bucket)
+ lister = BucketListResultSet(bucket=b, prefix=prefix, delimiter='/')
dirs = []
- for dir_fullpath in results.get('prefixes', []):
- dir_basename = dir_fullpath[subdir_length:]
- dirs.append(dir_basename[:-1]) # strip trailing slash
files = []
- for file_properties in results.get('items', []):
- file_fullpath = file_properties['name']
- file_basename = file_fullpath[subdir_length:]
- files.append(file_basename)
+ for item in lister:
+ t = type(item)
+ if t is Key:
+ files.append(item.key[prefix_length:])
+ elif t is Prefix:
+ dirs.append(item.name[prefix_length:-1])
return (dirs, files)
+
+
+def _config_file_as_dict(filepath):
+ """Reads a boto-style config file into a dict.
+
+ Parses all lines from the file of this form: key = value
+ TODO(epoger): Create unittest.
+
+ Params:
+ filepath: path to config file on local disk
+
+ Returns: contents of the config file, as a dictionary
+
+ Raises exception if file not found.
+ """
+ dic = {}
+ line_regex = re.compile('^\s*(\S+)\s*=\s*(\S+)\s*$')
+ with open(filepath) as f:
+ for line in f:
+ match = line_regex.match(line)
+ if match:
+ (key, value) = match.groups()
+ dic[key] = value
+ return dic
+
+
+def _create_connection(boto_file_path=os.path.join('~','.boto')):
+ """Returns a GSConnection object we can use to access Google Storage.
+
+ Params:
+ boto_file_path: full path (local-OS-style) on local disk where .boto
+ credentials file can be found
+
+ TODO(epoger): Change this module to be object-based, where __init__() reads
+ the boto file into boto_dict once instead of repeatedly for each operation.
+
+ TODO(epoger): if the file does not exist, rather than raising an exception,
+ create a GSConnection that can operate on public files.
+ """
+ boto_file_path = os.path.expanduser(boto_file_path)
+ print 'Reading boto file from %s' % boto_file_path
+ boto_dict = _config_file_as_dict(filepath=boto_file_path)
+ return GSConnection(
+ gs_access_key_id=boto_dict['gs_access_key_id'],
+ gs_secret_access_key=boto_dict['gs_secret_access_key'])
+
+
+def _makedirs_if_needed(path):
+ """ Creates a directory (and any parent directories needed), if it does not
+ exist yet.
+
+ Args:
+ path: full path of directory to create
+ """
+ try:
+ os.makedirs(path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+
+def _run_self_test():
+ bucket = 'chromium-skia-gm'
+ remote_dir = 'gs_utils_test/%d' % random.randint(0, sys.maxint)
+ subdir = 'subdir'
+ filenames_to_upload = ['file1', 'file2']
+
+ # Upload test files to Google Storage.
+ local_src_dir = tempfile.mkdtemp()
+ os.mkdir(os.path.join(local_src_dir, subdir))
+ try:
+ for filename in filenames_to_upload:
+ with open(os.path.join(local_src_dir, subdir, filename), 'w') as f:
+ f.write('contents of %s\n' % filename)
+ upload_file(source_path=os.path.join(local_src_dir, subdir, filename),
+ dest_bucket=bucket,
+ dest_path=posixpath.join(remote_dir, subdir, filename))
+ finally:
+ shutil.rmtree(local_src_dir)
+
+ # Get a list of the files we uploaded to Google Storage.
+ (dirs, files) = list_bucket_contents(
+ bucket=bucket, subdir=remote_dir)
+ assert dirs == [subdir]
+ assert files == []
+ (dirs, files) = list_bucket_contents(
+ bucket=bucket, subdir=posixpath.join(remote_dir, subdir))
+ assert dirs == []
+ assert files == filenames_to_upload
+
+ # Download the files we uploaded to Google Storage, and validate contents.
+ local_dest_dir = tempfile.mkdtemp()
+ try:
+ for filename in filenames_to_upload:
+ download_file(source_bucket=bucket,
+ source_path=posixpath.join(remote_dir, subdir, filename),
+ dest_path=os.path.join(local_dest_dir, subdir, filename),
+ create_subdirs_if_needed=True)
+ with open(os.path.join(local_dest_dir, subdir, filename)) as f:
+ file_contents = f.read()
+ assert file_contents == 'contents of %s\n' % filename
+ finally:
+ shutil.rmtree(local_dest_dir)
+
+ # Delete all the files we uploaded to Google Storage.
+ for filename in filenames_to_upload:
+ delete_file(bucket=bucket,
+ path=posixpath.join(remote_dir, subdir, filename))
+
+ # Confirm that we deleted all the files we uploaded to Google Storage.
+ (dirs, files) = list_bucket_contents(
+ bucket=bucket, subdir=posixpath.join(remote_dir, subdir))
+ assert dirs == []
+ assert files == []
+
+
+# TODO(epoger): How should we exercise this self-test?
+# I avoided using the standard unittest framework, because these Google Storage
+# operations are expensive and require .boto permissions.
+#
+# How can we automatically test this code without wasting too many resources
+# or needing .boto permissions?
+if __name__ == '__main__':
+ _run_self_test()
