add set_acl() and get_acl() to gs_utils.py
As needed for uploading SKPs and their renderings (see bug 2732)
BUG=skia:2732
R=rmistry@google.com
Review URL: https://codereview.chromium.org/390193002
diff --git a/py/utils/gs_utils.py b/py/utils/gs_utils.py
index 46c9bf1..afc3748 100755
--- a/py/utils/gs_utils.py
+++ b/py/utils/gs_utils.py
@@ -7,10 +7,12 @@
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
-Utilities for accessing Google Cloud Storage, using the boto library.
+Utilities for accessing Google Cloud Storage, using the boto library (wrapper
+for the XML API).
-See http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html
-for implementation tips.
+API/library references:
+- https://developers.google.com/storage/docs/reference-guide
+- http://googlecloudstorage.blogspot.com/2012/09/google-cloud-storage-tutorial-using-boto.html
"""
# pylint: enable=C0301
@@ -34,11 +36,36 @@
# We need to insert at the beginning of the path, to make sure that our
# imported versions are favored over others that might be in the path.
sys.path.insert(0, import_dirpath)
+from boto.gs import acl
from boto.gs.connection import GSConnection
from boto.gs.key import Key
from boto.s3.bucketlistresultset import BucketListResultSet
from boto.s3.prefix import Prefix
+# Permissions that may be set on each file in Google Storage.
+# See SupportedPermissions in
+# https://github.com/boto/boto/blob/develop/boto/gs/acl.py
+PERMISSION_NONE = None
+PERMISSION_OWNER = 'FULL_CONTROL'
+PERMISSION_READ = 'READ'
+PERMISSION_WRITE = 'WRITE'
+
+# Types of identifiers we can use to set ACLs.
+ID_TYPE_GROUP_BY_DOMAIN = acl.GROUP_BY_DOMAIN
+ID_TYPE_GROUP_BY_EMAIL = acl.GROUP_BY_EMAIL
+ID_TYPE_GROUP_BY_ID = acl.GROUP_BY_ID
+ID_TYPE_USER_BY_EMAIL = acl.USER_BY_EMAIL
+ID_TYPE_USER_BY_ID = acl.USER_BY_ID
+
+# Which field we get/set in ACL entries, depending on ID_TYPE.
+FIELD_BY_ID_TYPE = {
+ ID_TYPE_GROUP_BY_DOMAIN: 'domain',
+ ID_TYPE_GROUP_BY_EMAIL: 'email_address',
+ ID_TYPE_GROUP_BY_ID: 'id',
+ ID_TYPE_USER_BY_EMAIL: 'email_address',
+ ID_TYPE_USER_BY_ID: 'id',
+}
+
class GSUtils(object):
"""Utilities for accessing Google Cloud Storage, using the boto library."""
@@ -113,6 +140,96 @@
with open(dest_path, 'w') as f:
item.get_contents_to_file(fp=f)
+ def get_acl(self, bucket, path, id_type, id_value):
+ """Retrieve partial access permissions on a single file in Google Storage.
+
+ Various users who match this id_type/id_value pair may have access rights
+ other than that returned by this call, if they have been granted those
+ rights based on *other* id_types (e.g., perhaps they have group access
+ rights, beyond their individual access rights).
+
+ Params:
+ bucket: GS bucket
+ path: full path (Posix-style) to the file within that bucket
+ id_type: must be one of the ID_TYPE_* constants defined above
+ id_value: get permissions for users whose id_type field contains this
+ value
+
+ Returns: the PERMISSION_* constant which has been set for users matching
+ this id_type/id_value, on this file; or PERMISSION_NONE if no such
+ permissions have been set.
+ """
+ field = FIELD_BY_ID_TYPE[id_type]
+ conn = self._create_connection()
+ b = conn.get_bucket(bucket_name=bucket)
+ acls = b.get_acl(key_name=path)
+ matching_entries = [entry for entry in acls.entries.entry_list
+ if (entry.scope.type == id_type) and
+ (getattr(entry.scope, field) == id_value)]
+ if matching_entries:
+ assert len(matching_entries) == 1, '%d == 1' % len(matching_entries)
+ return matching_entries[0].permission
+ else:
+ return PERMISSION_NONE
+
+ def set_acl(self, bucket, path, id_type, id_value, permission):
+ """Set partial access permissions on a single file in Google Storage.
+
+ Note that a single set_acl() call will not guarantee what access rights any
+ given user will have on a given file, because permissions are additive.
+ (E.g., if you set READ permission for a group, but a member of that group
+ already has WRITE permission, that member will still have WRITE permission.)
+ TODO(epoger): Do we know that for sure? I *think* that's how it works...
+
+ If there is already a permission set on this file for this id_type/id_value
+ combination, this call will overwrite it.
+
+ Params:
+ bucket: GS bucket
+ path: full path (Posix-style) to the file within that bucket
+ id_type: must be one of the ID_TYPE_* constants defined above
+ id_value: add permission for users whose id_type field contains this value
+ permission: permission to add for users matching id_type/id_value;
+ must be one of the PERMISSION_* constants defined above.
+ If PERMISSION_NONE, then any permissions will be granted to this
+ particular id_type/id_value will be removed... but, given that
+ permissions are additive, specific users may still have access rights
+ based on permissions given to *other* id_type/id_value pairs.
+
+ Example Code:
+ bucket = 'gs://bucket-name'
+ path = 'path/to/file'
+ id_type = ID_TYPE_USER_BY_EMAIL
+ id_value = 'epoger@google.com'
+ set_acl(bucket, path, id_type, id_value, PERMISSION_READ)
+ assert PERMISSION_READ == get_acl(bucket, path, id_type, id_value)
+ set_acl(bucket, path, id_type, id_value, PERMISSION_WRITE)
+ assert PERMISSION_WRITE == get_acl(bucket, path, id_type, id_value)
+ """
+ field = FIELD_BY_ID_TYPE[id_type]
+ conn = self._create_connection()
+ b = conn.get_bucket(bucket_name=bucket)
+ acls = b.get_acl(key_name=path)
+
+ # Remove any existing entries that refer to the same id_type/id_value,
+ # because the API will fail if we try to set more than one.
+ matching_entries = [entry for entry in acls.entries.entry_list
+ if (entry.scope.type == id_type) and
+ (getattr(entry.scope, field) == id_value)]
+ if matching_entries:
+ assert len(matching_entries) == 1, '%d == 1' % len(matching_entries)
+ acls.entries.entry_list.remove(matching_entries[0])
+
+ # Add a new entry to the ACLs.
+ if permission != PERMISSION_NONE:
+ args = {'type': id_type, 'permission': permission}
+ args[field] = id_value
+ new_entry = acl.Entry(**args)
+ acls.entries.entry_list.append(new_entry)
+
+ # Finally, write back the modified ACLs.
+ b.set_acl(acl_or_str=acls, key_name=path)
+
def list_bucket_contents(self, bucket, subdir=None):
"""Returns files in the Google Storage bucket as a (dirs, files) tuple.
@@ -214,6 +331,41 @@
assert dirs == [], '%s == []' % dirs
assert files == filenames_to_upload, '%s == %s' % (files, filenames_to_upload)
+ # Manipulate ACLs on one of those files, and verify them.
+ # TODO(epoger): Test id_types other than ID_TYPE_GROUP_BY_DOMAIN ?
+ # TODO(epoger): Test setting multiple ACLs on the same file?
+ id_type = ID_TYPE_GROUP_BY_DOMAIN
+ id_value = 'google.com'
+ fullpath = posixpath.join(remote_dir, subdir, filenames_to_upload[0])
+ # Make sure ACL is empty to start with ...
+ gs.set_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value, permission=PERMISSION_NONE)
+ permission = gs.get_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value)
+ assert permission == PERMISSION_NONE, '%s == %s' % (
+ permission, PERMISSION_NONE)
+ # ... set it to OWNER ...
+ gs.set_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value, permission=PERMISSION_OWNER)
+ permission = gs.get_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value)
+ assert permission == PERMISSION_OWNER, '%s == %s' % (
+ permission, PERMISSION_OWNER)
+ # ... now set it to READ ...
+ gs.set_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value, permission=PERMISSION_READ)
+ permission = gs.get_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value)
+ assert permission == PERMISSION_READ, '%s == %s' % (
+ permission, PERMISSION_READ)
+ # ... and clear it again to finish.
+ gs.set_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value, permission=PERMISSION_NONE)
+ permission = gs.get_acl(bucket=bucket, path=fullpath,
+ id_type=id_type, id_value=id_value)
+ assert permission == PERMISSION_NONE, '%s == %s' % (
+ permission, PERMISSION_NONE)
+
# Download the files we uploaded to Google Storage, and validate contents.
local_dest_dir = tempfile.mkdtemp()
try:
