Add infrastructure for maintaining and using DEPS (#2684)
This adds in a script for updating DEPS and another for pulling in the
specified versions in the DEPS file.
DEPS entries that are not part of the documented build processs are
removed. Using the documented cmake process with them made for a very
sad state of one's checkout. If this removal breaks workflows we can
revist this, since the state is saved in git.
DEPS are rolled forward, so that this is taking use to a known good
state. These look like scary big rolls, but in reality the bots and
most people are running near HEAD for external/, so this integration
should already have been tested.
Summary of DEPS roll:
+ Rolling 3 dependencies
+ Roll external/effcee/ 04b624799..b83b58d17 (14 commits)
+ Roll external/googletest/ 98a0d007d..2f42d769a (576 commits)
+ Roll external/re2/ 6cf8ccd82..848dfb7e1 (90 commits)
+ Created with:
+ roll-dep external/effcee external/googletest external/re2 external/spirv-headers
Fixes #2665
diff --git a/DEPS b/DEPS
index db24e9b..f4557e9 100644
--- a/DEPS
+++ b/DEPS
@@ -1,174 +1,26 @@
use_relative_paths = True
vars = {
- 'chromium_git': 'https://chromium.googlesource.com',
'github': 'https://github.com',
- 'build_revision': '037f38ae0fe5e11b4f7c33b750fd7a1e9634a606',
- 'buildtools_revision': 'ab7b6a7b350dd15804c87c20ce78982811fdd76f',
- 'clang_revision': 'abe5e4f9dc0f1df848c7a0efa05256253e77a7b7',
- 'effcee_revision': '04b624799f5a9dbaf3fa1dbed2ba9dce2fc8dcf2',
- 'googletest_revision': '98a0d007d7092b72eea0e501bb9ad17908a1a036',
- 'testing_revision': '340252637e2e7c72c0901dcbeeacfff419e19b59',
- 're2_revision': '6cf8ccd82dbaab2668e9b13596c68183c9ecd13f',
+ 'effcee_revision': 'b83b58d177b797edd1f94c5f10837f2cc2863f0a',
+ 'googletest_revision': '2f42d769ad1b08742f7ccb5ad4dd357fc5ff248c',
+ 're2_revision': '848dfb7e1d7ba641d598cb66f81590f3999a555a',
'spirv_headers_revision': 'de99d4d834aeb51dd9f099baa285bd44fd04bb3d',
}
deps = {
- "build":
- Var('chromium_git') + "/chromium/src/build.git@" + Var('build_revision'),
-
- 'buildtools':
- Var('chromium_git') + '/chromium/buildtools.git@' +
- Var('buildtools_revision'),
-
- 'external/spirv-headers':
- Var('github') + '/KhronosGroup/SPIRV-Headers.git@' +
- Var('spirv_headers_revision'),
+ 'external/effcee':
+ Var('github') + '/google/effcee.git@' + Var('effcee_revision'),
'external/googletest':
Var('github') + '/google/googletest.git@' + Var('googletest_revision'),
- 'external/effcee':
- Var('github') + '/google/effcee.git@' + Var('effcee_revision'),
-
'external/re2':
Var('github') + '/google/re2.git@' + Var('re2_revision'),
- 'testing':
- Var('chromium_git') + '/chromium/src/testing@' +
- Var('testing_revision'),
-
- 'tools/clang':
- Var('chromium_git') + '/chromium/src/tools/clang@' + Var('clang_revision')
+ 'external/spirv-headers':
+ Var('github') + '/KhronosGroup/SPIRV-Headers.git@' +
+ Var('spirv_headers_revision'),
}
-recursedeps = [
- # buildtools provides clang_format, libc++, and libc++api
- 'buildtools',
-]
-
-hooks = [
- {
- 'name': 'gn_win',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=win32',
- '--no_auth',
- '--bucket', 'chromium-gn',
- '-s', 'SPIRV-Tools/buildtools/win/gn.exe.sha1',
- ],
- },
- {
- 'name': 'gn_mac',
- 'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=darwin',
- '--no_auth',
- '--bucket', 'chromium-gn',
- '-s', 'SPIRV-Tools/buildtools/mac/gn.sha1',
- ],
- },
- {
- 'name': 'gn_linux64',
- 'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=linux*',
- '--no_auth',
- '--bucket', 'chromium-gn',
- '-s', 'SPIRV-Tools/buildtools/linux64/gn.sha1',
- ],
- },
- # Pull clang-format binaries using checked-in hashes.
- {
- 'name': 'clang_format_win',
- 'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=win32',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'SPIRV-Tools/buildtools/win/clang-format.exe.sha1',
- ],
- },
- {
- 'name': 'clang_format_mac',
- 'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=darwin',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'SPIRV-Tools/buildtools/mac/clang-format.sha1',
- ],
- },
- {
- 'name': 'clang_format_linux',
- 'pattern': '.',
- 'action': [ 'download_from_google_storage',
- '--no_resume',
- '--platform=linux*',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'SPIRV-Tools/buildtools/linux64/clang-format.sha1',
- ],
- },
- {
- # Pull clang
- 'name': 'clang',
- 'pattern': '.',
- 'action': ['python',
- 'SPIRV-Tools/tools/clang/scripts/update.py'
- ],
- },
- {
- 'name': 'sysroot_arm',
- 'pattern': '.',
- 'condition': 'checkout_linux and checkout_arm',
- 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py',
- '--arch=arm'],
- },
- {
- 'name': 'sysroot_arm64',
- 'pattern': '.',
- 'condition': 'checkout_linux and checkout_arm64',
- 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py',
- '--arch=arm64'],
- },
- {
- 'name': 'sysroot_x86',
- 'pattern': '.',
- 'condition': 'checkout_linux and (checkout_x86 or checkout_x64)',
- 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py',
- '--arch=x86'],
- },
- {
- 'name': 'sysroot_mips',
- 'pattern': '.',
- 'condition': 'checkout_linux and checkout_mips',
- 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py',
- '--arch=mips'],
- },
- {
- 'name': 'sysroot_x64',
- 'pattern': '.',
- 'condition': 'checkout_linux and checkout_x64',
- 'action': ['python', 'SPIRV-Tools/build/linux/sysroot_scripts/install-sysroot.py',
- '--arch=x64'],
- },
- {
- # Update the Windows toolchain if necessary.
- 'name': 'win_toolchain',
- 'pattern': '.',
- 'condition': 'checkout_win',
- 'action': ['python', 'SPIRV-Tools/build/vs_toolchain.py', 'update', '--force'],
- },
- {
- # Update the Mac toolchain if necessary.
- 'name': 'mac_toolchain',
- 'pattern': '.',
- 'action': ['python', 'SPIRV-Tools/build/mac_toolchain.py'],
- },
-]
diff --git a/README.md b/README.md
index 9737f7e..8a5c85e 100644
--- a/README.md
+++ b/README.md
@@ -270,6 +270,10 @@
cmake [-G <platform-generator>] <spirv-dir>
```
+*Note*:
+The script `utils/git-sync-deps` can be used to checkout and/or update the
+contents of the repos under `external/` instead of manually maintaining them.
+
Once the build files have been generated, build using your preferred
development environment.
@@ -344,6 +348,13 @@
NDK_APP_OUT=`pwd`/app
```
+### Updating DEPS
+Occasionally the entries in DEPS will need to be updated. This is done on demand
+when there is a request to do this, often due to downstream breakages. There is
+a script `utils/roll_deps.sh` provided, which will generate a patch with the
+updated DEPS values. This will still need to be tested in your checkout to
+confirm that there are no integration issues that need to be resolved.
+
## Library
### Usage
diff --git a/utils/git-sync-deps b/utils/git-sync-deps
new file mode 100755
index 0000000..0575641
--- /dev/null
+++ b/utils/git-sync-deps
@@ -0,0 +1,282 @@
+#!/usr/bin/env python
+# Copyright 2014 Google Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Parse a DEPS file and git checkout all of the dependencies.
+
+Args:
+ An optional list of deps_os values.
+
+Environment Variables:
+ GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of
+ ['git', 'git.exe', 'git.bat'] in your default path.
+
+ GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
+ will use the file ../DEPS relative to this script's directory.
+
+ GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
+
+Git Config:
+ To disable syncing of a single repository:
+ cd path/to/repository
+ git config sync-deps.disable true
+
+ To re-enable sync:
+ cd path/to/repository
+ git config --unset sync-deps.disable
+"""
+
+
+import os
+import re
+import subprocess
+import sys
+import threading
+from builtins import bytes
+
+
+def git_executable():
+ """Find the git executable.
+
+ Returns:
+ A string suitable for passing to subprocess functions, or None.
+ """
+ envgit = os.environ.get('GIT_EXECUTABLE')
+ searchlist = ['git', 'git.exe', 'git.bat']
+ if envgit:
+ searchlist.insert(0, envgit)
+ with open(os.devnull, 'w') as devnull:
+ for git in searchlist:
+ try:
+ subprocess.call([git, '--version'], stdout=devnull)
+ except (OSError,):
+ continue
+ return git
+ return None
+
+
+DEFAULT_DEPS_PATH = os.path.normpath(
+ os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
+
+
+def usage(deps_file_path = None):
+ sys.stderr.write(
+ 'Usage: run to grab dependencies, with optional platform support:\n')
+ sys.stderr.write(' %s %s' % (sys.executable, __file__))
+ if deps_file_path:
+ parsed_deps = parse_file_to_dict(deps_file_path)
+ if 'deps_os' in parsed_deps:
+ for deps_os in parsed_deps['deps_os']:
+ sys.stderr.write(' [%s]' % deps_os)
+ sys.stderr.write('\n\n')
+ sys.stderr.write(__doc__)
+
+
+def git_repository_sync_is_disabled(git, directory):
+ try:
+ disable = subprocess.check_output(
+ [git, 'config', 'sync-deps.disable'], cwd=directory)
+ return disable.lower().strip() in ['true', '1', 'yes', 'on']
+ except subprocess.CalledProcessError:
+ return False
+
+
+def is_git_toplevel(git, directory):
+ """Return true iff the directory is the top level of a Git repository.
+
+ Args:
+ git (string) the git executable
+
+ directory (string) the path into which the repository
+ is expected to be checked out.
+ """
+ try:
+ toplevel = subprocess.check_output(
+ [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
+ return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel)
+ except subprocess.CalledProcessError:
+ return False
+
+
+def status(directory, checkoutable):
+ def truncate(s, length):
+ return s if len(s) <= length else s[:(length - 3)] + '...'
+ dlen = 36
+ directory = truncate(directory, dlen)
+ checkoutable = truncate(checkoutable, 40)
+ sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
+
+
+def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
+ """Checkout (and clone if needed) a Git repository.
+
+ Args:
+ git (string) the git executable
+
+ repo (string) the location of the repository, suitable
+ for passing to `git clone`.
+
+ checkoutable (string) a tag, branch, or commit, suitable for
+ passing to `git checkout`
+
+ directory (string) the path into which the repository
+ should be checked out.
+
+ verbose (boolean)
+
+ Raises an exception if any calls to git fail.
+ """
+ if not os.path.isdir(directory):
+ subprocess.check_call(
+ [git, 'clone', '--quiet', repo, directory])
+
+ if not is_git_toplevel(git, directory):
+ # if the directory exists, but isn't a git repo, you will modify
+ # the parent repostory, which isn't what you want.
+ sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
+ return
+
+ # Check to see if this repo is disabled. Quick return.
+ if git_repository_sync_is_disabled(git, directory):
+ sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
+ return
+
+ with open(os.devnull, 'w') as devnull:
+ # If this fails, we will fetch before trying again. Don't spam user
+ # with error infomation.
+ if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
+ cwd=directory, stderr=devnull):
+ # if this succeeds, skip slow `git fetch`.
+ if verbose:
+ status(directory, checkoutable) # Success.
+ return
+
+ # If the repo has changed, always force use of the correct repo.
+ # If origin already points to repo, this is a quick no-op.
+ subprocess.check_call(
+ [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
+
+ subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
+
+ subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
+
+ if verbose:
+ status(directory, checkoutable) # Success.
+
+
+def parse_file_to_dict(path):
+ dictionary = {}
+ contents = open(path).read()
+ # Need to convert Var() to vars[], so that the DEPS is actually Python. Var()
+ # comes from Autoroller using gclient which has a slightly different DEPS
+ # format.
+ contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents)
+ exec(contents, dictionary)
+ return dictionary
+
+
+def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
+ """Grab dependencies, with optional platform support.
+
+ Args:
+ deps_file_path (string) Path to the DEPS file.
+
+ command_line_os_requests (list of strings) Can be empty list.
+ List of strings that should each be a key in the deps_os
+ dictionary in the DEPS file.
+
+ Raises git Exceptions.
+ """
+ git = git_executable()
+ assert git
+
+ deps_file_directory = os.path.dirname(deps_file_path)
+ deps_file = parse_file_to_dict(deps_file_path)
+ dependencies = deps_file['deps'].copy()
+ os_specific_dependencies = deps_file.get('deps_os', dict())
+ if 'all' in command_line_os_requests:
+ for value in list(os_specific_dependencies.values()):
+ dependencies.update(value)
+ else:
+ for os_name in command_line_os_requests:
+ # Add OS-specific dependencies
+ if os_name in os_specific_dependencies:
+ dependencies.update(os_specific_dependencies[os_name])
+ for directory in dependencies:
+ for other_dir in dependencies:
+ if directory.startswith(other_dir + '/'):
+ raise Exception('%r is parent of %r' % (other_dir, directory))
+ list_of_arg_lists = []
+ for directory in sorted(dependencies):
+ if '@' in dependencies[directory]:
+ repo, checkoutable = dependencies[directory].split('@', 1)
+ else:
+ raise Exception("please specify commit or tag")
+
+ relative_directory = os.path.join(deps_file_directory, directory)
+
+ list_of_arg_lists.append(
+ (git, repo, checkoutable, relative_directory, verbose))
+
+ multithread(git_checkout_to_directory, list_of_arg_lists)
+
+ for directory in deps_file.get('recursedeps', []):
+ recursive_path = os.path.join(deps_file_directory, directory, 'DEPS')
+ git_sync_deps(recursive_path, command_line_os_requests, verbose)
+
+
+def multithread(function, list_of_arg_lists):
+ # for args in list_of_arg_lists:
+ # function(*args)
+ # return
+ threads = []
+ for args in list_of_arg_lists:
+ thread = threading.Thread(None, function, None, args)
+ thread.start()
+ threads.append(thread)
+ for thread in threads:
+ thread.join()
+
+
+def main(argv):
+ deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
+ verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
+
+ if '--help' in argv or '-h' in argv:
+ usage(deps_file_path)
+ return 1
+
+ git_sync_deps(deps_file_path, argv, verbose)
+ # subprocess.check_call(
+ # [sys.executable,
+ # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
+ return 0
+
+
+if __name__ == '__main__':
+ exit(main(sys.argv[1:]))
diff --git a/utils/roll_deps.sh b/utils/roll_deps.sh
new file mode 100755
index 0000000..622afc9
--- /dev/null
+++ b/utils/roll_deps.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Copyright (c) 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Attempts to roll all entries in DEPS to origin/master and creates a
+# commit.
+#
+# Depends on roll-dep from depot_path being in PATH.
+
+# This script assumes it's parent directory is the repo root.
+repo_path=$(dirname "$0")/..
+
+effcee_dir="external/effcee/"
+googletest_dir="external/googletest/"
+re2_dir="external/re2/"
+spirv_headers_dir="external/spirv-headers/"
+
+cd "$repo_path"
+
+roll-dep "$@" "${effcee_dir}" "${googletest_dir}" "${re2_dir}" "${spirv_headers_dir}"
