blob: b39597d1f99f0c5fc5b4e99e17beaa68fc1a0d71 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2020 The Emscripten Authors. All rights reserved.
# Emscripten is available under two separate licenses, the MIT license and the
# University of Illinois/NCSA Open Source License. Both these licenses can be
# found in the LICENSE file.
"""Updates the python binaries that we cache store at
http://storage.google.com/webassembly.
We only supply binaries for windows and macOS, but we do it very different ways for those two OSes.
Windows recipe:
1. Download the "embeddable zip file" version of python from python.org
2. Remove .pth file to work around https://bugs.python.org/issue34841
3. Download and install pywin32 in the `site-packages` directory
4. Re-zip and upload to storage.google.com
macOS recipe:
1. Clone cpython
2. Use homebrew to install and configure openssl (for static linking!)
3. Build cpython from source and use `make install` to create archive.
"""
import glob
import multiprocessing
import os
import platform
import urllib.request
import shutil
import subprocess
import sys
from subprocess import check_call
version = '3.9.2'
major_minor_version = '.'.join(version.split('.')[:2]) # e.g. '3.9.2' -> '3.9'
download_url = 'https://www.nuget.org/api/v2/package/python/%s' % version
revision = '4'
pywin32_version = '227'
pywin32_base = 'https://github.com/mhammond/pywin32/releases/download/b%s/' % pywin32_version
upload_base = 'gs://webassembly/emscripten-releases-builds/deps/'
def unzip_cmd():
# Use 7-Zip if available (https://www.7-zip.org/)
sevenzip = os.path.join(os.getenv('ProgramFiles', ''), '7-Zip', '7z.exe')
if os.path.isfile(sevenzip):
return [sevenzip, 'x']
# Fall back to 'unzip' tool
return ['unzip', '-q']
def zip_cmd():
# Use 7-Zip if available (https://www.7-zip.org/)
sevenzip = os.path.join(os.getenv('ProgramFiles', ''), '7-Zip', '7z.exe')
if os.path.isfile(sevenzip):
return [sevenzip, 'a', '-mx9']
# Fall back to 'zip' tool
return ['zip', '-rq']
def make_python_patch():
pywin32_filename = 'pywin32-%s.win-amd64-py%s.exe' % (pywin32_version, major_minor_version)
filename = 'python-%s-amd64.zip' % (version)
out_filename = 'python-%s-%s-amd64+pywin32.zip' % (version, revision)
if not os.path.exists(pywin32_filename):
url = pywin32_base + pywin32_filename
print('Downloading pywin32: ' + url)
urllib.request.urlretrieve(url, pywin32_filename)
if not os.path.exists(filename):
print('Downloading python: ' + download_url)
urllib.request.urlretrieve(download_url, filename)
os.mkdir('python-nuget')
check_call(unzip_cmd() + [os.path.abspath(filename)], cwd='python-nuget')
os.mkdir('pywin32')
rtn = subprocess.call(unzip_cmd() + [os.path.abspath(pywin32_filename)], cwd='pywin32')
assert rtn in [0, 1]
os.mkdir(os.path.join('python-nuget', 'lib'))
shutil.move(os.path.join('pywin32', 'PLATLIB'), os.path.join('python-nuget', 'toolss', 'Lib', 'site-packages'))
check_call(zip_cmd() + [os.path.join('..', '..', out_filename), '.'], cwd='python-nuget/tools')
# cleanup if everything went fine
shutil.rmtree('python-nuget')
shutil.rmtree('pywin32')
print('Created: %s' % out_filename)
if '--upload' in sys.argv:
upload_url = upload_base + out_filename
print('Uploading: ' + upload_url)
cmd = ['gsutil', 'cp', '-n', out_filename, upload_url]
print(' '.join(cmd))
check_call(cmd)
def build_python():
if sys.platform.startswith('darwin'):
# Take some rather drastic steps to link openssl and liblzma statically
# and avoid linking libintl completely.
osname = 'macos'
check_call(['brew', 'install', 'openssl', 'xz', 'pkg-config'])
if platform.machine() == 'x86_64':
prefix = '/usr/local'
min_macos_version = '10.11'
elif platform.machine() == 'arm64':
prefix = '/opt/homebrew'
min_macos_version = '11.0'
# Append '-x86_64' or '-arm64' depending on current arch. (TODO: Do
# this for Linux too, move this below?)
osname += '-' + platform.machine()
for f in [os.path.join(prefix, 'lib', 'libintl.dylib'),
os.path.join(prefix, 'include', 'libintl.h'),
os.path.join(prefix, 'opt', 'xz', 'lib', 'liblzma.dylib'),
os.path.join(prefix, 'opt', 'openssl', 'lib', 'libssl.dylib'),
os.path.join(prefix, 'opt', 'openssl', 'lib', 'libcrypto.dylib')]:
if os.path.exists(f):
os.remove(f)
os.environ['PKG_CONFIG_PATH'] = os.path.join(prefix, 'opt', 'openssl', 'lib', 'pkgconfig')
else:
osname = 'linux'
src_dir = 'cpython'
if not os.path.exists(src_dir):
check_call(['git', 'clone', 'https://github.com/python/cpython'])
check_call(['git', 'checkout', 'v' + version], cwd=src_dir)
env = os.environ
if sys.platform.startswith('darwin'):
# Specify the min OS version we want the build to work on
min_macos_version_line = '-mmacosx-version-min=' + min_macos_version
build_flags = min_macos_version_line + ' -Werror=partial-availability'
# Build against latest SDK, but issue an error if using any API that would not work on the min OS version
env = env.copy()
env['MACOSX_DEPLOYMENT_TARGET'] = min_macos_version
configure_args = ['CFLAGS=' + build_flags, 'CXXFLAGS=' + build_flags, 'LDFLAGS=' + min_macos_version_line]
else:
configure_args = []
check_call(['./configure'] + configure_args, cwd=src_dir, env=env)
check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir, env=env)
check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir, env=env)
install_dir = os.path.join(src_dir, 'install')
# Install requests module. This is needed in particualr on macOS to ensure
# SSL certificates are available (certifi in installed and used by requests).
pybin = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'python3')
pip = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'pip3')
check_call([pybin, '-m', 'ensurepip', '--upgrade'])
check_call([pybin, pip, 'install', 'requests'])
dirname = 'python-%s-%s' % (version, revision)
if os.path.isdir(dirname):
print('Erasing old build directory ' + dirname)
shutil.rmtree(dirname)
os.rename(os.path.join(install_dir, 'usr', 'local'), dirname)
tarball = 'python-%s-%s-%s.tar.gz' % (version, revision, osname)
shutil.rmtree(os.path.join(dirname, 'lib', 'python' + major_minor_version, 'test'))
shutil.rmtree(os.path.join(dirname, 'include'))
for lib in glob.glob(os.path.join(dirname, 'lib', 'lib*.a')):
os.remove(lib)
check_call(['tar', 'zcvf', tarball, dirname])
print('Created: %s' % tarball)
if '--upload' in sys.argv:
print('Uploading: ' + upload_base + tarball)
check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball])
def main():
if sys.platform.startswith('win') or '--win32' in sys.argv:
make_python_patch()
else:
build_python()
return 0
if __name__ == '__main__':
sys.exit(main())