Use bundled/embedded python3 binary on OSX (#561)
See https://github.com/emscripten-core/emscripten/issues/7198
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 47653df..f4a2db2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -54,9 +54,6 @@
- run:
name: Install cmake
command: brew install cmake
- - run:
- name: Install python 3
- command: brew install python3
- run: scripts/test.sh
- run:
name: test.py
diff --git a/emsdk b/emsdk
index 6859b54..eb24599 100755
--- a/emsdk
+++ b/emsdk
@@ -6,13 +6,13 @@
# Wrapper script that runs emsdk.py
-base_dir=$(dirname "$0")
-
-# Look for python3 first. This is especially important on macOS (See:
-# https://github.com/emscripten-core/emsdk/pull/273)
-python=$(which python3 2> /dev/null)
-if [ $? != 0 ]; then
- python=python
+if [ -z "$EMSDK_PYTHON" ]; then
+ # Look for python3 first. This is especially important on macOS (See:
+ # https://github.com/emscripten-core/emsdk/pull/273)
+ EMSDK_PYTHON=$(which python3 2> /dev/null)
+ if [ $? != 0 ]; then
+ EMSDK_PYTHON=python
+ fi
fi
-exec "$python" "$0.py" "$@"
+exec "$EMSDK_PYTHON" "$0.py" "$@"
diff --git a/emsdk_manifest.json b/emsdk_manifest.json
index 970c0cb..50e524e 100644
--- a/emsdk_manifest.json
+++ b/emsdk_manifest.json
@@ -289,6 +289,16 @@
"activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe"
},
{
+ "id": "python",
+ "version": "3.7.4",
+ "bitness": 64,
+ "arch": "x86_64",
+ "osx_url": "python-3.7.4-macos.tar.gz",
+ "activated_path": "%installation_dir%/bin",
+ "activated_cfg": "PYTHON='%installation_dir%/bin/python3'",
+ "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3"
+ },
+ {
"id": "java",
"version": "8.152",
"bitness": 32,
@@ -475,7 +485,7 @@
{
"version": "upstream-master",
"bitness": 64,
- "uses": ["llvm-git-master-64bit", "node-12.18.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
+ "uses": ["llvm-git-master-64bit", "node-12.18.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
"os": "osx"
},
{
@@ -505,7 +515,7 @@
{
"version": "fastcomp-master",
"bitness": 64,
- "uses": ["fastcomp-clang-master-64bit", "node-12.18.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
+ "uses": ["fastcomp-clang-master-64bit", "node-12.18.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
"os": "osx"
},
{
@@ -566,7 +576,7 @@
{
"version": "releases-upstream-%releases-tag%",
"bitness": 64,
- "uses": ["node-12.18.1-64bit", "releases-upstream-%releases-tag%-64bit"],
+ "uses": ["node-12.18.1-64bit", "python-3.7.4-64bit", "releases-upstream-%releases-tag%-64bit"],
"os": "osx",
"custom_install_script": "emscripten_npm_install"
},
@@ -587,7 +597,7 @@
{
"version": "releases-fastcomp-%releases-tag%",
"bitness": 64,
- "uses": ["node-12.18.1-64bit", "releases-fastcomp-%releases-tag%-64bit"],
+ "uses": ["node-12.18.1-64bit", "python-3.7.4-64bit", "releases-fastcomp-%releases-tag%-64bit"],
"os": "osx",
"custom_install_script": "emscripten_npm_install"
},
@@ -637,7 +647,7 @@
{
"version": "fastcomp-%precompiled_tag32%",
"bitness": 32,
- "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "emscripten-%precompiled_tag32%"],
+ "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-3.7.4-64bit", "emscripten-%precompiled_tag32%"],
"os": "osx",
"version_filter": [
["%precompiled_tag32%", ">", "1.37.22"]
@@ -646,7 +656,7 @@
{
"version": "fastcomp-%precompiled_tag64%",
"bitness": 64,
- "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "emscripten-%precompiled_tag64%"],
+ "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-3.7.4-64bit", "emscripten-%precompiled_tag64%"],
"os": "osx",
"version_filter": [
["%precompiled_tag64%", ">", "1.37.22"]
diff --git a/scripts/test.sh b/scripts/test.sh
index 1a4ae2f..73c0d42 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -8,4 +8,7 @@
./emsdk install latest
./emsdk activate latest
source ./emsdk_env.sh --build=Release
+# On mac and windows python3 should be in the path and point to the
+# bundled version.
+which python3
emcc -v
diff --git a/scripts/update_python.py b/scripts/update_python.py
index eed07ef..e1f81e3 100755
--- a/scripts/update_python.py
+++ b/scripts/update_python.py
@@ -7,22 +7,28 @@
"""Updates the python binaries that we cache store at
http://storage.google.com/webassembly.
-Currently this is windows only and we rely on the system python on other
-platforms.
+We only supply binaries for windows and macOS, but we do it very different ways for those two OSes.
-We currently bundle a version of python for windows use the following
-recipe:
+Windows recipe:
1. Download the "embeddable zip file" version of python from python.org
2. Remove .pth file to work around https://bugs.python.org/issue34841
3. Download and install pywin32 in the `site-packages` directory
4. Re-zip and upload to storage.google.com
+
+macOS recipe:
+ 1. Clone cpython
+ 2. Use homebrew to install and configure openssl (for static linking!)
+ 3. Build cpython from source and use `make install` to create archive.
"""
-import urllib.request
-import subprocess
+import glob
+import multiprocessing
import os
+import urllib.request
import shutil
+import subprocess
import sys
+from subprocess import check_call
version = '3.7.4'
base = 'https://www.python.org/ftp/python/%s/' % version
@@ -51,7 +57,7 @@
urllib.request.urlretrieve(download_url, filename)
os.mkdir('python-embed')
- subprocess.check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed')
+ check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed')
os.remove(os.path.join('python-embed', 'python37._pth'))
os.mkdir('pywin32')
@@ -61,23 +67,56 @@
os.mkdir(os.path.join('python-embed', 'lib'))
shutil.move(os.path.join('pywin32', 'PLATLIB'), os.path.join('python-embed', 'lib', 'site-packages'))
- subprocess.check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed')
+ check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed')
upload_url = upload_base + out_filename
print('Uploading: ' + upload_url)
cmd = ['gsutil', 'cp', '-n', out_filename, upload_url]
print(' '.join(cmd))
- subprocess.check_call(cmd)
+ check_call(cmd)
# cleanup if everything went fine
shutil.rmtree('python-embed')
shutil.rmtree('pywin32')
-def main():
- for arch in ('amd64', 'win32'):
- make_python_patch(arch)
+def build_python():
+ if sys.platform.startswith('darwin'):
+ osname = 'macos'
+ # Take some rather drastic steps to link openssl statically
+ check_call(['brew', 'install', 'openssl', 'pkg-config'])
+ os.remove('/usr/local/opt/openssl/lib/libssl.dylib')
+ os.remove('/usr/local/opt/openssl/lib/libcrypto.dylib')
+ os.environ['PKG_CONFIG_PATH'] = '/usr/local/opt/openssl/lib/pkgconfig/'
+ else:
+ osname = 'linux'
+ src_dir = 'cpython'
+ if not os.path.exists(src_dir):
+ check_call(['git', 'clone', 'https://github.com/python/cpython'])
+ check_call(['git', 'checkout', 'v' + version], cwd=src_dir)
+ check_call(['./configure'], cwd=src_dir)
+ check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir)
+ check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir)
+
+ install_dir = os.path.join(src_dir, 'install')
+ os.rename(os.path.join(install_dir, 'usr', 'local'), 'python-%s' % version)
+ tarball = 'python-%s-%s.tar.gz' % (version, osname)
+ shutil.rmtree(os.path.join('python-%s' % version, 'lib', 'python3.7', 'test'))
+ shutil.rmtree(os.path.join('python-%s' % version, 'include'))
+ for lib in glob.glob(os.path.join('python-%s' % version, 'lib', 'lib*.a')):
+ os.remove(lib)
+ check_call(['tar', 'zcvf', tarball, 'python-%s' % version])
+ print('Uploading: ' + upload_base + tarball)
+ check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball])
+
+
+def main():
+ if sys.platform.startswith('win'):
+ for arch in ('amd64', 'win32'):
+ make_python_patch(arch)
+ else:
+ build_python()
return 0
