Fix python scripts to work with python3 (#2426)
Updated script to work with python3 and python2.
Added required tools.
We added a section to the readme to mention the tools that are needed to
build and test spirv-tools. For the compiler, the compilers used by the
bots are mentioned.
The bots have been changed. The windows bots will not use python 3.6 for testing. The other bots will still use python 2.7. Both Python2 and Python3 will be tested.
Fixes #2407.
Fixes #1856.
diff --git a/.appveyor.yml b/.appveyor.yml
index a50c7c2..669de03 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -42,7 +42,7 @@
- set NINJA_URL="https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip"
- appveyor DownloadFile %NINJA_URL% -FileName ninja.zip
- 7z x ninja.zip -oC:\ninja > nul
- - set PATH=C:\ninja;%PATH%
+ - set PATH=C:\ninja;C:\Python36;%PATH%
before_build:
- git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers.git external/spirv-headers
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2dc6d3a..a5ecb90 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -176,7 +176,7 @@
endmacro()
endif()
-find_host_package(PythonInterp 2.7 REQUIRED)
+find_host_package(PythonInterp)
# Check for symbol exports on Linux.
# At the moment, this check will fail on the OSX build machines for the Android NDK.
diff --git a/README.md b/README.md
index 52c382e..bef9736 100644
--- a/README.md
+++ b/README.md
@@ -269,6 +269,36 @@
Once the build files have been generated, build using your preferred
development environment.
+### Tools you'll need
+
+For building and testing SPIRV-Tools, the following tools should be
+installed regardless of your OS:
+
+- [CMake](http://www.cmake.org/): for generating compilation targets. Version
+ 2.8.12 or later.
+- [Python](http://www.python.org/): for utility scripts and running the test
+suite. Version 2 or 3.
+
+We will be moving to Python3 only in the future. If you are using Python2, you
+will need to install Python-future:
+```pip install future
+```
+
+SPIRV-Tools is regularly tested with the the following compilers:
+
+On Linux
+- GCC version 4.8.5
+- Clang version 3.8
+
+On MacOS
+- AppleClang 10.0
+
+On Windows
+- Visual Studio 2015
+- Visual Studio 2017
+
+Other compilers or later versions may work, but they are not tested.
+
### CMake options
The following CMake options are supported:
diff --git a/kokoro/scripts/linux/build.sh b/kokoro/scripts/linux/build.sh
index d457539..d96d247 100644
--- a/kokoro/scripts/linux/build.sh
+++ b/kokoro/scripts/linux/build.sh
@@ -78,7 +78,7 @@
# Invoke the build.
BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
echo $(date): Starting build...
-cmake -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=install -DRE2_BUILD_TESTING=OFF $ADDITIONAL_CMAKE_FLAGS $CMAKE_C_CXX_COMPILER ..
+cmake -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=install -DRE2_BUILD_TESTING=OFF $ADDITIONAL_CMAKE_FLAGS $CMAKE_C_CXX_COMPILER ..
echo $(date): Build everything...
ninja
diff --git a/kokoro/scripts/windows/build.bat b/kokoro/scripts/windows/build.bat
index 1985419..715dd6a 100644
--- a/kokoro/scripts/windows/build.bat
+++ b/kokoro/scripts/windows/build.bat
@@ -21,8 +21,8 @@
set BUILD_TYPE=%1
set VS_VERSION=%2
-:: Force usage of python 2.7 rather than 3.6
-set PATH=C:\python27;%PATH%
+:: Force usage of python 3.6
+set PATH=C:\python36;%PATH%
cd %SRC%
git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers
diff --git a/test/tools/expect.py b/test/tools/expect.py
index c959650..5844882 100755
--- a/test/tools/expect.py
+++ b/test/tools/expect.py
@@ -18,15 +18,18 @@
methods in the mixin classes.
"""
import difflib
+import functools
import os
import re
import subprocess
+import traceback
from spirv_test_framework import SpirvTest
-
+from builtins import bytes
def convert_to_unix_line_endings(source):
"""Converts all line endings in source to be unix line endings."""
- return source.replace('\r\n', '\n').replace('\r', '\n')
+ result = source.replace('\r\n', '\n').replace('\r', '\n')
+ return result
def substitute_file_extension(filename, extension):
@@ -133,7 +136,7 @@
word = binary[index * 4:(index + 1) * 4]
if little_endian:
word = reversed(word)
- return reduce(lambda w, b: (w << 8) | ord(b), word, 0)
+ return functools.reduce(lambda w, b: (w << 8) | b, word, 0)
def check_endianness(binary):
"""Checks the endianness of the given SPIR-V binary.
@@ -479,7 +482,7 @@
if not status.stderr:
return False, 'Expected error message, but no output on stderr'
if self.expected_error_substr not in convert_to_unix_line_endings(
- status.stderr):
+ status.stderr.decode('utf8')):
return False, ('Incorrect stderr output:\n{act}\n'
'Expected substring not found in stderr:\n{exp}'.format(
act=status.stderr, exp=self.expected_error_substr))
@@ -499,7 +502,7 @@
' command execution')
if not status.stderr:
return False, 'Expected warning message, but no output on stderr'
- if self.expected_warning != convert_to_unix_line_endings(status.stderr):
+ if self.expected_warning != convert_to_unix_line_endings(status.stderr.decode('utf8')):
return False, ('Incorrect stderr output:\n{act}\n'
'Expected:\n{exp}'.format(
act=status.stderr, exp=self.expected_warning))
@@ -559,16 +562,16 @@
if not status.stdout:
return False, 'Expected something on stdout'
elif type(self.expected_stdout) == str:
- if self.expected_stdout != convert_to_unix_line_endings(status.stdout):
+ if self.expected_stdout != convert_to_unix_line_endings(status.stdout.decode('utf8')):
return False, ('Incorrect stdout output:\n{ac}\n'
'Expected:\n{ex}'.format(
ac=status.stdout, ex=self.expected_stdout))
else:
- if not self.expected_stdout.search(
- convert_to_unix_line_endings(status.stdout)):
+ converted = convert_to_unix_line_endings(status.stdout.decode('utf8'))
+ if not self.expected_stdout.search(converted):
return False, ('Incorrect stdout output:\n{ac}\n'
'Expected to match regex:\n{ex}'.format(
- ac=status.stdout, ex=self.expected_stdout.pattern))
+ ac=status.stdout.decode('utf8'), ex=self.expected_stdout.pattern))
return True, ''
@@ -593,13 +596,13 @@
if not status.stderr:
return False, 'Expected something on stderr'
elif type(self.expected_stderr) == str:
- if self.expected_stderr != convert_to_unix_line_endings(status.stderr):
+ if self.expected_stderr != convert_to_unix_line_endings(status.stderr.decode('utf8')):
return False, ('Incorrect stderr output:\n{ac}\n'
'Expected:\n{ex}'.format(
ac=status.stderr, ex=self.expected_stderr))
else:
if not self.expected_stderr.search(
- convert_to_unix_line_endings(status.stderr)):
+ convert_to_unix_line_endings(status.stderr.decode('utf8'))):
return False, ('Incorrect stderr output:\n{ac}\n'
'Expected to match regex:\n{ex}'.format(
ac=status.stderr, ex=self.expected_stderr.pattern))
@@ -664,7 +667,7 @@
# Collect all the output lines containing a pass name.
pass_names = []
pass_name_re = re.compile(r'.*IR before pass (?P<pass_name>[\S]+)')
- for line in status.stderr.splitlines():
+ for line in status.stderr.decode('utf8').splitlines():
match = pass_name_re.match(line)
if match:
pass_names.append(match.group('pass_name'))
