#!/bin/bash

# Copyright (c) 2016-2026 The Brenwill Workshop Ltd.
#
# runcts - This script simplifies running Vulkan CTS tests on MoltenVK.
#
# CTS will save temporary output files to the directory this script is located in. It is 
# recommended that you copy this file to a location where your testing files will be kept.
# By default, this should be a directory beside the VK-GL-CTS directory. See the description 
# of the --cts option below for information about locations.
#
# Executing a large case list file of CTS tests can take a long time. It is recommended
# that you run this using caffeinate, as follows:
#     caffeinate -is ./runcts [-o path] [--cts path] [-p] [--portability] case_list_file
#
#
# macOS usage: ./runcts [-o path] [--cts path] [-p] [--portability] case_list_file
#
#     case_list_file
#         The path to the file that contains a list of the CTS test cases to run.
#
#     -o path
#         The path to the file to write the test results. If this option is not provided, 
#         the results will be written to an output text file with the same name as the 
#         case_list_file, with "-results.txt" appended. CTS will also output several
#         working temporary files into the directory holding this script.
#
#     -fo path
#         The path to the file to write a list of the failed CTS tests. If this option
#         is not provided, the list of failed tests will be written to an output text
#         file with the same name as the case_list_file, with "-fails.txt" appended.
#
#     --cts path
#         The path to the directory containing the built CTS executable.
#         If this parameter is not provided, it defaults to:
#         "../../VK-GL-CTS/build/external/vulkancts/modules/vulkan/Debug".
#
#      -cMm
#         Test conformance against Vulkan version M.m, with no additional extensions,
#         under the Vulkan Portability initiative.
#         In neither this nor the -p option is specified, the tests are run against the
#         latest Vulkan version, plus all extensions, that are supported by MoltenVK.
#
#      -p
#         Same as the --portability option.
#
#      --portability
#         Indicates that this testing is for testing conformance with the 
#         Vulkan Portability initiative. MoltenVK will advertise support for
#         only Vulkan 1.0, and only the following extensions:
#             VK_KHR_get_physical_device_properties2
#             VK_KHR_portability_subset
#

cts_vk_dir="../../VK-GL-CTS/build/external/vulkancts/modules/vulkan/Debug"
caselist_file=""
results_file=""
failures_file=""
is_portability=""
conformance_version=""

while (( "$#" )); do
  case "$1" in
       -o)
         results_file="${2}"
         shift 2
         ;;
       -fo)
         failures_file="${2}"
         shift 2
         ;;
       --cts)
         cts_vk_dir="${2}"
         shift 2
         ;;
       -p | --portability)
         is_portability="Y"
         shift 1
         ;;
       -c14)
         conformance_version="14"
         shift 1
         ;;
       -c13)
         conformance_version="13"
         shift 1
         ;;
       -c12)
         conformance_version="12"
         shift 1
         ;;
       -c11)
         conformance_version="11"
         shift 1
         ;;
       -c10 | -p | --portability)
         conformance_version="10"
         shift 1
         ;;
       -*|--*=)
         echo "Error: Unsupported option $1. See usage instructions in body of this script." >&2
         exit 1
         ;;
       *)
         caselist_file="${1}"
         shift 1
         ;;
  esac
done

if [ "${caselist_file}" == "" ]; then 
	echo "Error: No caselist file specified. See usage instructions in body of this script." >&2
	exit 1
fi

if [ "${results_file}" == "" ]; then
	results_file="${caselist_file}-results.txt"
fi

if [ "${failures_file}" == "" ]; then
	failures_file="${caselist_file}-fails.txt"
fi


# -------------- MoltenVK configuration --------------------

# As documented above, the portability option restricts to Vulkan 1.0 and a very limited set of extensions.
# The values used here are documented in MoltenVK_Configuration_Parameters.md.
# - MVK_CONFIG_API_VERSION_TO_ADVERTISE = 4194304 (VK_API_VERSION_1_0).
# - MVK_CONFIG_ADVERTISE_EXTENSIONS = 4 (VK_KHR_portability_subset and prerequistes).
if [ "${is_portability}" != "" ]; then
	export MVK_CONFIG_API_VERSION_TO_ADVERTISE=10
	export MVK_CONFIG_ADVERTISE_EXTENSIONS=4
	echo "Testing against Vulkan portability ${conformance_version} with Vulkan portability extension."
fi

# When testing conformance to a particular Vulkan version,
# advertise that version, and disable all extensions except portability.
if [ "${conformance_version}" != "" ]; then
	export MVK_CONFIG_API_VERSION_TO_ADVERTISE="${conformance_version}"
	export MVK_CONFIG_ADVERTISE_EXTENSIONS=4
	echo "Testing against Vulkan ${conformance_version} with only Vulkan portability extension."
fi

# ----- System settings ------
export DYLD_LIBRARY_PATH="/usr/local/lib"

# ----- Metal validation settings ------
export METAL_DEVICE_WRAPPER_TYPE=1
export METAL_ERROR_MODE=3
export METAL_DEBUG_ERROR_MODE=3

# ----- MoltenVK config settings ------
export MVK_CONFIG_LOG_LEVEL=1	#(1 = Errors only, 3 = Info)
export MVK_CONFIG_DEBUG=0

# Additional MoltenVK configuration can be set here by editing below.
# The values used here are documented in MoltenVK_Configuration_Parameters.md.
export MVK_CONFIG_RESUME_LOST_DEVICE=1
export MVK_CONFIG_FAST_MATH_ENABLED=1
export MVK_CONFIG_FORCE_LOW_POWER_GPU=0
export MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS=1
export MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE=2          #(2 = MTLEvents always)
export MVK_CONFIG_SHADER_COMPRESSION_ALGORITHM=0        #(2 = ZLIB, 3 = LZ4)
export MVK_CONFIG_PERFORMANCE_TRACKING=0
export MVK_CONFIG_ACTIVITY_PERFORMANCE_LOGGING_STYLE=3  #(2 = Device lifetime, 3 = Process lifetime)

# -------------- Operation --------------------

echo Testing started at `date +%r`
start_time=${SECONDS}

"${cts_vk_dir}/deqp-vk"                    \
--deqp-archive-dir="${cts_vk_dir}/.."      \
--deqp-log-filename="/dev/null"            \
--deqp-log-images=disable                  \
--deqp-log-shader-sources=disable          \
--deqp-shadercache=disable                 \
--deqp-log-decompiled-spirv=disable        \
--deqp-log-flush=disable                   \
--deqp-caselist-file="${caselist_file}"    \
&> "${results_file}"

python3 ./get_failing_cts_tests.py -q -o "${failures_file}" "${results_file}"

echo Testing complete in $(($SECONDS - $start_time)) seconds at `date +%r`