| // Copyright 2025 The Khronos Group Inc. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #include "fnvar.h" | |
| #include <initializer_list> | |
| #include <memory> | |
| #include <sstream> | |
| #include "source/opt/instruction.h" | |
| namespace spvtools { | |
| using opt::Function; | |
| using opt::Instruction; | |
| using opt::analysis::Type; | |
| namespace { | |
| // Helper functions | |
| // Parses a CSV source string for the purpose of this extension. | |
| // | |
| // Required columns must be known in advance and supplied as the required_cols | |
| // argument -- this is used for error checking. Values are assumed to be | |
| // separated by CSV_SEP. The input source string is assumed to be the output of | |
| // io::ReadTextFile and no other validation, apart from the CSV parsing, is | |
| // performed. | |
| // | |
| // Returns true on success, false on error (with error message stored in | |
| // err_msg). | |
| bool ParseCsv(const std::string& source, | |
| const std::vector<std::string>& required_cols, | |
| std::stringstream& err_msg, | |
| std::vector<std::vector<std::string>>& result) { | |
| std::stringstream fn_variants_csv_stream(source); | |
| std::string line; | |
| std::vector<std::string> columns; | |
| constexpr char CSV_SEP = ','; | |
| bool first_line = true; | |
| while (std::getline(fn_variants_csv_stream, line, '\n')) { | |
| if (line.empty()) { | |
| continue; | |
| } | |
| std::vector<std::string> vals; | |
| std::string val; | |
| std::stringstream line_stream(line); | |
| auto* vec = first_line ? &columns : &vals; | |
| while (std::getline(line_stream, val, CSV_SEP)) { | |
| vec->push_back(val); | |
| } | |
| if (!line_stream && val.empty()) { | |
| vec->push_back(""); | |
| } | |
| if (!first_line) { | |
| if (vals.size() != columns.size()) { | |
| err_msg << "Number of values does not match the number of columns. " | |
| "Offending line:\n" | |
| << line; | |
| return false; | |
| } | |
| result.push_back(vals); | |
| } | |
| first_line = false; | |
| } | |
| // check if required columns match actual columns (ordering matters) | |
| if (columns.size() != required_cols.size()) { | |
| err_msg << "Invalid number of CSV columns: " << columns.size() | |
| << ", expected " << required_cols.size() << "."; | |
| return false; | |
| } | |
| for (size_t i = 0; i < columns.size(); ++i) { | |
| if (columns[i] != required_cols[i]) { | |
| err_msg << "Invalid name of column " << i + 1 << ". Expected '" | |
| << required_cols[i] << "', got '" << columns[i] << "'."; | |
| return false; | |
| } | |
| } | |
| return true; | |
| } | |
| // Annotate ID with ConditionalINTEL decoration | |
| void DecorateConditional(IRContext* context, uint32_t id_to_decorate, | |
| uint32_t spec_const_id) { | |
| auto decor_instr = | |
| std::make_unique<Instruction>(context, spv::Op::OpDecorate); | |
| decor_instr->AddOperand({SPV_OPERAND_TYPE_ID, {id_to_decorate}}); | |
| decor_instr->AddOperand({SPV_OPERAND_TYPE_DECORATION, | |
| {uint32_t(spv::Decoration::ConditionalINTEL)}}); | |
| decor_instr->AddOperand({SPV_OPERAND_TYPE_ID, {spec_const_id}}); | |
| context->module()->AddAnnotationInst(std::move(decor_instr)); | |
| } | |
| // Finds entry point corresponding to a function | |
| // | |
| // Returns null if not found, otherwise returns pointer to the EP Instruction. | |
| Instruction* FindEntryPoint(const Instruction& fn_inst) { | |
| auto* mod = fn_inst.context()->module(); | |
| for (auto& entry_point : mod->entry_points()) { | |
| const int ep_i = | |
| entry_point.opcode() == spv::Op::OpConditionalEntryPointINTEL ? 2 : 1; | |
| if (entry_point.GetOperand(ep_i).AsId() == fn_inst.result_id()) { | |
| return &entry_point; | |
| } | |
| } | |
| return nullptr; | |
| } | |
| // If the function has an entry point, converts it to a conditional one | |
| void ConvertEPToConditional(Module* module, const Function& fn, | |
| uint32_t spec_const_id) { | |
| for (const auto& ep_inst : module->entry_points()) { | |
| if (ep_inst.opcode() == spv::Op::OpEntryPoint) { | |
| auto* entry_point = FindEntryPoint(fn.DefInst()); | |
| if (entry_point != nullptr) { | |
| std::vector<opt::Operand> old_operands; | |
| for (auto operand : *entry_point) { | |
| old_operands.push_back(operand); | |
| } | |
| entry_point->ToNop(); | |
| entry_point->SetOpcode(spv::Op::OpConditionalEntryPointINTEL); | |
| entry_point->AddOperand({SPV_OPERAND_TYPE_ID, {spec_const_id}}); | |
| for (auto old_operand : old_operands) { | |
| entry_point->AddOperand(old_operand); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| // Finds ID of a bool type (returns 0 if not found) | |
| uint32_t FindIdOfBoolType(const Module* const mod) { | |
| return mod->context()->get_type_mgr()->GetBoolTypeId(); | |
| } | |
| // Combines IDs using OpSpecConstantOp with the operation defined by cmp_op. | |
| // | |
| // Returns the ID of the final result. If there are no IDs, returns 0. If there | |
| // is one ID, does not generate any instructions and returns the ID. | |
| uint32_t CombineIds(IRContext* const context, const std::vector<uint32_t>& ids, | |
| spv::Op cmp_op) { | |
| if (ids.empty()) { | |
| return 0; | |
| } else if (ids.size() == 1) { | |
| return ids[0]; | |
| } else { | |
| uint32_t bool_id = FindIdOfBoolType(context->module()); | |
| assert(bool_id != 0); | |
| uint32_t prev_spec_const_id = ids[0]; | |
| for (size_t i = 1; i < ids.size(); ++i) { | |
| const uint32_t id = ids[i]; | |
| const uint32_t spec_const_op_id = context->TakeNextId(); | |
| auto inst = std::make_unique<Instruction>( | |
| context, spv::Op::OpSpecConstantOp, bool_id, spec_const_op_id, | |
| std::initializer_list<opt::Operand>{ | |
| {SPV_OPERAND_TYPE_SPEC_CONSTANT_OP_NUMBER, {(uint32_t)(cmp_op)}}, | |
| {SPV_OPERAND_TYPE_ID, {prev_spec_const_id}}, | |
| {SPV_OPERAND_TYPE_ID, {id}}}); | |
| context->module()->AddType(std::move(inst)); | |
| prev_spec_const_id = spec_const_op_id; | |
| } | |
| return prev_spec_const_id; | |
| } | |
| } | |
| // Returns whether instruction can be shared between variant modules and | |
| // combined using spec constants (such as conditional capabilities). | |
| bool CanBeFnVarCombined(const Instruction* inst) { | |
| const spv::Op opcode = inst->opcode(); | |
| if ((opcode != spv::Op::OpExtInstImport) && | |
| (opcode != spv::Op::OpCapability) && (opcode != spv::Op::OpExtension) && | |
| !spvOpcodeGeneratesType(opcode)) { | |
| return false; | |
| } | |
| if ((opcode == spv::Op::OpCapability) && | |
| ((inst->GetSingleWordOperand(0) == | |
| static_cast<uint32_t>(spv::Capability::FunctionVariantsINTEL)) || | |
| (inst->GetSingleWordOperand(0) == | |
| static_cast<uint32_t>(spv::Capability::SpecConditionalINTEL)))) { | |
| // Always enabled | |
| return false; | |
| } | |
| if ((opcode == spv::Op::OpExtension) && | |
| (inst->GetOperand(0).AsString() == FNVAR_EXT_NAME)) { | |
| // Always enabled | |
| return false; | |
| } | |
| return true; | |
| } | |
| // Calculates hash of an instruction. | |
| // | |
| // Applicable only to instructions that can be combined (ie. with | |
| // CanBeFnVarCombined being true) and from those, hash can be only computed for | |
| // selected instructions. Computing hash from other instruction is unsupported. | |
| size_t HashInst(const Instruction* inst) { | |
| if (CanBeFnVarCombined(inst)) { | |
| if (spvOpcodeGeneratesType(inst->opcode())) { | |
| const Type* t = | |
| inst->context()->get_type_mgr()->GetType(inst->result_id()); | |
| assert(t != nullptr); | |
| return t->HashValue(); | |
| } | |
| if (inst->opcode() == spv::Op::OpExtension) { | |
| const auto name = inst->GetOperand(0).AsString(); | |
| return std::hash<std::string>()(name); | |
| } | |
| if (inst->opcode() == spv::Op::OpCapability) { | |
| const auto cap = inst->GetSingleWordOperand(0); | |
| return std::hash<uint32_t>()(cap); | |
| } | |
| if (inst->opcode() == spv::Op::OpExtInstImport) { | |
| const auto name = inst->GetOperand(1).AsString(); | |
| return std::hash<std::string>()(name); | |
| } | |
| } | |
| assert(false && "Unsupported instruction hash"); | |
| return std::hash<const Instruction*>()(inst); | |
| } | |
| std::string GetFnName(const Instruction& fn_inst) { | |
| // Check entry point | |
| const auto* ep_inst = FindEntryPoint(fn_inst); | |
| if (ep_inst != nullptr) { | |
| const int name_i = | |
| ep_inst->opcode() == spv::Op::OpConditionalEntryPointINTEL ? 3 : 2; | |
| return ep_inst->GetOperand(name_i).AsString(); | |
| } | |
| // Check name of export linkage attribute decoration | |
| const auto* decor_mgr = fn_inst.context()->get_decoration_mgr(); | |
| for (const auto* inst : | |
| decor_mgr->GetDecorationsFor(fn_inst.result_id(), true)) { | |
| const auto decoration = inst->GetOperand(1); | |
| if ((decoration.type == SPV_OPERAND_TYPE_DECORATION) && | |
| (decoration.words.size() == 1) && | |
| (decoration.words[0] == | |
| static_cast<uint32_t>(spv::Decoration::LinkageAttributes))) { | |
| const auto linkage = inst->GetOperand(3); | |
| if ((linkage.type == SPV_OPERAND_TYPE_LINKAGE_TYPE) && | |
| (linkage.words.size() == 1) && | |
| (linkage.words[0] == | |
| static_cast<uint32_t>(spv::LinkageType::Export))) { | |
| // decorates fn with LinkageAttribute and Export linkage type -> get the | |
| // name | |
| return inst->GetOperand(2).AsString(); | |
| } | |
| } | |
| } | |
| return ""; | |
| } | |
| uint32_t FindSpecConstByName(const Module* mod, std::string name) { | |
| for (const auto* const_inst : mod->context()->GetConstants()) { | |
| if (opt::IsSpecConstantInst(const_inst->opcode())) { | |
| const auto id = const_inst->result_id(); | |
| for (const auto& name_inst : mod->debugs2()) { | |
| if ((name_inst.opcode() == spv::Op::OpName) && | |
| (name_inst.GetOperand(0).AsId() == id) && | |
| (name_inst.GetOperand(1).AsString() == name)) { | |
| return id; | |
| } | |
| } | |
| } | |
| } | |
| return 0; | |
| } | |
| uint32_t CombineVariantDefs(const std::vector<VariantDef>& variant_defs, | |
| const std::vector<size_t> var_ids, | |
| IRContext* context, | |
| std::map<std::vector<size_t>, uint32_t>& cache) { | |
| assert(var_ids.size() <= variant_defs.size()); | |
| uint32_t spec_const_comb_id = 0; | |
| if (var_ids.size() != variant_defs.size()) { | |
| // if not used by all variants | |
| if (cache.find(var_ids) == cache.end()) { | |
| // cache variant combinations | |
| std::vector<uint32_t> spec_const_ids; | |
| for (const auto& var_id : var_ids) { | |
| const auto var_name = variant_defs[var_id].GetName(); | |
| const auto var_spec_id = | |
| FindSpecConstByName(context->module(), var_name); | |
| spec_const_ids.push_back(var_spec_id); | |
| } | |
| spec_const_comb_id = | |
| CombineIds(context, spec_const_ids, spv::Op::OpLogicalOr); | |
| assert(spec_const_comb_id != 0); | |
| cache.insert({var_ids, spec_const_comb_id}); | |
| } else { | |
| spec_const_comb_id = cache[var_ids]; | |
| } | |
| } | |
| return spec_const_comb_id; | |
| } | |
| bool strToInt(std::string s, uint32_t* x) { | |
| for (const char& c : s) { | |
| if (c < '0' || c > '9') { | |
| return false; | |
| } | |
| } | |
| if (!(std::stringstream(s) >> *x)) { | |
| return false; | |
| } | |
| return true; | |
| } | |
| } // anonymous namespace | |
| bool VariantDefs::ProcessFnVar(const LinkerOptions& options, | |
| const std::vector<Module*>& modules) { | |
| assert(variant_defs_.empty()); | |
| assert(modules.size() == options.GetInFiles().size()); | |
| for (size_t i = 0; i < modules.size(); ++i) { | |
| const auto* feat_mgr = modules[i]->context()->get_feature_mgr(); | |
| if ((feat_mgr->HasCapability(spv::Capability::FunctionVariantsINTEL)) || | |
| (feat_mgr->HasCapability(spv::Capability::SpecConditionalINTEL)) || | |
| (feat_mgr->HasExtension(kSPV_INTEL_function_variants))) { | |
| // In principle, it can be done but it's complicated due to having to | |
| // combine the existing conditionals with the new ones. For example, | |
| // conditional capabilities would need to become "doubly-conditional". | |
| err_ << "Creating multitarget modules from multitarget modules is not " | |
| "supported. Offending file: " | |
| << options.GetInFiles()[i]; | |
| return false; | |
| } | |
| } | |
| std::vector<std::vector<std::string>> target_rows; | |
| std::vector<std::vector<std::string>> architecture_rows; | |
| if (!options.GetFnVarTargetsCsv().empty()) { | |
| const std::vector<std::string> tgt_cols = {"module", "target", "features"}; | |
| if (!ParseCsv(options.GetFnVarTargetsCsv(), tgt_cols, err_, target_rows)) { | |
| return false; | |
| } | |
| } | |
| if (!options.GetFnVarArchitecturesCsv().empty()) { | |
| const std::vector<std::string> arch_cols = {"module", "category", "family", | |
| "op", "architecture"}; | |
| if (!ParseCsv(options.GetFnVarArchitecturesCsv(), arch_cols, err_, | |
| architecture_rows)) { | |
| return false; | |
| } | |
| } | |
| // check that all modules defined in the CSV exist | |
| for (const auto& tgt_vals : target_rows) { | |
| bool found = false; | |
| for (const auto& in_file : options.GetInFiles()) { | |
| if (tgt_vals[0] == in_file) { | |
| found = true; | |
| } | |
| } | |
| if (!found) { | |
| err_ << "Module '" << tgt_vals[0] | |
| << "' found in targets CSV not passed to the CLI."; | |
| return false; | |
| } | |
| } | |
| for (const auto& arch_vals : architecture_rows) { | |
| bool found = false; | |
| for (const auto& in_file : options.GetInFiles()) { | |
| if (arch_vals[0] == in_file) { | |
| found = true; | |
| } | |
| } | |
| if (!found) { | |
| err_ << "Module '" << arch_vals[0] | |
| << "' found in architectures CSV not passed to the CLI."; | |
| return false; | |
| } | |
| } | |
| // create per-module variant defs | |
| for (size_t i = 0; i < modules.size(); ++i) { | |
| // first module passed to the CLI is considered the base module | |
| bool is_base = i == 0; | |
| const auto name = options.GetInFiles()[i]; | |
| auto variant_def = VariantDef(is_base, name, modules[i]); | |
| for (const auto& arch_row : architecture_rows) { | |
| const auto row_name = arch_row[0]; | |
| if (row_name == name) { | |
| uint32_t category, family, op, architecture; | |
| if (!strToInt(arch_row[1], &category)) { | |
| err_ << "Error converting " << arch_row[1] | |
| << " to architecture category."; | |
| return false; | |
| } | |
| if (!strToInt(arch_row[2], &family)) { | |
| err_ << "Error converting " << arch_row[2] | |
| << " to architecture family."; | |
| return false; | |
| } | |
| if (!strToInt(arch_row[3], &op)) { | |
| err_ << "Error converting " << arch_row[3] << " to architecture op."; | |
| return false; | |
| } | |
| if (!strToInt(arch_row[4], &architecture)) { | |
| err_ << "Error converting " << arch_row[4] << " to architecture."; | |
| return false; | |
| } | |
| variant_def.AddArchDef(category, family, op, architecture); | |
| } | |
| } | |
| for (const auto& tgt_row : target_rows) { | |
| const auto row_name = tgt_row[0]; | |
| if (row_name == name) { | |
| uint32_t target; | |
| std::vector<uint32_t> features; | |
| if (!strToInt(tgt_row[1], &target)) { | |
| err_ << "Error converting " << tgt_row[1] << " to target."; | |
| return false; | |
| } | |
| // get features as FEAT_SEP-delimited integers | |
| std::stringstream feat_stream(tgt_row[2]); | |
| std::string feat; | |
| while (std::getline(feat_stream, feat, FEAT_SEP)) { | |
| uint32_t ufeat; | |
| // if (!(std::stringstream(feat) >> ufeat)) { | |
| if (!strToInt(feat, &ufeat)) { | |
| err_ << "Error converting " << feat << " in " << tgt_row[2] | |
| << " to target feature."; | |
| return false; | |
| } | |
| features.push_back(ufeat); | |
| } | |
| variant_def.AddTgtDef(target, features); | |
| } | |
| } | |
| if (options.GetHasFnVarCapabilities()) { | |
| variant_def.InferCapabilities(); | |
| } | |
| variant_defs_.push_back(variant_def); | |
| } | |
| return true; | |
| } | |
| bool VariantDefs::ProcessVariantDefs() { | |
| EnsureBoolType(); | |
| CollectVarInsts(); | |
| if (!GenerateFnVarConstants()) { | |
| return false; | |
| } | |
| CollectBaseFnCalls(); | |
| return true; | |
| } | |
| void VariantDefs::GenerateHeader(IRContext* linked_context) { | |
| linked_context->AddCapability(spv::Capability::SpecConditionalINTEL); | |
| linked_context->AddCapability(spv::Capability::FunctionVariantsINTEL); | |
| linked_context->AddExtension(std::string(FNVAR_EXT_NAME)); | |
| // Specifies used registry version | |
| auto inst = | |
| std::make_unique<Instruction>(linked_context, spv::Op::OpModuleProcessed); | |
| std::stringstream line; | |
| line << "SPV_INTEL_function_variants registry version " | |
| << FNVAR_REGISTRY_VERSION; | |
| inst->AddOperand( | |
| {SPV_OPERAND_TYPE_LITERAL_STRING, utils::MakeVector(line.str())}); | |
| linked_context->AddDebug3Inst(std::move(inst)); | |
| } | |
| void VariantDefs::CombineVariantInstructions(IRContext* linked_context) { | |
| CombineBaseFnCalls(linked_context); | |
| CombineInstructions(linked_context); | |
| } | |
| void VariantDefs::EnsureBoolType() { | |
| for (auto& variant_def : variant_defs_) { | |
| Module* module = variant_def.GetModule(); | |
| IRContext* context = module->context(); | |
| uint32_t bool_id = FindIdOfBoolType(module); | |
| if (bool_id == 0) { | |
| bool_id = context->TakeNextId(); | |
| auto variant_bool = std::make_unique<Instruction>( | |
| context, spv::Op::OpTypeBool, 0, bool_id, | |
| std::initializer_list<opt::Operand>{}); | |
| module->AddType(std::move(variant_bool)); | |
| } | |
| } | |
| } | |
| void VariantDefs::CollectVarInsts() { | |
| for (size_t i = 0; i < variant_defs_.size(); ++i) { | |
| const auto variant_def = variant_defs_[i]; | |
| const auto* var_mod = variant_def.GetModule(); | |
| var_mod->ForEachInst([this, &i](const Instruction* inst) { | |
| if (CanBeFnVarCombined(inst)) { | |
| const size_t inst_hash = HashInst(inst); | |
| if (fnvar_usage_.find(inst_hash) == fnvar_usage_.end()) { | |
| fnvar_usage_.insert({inst_hash, {i}}); | |
| } else { | |
| assert(fnvar_usage_[inst_hash].size() < variant_defs_.size()); | |
| fnvar_usage_[inst_hash].push_back(i); | |
| } | |
| } | |
| }); | |
| } | |
| } | |
| bool VariantDefs::GenerateFnVarConstants() { | |
| assert(variant_defs_.size() > 0); | |
| assert(variant_defs_[0].IsBase()); | |
| if (variant_defs_.size() == 1) { | |
| return true; | |
| } | |
| for (auto& variant_def : variant_defs_) { | |
| Module* module = variant_def.GetModule(); | |
| IRContext* context = module->context(); | |
| uint32_t bool_id = FindIdOfBoolType(module); | |
| if (bool_id == 0) { | |
| // add a bool type if not present already | |
| bool_id = context->TakeNextId(); | |
| auto variant_bool = std::make_unique<Instruction>( | |
| context, spv::Op::OpTypeBool, 0, bool_id, | |
| std::initializer_list<opt::Operand>{}); | |
| module->AddType(std::move(variant_bool)); | |
| } | |
| // Spec constant architecture and target | |
| std::vector<uint32_t> spec_const_arch_ids; | |
| for (const auto& arch_def : variant_def.GetArchDefs()) { | |
| const uint32_t spec_const_arch_id = context->TakeNextId(); | |
| spec_const_arch_ids.push_back(spec_const_arch_id); | |
| auto inst = std::make_unique<Instruction>( | |
| context, spv::Op::OpSpecConstantArchitectureINTEL, bool_id, | |
| spec_const_arch_id, | |
| std::initializer_list<opt::Operand>{ | |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.category}}, | |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.family}}, | |
| // Using spec op opcode here expects then next operand to be | |
| // a type: | |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.op}}, | |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.architecture}}, | |
| }); | |
| module->AddType(std::move(inst)); | |
| } | |
| std::vector<uint32_t> spec_const_tgt_ids; | |
| for (const auto& tgt_def : variant_def.GetTgtDefs()) { | |
| const uint32_t spec_const_tgt_id = context->TakeNextId(); | |
| spec_const_tgt_ids.push_back(spec_const_tgt_id); | |
| auto inst = std::make_unique<Instruction>( | |
| context, spv::Op::OpSpecConstantTargetINTEL, bool_id, | |
| spec_const_tgt_id, | |
| std::initializer_list<opt::Operand>{ | |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {tgt_def.target}}, | |
| }); | |
| for (const auto& feat : tgt_def.features) { | |
| inst->AddOperand({SPV_OPERAND_TYPE_LITERAL_INTEGER, {feat}}); | |
| } | |
| module->AddType(std::move(inst)); | |
| } | |
| std::vector<uint32_t> spec_const_ids; | |
| // Spec constant capabilities | |
| const auto variant_capabilities = variant_def.GetCapabilities(); | |
| if (!variant_capabilities.empty()) { | |
| const uint32_t spec_const_cap_id = context->TakeNextId(); | |
| auto inst = std::make_unique<Instruction>( | |
| context, spv::Op::OpSpecConstantCapabilitiesINTEL, bool_id, | |
| spec_const_cap_id, std::initializer_list<opt::Operand>{}); | |
| for (const auto& cap : variant_capabilities) { | |
| inst->AddOperand({SPV_OPERAND_TYPE_CAPABILITY, {uint32_t(cap)}}); | |
| } | |
| module->AddType(std::move(inst)); | |
| spec_const_ids.push_back(spec_const_cap_id); | |
| } | |
| // Combine architectures such that, for the same module, those with the same | |
| // category and family are combined with AND and different cat/fam are | |
| // combined with OR. | |
| // This lets you create combinations like "architecture between X and Y". | |
| // map (category, family) -> IDs | |
| std::map<std::pair<uint32_t, uint32_t>, std::vector<uint32_t>> arch_map_and; | |
| for (size_t i = 0; i < spec_const_arch_ids.size(); ++i) { | |
| const auto& arch_def = variant_def.GetArchDefs()[i]; | |
| const auto id = spec_const_arch_ids[i]; | |
| const auto key = std::make_pair(arch_def.category, arch_def.family); | |
| if (arch_map_and.find(key) == arch_map_and.end()) { | |
| arch_map_and[key] = {id}; | |
| } else { | |
| arch_map_and[key].push_back(id); | |
| } | |
| } | |
| std::vector<uint32_t> arch_ids_or; | |
| for (const auto& it : arch_map_and) { | |
| const auto id = CombineIds(context, it.second, spv::Op::OpLogicalAnd); | |
| if (id > 0) { | |
| arch_ids_or.push_back(id); | |
| } | |
| } | |
| const uint32_t spec_const_arch_id = | |
| CombineIds(context, arch_ids_or, spv::Op::OpLogicalOr); | |
| if (spec_const_arch_id > 0) { | |
| spec_const_ids.push_back(spec_const_arch_id); | |
| } | |
| const uint32_t spec_const_tgt_id = | |
| CombineIds(context, spec_const_tgt_ids, spv::Op::OpLogicalOr); | |
| if (spec_const_tgt_id > 0) { | |
| spec_const_ids.push_back(spec_const_tgt_id); | |
| } | |
| uint32_t combined_spec_const_id = | |
| CombineIds(context, spec_const_ids, spv::Op::OpLogicalAnd); | |
| if (combined_spec_const_id == 0) { | |
| // If the variant module has no constraints, use SpecConstantTrue | |
| combined_spec_const_id = context->TakeNextId(); | |
| auto inst = std::make_unique<Instruction>( | |
| context, spv::Op::OpSpecConstantTrue, bool_id, combined_spec_const_id, | |
| std::initializer_list<opt::Operand>{}); | |
| context->module()->AddType(std::move(inst)); | |
| } | |
| assert(combined_spec_const_id != 0); | |
| // Add a name the combined boolean ID so we can look it up after the IDs are | |
| // shifted | |
| auto inst = std::make_unique<Instruction>(context, spv::Op::OpName); | |
| inst->AddOperand({SPV_OPERAND_TYPE_ID, {combined_spec_const_id}}); | |
| std::vector<uint32_t> str_words; | |
| utils::AppendToVector(variant_def.GetName(), &str_words); | |
| inst->AddOperand({SPV_OPERAND_TYPE_LITERAL_STRING, {str_words}}); | |
| module->AddDebug2Inst(std::move(inst)); | |
| // Annotate all instructions in the types section (eg. constants) with | |
| // ConditionalINTEL, unless they can be shared between variant_defs_ (eg. | |
| // types). Spec constants are excluded because they might have been | |
| // generated by this extension. | |
| for (const auto& type_inst : module->types_values()) { | |
| if (!CanBeFnVarCombined(&type_inst) && | |
| !spvOpcodeIsSpecConstant(type_inst.opcode())) { | |
| DecorateConditional(context, type_inst.result_id(), | |
| combined_spec_const_id); | |
| } | |
| } | |
| } | |
| // Annotate functions with ConditionalINTEL | |
| for (const auto& base_fn : *variant_defs_[0].GetModule()) { | |
| // For each function of the base module, find matching variant functions in | |
| // other modules | |
| auto base_fn_name = GetFnName(base_fn.DefInst()); | |
| if (base_fn_name.empty()) { | |
| err_ << "Could not find name of a function " << base_fn.result_id() | |
| << " in a base module " << variant_defs_[0].GetName() | |
| << ". To be usable by SPV_INTEL_function_variants, a function " | |
| "must either have an entry point or an export " | |
| "LinkAttribute decoration."; | |
| return false; | |
| } | |
| bool base_fn_needs_conditional = false; | |
| for (size_t i = 1; i < variant_defs_.size(); ++i) { | |
| const auto& variant_def = variant_defs_[i]; | |
| auto* variant_module = variant_def.GetModule(); | |
| auto* variant_context = variant_module->context(); | |
| for (const auto& var_fn : *variant_module) { | |
| auto var_fn_name = GetFnName(var_fn.DefInst()); | |
| if (var_fn_name.empty()) { | |
| err_ << "Could not find name of a function " << var_fn.result_id() | |
| << " in a base module " << variant_def.GetName() | |
| << ". To be usable by SPV_INTEL_function_variants, a function " | |
| "must either have an entry point or an export " | |
| "LinkAttribute decoration."; | |
| return false; | |
| } | |
| if (base_fn_name == var_fn_name) { | |
| base_fn_needs_conditional = true; | |
| } | |
| // each function in a variant module gets a ConditionalINTEL decoration | |
| uint32_t spec_const_id = | |
| FindSpecConstByName(variant_module, variant_def.GetName()); | |
| assert(spec_const_id != 0); | |
| DecorateConditional(variant_context, var_fn.result_id(), spec_const_id); | |
| ConvertEPToConditional(variant_module, var_fn, spec_const_id); | |
| } | |
| } | |
| if (base_fn_needs_conditional) { | |
| // only a base function that has a variant in another module gets a | |
| // ConditionalINTEL decoration, the others are common for all | |
| // variant_defs_ | |
| auto* base_module = variant_defs_[0].GetModule(); | |
| auto* base_context = base_module->context(); | |
| uint32_t spec_const_id = | |
| FindSpecConstByName(base_module, variant_defs_[0].GetName()); | |
| assert(spec_const_id != 0); | |
| DecorateConditional(base_context, base_fn.result_id(), spec_const_id); | |
| ConvertEPToConditional(base_module, base_fn, spec_const_id); | |
| } | |
| } | |
| return true; | |
| } | |
| void VariantDefs::CollectBaseFnCalls() { | |
| auto* base_mod = variant_defs_[0].GetModule(); | |
| assert(variant_defs_[0].IsBase()); | |
| const auto* base_def_use_mgr = base_mod->context()->get_def_use_mgr(); | |
| base_mod->ForEachInst([this, &base_def_use_mgr](const Instruction* inst) { | |
| if (inst->opcode() == spv::Op::OpFunctionCall) { | |
| // For each function call in base module, get the function name | |
| const auto fn_id = inst->GetOperand(2).AsId(); | |
| const auto* called_fn_inst = base_def_use_mgr->GetDef(fn_id); | |
| assert(called_fn_inst != nullptr); | |
| const auto called_fn_name = GetFnName(*called_fn_inst); | |
| assert(!called_fn_name.empty()); | |
| std::vector<std::pair<std::string, const opt::Function*>> called_fns; | |
| for (size_t i = 1; i < variant_defs_.size(); ++i) { | |
| // ... then see in which variant the called function was defined | |
| const auto& variant_def = variant_defs_[i]; | |
| assert(!variant_def.IsBase()); | |
| for (const auto& fn : *variant_def.GetModule()) { | |
| const auto fn_name = GetFnName(fn.DefInst()); | |
| if (fn_name == called_fn_name) { | |
| called_fns.push_back(std::make_pair(variant_def.GetName(), &fn)); | |
| } | |
| } | |
| } | |
| if (!called_fns.empty()) { | |
| base_fn_calls_[inst->result_id()] = called_fns; | |
| } | |
| } | |
| }); | |
| } | |
| void VariantDefs::CombineBaseFnCalls(IRContext* linked_context) { | |
| for (auto kv : base_fn_calls_) { | |
| const uint32_t call_id = kv.first; | |
| const auto called_fns = kv.second; | |
| if (called_fns.empty()) { | |
| return; | |
| } | |
| opt::BasicBlock* fn_call_bb = linked_context->get_instr_block(call_id); | |
| Instruction* found_call_inst = nullptr; | |
| auto bb_iter = fn_call_bb->begin(); | |
| while (bb_iter != fn_call_bb->end() && found_call_inst == nullptr) { | |
| if (bb_iter->HasResultId() && bb_iter->result_id() == call_id) { | |
| found_call_inst = &*bb_iter; | |
| } | |
| ++bb_iter; | |
| } | |
| if (found_call_inst == nullptr) { | |
| return; | |
| } | |
| const auto base_spec_const_id = FindSpecConstByName( | |
| variant_defs_[0].GetModule(), variant_defs_[0].GetName()); | |
| const auto base_type_op = found_call_inst->context() | |
| ->get_def_use_mgr() | |
| ->GetDef(found_call_inst->type_id()) | |
| ->opcode(); | |
| const auto base_call_id = found_call_inst->result_id(); | |
| // decorate the base call with ConditionalINTEL | |
| DecorateConditional(linked_context, base_call_id, base_spec_const_id); | |
| // Add OpFunctionCall for each variant | |
| Instruction* last_inst = found_call_inst; | |
| std::vector<std::pair<uint32_t, uint32_t>> var_call_ids; | |
| for (const auto& kv2 : called_fns) { | |
| const std::string var_name = kv2.first; | |
| const opt::Function* fn = kv2.second; | |
| const uint32_t spec_const_id = | |
| FindSpecConstByName(linked_context->module(), var_name); | |
| assert(spec_const_id != 0); | |
| const uint32_t var_call_id = linked_context->TakeNextId(); | |
| var_call_ids.push_back(std::make_pair(spec_const_id, var_call_id)); | |
| auto* var_call_inst = found_call_inst->Clone(linked_context); | |
| var_call_inst->SetResultId(var_call_id); | |
| var_call_inst->SetOperand(2, {fn->result_id()}); | |
| var_call_inst->InsertAfter(last_inst); | |
| linked_context->set_instr_block(var_call_inst, fn_call_bb); | |
| last_inst = var_call_inst; | |
| // decorate the variant call with ConditionalINTEL | |
| DecorateConditional(linked_context, var_call_id, spec_const_id); | |
| } | |
| if (base_type_op != spv::Op::OpTypeVoid) { | |
| // Add OpConditionalCopyObjectINTEL combining the function calls | |
| const uint32_t result_id = linked_context->TakeNextId(); | |
| auto conditional_copy_inst = new Instruction( | |
| linked_context, spv::Op::OpConditionalCopyObjectINTEL, | |
| found_call_inst->type_id(), result_id, | |
| {{SPV_OPERAND_TYPE_ID, {base_spec_const_id}}, | |
| {SPV_OPERAND_TYPE_ID, {found_call_inst->result_id()}}}); | |
| for (const auto& kv3 : var_call_ids) { | |
| const auto spec_const_id = kv3.first; | |
| const auto var_call_id = kv3.second; | |
| conditional_copy_inst->AddOperand( | |
| {SPV_OPERAND_TYPE_ID, {spec_const_id}}); | |
| conditional_copy_inst->AddOperand({SPV_OPERAND_TYPE_ID, {var_call_id}}); | |
| } | |
| conditional_copy_inst->InsertAfter(last_inst); | |
| linked_context->set_instr_block(conditional_copy_inst, fn_call_bb); | |
| last_inst = conditional_copy_inst; | |
| // In all remaining instructions within the basic block, replace all | |
| // usages of the base call ID with the result of | |
| // OpConditionalCopyObjectINTEL | |
| do { | |
| last_inst = last_inst->NextNode(); | |
| last_inst->ForEachInId([base_call_id, result_id](uint32_t* id) { | |
| if (*id == base_call_id) { | |
| *id = result_id; | |
| } | |
| }); | |
| } while (last_inst != nullptr && *last_inst != *fn_call_bb->tail()); | |
| } | |
| } | |
| // Combine spec consts for the base module (base module is activated if all | |
| // variant defs are inactive AND the base module constraints are satisfied) | |
| std::vector<uint32_t> var_spec_const_ids; | |
| for (const auto& variant_def : variant_defs_) { | |
| if (variant_def.IsBase()) { | |
| continue; | |
| } | |
| const auto id = | |
| FindSpecConstByName(linked_context->module(), variant_def.GetName()); | |
| assert(id != 0); | |
| var_spec_const_ids.push_back(id); | |
| } | |
| const uint32_t base_or_id = | |
| CombineIds(linked_context, var_spec_const_ids, spv::Op::OpLogicalOr); | |
| if (base_or_id != 0) { | |
| const uint32_t bool_id = FindIdOfBoolType(linked_context->module()); | |
| assert(bool_id != 0); | |
| const uint32_t base_not_id = linked_context->TakeNextId(); | |
| auto spec_const_op_inst = std::make_unique<Instruction>( | |
| linked_context, spv::Op::OpSpecConstantOp, bool_id, base_not_id, | |
| std::initializer_list<opt::Operand>{ | |
| {SPV_OPERAND_TYPE_SPEC_CONSTANT_OP_NUMBER, | |
| {(uint32_t)(spv::Op::OpLogicalNot)}}, | |
| {SPV_OPERAND_TYPE_ID, {base_or_id}}}); | |
| linked_context->module()->AddType(std::move(spec_const_op_inst)); | |
| // Update any ConditionalINTEL annotations, names and entry points | |
| // referencing the old spec const ID to use the new one | |
| const uint32_t old_base_spec_const_id = FindSpecConstByName( | |
| linked_context->module(), variant_defs_[0].GetName()); | |
| assert(old_base_spec_const_id != 0); | |
| const uint32_t base_spec_const_id = | |
| CombineIds(linked_context, {old_base_spec_const_id, base_not_id}, | |
| spv::Op::OpLogicalAnd); | |
| for (auto& annot_inst : linked_context->module()->annotations()) { | |
| if ((annot_inst.GetSingleWordOperand(1) == | |
| uint32_t(spv::Decoration::ConditionalINTEL)) && | |
| (annot_inst.GetOperand(2).AsId() == old_base_spec_const_id)) { | |
| annot_inst.SetOperand(2, {base_spec_const_id}); | |
| } | |
| } | |
| for (auto& name_inst : linked_context->module()->debugs2()) { | |
| if ((name_inst.opcode() == spv::Op::OpName) && | |
| (name_inst.GetOperand(0).AsId() == old_base_spec_const_id)) { | |
| name_inst.SetOperand(0, {base_spec_const_id}); | |
| } | |
| } | |
| for (auto& ep_inst : linked_context->module()->entry_points()) { | |
| if ((ep_inst.opcode() == spv::Op::OpConditionalEntryPointINTEL) && | |
| (ep_inst.GetOperand(0).AsId() == old_base_spec_const_id)) { | |
| ep_inst.SetOperand(0, {base_spec_const_id}); | |
| } | |
| } | |
| linked_context->module()->ForEachInst( | |
| [old_base_spec_const_id, base_spec_const_id](Instruction* inst) { | |
| if (inst->opcode() == spv::Op::OpConditionalCopyObjectINTEL) { | |
| inst->ForEachInId( | |
| [old_base_spec_const_id, base_spec_const_id](uint32_t* id) { | |
| if (*id == old_base_spec_const_id) { | |
| *id = base_spec_const_id; | |
| } | |
| }); | |
| } | |
| }); | |
| } | |
| } | |
| void VariantDefs::CombineInstructions(IRContext* linked_context) { | |
| // cache for existing variant ID combinations | |
| std::map<std::vector<size_t>, uint32_t> spec_const_comb_ids; | |
| linked_context->module()->ForEachInst( | |
| [this, &linked_context, &spec_const_comb_ids](Instruction* inst) { | |
| if (!CanBeFnVarCombined(inst)) { | |
| return; | |
| } | |
| const size_t inst_hash = HashInst(inst); | |
| if (fnvar_usage_.find(inst_hash) != fnvar_usage_.end()) { | |
| const std::vector<size_t> var_ids = fnvar_usage_[inst_hash]; | |
| const uint32_t spec_const_comb_id = CombineVariantDefs( | |
| variant_defs_, var_ids, linked_context, spec_const_comb_ids); | |
| if (spec_const_comb_id != 0) { | |
| if (inst->HasResultId()) { | |
| DecorateConditional(linked_context, inst->result_id(), | |
| spec_const_comb_id); | |
| } else if (inst->opcode() == spv::Op::OpCapability) { | |
| const uint32_t cap = inst->GetSingleWordOperand(0); | |
| inst->SetOpcode(spv::Op::OpConditionalCapabilityINTEL); | |
| inst->SetInOperands({{SPV_OPERAND_TYPE_ID, {spec_const_comb_id}}, | |
| {SPV_OPERAND_TYPE_CAPABILITY, {cap}}}); | |
| } else if (inst->opcode() == spv::Op::OpExtension) { | |
| const std::string ext_name = inst->GetOperand(0).AsString(); | |
| inst->SetOpcode(spv::Op::OpConditionalExtensionINTEL); | |
| inst->SetInOperands({{SPV_OPERAND_TYPE_ID, {spec_const_comb_id}}, | |
| {SPV_OPERAND_TYPE_LITERAL_STRING, | |
| {utils::MakeVector(ext_name)}}}); | |
| } else { | |
| assert(false && "Unsupported"); | |
| } | |
| } | |
| } | |
| }); | |
| } | |
| } // namespace spvtools |