| // Copyright 2025 The Khronos Group Inc. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "fnvar.h" |
| |
| #include <initializer_list> |
| #include <memory> |
| #include <sstream> |
| |
| #include "source/opt/instruction.h" |
| |
| namespace spvtools { |
| |
| using opt::Function; |
| using opt::Instruction; |
| using opt::analysis::Type; |
| |
| namespace { |
| // Helper functions |
| |
| // Parses a CSV source string for the purpose of this extension. |
| // |
| // Required columns must be known in advance and supplied as the required_cols |
| // argument -- this is used for error checking. Values are assumed to be |
| // separated by CSV_SEP. The input source string is assumed to be the output of |
| // io::ReadTextFile and no other validation, apart from the CSV parsing, is |
| // performed. |
| // |
| // Returns true on success, false on error (with error message stored in |
| // err_msg). |
| bool ParseCsv(const std::string& source, |
| const std::vector<std::string>& required_cols, |
| std::stringstream& err_msg, |
| std::vector<std::vector<std::string>>& result) { |
| std::stringstream fn_variants_csv_stream(source); |
| std::string line; |
| std::vector<std::string> columns; |
| constexpr char CSV_SEP = ','; |
| bool first_line = true; |
| |
| while (std::getline(fn_variants_csv_stream, line, '\n')) { |
| if (line.empty()) { |
| continue; |
| } |
| |
| std::vector<std::string> vals; |
| std::string val; |
| std::stringstream line_stream(line); |
| auto* vec = first_line ? &columns : &vals; |
| |
| while (std::getline(line_stream, val, CSV_SEP)) { |
| vec->push_back(val); |
| } |
| |
| if (!line_stream && val.empty()) { |
| vec->push_back(""); |
| } |
| |
| if (!first_line) { |
| if (vals.size() != columns.size()) { |
| err_msg << "Number of values does not match the number of columns. " |
| "Offending line:\n" |
| << line; |
| return false; |
| } |
| result.push_back(vals); |
| } |
| |
| first_line = false; |
| } |
| |
| // check if required columns match actual columns (ordering matters) |
| |
| if (columns.size() != required_cols.size()) { |
| err_msg << "Invalid number of CSV columns: " << columns.size() |
| << ", expected " << required_cols.size() << "."; |
| return false; |
| } |
| |
| for (size_t i = 0; i < columns.size(); ++i) { |
| if (columns[i] != required_cols[i]) { |
| err_msg << "Invalid name of column " << i + 1 << ". Expected '" |
| << required_cols[i] << "', got '" << columns[i] << "'."; |
| return false; |
| } |
| } |
| |
| return true; |
| } |
| |
| // Annotate ID with ConditionalINTEL decoration |
| void DecorateConditional(IRContext* context, uint32_t id_to_decorate, |
| uint32_t spec_const_id) { |
| auto decor_instr = |
| std::make_unique<Instruction>(context, spv::Op::OpDecorate); |
| decor_instr->AddOperand({SPV_OPERAND_TYPE_ID, {id_to_decorate}}); |
| decor_instr->AddOperand({SPV_OPERAND_TYPE_DECORATION, |
| {uint32_t(spv::Decoration::ConditionalINTEL)}}); |
| decor_instr->AddOperand({SPV_OPERAND_TYPE_ID, {spec_const_id}}); |
| context->module()->AddAnnotationInst(std::move(decor_instr)); |
| } |
| |
| // Finds entry point corresponding to a function |
| // |
| // Returns null if not found, otherwise returns pointer to the EP Instruction. |
| Instruction* FindEntryPoint(const Instruction& fn_inst) { |
| auto* mod = fn_inst.context()->module(); |
| for (auto& entry_point : mod->entry_points()) { |
| const int ep_i = |
| entry_point.opcode() == spv::Op::OpConditionalEntryPointINTEL ? 2 : 1; |
| if (entry_point.GetOperand(ep_i).AsId() == fn_inst.result_id()) { |
| return &entry_point; |
| } |
| } |
| return nullptr; |
| } |
| |
| // If the function has an entry point, converts it to a conditional one |
| void ConvertEPToConditional(Module* module, const Function& fn, |
| uint32_t spec_const_id) { |
| for (const auto& ep_inst : module->entry_points()) { |
| if (ep_inst.opcode() == spv::Op::OpEntryPoint) { |
| auto* entry_point = FindEntryPoint(fn.DefInst()); |
| if (entry_point != nullptr) { |
| std::vector<opt::Operand> old_operands; |
| for (auto operand : *entry_point) { |
| old_operands.push_back(operand); |
| } |
| entry_point->ToNop(); |
| entry_point->SetOpcode(spv::Op::OpConditionalEntryPointINTEL); |
| entry_point->AddOperand({SPV_OPERAND_TYPE_ID, {spec_const_id}}); |
| for (auto old_operand : old_operands) { |
| entry_point->AddOperand(old_operand); |
| } |
| } |
| } |
| } |
| } |
| |
| // Finds ID of a bool type (returns 0 if not found) |
| uint32_t FindIdOfBoolType(const Module* const mod) { |
| return mod->context()->get_type_mgr()->GetBoolTypeId(); |
| } |
| |
| // Combines IDs using OpSpecConstantOp with the operation defined by cmp_op. |
| // |
| // Returns the ID of the final result. If there are no IDs, returns 0. If there |
| // is one ID, does not generate any instructions and returns the ID. |
| uint32_t CombineIds(IRContext* const context, const std::vector<uint32_t>& ids, |
| spv::Op cmp_op) { |
| if (ids.empty()) { |
| return 0; |
| } else if (ids.size() == 1) { |
| return ids[0]; |
| } else { |
| uint32_t bool_id = FindIdOfBoolType(context->module()); |
| assert(bool_id != 0); |
| |
| uint32_t prev_spec_const_id = ids[0]; |
| |
| for (size_t i = 1; i < ids.size(); ++i) { |
| const uint32_t id = ids[i]; |
| const uint32_t spec_const_op_id = context->TakeNextId(); |
| |
| auto inst = std::make_unique<Instruction>( |
| context, spv::Op::OpSpecConstantOp, bool_id, spec_const_op_id, |
| std::initializer_list<opt::Operand>{ |
| {SPV_OPERAND_TYPE_SPEC_CONSTANT_OP_NUMBER, {(uint32_t)(cmp_op)}}, |
| {SPV_OPERAND_TYPE_ID, {prev_spec_const_id}}, |
| {SPV_OPERAND_TYPE_ID, {id}}}); |
| context->module()->AddType(std::move(inst)); |
| |
| prev_spec_const_id = spec_const_op_id; |
| } |
| |
| return prev_spec_const_id; |
| } |
| } |
| |
| // Returns whether instruction can be shared between variant modules and |
| // combined using spec constants (such as conditional capabilities). |
| bool CanBeFnVarCombined(const Instruction* inst) { |
| const spv::Op opcode = inst->opcode(); |
| |
| if ((opcode != spv::Op::OpExtInstImport) && |
| (opcode != spv::Op::OpCapability) && (opcode != spv::Op::OpExtension) && |
| !spvOpcodeGeneratesType(opcode)) { |
| return false; |
| } |
| |
| if ((opcode == spv::Op::OpCapability) && |
| ((inst->GetSingleWordOperand(0) == |
| static_cast<uint32_t>(spv::Capability::FunctionVariantsINTEL)) || |
| (inst->GetSingleWordOperand(0) == |
| static_cast<uint32_t>(spv::Capability::SpecConditionalINTEL)))) { |
| // Always enabled |
| return false; |
| } |
| |
| if ((opcode == spv::Op::OpExtension) && |
| (inst->GetOperand(0).AsString() == FNVAR_EXT_NAME)) { |
| // Always enabled |
| return false; |
| } |
| |
| return true; |
| } |
| |
| // Calculates hash of an instruction. |
| // |
| // Applicable only to instructions that can be combined (ie. with |
| // CanBeFnVarCombined being true) and from those, hash can be only computed for |
| // selected instructions. Computing hash from other instruction is unsupported. |
| size_t HashInst(const Instruction* inst) { |
| if (CanBeFnVarCombined(inst)) { |
| if (spvOpcodeGeneratesType(inst->opcode())) { |
| const Type* t = |
| inst->context()->get_type_mgr()->GetType(inst->result_id()); |
| assert(t != nullptr); |
| return t->HashValue(); |
| } |
| |
| if (inst->opcode() == spv::Op::OpExtension) { |
| const auto name = inst->GetOperand(0).AsString(); |
| return std::hash<std::string>()(name); |
| } |
| |
| if (inst->opcode() == spv::Op::OpCapability) { |
| const auto cap = inst->GetSingleWordOperand(0); |
| return std::hash<uint32_t>()(cap); |
| } |
| |
| if (inst->opcode() == spv::Op::OpExtInstImport) { |
| const auto name = inst->GetOperand(1).AsString(); |
| return std::hash<std::string>()(name); |
| } |
| } |
| |
| assert(false && "Unsupported instruction hash"); |
| return std::hash<const Instruction*>()(inst); |
| } |
| |
| std::string GetFnName(const Instruction& fn_inst) { |
| // Check entry point |
| const auto* ep_inst = FindEntryPoint(fn_inst); |
| if (ep_inst != nullptr) { |
| const int name_i = |
| ep_inst->opcode() == spv::Op::OpConditionalEntryPointINTEL ? 3 : 2; |
| return ep_inst->GetOperand(name_i).AsString(); |
| } |
| |
| // Check name of export linkage attribute decoration |
| const auto* decor_mgr = fn_inst.context()->get_decoration_mgr(); |
| for (const auto* inst : |
| decor_mgr->GetDecorationsFor(fn_inst.result_id(), true)) { |
| const auto decoration = inst->GetOperand(1); |
| if ((decoration.type == SPV_OPERAND_TYPE_DECORATION) && |
| (decoration.words.size() == 1) && |
| (decoration.words[0] == |
| static_cast<uint32_t>(spv::Decoration::LinkageAttributes))) { |
| const auto linkage = inst->GetOperand(3); |
| if ((linkage.type == SPV_OPERAND_TYPE_LINKAGE_TYPE) && |
| (linkage.words.size() == 1) && |
| (linkage.words[0] == |
| static_cast<uint32_t>(spv::LinkageType::Export))) { |
| // decorates fn with LinkageAttribute and Export linkage type -> get the |
| // name |
| return inst->GetOperand(2).AsString(); |
| } |
| } |
| } |
| |
| return ""; |
| } |
| |
| uint32_t FindSpecConstByName(const Module* mod, std::string name) { |
| for (const auto* const_inst : mod->context()->GetConstants()) { |
| if (opt::IsSpecConstantInst(const_inst->opcode())) { |
| const auto id = const_inst->result_id(); |
| for (const auto& name_inst : mod->debugs2()) { |
| if ((name_inst.opcode() == spv::Op::OpName) && |
| (name_inst.GetOperand(0).AsId() == id) && |
| (name_inst.GetOperand(1).AsString() == name)) { |
| return id; |
| } |
| } |
| } |
| } |
| return 0; |
| } |
| |
| uint32_t CombineVariantDefs(const std::vector<VariantDef>& variant_defs, |
| const std::vector<size_t> var_ids, |
| IRContext* context, |
| std::map<std::vector<size_t>, uint32_t>& cache) { |
| assert(var_ids.size() <= variant_defs.size()); |
| uint32_t spec_const_comb_id = 0; |
| if (var_ids.size() != variant_defs.size()) { |
| // if not used by all variants |
| if (cache.find(var_ids) == cache.end()) { |
| // cache variant combinations |
| std::vector<uint32_t> spec_const_ids; |
| for (const auto& var_id : var_ids) { |
| const auto var_name = variant_defs[var_id].GetName(); |
| const auto var_spec_id = |
| FindSpecConstByName(context->module(), var_name); |
| spec_const_ids.push_back(var_spec_id); |
| } |
| spec_const_comb_id = |
| CombineIds(context, spec_const_ids, spv::Op::OpLogicalOr); |
| assert(spec_const_comb_id != 0); |
| cache.insert({var_ids, spec_const_comb_id}); |
| } else { |
| spec_const_comb_id = cache[var_ids]; |
| } |
| } |
| return spec_const_comb_id; |
| } |
| |
| bool strToInt(std::string s, uint32_t* x) { |
| for (const char& c : s) { |
| if (c < '0' || c > '9') { |
| return false; |
| } |
| } |
| if (!(std::stringstream(s) >> *x)) { |
| return false; |
| } |
| return true; |
| } |
| |
| } // anonymous namespace |
| |
| bool VariantDefs::ProcessFnVar(const LinkerOptions& options, |
| const std::vector<Module*>& modules) { |
| assert(variant_defs_.empty()); |
| assert(modules.size() == options.GetInFiles().size()); |
| |
| for (size_t i = 0; i < modules.size(); ++i) { |
| const auto* feat_mgr = modules[i]->context()->get_feature_mgr(); |
| if ((feat_mgr->HasCapability(spv::Capability::FunctionVariantsINTEL)) || |
| (feat_mgr->HasCapability(spv::Capability::SpecConditionalINTEL)) || |
| (feat_mgr->HasExtension(kSPV_INTEL_function_variants))) { |
| // In principle, it can be done but it's complicated due to having to |
| // combine the existing conditionals with the new ones. For example, |
| // conditional capabilities would need to become "doubly-conditional". |
| err_ << "Creating multitarget modules from multitarget modules is not " |
| "supported. Offending file: " |
| << options.GetInFiles()[i]; |
| return false; |
| } |
| } |
| |
| std::vector<std::vector<std::string>> target_rows; |
| std::vector<std::vector<std::string>> architecture_rows; |
| |
| if (!options.GetFnVarTargetsCsv().empty()) { |
| const std::vector<std::string> tgt_cols = {"module", "target", "features"}; |
| if (!ParseCsv(options.GetFnVarTargetsCsv(), tgt_cols, err_, target_rows)) { |
| return false; |
| } |
| } |
| |
| if (!options.GetFnVarArchitecturesCsv().empty()) { |
| const std::vector<std::string> arch_cols = {"module", "category", "family", |
| "op", "architecture"}; |
| if (!ParseCsv(options.GetFnVarArchitecturesCsv(), arch_cols, err_, |
| architecture_rows)) { |
| return false; |
| } |
| } |
| |
| // check that all modules defined in the CSV exist |
| |
| for (const auto& tgt_vals : target_rows) { |
| bool found = false; |
| for (const auto& in_file : options.GetInFiles()) { |
| if (tgt_vals[0] == in_file) { |
| found = true; |
| } |
| } |
| if (!found) { |
| err_ << "Module '" << tgt_vals[0] |
| << "' found in targets CSV not passed to the CLI."; |
| return false; |
| } |
| } |
| |
| for (const auto& arch_vals : architecture_rows) { |
| bool found = false; |
| for (const auto& in_file : options.GetInFiles()) { |
| if (arch_vals[0] == in_file) { |
| found = true; |
| } |
| } |
| if (!found) { |
| err_ << "Module '" << arch_vals[0] |
| << "' found in architectures CSV not passed to the CLI."; |
| return false; |
| } |
| } |
| |
| // create per-module variant defs |
| |
| for (size_t i = 0; i < modules.size(); ++i) { |
| // first module passed to the CLI is considered the base module |
| bool is_base = i == 0; |
| const auto name = options.GetInFiles()[i]; |
| auto variant_def = VariantDef(is_base, name, modules[i]); |
| |
| for (const auto& arch_row : architecture_rows) { |
| const auto row_name = arch_row[0]; |
| if (row_name == name) { |
| uint32_t category, family, op, architecture; |
| |
| if (!strToInt(arch_row[1], &category)) { |
| err_ << "Error converting " << arch_row[1] |
| << " to architecture category."; |
| return false; |
| } |
| if (!strToInt(arch_row[2], &family)) { |
| err_ << "Error converting " << arch_row[2] |
| << " to architecture family."; |
| return false; |
| } |
| if (!strToInt(arch_row[3], &op)) { |
| err_ << "Error converting " << arch_row[3] << " to architecture op."; |
| return false; |
| } |
| if (!strToInt(arch_row[4], &architecture)) { |
| err_ << "Error converting " << arch_row[4] << " to architecture."; |
| return false; |
| } |
| |
| variant_def.AddArchDef(category, family, op, architecture); |
| } |
| } |
| |
| for (const auto& tgt_row : target_rows) { |
| const auto row_name = tgt_row[0]; |
| if (row_name == name) { |
| uint32_t target; |
| std::vector<uint32_t> features; |
| |
| if (!strToInt(tgt_row[1], &target)) { |
| err_ << "Error converting " << tgt_row[1] << " to target."; |
| return false; |
| } |
| |
| // get features as FEAT_SEP-delimited integers |
| |
| std::stringstream feat_stream(tgt_row[2]); |
| std::string feat; |
| while (std::getline(feat_stream, feat, FEAT_SEP)) { |
| uint32_t ufeat; |
| // if (!(std::stringstream(feat) >> ufeat)) { |
| if (!strToInt(feat, &ufeat)) { |
| err_ << "Error converting " << feat << " in " << tgt_row[2] |
| << " to target feature."; |
| return false; |
| } |
| features.push_back(ufeat); |
| } |
| |
| variant_def.AddTgtDef(target, features); |
| } |
| } |
| |
| if (options.GetHasFnVarCapabilities()) { |
| variant_def.InferCapabilities(); |
| } |
| |
| variant_defs_.push_back(variant_def); |
| } |
| |
| return true; |
| } |
| |
| bool VariantDefs::ProcessVariantDefs() { |
| EnsureBoolType(); |
| CollectVarInsts(); |
| if (!GenerateFnVarConstants()) { |
| return false; |
| } |
| CollectBaseFnCalls(); |
| return true; |
| } |
| |
| void VariantDefs::GenerateHeader(IRContext* linked_context) { |
| linked_context->AddCapability(spv::Capability::SpecConditionalINTEL); |
| linked_context->AddCapability(spv::Capability::FunctionVariantsINTEL); |
| linked_context->AddExtension(std::string(FNVAR_EXT_NAME)); |
| |
| // Specifies used registry version |
| auto inst = |
| std::make_unique<Instruction>(linked_context, spv::Op::OpModuleProcessed); |
| std::stringstream line; |
| line << "SPV_INTEL_function_variants registry version " |
| << FNVAR_REGISTRY_VERSION; |
| inst->AddOperand( |
| {SPV_OPERAND_TYPE_LITERAL_STRING, utils::MakeVector(line.str())}); |
| linked_context->AddDebug3Inst(std::move(inst)); |
| } |
| |
| void VariantDefs::CombineVariantInstructions(IRContext* linked_context) { |
| CombineBaseFnCalls(linked_context); |
| CombineInstructions(linked_context); |
| } |
| |
| void VariantDefs::EnsureBoolType() { |
| for (auto& variant_def : variant_defs_) { |
| Module* module = variant_def.GetModule(); |
| IRContext* context = module->context(); |
| |
| uint32_t bool_id = FindIdOfBoolType(module); |
| if (bool_id == 0) { |
| bool_id = context->TakeNextId(); |
| auto variant_bool = std::make_unique<Instruction>( |
| context, spv::Op::OpTypeBool, 0, bool_id, |
| std::initializer_list<opt::Operand>{}); |
| module->AddType(std::move(variant_bool)); |
| } |
| } |
| } |
| |
| void VariantDefs::CollectVarInsts() { |
| for (size_t i = 0; i < variant_defs_.size(); ++i) { |
| const auto variant_def = variant_defs_[i]; |
| const auto* var_mod = variant_def.GetModule(); |
| |
| var_mod->ForEachInst([this, &i](const Instruction* inst) { |
| if (CanBeFnVarCombined(inst)) { |
| const size_t inst_hash = HashInst(inst); |
| if (fnvar_usage_.find(inst_hash) == fnvar_usage_.end()) { |
| fnvar_usage_.insert({inst_hash, {i}}); |
| } else { |
| assert(fnvar_usage_[inst_hash].size() < variant_defs_.size()); |
| fnvar_usage_[inst_hash].push_back(i); |
| } |
| } |
| }); |
| } |
| } |
| |
| bool VariantDefs::GenerateFnVarConstants() { |
| assert(variant_defs_.size() > 0); |
| assert(variant_defs_[0].IsBase()); |
| |
| if (variant_defs_.size() == 1) { |
| return true; |
| } |
| |
| for (auto& variant_def : variant_defs_) { |
| Module* module = variant_def.GetModule(); |
| IRContext* context = module->context(); |
| |
| uint32_t bool_id = FindIdOfBoolType(module); |
| if (bool_id == 0) { |
| // add a bool type if not present already |
| bool_id = context->TakeNextId(); |
| auto variant_bool = std::make_unique<Instruction>( |
| context, spv::Op::OpTypeBool, 0, bool_id, |
| std::initializer_list<opt::Operand>{}); |
| module->AddType(std::move(variant_bool)); |
| } |
| |
| // Spec constant architecture and target |
| |
| std::vector<uint32_t> spec_const_arch_ids; |
| for (const auto& arch_def : variant_def.GetArchDefs()) { |
| const uint32_t spec_const_arch_id = context->TakeNextId(); |
| spec_const_arch_ids.push_back(spec_const_arch_id); |
| |
| auto inst = std::make_unique<Instruction>( |
| context, spv::Op::OpSpecConstantArchitectureINTEL, bool_id, |
| spec_const_arch_id, |
| std::initializer_list<opt::Operand>{ |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.category}}, |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.family}}, |
| // Using spec op opcode here expects then next operand to be |
| // a type: |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.op}}, |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {arch_def.architecture}}, |
| }); |
| module->AddType(std::move(inst)); |
| } |
| |
| std::vector<uint32_t> spec_const_tgt_ids; |
| for (const auto& tgt_def : variant_def.GetTgtDefs()) { |
| const uint32_t spec_const_tgt_id = context->TakeNextId(); |
| spec_const_tgt_ids.push_back(spec_const_tgt_id); |
| |
| auto inst = std::make_unique<Instruction>( |
| context, spv::Op::OpSpecConstantTargetINTEL, bool_id, |
| spec_const_tgt_id, |
| std::initializer_list<opt::Operand>{ |
| {SPV_OPERAND_TYPE_LITERAL_INTEGER, {tgt_def.target}}, |
| }); |
| for (const auto& feat : tgt_def.features) { |
| inst->AddOperand({SPV_OPERAND_TYPE_LITERAL_INTEGER, {feat}}); |
| } |
| module->AddType(std::move(inst)); |
| } |
| |
| std::vector<uint32_t> spec_const_ids; |
| |
| // Spec constant capabilities |
| |
| const auto variant_capabilities = variant_def.GetCapabilities(); |
| if (!variant_capabilities.empty()) { |
| const uint32_t spec_const_cap_id = context->TakeNextId(); |
| auto inst = std::make_unique<Instruction>( |
| context, spv::Op::OpSpecConstantCapabilitiesINTEL, bool_id, |
| spec_const_cap_id, std::initializer_list<opt::Operand>{}); |
| for (const auto& cap : variant_capabilities) { |
| inst->AddOperand({SPV_OPERAND_TYPE_CAPABILITY, {uint32_t(cap)}}); |
| } |
| module->AddType(std::move(inst)); |
| spec_const_ids.push_back(spec_const_cap_id); |
| } |
| |
| // Combine architectures such that, for the same module, those with the same |
| // category and family are combined with AND and different cat/fam are |
| // combined with OR. |
| // This lets you create combinations like "architecture between X and Y". |
| |
| // map (category, family) -> IDs |
| std::map<std::pair<uint32_t, uint32_t>, std::vector<uint32_t>> arch_map_and; |
| |
| for (size_t i = 0; i < spec_const_arch_ids.size(); ++i) { |
| const auto& arch_def = variant_def.GetArchDefs()[i]; |
| const auto id = spec_const_arch_ids[i]; |
| const auto key = std::make_pair(arch_def.category, arch_def.family); |
| if (arch_map_and.find(key) == arch_map_and.end()) { |
| arch_map_and[key] = {id}; |
| } else { |
| arch_map_and[key].push_back(id); |
| } |
| } |
| |
| std::vector<uint32_t> arch_ids_or; |
| for (const auto& it : arch_map_and) { |
| const auto id = CombineIds(context, it.second, spv::Op::OpLogicalAnd); |
| if (id > 0) { |
| arch_ids_or.push_back(id); |
| } |
| } |
| |
| const uint32_t spec_const_arch_id = |
| CombineIds(context, arch_ids_or, spv::Op::OpLogicalOr); |
| if (spec_const_arch_id > 0) { |
| spec_const_ids.push_back(spec_const_arch_id); |
| } |
| |
| const uint32_t spec_const_tgt_id = |
| CombineIds(context, spec_const_tgt_ids, spv::Op::OpLogicalOr); |
| if (spec_const_tgt_id > 0) { |
| spec_const_ids.push_back(spec_const_tgt_id); |
| } |
| |
| uint32_t combined_spec_const_id = |
| CombineIds(context, spec_const_ids, spv::Op::OpLogicalAnd); |
| if (combined_spec_const_id == 0) { |
| // If the variant module has no constraints, use SpecConstantTrue |
| combined_spec_const_id = context->TakeNextId(); |
| auto inst = std::make_unique<Instruction>( |
| context, spv::Op::OpSpecConstantTrue, bool_id, combined_spec_const_id, |
| std::initializer_list<opt::Operand>{}); |
| context->module()->AddType(std::move(inst)); |
| } |
| assert(combined_spec_const_id != 0); |
| |
| // Add a name the combined boolean ID so we can look it up after the IDs are |
| // shifted |
| auto inst = std::make_unique<Instruction>(context, spv::Op::OpName); |
| inst->AddOperand({SPV_OPERAND_TYPE_ID, {combined_spec_const_id}}); |
| std::vector<uint32_t> str_words; |
| utils::AppendToVector(variant_def.GetName(), &str_words); |
| inst->AddOperand({SPV_OPERAND_TYPE_LITERAL_STRING, {str_words}}); |
| module->AddDebug2Inst(std::move(inst)); |
| |
| // Annotate all instructions in the types section (eg. constants) with |
| // ConditionalINTEL, unless they can be shared between variant_defs_ (eg. |
| // types). Spec constants are excluded because they might have been |
| // generated by this extension. |
| for (const auto& type_inst : module->types_values()) { |
| if (!CanBeFnVarCombined(&type_inst) && |
| !spvOpcodeIsSpecConstant(type_inst.opcode())) { |
| DecorateConditional(context, type_inst.result_id(), |
| combined_spec_const_id); |
| } |
| } |
| } |
| |
| // Annotate functions with ConditionalINTEL |
| |
| for (const auto& base_fn : *variant_defs_[0].GetModule()) { |
| // For each function of the base module, find matching variant functions in |
| // other modules |
| |
| auto base_fn_name = GetFnName(base_fn.DefInst()); |
| if (base_fn_name.empty()) { |
| err_ << "Could not find name of a function " << base_fn.result_id() |
| << " in a base module " << variant_defs_[0].GetName() |
| << ". To be usable by SPV_INTEL_function_variants, a function " |
| "must either have an entry point or an export " |
| "LinkAttribute decoration."; |
| return false; |
| } |
| |
| bool base_fn_needs_conditional = false; |
| for (size_t i = 1; i < variant_defs_.size(); ++i) { |
| const auto& variant_def = variant_defs_[i]; |
| auto* variant_module = variant_def.GetModule(); |
| auto* variant_context = variant_module->context(); |
| |
| for (const auto& var_fn : *variant_module) { |
| auto var_fn_name = GetFnName(var_fn.DefInst()); |
| if (var_fn_name.empty()) { |
| err_ << "Could not find name of a function " << var_fn.result_id() |
| << " in a base module " << variant_def.GetName() |
| << ". To be usable by SPV_INTEL_function_variants, a function " |
| "must either have an entry point or an export " |
| "LinkAttribute decoration."; |
| return false; |
| } |
| |
| if (base_fn_name == var_fn_name) { |
| base_fn_needs_conditional = true; |
| } |
| |
| // each function in a variant module gets a ConditionalINTEL decoration |
| |
| uint32_t spec_const_id = |
| FindSpecConstByName(variant_module, variant_def.GetName()); |
| assert(spec_const_id != 0); |
| DecorateConditional(variant_context, var_fn.result_id(), spec_const_id); |
| ConvertEPToConditional(variant_module, var_fn, spec_const_id); |
| } |
| } |
| |
| if (base_fn_needs_conditional) { |
| // only a base function that has a variant in another module gets a |
| // ConditionalINTEL decoration, the others are common for all |
| // variant_defs_ |
| auto* base_module = variant_defs_[0].GetModule(); |
| auto* base_context = base_module->context(); |
| uint32_t spec_const_id = |
| FindSpecConstByName(base_module, variant_defs_[0].GetName()); |
| assert(spec_const_id != 0); |
| DecorateConditional(base_context, base_fn.result_id(), spec_const_id); |
| ConvertEPToConditional(base_module, base_fn, spec_const_id); |
| } |
| } |
| |
| return true; |
| } |
| |
| void VariantDefs::CollectBaseFnCalls() { |
| auto* base_mod = variant_defs_[0].GetModule(); |
| assert(variant_defs_[0].IsBase()); |
| const auto* base_def_use_mgr = base_mod->context()->get_def_use_mgr(); |
| |
| base_mod->ForEachInst([this, &base_def_use_mgr](const Instruction* inst) { |
| if (inst->opcode() == spv::Op::OpFunctionCall) { |
| // For each function call in base module, get the function name |
| const auto fn_id = inst->GetOperand(2).AsId(); |
| const auto* called_fn_inst = base_def_use_mgr->GetDef(fn_id); |
| assert(called_fn_inst != nullptr); |
| const auto called_fn_name = GetFnName(*called_fn_inst); |
| assert(!called_fn_name.empty()); |
| |
| std::vector<std::pair<std::string, const opt::Function*>> called_fns; |
| for (size_t i = 1; i < variant_defs_.size(); ++i) { |
| // ... then see in which variant the called function was defined |
| const auto& variant_def = variant_defs_[i]; |
| assert(!variant_def.IsBase()); |
| |
| for (const auto& fn : *variant_def.GetModule()) { |
| const auto fn_name = GetFnName(fn.DefInst()); |
| if (fn_name == called_fn_name) { |
| called_fns.push_back(std::make_pair(variant_def.GetName(), &fn)); |
| } |
| } |
| } |
| |
| if (!called_fns.empty()) { |
| base_fn_calls_[inst->result_id()] = called_fns; |
| } |
| } |
| }); |
| } |
| |
| void VariantDefs::CombineBaseFnCalls(IRContext* linked_context) { |
| for (auto kv : base_fn_calls_) { |
| const uint32_t call_id = kv.first; |
| const auto called_fns = kv.second; |
| |
| if (called_fns.empty()) { |
| return; |
| } |
| |
| opt::BasicBlock* fn_call_bb = linked_context->get_instr_block(call_id); |
| |
| Instruction* found_call_inst = nullptr; |
| auto bb_iter = fn_call_bb->begin(); |
| while (bb_iter != fn_call_bb->end() && found_call_inst == nullptr) { |
| if (bb_iter->HasResultId() && bb_iter->result_id() == call_id) { |
| found_call_inst = &*bb_iter; |
| } |
| ++bb_iter; |
| } |
| |
| if (found_call_inst == nullptr) { |
| return; |
| } |
| |
| const auto base_spec_const_id = FindSpecConstByName( |
| variant_defs_[0].GetModule(), variant_defs_[0].GetName()); |
| const auto base_type_op = found_call_inst->context() |
| ->get_def_use_mgr() |
| ->GetDef(found_call_inst->type_id()) |
| ->opcode(); |
| const auto base_call_id = found_call_inst->result_id(); |
| |
| // decorate the base call with ConditionalINTEL |
| DecorateConditional(linked_context, base_call_id, base_spec_const_id); |
| |
| // Add OpFunctionCall for each variant |
| Instruction* last_inst = found_call_inst; |
| std::vector<std::pair<uint32_t, uint32_t>> var_call_ids; |
| for (const auto& kv2 : called_fns) { |
| const std::string var_name = kv2.first; |
| const opt::Function* fn = kv2.second; |
| const uint32_t spec_const_id = |
| FindSpecConstByName(linked_context->module(), var_name); |
| assert(spec_const_id != 0); |
| const uint32_t var_call_id = linked_context->TakeNextId(); |
| var_call_ids.push_back(std::make_pair(spec_const_id, var_call_id)); |
| |
| auto* var_call_inst = found_call_inst->Clone(linked_context); |
| var_call_inst->SetResultId(var_call_id); |
| var_call_inst->SetOperand(2, {fn->result_id()}); |
| var_call_inst->InsertAfter(last_inst); |
| linked_context->set_instr_block(var_call_inst, fn_call_bb); |
| last_inst = var_call_inst; |
| |
| // decorate the variant call with ConditionalINTEL |
| DecorateConditional(linked_context, var_call_id, spec_const_id); |
| } |
| |
| if (base_type_op != spv::Op::OpTypeVoid) { |
| // Add OpConditionalCopyObjectINTEL combining the function calls |
| const uint32_t result_id = linked_context->TakeNextId(); |
| auto conditional_copy_inst = new Instruction( |
| linked_context, spv::Op::OpConditionalCopyObjectINTEL, |
| found_call_inst->type_id(), result_id, |
| {{SPV_OPERAND_TYPE_ID, {base_spec_const_id}}, |
| {SPV_OPERAND_TYPE_ID, {found_call_inst->result_id()}}}); |
| |
| for (const auto& kv3 : var_call_ids) { |
| const auto spec_const_id = kv3.first; |
| const auto var_call_id = kv3.second; |
| conditional_copy_inst->AddOperand( |
| {SPV_OPERAND_TYPE_ID, {spec_const_id}}); |
| conditional_copy_inst->AddOperand({SPV_OPERAND_TYPE_ID, {var_call_id}}); |
| } |
| conditional_copy_inst->InsertAfter(last_inst); |
| linked_context->set_instr_block(conditional_copy_inst, fn_call_bb); |
| last_inst = conditional_copy_inst; |
| |
| // In all remaining instructions within the basic block, replace all |
| // usages of the base call ID with the result of |
| // OpConditionalCopyObjectINTEL |
| do { |
| last_inst = last_inst->NextNode(); |
| last_inst->ForEachInId([base_call_id, result_id](uint32_t* id) { |
| if (*id == base_call_id) { |
| *id = result_id; |
| } |
| }); |
| } while (last_inst != nullptr && *last_inst != *fn_call_bb->tail()); |
| } |
| } |
| |
| // Combine spec consts for the base module (base module is activated if all |
| // variant defs are inactive AND the base module constraints are satisfied) |
| |
| std::vector<uint32_t> var_spec_const_ids; |
| for (const auto& variant_def : variant_defs_) { |
| if (variant_def.IsBase()) { |
| continue; |
| } |
| |
| const auto id = |
| FindSpecConstByName(linked_context->module(), variant_def.GetName()); |
| assert(id != 0); |
| var_spec_const_ids.push_back(id); |
| } |
| const uint32_t base_or_id = |
| CombineIds(linked_context, var_spec_const_ids, spv::Op::OpLogicalOr); |
| |
| if (base_or_id != 0) { |
| const uint32_t bool_id = FindIdOfBoolType(linked_context->module()); |
| assert(bool_id != 0); |
| |
| const uint32_t base_not_id = linked_context->TakeNextId(); |
| auto spec_const_op_inst = std::make_unique<Instruction>( |
| linked_context, spv::Op::OpSpecConstantOp, bool_id, base_not_id, |
| std::initializer_list<opt::Operand>{ |
| {SPV_OPERAND_TYPE_SPEC_CONSTANT_OP_NUMBER, |
| {(uint32_t)(spv::Op::OpLogicalNot)}}, |
| {SPV_OPERAND_TYPE_ID, {base_or_id}}}); |
| linked_context->module()->AddType(std::move(spec_const_op_inst)); |
| |
| // Update any ConditionalINTEL annotations, names and entry points |
| // referencing the old spec const ID to use the new one |
| |
| const uint32_t old_base_spec_const_id = FindSpecConstByName( |
| linked_context->module(), variant_defs_[0].GetName()); |
| assert(old_base_spec_const_id != 0); |
| const uint32_t base_spec_const_id = |
| CombineIds(linked_context, {old_base_spec_const_id, base_not_id}, |
| spv::Op::OpLogicalAnd); |
| |
| for (auto& annot_inst : linked_context->module()->annotations()) { |
| if ((annot_inst.GetSingleWordOperand(1) == |
| uint32_t(spv::Decoration::ConditionalINTEL)) && |
| (annot_inst.GetOperand(2).AsId() == old_base_spec_const_id)) { |
| annot_inst.SetOperand(2, {base_spec_const_id}); |
| } |
| } |
| |
| for (auto& name_inst : linked_context->module()->debugs2()) { |
| if ((name_inst.opcode() == spv::Op::OpName) && |
| (name_inst.GetOperand(0).AsId() == old_base_spec_const_id)) { |
| name_inst.SetOperand(0, {base_spec_const_id}); |
| } |
| } |
| |
| for (auto& ep_inst : linked_context->module()->entry_points()) { |
| if ((ep_inst.opcode() == spv::Op::OpConditionalEntryPointINTEL) && |
| (ep_inst.GetOperand(0).AsId() == old_base_spec_const_id)) { |
| ep_inst.SetOperand(0, {base_spec_const_id}); |
| } |
| } |
| |
| linked_context->module()->ForEachInst( |
| [old_base_spec_const_id, base_spec_const_id](Instruction* inst) { |
| if (inst->opcode() == spv::Op::OpConditionalCopyObjectINTEL) { |
| inst->ForEachInId( |
| [old_base_spec_const_id, base_spec_const_id](uint32_t* id) { |
| if (*id == old_base_spec_const_id) { |
| *id = base_spec_const_id; |
| } |
| }); |
| } |
| }); |
| } |
| } |
| |
| void VariantDefs::CombineInstructions(IRContext* linked_context) { |
| // cache for existing variant ID combinations |
| std::map<std::vector<size_t>, uint32_t> spec_const_comb_ids; |
| |
| linked_context->module()->ForEachInst( |
| [this, &linked_context, &spec_const_comb_ids](Instruction* inst) { |
| if (!CanBeFnVarCombined(inst)) { |
| return; |
| } |
| |
| const size_t inst_hash = HashInst(inst); |
| if (fnvar_usage_.find(inst_hash) != fnvar_usage_.end()) { |
| const std::vector<size_t> var_ids = fnvar_usage_[inst_hash]; |
| const uint32_t spec_const_comb_id = CombineVariantDefs( |
| variant_defs_, var_ids, linked_context, spec_const_comb_ids); |
| if (spec_const_comb_id != 0) { |
| if (inst->HasResultId()) { |
| DecorateConditional(linked_context, inst->result_id(), |
| spec_const_comb_id); |
| } else if (inst->opcode() == spv::Op::OpCapability) { |
| const uint32_t cap = inst->GetSingleWordOperand(0); |
| inst->SetOpcode(spv::Op::OpConditionalCapabilityINTEL); |
| inst->SetInOperands({{SPV_OPERAND_TYPE_ID, {spec_const_comb_id}}, |
| {SPV_OPERAND_TYPE_CAPABILITY, {cap}}}); |
| } else if (inst->opcode() == spv::Op::OpExtension) { |
| const std::string ext_name = inst->GetOperand(0).AsString(); |
| inst->SetOpcode(spv::Op::OpConditionalExtensionINTEL); |
| inst->SetInOperands({{SPV_OPERAND_TYPE_ID, {spec_const_comb_id}}, |
| {SPV_OPERAND_TYPE_LITERAL_STRING, |
| {utils::MakeVector(ext_name)}}}); |
| } else { |
| assert(false && "Unsupported"); |
| } |
| } |
| } |
| }); |
| } |
| |
| } // namespace spvtools |
