| #pragma once |
| #ifndef XBYAK_XBYAK_H_ |
| #define XBYAK_XBYAK_H_ |
| /*! |
| @file xbyak.h |
| @brief Xbyak ; JIT assembler for x86(IA32)/x64 by C++ |
| @author herumi |
| @url https://github.com/herumi/xbyak |
| @note modified new BSD license |
| http://opensource.org/licenses/BSD-3-Clause |
| */ |
| #if (not +0) && !defined(XBYAK_NO_OP_NAMES) // trick to detect whether 'not' is operator or not |
| #define XBYAK_NO_OP_NAMES |
| #endif |
| |
| #include <stdio.h> // for debug print |
| #include <assert.h> |
| #include <list> |
| #include <string> |
| #include <algorithm> |
| #ifndef NDEBUG |
| #include <iostream> |
| #endif |
| |
| // #define XBYAK_DISABLE_AVX512 |
| |
| #if !defined(XBYAK_USE_MMAP_ALLOCATOR) && !defined(XBYAK_DONT_USE_MMAP_ALLOCATOR) |
| #define XBYAK_USE_MMAP_ALLOCATOR |
| #endif |
| #if !defined(__GNUC__) || defined(__MINGW32__) |
| #undef XBYAK_USE_MMAP_ALLOCATOR |
| #endif |
| |
| #ifdef __GNUC__ |
| #define XBYAK_GNUC_PREREQ(major, minor) ((__GNUC__) * 100 + (__GNUC_MINOR__) >= (major) * 100 + (minor)) |
| #else |
| #define XBYAK_GNUC_PREREQ(major, minor) 0 |
| #endif |
| |
| // This covers -std=(gnu|c)++(0x|11|1y), -stdlib=libc++, and modern Microsoft. |
| #if ((defined(_MSC_VER) && (_MSC_VER >= 1600)) || defined(_LIBCPP_VERSION) ||\ |
| ((__cplusplus >= 201103) || defined(__GXX_EXPERIMENTAL_CXX0X__))) |
| #include <unordered_set> |
| #define XBYAK_STD_UNORDERED_SET std::unordered_set |
| #include <unordered_map> |
| #define XBYAK_STD_UNORDERED_MAP std::unordered_map |
| #define XBYAK_STD_UNORDERED_MULTIMAP std::unordered_multimap |
| |
| /* |
| Clang/llvm-gcc and ICC-EDG in 'GCC-mode' always claim to be GCC 4.2, using |
| libstdcxx 20070719 (from GCC 4.2.1, the last GPL 2 version). |
| */ |
| #elif XBYAK_GNUC_PREREQ(4, 5) || (XBYAK_GNUC_PREREQ(4, 2) && __GLIBCXX__ >= 20070719) || defined(__INTEL_COMPILER) || defined(__llvm__) |
| #include <tr1/unordered_set> |
| #define XBYAK_STD_UNORDERED_SET std::tr1::unordered_set |
| #include <tr1/unordered_map> |
| #define XBYAK_STD_UNORDERED_MAP std::tr1::unordered_map |
| #define XBYAK_STD_UNORDERED_MULTIMAP std::tr1::unordered_multimap |
| |
| #elif defined(_MSC_VER) && (_MSC_VER >= 1500) && (_MSC_VER < 1600) |
| #include <unordered_set> |
| #define XBYAK_STD_UNORDERED_SET std::tr1::unordered_set |
| #include <unordered_map> |
| #define XBYAK_STD_UNORDERED_MAP std::tr1::unordered_map |
| #define XBYAK_STD_UNORDERED_MULTIMAP std::tr1::unordered_multimap |
| |
| #else |
| #include <set> |
| #define XBYAK_STD_UNORDERED_SET std::set |
| #include <map> |
| #define XBYAK_STD_UNORDERED_MAP std::map |
| #define XBYAK_STD_UNORDERED_MULTIMAP std::multimap |
| #endif |
| #ifdef _WIN32 |
| #ifndef WIN32_LEAN_AND_MEAN |
| #define WIN32_LEAN_AND_MEAN |
| #endif |
| #include <windows.h> |
| #include <malloc.h> |
| #ifdef _MSC_VER |
| #define XBYAK_TLS __declspec(thread) |
| #else |
| #define XBYAK_TLS __thread |
| #endif |
| #elif defined(__GNUC__) |
| #include <unistd.h> |
| #include <sys/mman.h> |
| #include <stdlib.h> |
| #define XBYAK_TLS __thread |
| #endif |
| #if defined(__APPLE__) && !defined(XBYAK_DONT_USE_MAP_JIT) |
| #define XBYAK_USE_MAP_JIT |
| #include <sys/sysctl.h> |
| #ifndef MAP_JIT |
| #define MAP_JIT 0x800 |
| #endif |
| #endif |
| #if !defined(_MSC_VER) || (_MSC_VER >= 1600) |
| #include <stdint.h> |
| #endif |
| |
| // MFD_CLOEXEC defined only linux 3.17 or later. |
| // Android wraps the memfd_create syscall from API version 30. |
| #if !defined(MFD_CLOEXEC) || (defined(__ANDROID__) && __ANDROID_API__ < 30) |
| #undef XBYAK_USE_MEMFD |
| #endif |
| |
| #if defined(_WIN64) || defined(__MINGW64__) || (defined(__CYGWIN__) && defined(__x86_64__)) |
| #define XBYAK64_WIN |
| #elif defined(__x86_64__) |
| #define XBYAK64_GCC |
| #endif |
| #if !defined(XBYAK64) && !defined(XBYAK32) |
| #if defined(XBYAK64_GCC) || defined(XBYAK64_WIN) |
| #define XBYAK64 |
| #else |
| #define XBYAK32 |
| #endif |
| #endif |
| |
| #if (__cplusplus >= 201103) || (defined(_MSC_VER) && _MSC_VER >= 1900) |
| #undef XBYAK_TLS |
| #define XBYAK_TLS thread_local |
| #define XBYAK_VARIADIC_TEMPLATE |
| #define XBYAK_NOEXCEPT noexcept |
| #else |
| #define XBYAK_NOEXCEPT throw() |
| #endif |
| |
| // require c++14 or later |
| // Visual Studio 2017 version 15.0 or later |
| // g++-6 or later |
| #if ((__cplusplus >= 201402L) && !(!defined(__clang__) && defined(__GNUC__) && (__GNUC__ <= 5))) || (defined(_MSC_VER) && _MSC_VER >= 1910) |
| #define XBYAK_CONSTEXPR constexpr |
| #else |
| #define XBYAK_CONSTEXPR |
| #endif |
| |
| #ifdef _MSC_VER |
| #pragma warning(push) |
| #pragma warning(disable : 4514) /* remove inline function */ |
| #pragma warning(disable : 4786) /* identifier is too long */ |
| #pragma warning(disable : 4503) /* name is too long */ |
| #pragma warning(disable : 4127) /* constant expresison */ |
| #endif |
| |
| // disable -Warray-bounds because it may be a bug of gcc. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104603 |
| #if defined(__GNUC__) && !defined(__clang__) |
| #define XBYAK_DISABLE_WARNING_ARRAY_BOUNDS |
| #pragma GCC diagnostic push |
| #pragma GCC diagnostic ignored "-Warray-bounds" |
| #endif |
| |
| namespace Xbyak { |
| |
| enum { |
| DEFAULT_MAX_CODE_SIZE = 4096, |
| VERSION = 0x7030 /* 0xABCD = A.BC(.D) */ |
| }; |
| |
| #ifndef MIE_INTEGER_TYPE_DEFINED |
| #define MIE_INTEGER_TYPE_DEFINED |
| // for backward compatibility |
| typedef uint64_t uint64; |
| typedef int64_t sint64; |
| typedef uint32_t uint32; |
| typedef uint16_t uint16; |
| typedef uint8_t uint8; |
| #endif |
| |
| #ifndef MIE_ALIGN |
| #ifdef _MSC_VER |
| #define MIE_ALIGN(x) __declspec(align(x)) |
| #else |
| #define MIE_ALIGN(x) __attribute__((aligned(x))) |
| #endif |
| #endif |
| #ifndef MIE_PACK // for shufps |
| #define MIE_PACK(x, y, z, w) ((x) * 64 + (y) * 16 + (z) * 4 + (w)) |
| #endif |
| |
| enum { |
| ERR_NONE = 0, |
| ERR_BAD_ADDRESSING, |
| ERR_CODE_IS_TOO_BIG, |
| ERR_BAD_SCALE, |
| ERR_ESP_CANT_BE_INDEX, |
| ERR_BAD_COMBINATION, |
| ERR_BAD_SIZE_OF_REGISTER, |
| ERR_IMM_IS_TOO_BIG, |
| ERR_BAD_ALIGN, |
| ERR_LABEL_IS_REDEFINED, |
| ERR_LABEL_IS_TOO_FAR, |
| ERR_LABEL_IS_NOT_FOUND, |
| ERR_CODE_ISNOT_COPYABLE, |
| ERR_BAD_PARAMETER, |
| ERR_CANT_PROTECT, |
| ERR_CANT_USE_64BIT_DISP, |
| ERR_OFFSET_IS_TOO_BIG, |
| ERR_MEM_SIZE_IS_NOT_SPECIFIED, |
| ERR_BAD_MEM_SIZE, |
| ERR_BAD_ST_COMBINATION, |
| ERR_OVER_LOCAL_LABEL, // not used |
| ERR_UNDER_LOCAL_LABEL, |
| ERR_CANT_ALLOC, |
| ERR_ONLY_T_NEAR_IS_SUPPORTED_IN_AUTO_GROW, |
| ERR_BAD_PROTECT_MODE, |
| ERR_BAD_PNUM, |
| ERR_BAD_TNUM, |
| ERR_BAD_VSIB_ADDRESSING, |
| ERR_CANT_CONVERT, |
| ERR_LABEL_ISNOT_SET_BY_L, |
| ERR_LABEL_IS_ALREADY_SET_BY_L, |
| ERR_BAD_LABEL_STR, |
| ERR_MUNMAP, |
| ERR_OPMASK_IS_ALREADY_SET, |
| ERR_ROUNDING_IS_ALREADY_SET, |
| ERR_K0_IS_INVALID, |
| ERR_EVEX_IS_INVALID, |
| ERR_SAE_IS_INVALID, |
| ERR_ER_IS_INVALID, |
| ERR_INVALID_BROADCAST, |
| ERR_INVALID_OPMASK_WITH_MEMORY, |
| ERR_INVALID_ZERO, |
| ERR_INVALID_RIP_IN_AUTO_GROW, |
| ERR_INVALID_MIB_ADDRESS, |
| ERR_X2APIC_IS_NOT_SUPPORTED, |
| ERR_NOT_SUPPORTED, |
| ERR_SAME_REGS_ARE_INVALID, |
| ERR_INVALID_NF, |
| ERR_INVALID_ZU, |
| ERR_CANT_USE_REX2, |
| ERR_INVALID_DFV, |
| ERR_INVALID_REG_IDX, |
| ERR_INTERNAL // Put it at last. |
| }; |
| |
| inline const char *ConvertErrorToString(int err) |
| { |
| static const char *errTbl[] = { |
| "none", |
| "bad addressing", |
| "code is too big", |
| "bad scale", |
| "esp can't be index", |
| "bad combination", |
| "bad size of register", |
| "imm is too big", |
| "bad align", |
| "label is redefined", |
| "label is too far", |
| "label is not found", |
| "code is not copyable", |
| "bad parameter", |
| "can't protect", |
| "can't use 64bit disp(use (void*))", |
| "offset is too big", |
| "MEM size is not specified", |
| "bad mem size", |
| "bad st combination", |
| "over local label", |
| "under local label", |
| "can't alloc", |
| "T_SHORT is not supported in AutoGrow", |
| "bad protect mode", |
| "bad pNum", |
| "bad tNum", |
| "bad vsib addressing", |
| "can't convert", |
| "label is not set by L()", |
| "label is already set by L()", |
| "bad label string", |
| "err munmap", |
| "opmask is already set", |
| "rounding is already set", |
| "k0 is invalid", |
| "evex is invalid", |
| "sae(suppress all exceptions) is invalid", |
| "er(embedded rounding) is invalid", |
| "invalid broadcast", |
| "invalid opmask with memory", |
| "invalid zero", |
| "invalid rip in AutoGrow", |
| "invalid mib address", |
| "x2APIC is not supported", |
| "not supported", |
| "same regs are invalid", |
| "invalid NF", |
| "invalid ZU", |
| "can't use rex2", |
| "invalid dfv", |
| "invalid reg index", |
| "internal error" |
| }; |
| assert(ERR_INTERNAL + 1 == sizeof(errTbl) / sizeof(*errTbl)); |
| return err <= ERR_INTERNAL ? errTbl[err] : "unknown err"; |
| } |
| |
| #ifdef XBYAK_NO_EXCEPTION |
| namespace local { |
| |
| inline int& GetErrorRef() { |
| static XBYAK_TLS int err = 0; |
| return err; |
| } |
| |
| inline void SetError(int err) { |
| if (local::GetErrorRef()) return; // keep the first err code |
| local::GetErrorRef() = err; |
| } |
| |
| } // local |
| |
| inline void ClearError() { |
| local::GetErrorRef() = 0; |
| } |
| inline int GetError() { return Xbyak::local::GetErrorRef(); } |
| |
| #define XBYAK_THROW(err) { Xbyak::local::SetError(err); return; } |
| #define XBYAK_THROW_RET(err, r) { Xbyak::local::SetError(err); return r; } |
| |
| #else |
| class Error : public std::exception { |
| int err_; |
| public: |
| explicit Error(int err) : err_(err) |
| { |
| if (err_ < 0 || err_ > ERR_INTERNAL) { |
| err_ = ERR_INTERNAL; |
| } |
| } |
| operator int() const { return err_; } |
| const char *what() const XBYAK_NOEXCEPT |
| { |
| return ConvertErrorToString(err_); |
| } |
| }; |
| |
| // dummy functions |
| inline void ClearError() { } |
| inline int GetError() { return 0; } |
| |
| inline const char *ConvertErrorToString(const Error& err) |
| { |
| return err.what(); |
| } |
| |
| #define XBYAK_THROW(err) { throw Error(err); } |
| #define XBYAK_THROW_RET(err, r) { throw Error(err); } |
| |
| #endif |
| |
| inline void *AlignedMalloc(size_t size, size_t alignment) |
| { |
| #ifdef __MINGW32__ |
| return __mingw_aligned_malloc(size, alignment); |
| #elif defined(_WIN32) |
| return _aligned_malloc(size, alignment); |
| #else |
| void *p; |
| int ret = posix_memalign(&p, alignment, size); |
| return (ret == 0) ? p : 0; |
| #endif |
| } |
| |
| inline void AlignedFree(void *p) |
| { |
| #ifdef __MINGW32__ |
| __mingw_aligned_free(p); |
| #elif defined(_MSC_VER) |
| _aligned_free(p); |
| #else |
| free(p); |
| #endif |
| } |
| |
| template<class To, class From> |
| inline const To CastTo(From p) XBYAK_NOEXCEPT |
| { |
| return (const To)(size_t)(p); |
| } |
| namespace inner { |
| |
| #ifdef _WIN32 |
| struct SystemInfo { |
| SYSTEM_INFO info; |
| SystemInfo() |
| { |
| GetSystemInfo(&info); |
| } |
| }; |
| #endif |
| //static const size_t ALIGN_PAGE_SIZE = 4096; |
| inline size_t getPageSize() |
| { |
| #ifdef _WIN32 |
| static const SystemInfo si; |
| return si.info.dwPageSize; |
| #else |
| #ifdef __GNUC__ |
| static const long pageSize = sysconf(_SC_PAGESIZE); |
| if (pageSize > 0) { |
| return (size_t)pageSize; |
| } |
| #endif |
| return 4096; |
| #endif |
| } |
| |
| inline bool IsInDisp8(uint32_t x) { return 0xFFFFFF80 <= x || x <= 0x7F; } |
| inline bool IsInInt32(uint64_t x) { return ~uint64_t(0x7fffffffu) <= x || x <= 0x7FFFFFFFU; } |
| |
| inline uint32_t VerifyInInt32(uint64_t x) |
| { |
| #if defined(XBYAK64) && !defined(__ILP32__) |
| if (!IsInInt32(x)) XBYAK_THROW_RET(ERR_OFFSET_IS_TOO_BIG, 0) |
| #endif |
| return static_cast<uint32_t>(x); |
| } |
| |
| enum LabelMode { |
| LasIs, // as is |
| Labs, // absolute |
| LaddTop // (addr + top) for mov(reg, label) with AutoGrow |
| }; |
| |
| } // inner |
| |
| /* |
| custom allocator |
| */ |
| struct Allocator { |
| explicit Allocator(const std::string& = "") {} // same interface with MmapAllocator |
| virtual uint8_t *alloc(size_t size) { return reinterpret_cast<uint8_t*>(AlignedMalloc(size, inner::getPageSize())); } |
| virtual void free(uint8_t *p) { AlignedFree(p); } |
| virtual ~Allocator() {} |
| /* override to return false if you call protect() manually */ |
| virtual bool useProtect() const { return true; } |
| }; |
| |
| #ifdef XBYAK_USE_MMAP_ALLOCATOR |
| #ifdef XBYAK_USE_MAP_JIT |
| namespace util { |
| |
| inline int getMacOsVersionPure() |
| { |
| char buf[64]; |
| size_t size = sizeof(buf); |
| int err = sysctlbyname("kern.osrelease", buf, &size, NULL, 0); |
| if (err != 0) return 0; |
| char *endp; |
| int major = strtol(buf, &endp, 10); |
| if (*endp != '.') return 0; |
| return major; |
| } |
| |
| inline int getMacOsVersion() |
| { |
| static const int version = getMacOsVersionPure(); |
| return version; |
| } |
| |
| } // util |
| #endif |
| class MmapAllocator : public Allocator { |
| struct Allocation { |
| size_t size; |
| #if defined(XBYAK_USE_MEMFD) |
| // fd_ is only used with XBYAK_USE_MEMFD. We keep the file open |
| // during the lifetime of each allocation in order to support |
| // checkpoint/restore by unprivileged users. |
| int fd; |
| #endif |
| }; |
| const std::string name_; // only used with XBYAK_USE_MEMFD |
| typedef XBYAK_STD_UNORDERED_MAP<uintptr_t, Allocation> AllocationList; |
| AllocationList allocList_; |
| public: |
| explicit MmapAllocator(const std::string& name = "xbyak") : name_(name) {} |
| uint8_t *alloc(size_t size) |
| { |
| const size_t alignedSizeM1 = inner::getPageSize() - 1; |
| size = (size + alignedSizeM1) & ~alignedSizeM1; |
| #if defined(MAP_ANONYMOUS) |
| int mode = MAP_PRIVATE | MAP_ANONYMOUS; |
| #elif defined(MAP_ANON) |
| int mode = MAP_PRIVATE | MAP_ANON; |
| #else |
| #error "not supported" |
| #endif |
| #if defined(XBYAK_USE_MAP_JIT) |
| const int mojaveVersion = 18; |
| if (util::getMacOsVersion() >= mojaveVersion) mode |= MAP_JIT; |
| #endif |
| int fd = -1; |
| #if defined(XBYAK_USE_MEMFD) |
| fd = memfd_create(name_.c_str(), MFD_CLOEXEC); |
| if (fd != -1) { |
| mode = MAP_SHARED; |
| if (ftruncate(fd, size) != 0) { |
| close(fd); |
| XBYAK_THROW_RET(ERR_CANT_ALLOC, 0) |
| } |
| } |
| #endif |
| void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, mode, fd, 0); |
| if (p == MAP_FAILED) { |
| if (fd != -1) close(fd); |
| XBYAK_THROW_RET(ERR_CANT_ALLOC, 0) |
| } |
| assert(p); |
| Allocation &alloc = allocList_[(uintptr_t)p]; |
| alloc.size = size; |
| #if defined(XBYAK_USE_MEMFD) |
| alloc.fd = fd; |
| #endif |
| return (uint8_t*)p; |
| } |
| void free(uint8_t *p) |
| { |
| if (p == 0) return; |
| AllocationList::iterator i = allocList_.find((uintptr_t)p); |
| if (i == allocList_.end()) XBYAK_THROW(ERR_BAD_PARAMETER) |
| if (munmap((void*)i->first, i->second.size) < 0) XBYAK_THROW(ERR_MUNMAP) |
| #if defined(XBYAK_USE_MEMFD) |
| if (i->second.fd != -1) close(i->second.fd); |
| #endif |
| allocList_.erase(i); |
| } |
| }; |
| #else |
| typedef Allocator MmapAllocator; |
| #endif |
| |
| class Address; |
| class Reg; |
| |
| struct ApxFlagNF {}; |
| struct ApxFlagZU {}; |
| |
| // dfv (default flags value) is or operation of these flags |
| static const int T_of = 8; |
| static const int T_sf = 4; |
| static const int T_zf = 2; |
| static const int T_cf = 1; |
| |
| class Operand { |
| static const uint8_t EXT8BIT = 0x20; |
| unsigned int idx_:6; // 0..31 + EXT8BIT = 1 if spl/bpl/sil/dil |
| unsigned int kind_:10; |
| unsigned int bit_:14; |
| protected: |
| unsigned int zero_:1; |
| unsigned int mask_:3; |
| unsigned int rounding_:3; |
| unsigned int NF_:1; |
| unsigned int ZU_:1; // ND=ZU |
| void setIdx(int idx) { idx_ = idx; } |
| public: |
| enum Kind { |
| NONE = 0, |
| MEM = 1 << 0, |
| REG = 1 << 1, |
| MMX = 1 << 2, |
| FPU = 1 << 3, |
| XMM = 1 << 4, |
| YMM = 1 << 5, |
| ZMM = 1 << 6, |
| OPMASK = 1 << 7, |
| BNDREG = 1 << 8, |
| TMM = 1 << 9 |
| }; |
| enum Code { |
| #ifdef XBYAK64 |
| RAX = 0, RCX, RDX, RBX, RSP, RBP, RSI, RDI, R8, R9, R10, R11, R12, R13, R14, R15, |
| R16, R17, R18, R19, R20, R21, R22, R23, R24, R25, R26, R27, R28, R29, R30, R31, |
| R8D = 8, R9D, R10D, R11D, R12D, R13D, R14D, R15D, |
| R16D, R17D, R18D, R19D, R20D, R21D, R22D, R23D, R24D, R25D, R26D, R27D, R28D, R29D, R30D, R31D, |
| R8W = 8, R9W, R10W, R11W, R12W, R13W, R14W, R15W, |
| R16W, R17W, R18W, R19W, R20W, R21W, R22W, R23W, R24W, R25W, R26W, R27W, R28W, R29W, R30W, R31W, |
| R8B = 8, R9B, R10B, R11B, R12B, R13B, R14B, R15B, |
| R16B, R17B, R18B, R19B, R20B, R21B, R22B, R23B, R24B, R25B, R26B, R27B, R28B, R29B, R30B, R31B, |
| SPL = 4, BPL, SIL, DIL, |
| #endif |
| EAX = 0, ECX, EDX, EBX, ESP, EBP, ESI, EDI, |
| AX = 0, CX, DX, BX, SP, BP, SI, DI, |
| AL = 0, CL, DL, BL, AH, CH, DH, BH |
| }; |
| XBYAK_CONSTEXPR Operand() : idx_(0), kind_(0), bit_(0), zero_(0), mask_(0), rounding_(0), NF_(0), ZU_(0) { } |
| XBYAK_CONSTEXPR Operand(int idx, Kind kind, int bit, bool ext8bit = 0) |
| : idx_(static_cast<uint8_t>(idx | (ext8bit ? EXT8BIT : 0))) |
| , kind_(kind) |
| , bit_(bit) |
| , zero_(0), mask_(0), rounding_(0), NF_(0), ZU_(0) |
| { |
| assert((bit_ & (bit_ - 1)) == 0); // bit must be power of two |
| } |
| XBYAK_CONSTEXPR Kind getKind() const { return static_cast<Kind>(kind_); } |
| XBYAK_CONSTEXPR int getIdx() const { return idx_ & (EXT8BIT - 1); } |
| XBYAK_CONSTEXPR bool hasIdxBit(int bit) const { return idx_ & (1<<bit); } |
| XBYAK_CONSTEXPR bool isNone() const { return kind_ == 0; } |
| XBYAK_CONSTEXPR bool isMMX() const { return is(MMX); } |
| XBYAK_CONSTEXPR bool isXMM() const { return is(XMM); } |
| XBYAK_CONSTEXPR bool isYMM() const { return is(YMM); } |
| XBYAK_CONSTEXPR bool isZMM() const { return is(ZMM); } |
| XBYAK_CONSTEXPR bool isSIMD() const { return is(XMM|YMM|ZMM); } |
| XBYAK_CONSTEXPR bool isTMM() const { return is(TMM); } |
| XBYAK_CONSTEXPR bool isXMEM() const { return is(XMM | MEM); } |
| XBYAK_CONSTEXPR bool isYMEM() const { return is(YMM | MEM); } |
| XBYAK_CONSTEXPR bool isZMEM() const { return is(ZMM | MEM); } |
| XBYAK_CONSTEXPR bool isOPMASK() const { return is(OPMASK); } |
| XBYAK_CONSTEXPR bool isBNDREG() const { return is(BNDREG); } |
| XBYAK_CONSTEXPR bool isREG(int bit = 0) const { return is(REG, bit); } |
| XBYAK_CONSTEXPR bool isMEM(int bit = 0) const { return is(MEM, bit); } |
| XBYAK_CONSTEXPR bool isFPU() const { return is(FPU); } |
| XBYAK_CONSTEXPR bool isExt8bit() const { return (idx_ & EXT8BIT) != 0; } |
| XBYAK_CONSTEXPR bool isExtIdx() const { return (getIdx() & 8) != 0; } |
| XBYAK_CONSTEXPR bool isExtIdx2() const { return (getIdx() & 16) != 0; } |
| XBYAK_CONSTEXPR bool hasEvex() const { return isZMM() || isExtIdx2() || getOpmaskIdx() || getRounding(); } |
| XBYAK_CONSTEXPR bool hasRex() const { return isExt8bit() || isREG(64) || isExtIdx(); } |
| XBYAK_CONSTEXPR bool hasRex2() const; |
| XBYAK_CONSTEXPR bool hasRex2NF() const { return hasRex2() || NF_; } |
| XBYAK_CONSTEXPR bool hasRex2NFZU() const { return hasRex2() || NF_ || ZU_; } |
| XBYAK_CONSTEXPR bool hasZero() const { return zero_; } |
| XBYAK_CONSTEXPR int getOpmaskIdx() const { return mask_; } |
| XBYAK_CONSTEXPR int getRounding() const { return rounding_; } |
| void setKind(Kind kind) |
| { |
| if ((kind & (XMM|YMM|ZMM|TMM)) == 0) return; |
| kind_ = kind; |
| bit_ = kind == XMM ? 128 : kind == YMM ? 256 : kind == ZMM ? 512 : 8192; |
| } |
| // err if MMX/FPU/OPMASK/BNDREG |
| void setBit(int bit); |
| void setOpmaskIdx(int idx, bool /*ignore_idx0*/ = true) |
| { |
| if (mask_) XBYAK_THROW(ERR_OPMASK_IS_ALREADY_SET) |
| mask_ = idx; |
| } |
| void setRounding(int idx) |
| { |
| if (rounding_) XBYAK_THROW(ERR_ROUNDING_IS_ALREADY_SET) |
| rounding_ = idx; |
| } |
| void setZero() { zero_ = true; } |
| void setNF() { NF_ = true; } |
| int getNF() const { return NF_; } |
| void setZU() { ZU_ = true; } |
| int getZU() const { return ZU_; } |
| // ah, ch, dh, bh? |
| bool isHigh8bit() const |
| { |
| if (!isBit(8)) return false; |
| if (isExt8bit()) return false; |
| const int idx = getIdx(); |
| return AH <= idx && idx <= BH; |
| } |
| // any bit is accetable if bit == 0 |
| XBYAK_CONSTEXPR bool is(int kind, uint32_t bit = 0) const |
| { |
| return (kind == 0 || (kind_ & kind)) && (bit == 0 || (bit_ & bit)); // cf. you can set (8|16) |
| } |
| XBYAK_CONSTEXPR bool isBit(uint32_t bit) const { return (bit_ & bit) != 0; } |
| XBYAK_CONSTEXPR uint32_t getBit() const { return bit_; } |
| const char *toString() const |
| { |
| const int idx = getIdx(); |
| if (kind_ == REG) { |
| if (isExt8bit()) { |
| static const char *tbl[4] = { "spl", "bpl", "sil", "dil" }; |
| return tbl[idx - 4]; |
| } |
| static const char *tbl[4][32] = { |
| { "al", "cl", "dl", "bl", "ah", "ch", "dh", "bh", "r8b", "r9b", "r10b", "r11b", "r12b", "r13b", "r14b", "r15b", |
| "r16b", "r17b", "r18b", "r19b", "r20b", "r21b", "r22b", "r23b", "r24b", "r25b", "r26b", "r27b", "r28b", "r29b", "r30b", "r31b", |
| }, |
| { "ax", "cx", "dx", "bx", "sp", "bp", "si", "di", "r8w", "r9w", "r10w", "r11w", "r12w", "r13w", "r14w", "r15w", |
| "r16w", "r17w", "r18w", "r19w", "r20w", "r21w", "r22w", "r23w", "r24w", "r25w", "r26w", "r27w", "r28w", "r29w", "r30w", "r31w", |
| }, |
| { "eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi", "r8d", "r9d", "r10d", "r11d", "r12d", "r13d", "r14d", "r15d", |
| "r16d", "r17d", "r18d", "r19d", "r20d", "r21d", "r22d", "r23d", "r24d", "r25d", "r26d", "r27d", "r28d", "r29d", "r30d", "r31d", |
| }, |
| { "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", |
| "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31", |
| }, |
| }; |
| return tbl[bit_ == 8 ? 0 : bit_ == 16 ? 1 : bit_ == 32 ? 2 : 3][idx]; |
| } else if (isOPMASK()) { |
| static const char *tbl[8] = { "k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7" }; |
| return tbl[idx]; |
| } else if (isTMM()) { |
| static const char *tbl[8] = { |
| "tmm0", "tmm1", "tmm2", "tmm3", "tmm4", "tmm5", "tmm6", "tmm7" |
| }; |
| return tbl[idx]; |
| } else if (isZMM()) { |
| static const char *tbl[32] = { |
| "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", |
| "zmm16", "zmm17", "zmm18", "zmm19", "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", "zmm27", "zmm28", "zmm29", "zmm30", "zmm31" |
| }; |
| return tbl[idx]; |
| } else if (isYMM()) { |
| static const char *tbl[32] = { |
| "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15", |
| "ymm16", "ymm17", "ymm18", "ymm19", "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31" |
| }; |
| return tbl[idx]; |
| } else if (isXMM()) { |
| static const char *tbl[32] = { |
| "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", |
| "xmm16", "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "xmm24", "xmm25", "xmm26", "xmm27", "xmm28", "xmm29", "xmm30", "xmm31" |
| }; |
| return tbl[idx]; |
| } else if (isMMX()) { |
| static const char *tbl[8] = { "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7" }; |
| return tbl[idx]; |
| } else if (isFPU()) { |
| static const char *tbl[8] = { "st0", "st1", "st2", "st3", "st4", "st5", "st6", "st7" }; |
| return tbl[idx]; |
| } else if (isBNDREG()) { |
| static const char *tbl[4] = { "bnd0", "bnd1", "bnd2", "bnd3" }; |
| return tbl[idx]; |
| } |
| XBYAK_THROW_RET(ERR_INTERNAL, 0); |
| } |
| bool isEqualIfNotInherited(const Operand& rhs) const { return idx_ == rhs.idx_ && kind_ == rhs.kind_ && bit_ == rhs.bit_ && zero_ == rhs.zero_ && mask_ == rhs.mask_ && rounding_ == rhs.rounding_; } |
| bool operator==(const Operand& rhs) const; |
| bool operator!=(const Operand& rhs) const { return !operator==(rhs); } |
| const Address& getAddress() const; |
| const Reg& getReg() const; |
| }; |
| |
| inline void Operand::setBit(int bit) |
| { |
| if (bit != 8 && bit != 16 && bit != 32 && bit != 64 && bit != 128 && bit != 256 && bit != 512 && bit != 8192) goto ERR; |
| if (isBit(bit)) return; |
| if (is(MEM | OPMASK)) { |
| bit_ = bit; |
| return; |
| } |
| if (is(REG | XMM | YMM | ZMM | TMM)) { |
| int idx = getIdx(); |
| // err if converting ah, bh, ch, dh |
| if (isREG(8) && (4 <= idx && idx < 8) && !isExt8bit()) goto ERR; |
| Kind kind = REG; |
| switch (bit) { |
| case 8: |
| #ifdef XBYAK32 |
| if (idx >= 4) goto ERR; |
| #else |
| if (idx >= 32) goto ERR; |
| if (4 <= idx && idx < 8) idx |= EXT8BIT; |
| #endif |
| break; |
| case 16: |
| case 32: |
| case 64: |
| #ifdef XBYAK32 |
| if (idx >= 16) goto ERR; |
| #else |
| if (idx >= 32) goto ERR; |
| #endif |
| break; |
| case 128: kind = XMM; break; |
| case 256: kind = YMM; break; |
| case 512: kind = ZMM; break; |
| case 8192: kind = TMM; break; |
| } |
| idx_ = idx; |
| kind_ = kind; |
| bit_ = bit; |
| if (bit >= 128) return; // keep mask_ and rounding_ |
| mask_ = 0; |
| rounding_ = 0; |
| return; |
| } |
| ERR: |
| XBYAK_THROW(ERR_CANT_CONVERT) |
| } |
| |
| class Label; |
| |
| struct Reg8; |
| struct Reg16; |
| struct Reg32; |
| #ifdef XBYAK64 |
| struct Reg64; |
| #endif |
| class Reg : public Operand { |
| public: |
| XBYAK_CONSTEXPR Reg() { } |
| XBYAK_CONSTEXPR Reg(int idx, Kind kind, int bit = 0, bool ext8bit = false) : Operand(idx, kind, bit, ext8bit) { } |
| // convert to Reg8/Reg16/Reg32/Reg64/XMM/YMM/ZMM |
| Reg changeBit(int bit) const { Reg r(*this); r.setBit(bit); return r; } |
| Reg8 cvt8() const; |
| Reg16 cvt16() const; |
| Reg32 cvt32() const; |
| #ifdef XBYAK64 |
| Reg64 cvt64() const; |
| #endif |
| Reg operator|(const ApxFlagNF&) const { Reg r(*this); r.setNF(); return r; } |
| Reg operator|(const ApxFlagZU&) const { Reg r(*this); r.setZU(); return r; } |
| }; |
| |
| inline const Reg& Operand::getReg() const |
| { |
| assert(!isMEM()); |
| return static_cast<const Reg&>(*this); |
| } |
| |
| struct Reg8 : public Reg { |
| explicit XBYAK_CONSTEXPR Reg8(int idx = 0, bool ext8bit = false) : Reg(idx, Operand::REG, 8, ext8bit) { } |
| }; |
| |
| struct Reg16 : public Reg { |
| explicit XBYAK_CONSTEXPR Reg16(int idx = 0) : Reg(idx, Operand::REG, 16) { } |
| }; |
| |
| struct Mmx : public Reg { |
| explicit XBYAK_CONSTEXPR Mmx(int idx = 0, Kind kind = Operand::MMX, int bit = 64) : Reg(idx, kind, bit) { } |
| }; |
| |
| struct EvexModifierRounding { |
| enum { |
| T_RN_SAE = 1, |
| T_RD_SAE = 2, |
| T_RU_SAE = 3, |
| T_RZ_SAE = 4, |
| T_SAE = 5 |
| }; |
| explicit XBYAK_CONSTEXPR EvexModifierRounding(int rounding) : rounding(rounding) {} |
| int rounding; |
| }; |
| struct EvexModifierZero{ XBYAK_CONSTEXPR EvexModifierZero() {}}; |
| |
| struct Xmm : public Mmx { |
| explicit XBYAK_CONSTEXPR Xmm(int idx = 0, Kind kind = Operand::XMM, int bit = 128) : Mmx(idx, kind, bit) { } |
| XBYAK_CONSTEXPR Xmm(Kind kind, int idx) : Mmx(idx, kind, kind == XMM ? 128 : kind == YMM ? 256 : 512) { } |
| Xmm operator|(const EvexModifierRounding& emr) const { Xmm r(*this); r.setRounding(emr.rounding); return r; } |
| Xmm copyAndSetIdx(int idx) const { Xmm ret(*this); ret.setIdx(idx); return ret; } |
| Xmm copyAndSetKind(Operand::Kind kind) const { Xmm ret(*this); ret.setKind(kind); return ret; } |
| }; |
| |
| struct Ymm : public Xmm { |
| explicit XBYAK_CONSTEXPR Ymm(int idx = 0, Kind kind = Operand::YMM, int bit = 256) : Xmm(idx, kind, bit) { } |
| Ymm operator|(const EvexModifierRounding& emr) const { Ymm r(*this); r.setRounding(emr.rounding); return r; } |
| }; |
| |
| struct Zmm : public Ymm { |
| explicit XBYAK_CONSTEXPR Zmm(int idx = 0) : Ymm(idx, Operand::ZMM, 512) { } |
| Zmm operator|(const EvexModifierRounding& emr) const { Zmm r(*this); r.setRounding(emr.rounding); return r; } |
| }; |
| |
| #ifdef XBYAK64 |
| struct Tmm : public Reg { |
| explicit XBYAK_CONSTEXPR Tmm(int idx = 0, Kind kind = Operand::TMM, int bit = 8192) : Reg(idx, kind, bit) { } |
| }; |
| #endif |
| |
| struct Opmask : public Reg { |
| explicit XBYAK_CONSTEXPR Opmask(int idx = 0) : Reg(idx, Operand::OPMASK, 64) {} |
| }; |
| |
| struct BoundsReg : public Reg { |
| explicit XBYAK_CONSTEXPR BoundsReg(int idx = 0) : Reg(idx, Operand::BNDREG, 128) {} |
| }; |
| |
| template<class T>T operator|(const T& x, const Opmask& k) { T r(x); r.setOpmaskIdx(k.getIdx()); return r; } |
| template<class T>T operator|(const T& x, const EvexModifierZero&) { T r(x); r.setZero(); return r; } |
| template<class T>T operator|(const T& x, const EvexModifierRounding& emr) { T r(x); r.setRounding(emr.rounding); return r; } |
| |
| struct Fpu : public Reg { |
| explicit XBYAK_CONSTEXPR Fpu(int idx = 0) : Reg(idx, Operand::FPU, 32) { } |
| }; |
| |
| struct Reg32e : public Reg { |
| explicit XBYAK_CONSTEXPR Reg32e(int idx, int bit) : Reg(idx, Operand::REG, bit) {} |
| Reg32e operator|(const ApxFlagNF&) const { Reg32e r(*this); r.setNF(); return r; } |
| Reg32e operator|(const ApxFlagZU&) const { Reg32e r(*this); r.setZU(); return r; } |
| }; |
| struct Reg32 : public Reg32e { |
| explicit XBYAK_CONSTEXPR Reg32(int idx = 0) : Reg32e(idx, 32) {} |
| }; |
| #ifdef XBYAK64 |
| struct Reg64 : public Reg32e { |
| explicit XBYAK_CONSTEXPR Reg64(int idx = 0) : Reg32e(idx, 64) {} |
| }; |
| struct RegRip { |
| int64_t disp_; |
| const Label* label_; |
| bool isAddr_; |
| explicit XBYAK_CONSTEXPR RegRip(int64_t disp = 0, const Label* label = 0, bool isAddr = false) : disp_(disp), label_(label), isAddr_(isAddr) {} |
| friend const RegRip operator+(const RegRip& r, int disp) { |
| return RegRip(r.disp_ + disp, r.label_, r.isAddr_); |
| } |
| friend const RegRip operator-(const RegRip& r, int disp) { |
| return RegRip(r.disp_ - disp, r.label_, r.isAddr_); |
| } |
| friend const RegRip operator+(const RegRip& r, int64_t disp) { |
| return RegRip(r.disp_ + disp, r.label_, r.isAddr_); |
| } |
| friend const RegRip operator-(const RegRip& r, int64_t disp) { |
| return RegRip(r.disp_ - disp, r.label_, r.isAddr_); |
| } |
| friend const RegRip operator+(const RegRip& r, const Label& label) { |
| if (r.label_ || r.isAddr_) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegRip()); |
| return RegRip(r.disp_, &label); |
| } |
| friend const RegRip operator+(const RegRip& r, const void *addr) { |
| if (r.label_ || r.isAddr_) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegRip()); |
| return RegRip(r.disp_ + (int64_t)addr, 0, true); |
| } |
| }; |
| #endif |
| |
| inline Reg8 Reg::cvt8() const |
| { |
| Reg r = changeBit(8); return Reg8(r.getIdx(), r.isExt8bit()); |
| } |
| |
| inline Reg16 Reg::cvt16() const |
| { |
| return Reg16(changeBit(16).getIdx()); |
| } |
| |
| inline Reg32 Reg::cvt32() const |
| { |
| return Reg32(changeBit(32).getIdx()); |
| } |
| |
| #ifdef XBYAK64 |
| inline Reg64 Reg::cvt64() const |
| { |
| return Reg64(changeBit(64).getIdx()); |
| } |
| #endif |
| |
| #ifndef XBYAK_DISABLE_SEGMENT |
| // not derived from Reg |
| class Segment { |
| int idx_; |
| public: |
| enum { |
| es, cs, ss, ds, fs, gs |
| }; |
| explicit XBYAK_CONSTEXPR Segment(int idx) : idx_(idx) { assert(0 <= idx_ && idx_ < 6); } |
| int getIdx() const { return idx_; } |
| const char *toString() const |
| { |
| static const char tbl[][3] = { |
| "es", "cs", "ss", "ds", "fs", "gs" |
| }; |
| return tbl[idx_]; |
| } |
| }; |
| #endif |
| |
| class RegExp { |
| public: |
| #ifdef XBYAK64 |
| enum { i32e = 32 | 64 }; |
| #else |
| enum { i32e = 32 }; |
| #endif |
| XBYAK_CONSTEXPR RegExp(size_t disp = 0) : scale_(0), disp_(disp) { } |
| XBYAK_CONSTEXPR RegExp(const Reg& r, int scale = 1) |
| : scale_(scale) |
| , disp_(0) |
| { |
| if (!r.isREG(i32e) && !r.is(Reg::XMM|Reg::YMM|Reg::ZMM|Reg::TMM)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| if (scale == 0) return; |
| if (scale != 1 && scale != 2 && scale != 4 && scale != 8) XBYAK_THROW(ERR_BAD_SCALE) |
| if (r.getBit() >= 128 || scale != 1) { // xmm/ymm is always index |
| index_ = r; |
| } else { |
| base_ = r; |
| } |
| } |
| bool isVsib(int bit = 128 | 256 | 512) const { return index_.isBit(bit); } |
| RegExp optimize() const |
| { |
| RegExp exp = *this; |
| // [reg * 2] => [reg + reg] |
| if (index_.isBit(i32e) && !base_.getBit() && scale_ == 2) { |
| exp.base_ = index_; |
| exp.scale_ = 1; |
| } |
| return exp; |
| } |
| bool operator==(const RegExp& rhs) const |
| { |
| return base_ == rhs.base_ && index_ == rhs.index_ && disp_ == rhs.disp_ && scale_ == rhs.scale_; |
| } |
| const Reg& getBase() const { return base_; } |
| const Reg& getIndex() const { return index_; } |
| int getScale() const { return scale_; } |
| size_t getDisp() const { return disp_; } |
| XBYAK_CONSTEXPR void verify() const |
| { |
| if (base_.getBit() >= 128) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| if (index_.getBit() && index_.getBit() <= 64) { |
| if (index_.getIdx() == Operand::ESP) XBYAK_THROW(ERR_ESP_CANT_BE_INDEX) |
| if (base_.getBit() && base_.getBit() != index_.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| } |
| } |
| friend RegExp operator+(const RegExp& a, const RegExp& b); |
| friend RegExp operator-(const RegExp& e, size_t disp); |
| private: |
| /* |
| [base_ + index_ * scale_ + disp_] |
| base : Reg32e, index : Reg32e(w/o esp), Xmm, Ymm |
| */ |
| Reg base_; |
| Reg index_; |
| int scale_; |
| size_t disp_; |
| }; |
| |
| inline RegExp operator+(const RegExp& a, const RegExp& b) |
| { |
| if (a.index_.getBit() && b.index_.getBit()) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) |
| RegExp ret = a; |
| if (!ret.index_.getBit()) { ret.index_ = b.index_; ret.scale_ = b.scale_; } |
| if (b.base_.getBit()) { |
| if (ret.base_.getBit()) { |
| if (ret.index_.getBit()) XBYAK_THROW_RET(ERR_BAD_ADDRESSING, RegExp()) |
| // base + base => base + index * 1 |
| ret.index_ = b.base_; |
| // [reg + esp] => [esp + reg] |
| if (ret.index_.getIdx() == Operand::ESP) std::swap(ret.base_, ret.index_); |
| ret.scale_ = 1; |
| } else { |
| ret.base_ = b.base_; |
| } |
| } |
| ret.disp_ += b.disp_; |
| return ret; |
| } |
| inline RegExp operator*(const Reg& r, int scale) |
| { |
| return RegExp(r, scale); |
| } |
| inline RegExp operator*(int scale, const Reg& r) |
| { |
| return r * scale; |
| } |
| inline RegExp operator-(const RegExp& e, size_t disp) |
| { |
| RegExp ret = e; |
| ret.disp_ -= disp; |
| return ret; |
| } |
| |
| // 2nd parameter for constructor of CodeArray(maxSize, userPtr, alloc) |
| void *const AutoGrow = (void*)1; //-V566 |
| void *const DontSetProtectRWE = (void*)2; //-V566 |
| |
| class CodeArray { |
| enum Type { |
| USER_BUF = 1, // use userPtr(non alignment, non protect) |
| ALLOC_BUF, // use new(alignment, protect) |
| AUTO_GROW // automatically move and grow memory if necessary |
| }; |
| CodeArray(const CodeArray& rhs); |
| void operator=(const CodeArray&); |
| bool isAllocType() const { return type_ == ALLOC_BUF || type_ == AUTO_GROW; } |
| struct AddrInfo { |
| size_t codeOffset; // position to write |
| size_t jmpAddr; // value to write |
| int jmpSize; // size of jmpAddr |
| inner::LabelMode mode; |
| AddrInfo(size_t _codeOffset, size_t _jmpAddr, int _jmpSize, inner::LabelMode _mode) |
| : codeOffset(_codeOffset), jmpAddr(_jmpAddr), jmpSize(_jmpSize), mode(_mode) {} |
| uint64_t getVal(const uint8_t *top) const |
| { |
| uint64_t disp = (mode == inner::LaddTop) ? jmpAddr + size_t(top) : (mode == inner::LasIs) ? jmpAddr : jmpAddr - size_t(top); |
| if (jmpSize == 4) disp = inner::VerifyInInt32(disp); |
| return disp; |
| } |
| }; |
| typedef std::list<AddrInfo> AddrInfoList; |
| AddrInfoList addrInfoList_; |
| const Type type_; |
| #ifdef XBYAK_USE_MMAP_ALLOCATOR |
| MmapAllocator defaultAllocator_; |
| #else |
| Allocator defaultAllocator_; |
| #endif |
| Allocator *alloc_; |
| protected: |
| size_t maxSize_; |
| uint8_t *top_; |
| size_t size_; |
| bool isCalledCalcJmpAddress_; |
| |
| bool useProtect() const { return alloc_->useProtect(); } |
| /* |
| allocate new memory and copy old data to the new area |
| */ |
| void growMemory() |
| { |
| const size_t newSize = (std::max<size_t>)(DEFAULT_MAX_CODE_SIZE, maxSize_ * 2); |
| uint8_t *newTop = alloc_->alloc(newSize); |
| if (newTop == 0) XBYAK_THROW(ERR_CANT_ALLOC) |
| for (size_t i = 0; i < size_; i++) newTop[i] = top_[i]; |
| alloc_->free(top_); |
| top_ = newTop; |
| maxSize_ = newSize; |
| } |
| /* |
| calc jmp address for AutoGrow mode |
| */ |
| void calcJmpAddress() |
| { |
| if (isCalledCalcJmpAddress_) return; |
| for (AddrInfoList::const_iterator i = addrInfoList_.begin(), ie = addrInfoList_.end(); i != ie; ++i) { |
| uint64_t disp = i->getVal(top_); |
| rewrite(i->codeOffset, disp, i->jmpSize); |
| } |
| isCalledCalcJmpAddress_ = true; |
| } |
| public: |
| enum ProtectMode { |
| PROTECT_RW = 0, // read/write |
| PROTECT_RWE = 1, // read/write/exec |
| PROTECT_RE = 2 // read/exec |
| }; |
| explicit CodeArray(size_t maxSize, void *userPtr = 0, Allocator *allocator = 0) |
| : type_(userPtr == AutoGrow ? AUTO_GROW : (userPtr == 0 || userPtr == DontSetProtectRWE) ? ALLOC_BUF : USER_BUF) |
| , alloc_(allocator ? allocator : (Allocator*)&defaultAllocator_) |
| , maxSize_(maxSize) |
| , top_(type_ == USER_BUF ? reinterpret_cast<uint8_t*>(userPtr) : alloc_->alloc((std::max<size_t>)(maxSize, 1))) |
| , size_(0) |
| , isCalledCalcJmpAddress_(false) |
| { |
| if (maxSize_ > 0 && top_ == 0) XBYAK_THROW(ERR_CANT_ALLOC) |
| if ((type_ == ALLOC_BUF && userPtr != DontSetProtectRWE && useProtect()) && !setProtectMode(PROTECT_RWE, false)) { |
| alloc_->free(top_); |
| XBYAK_THROW(ERR_CANT_PROTECT) |
| } |
| } |
| virtual ~CodeArray() |
| { |
| if (isAllocType()) { |
| if (useProtect()) setProtectModeRW(false); |
| alloc_->free(top_); |
| } |
| } |
| bool setProtectMode(ProtectMode mode, bool throwException = true) |
| { |
| bool isOK = protect(top_, maxSize_, mode); |
| if (isOK) return true; |
| if (throwException) XBYAK_THROW_RET(ERR_CANT_PROTECT, false) |
| return false; |
| } |
| bool setProtectModeRE(bool throwException = true) { return setProtectMode(PROTECT_RE, throwException); } |
| bool setProtectModeRW(bool throwException = true) { return setProtectMode(PROTECT_RW, throwException); } |
| void resetSize() |
| { |
| size_ = 0; |
| addrInfoList_.clear(); |
| isCalledCalcJmpAddress_ = false; |
| } |
| void db(int code) |
| { |
| if (size_ >= maxSize_) { |
| if (type_ == AUTO_GROW) { |
| growMemory(); |
| } else { |
| XBYAK_THROW(ERR_CODE_IS_TOO_BIG) |
| } |
| } |
| top_[size_++] = static_cast<uint8_t>(code); |
| } |
| void db(const uint8_t *code, size_t codeSize) |
| { |
| for (size_t i = 0; i < codeSize; i++) db(code[i]); |
| } |
| void db(uint64_t code, size_t codeSize) |
| { |
| if (codeSize > 8) XBYAK_THROW(ERR_BAD_PARAMETER) |
| for (size_t i = 0; i < codeSize; i++) db(static_cast<uint8_t>(code >> (i * 8))); |
| } |
| void dw(uint32_t code) { db(code, 2); } |
| void dd(uint32_t code) { db(code, 4); } |
| void dq(uint64_t code) { db(code, 8); } |
| const uint8_t *getCode() const { return top_; } |
| template<class F> |
| const F getCode() const { return reinterpret_cast<F>(top_); } |
| const uint8_t *getCurr() const { return &top_[size_]; } |
| template<class F> |
| const F getCurr() const { return reinterpret_cast<F>(&top_[size_]); } |
| size_t getSize() const { return size_; } |
| void setSize(size_t size) |
| { |
| if (size > maxSize_) XBYAK_THROW(ERR_OFFSET_IS_TOO_BIG) |
| size_ = size; |
| } |
| void dump() const |
| { |
| const uint8_t *p = getCode(); |
| size_t bufSize = getSize(); |
| size_t remain = bufSize; |
| for (int i = 0; i < 4; i++) { |
| size_t disp = 16; |
| if (remain < 16) { |
| disp = remain; |
| } |
| for (size_t j = 0; j < 16; j++) { |
| if (j < disp) { |
| printf("%02X", p[i * 16 + j]); |
| } |
| } |
| putchar('\n'); |
| remain -= disp; |
| if (remain == 0) { |
| break; |
| } |
| } |
| } |
| /* |
| @param offset [in] offset from top |
| @param disp [in] offset from the next of jmp |
| @param size [in] write size(1, 2, 4, 8) |
| */ |
| void rewrite(size_t offset, uint64_t disp, size_t size) |
| { |
| assert(offset < maxSize_); |
| if (size != 1 && size != 2 && size != 4 && size != 8) XBYAK_THROW(ERR_BAD_PARAMETER) |
| uint8_t *const data = top_ + offset; |
| for (size_t i = 0; i < size; i++) { |
| data[i] = static_cast<uint8_t>(disp >> (i * 8)); |
| } |
| } |
| void save(size_t offset, size_t val, int size, inner::LabelMode mode) |
| { |
| addrInfoList_.push_back(AddrInfo(offset, val, size, mode)); |
| } |
| bool isAutoGrow() const { return type_ == AUTO_GROW; } |
| bool isCalledCalcJmpAddress() const { return isCalledCalcJmpAddress_; } |
| /** |
| change exec permission of memory |
| @param addr [in] buffer address |
| @param size [in] buffer size |
| @param protectMode [in] mode(RW/RWE/RE) |
| @return true(success), false(failure) |
| */ |
| static inline bool protect(const void *addr, size_t size, int protectMode) |
| { |
| #if defined(_WIN32) |
| const DWORD c_rw = PAGE_READWRITE; |
| const DWORD c_rwe = PAGE_EXECUTE_READWRITE; |
| const DWORD c_re = PAGE_EXECUTE_READ; |
| DWORD mode; |
| #else |
| const int c_rw = PROT_READ | PROT_WRITE; |
| const int c_rwe = PROT_READ | PROT_WRITE | PROT_EXEC; |
| const int c_re = PROT_READ | PROT_EXEC; |
| int mode; |
| #endif |
| switch (protectMode) { |
| case PROTECT_RW: mode = c_rw; break; |
| case PROTECT_RWE: mode = c_rwe; break; |
| case PROTECT_RE: mode = c_re; break; |
| default: |
| return false; |
| } |
| #if defined(_WIN32) |
| DWORD oldProtect; |
| return VirtualProtect(const_cast<void*>(addr), size, mode, &oldProtect) != 0; |
| #elif defined(__GNUC__) |
| size_t pageSize = sysconf(_SC_PAGESIZE); |
| size_t iaddr = reinterpret_cast<size_t>(addr); |
| size_t roundAddr = iaddr & ~(pageSize - static_cast<size_t>(1)); |
| return mprotect(reinterpret_cast<void*>(roundAddr), size + (iaddr - roundAddr), mode) == 0; |
| #else |
| return true; |
| #endif |
| } |
| /** |
| get aligned memory pointer |
| @param addr [in] address |
| @param alignedSize [in] power of two |
| @return aligned addr by alingedSize |
| */ |
| static inline uint8_t *getAlignedAddress(uint8_t *addr, size_t alignedSize = 16) |
| { |
| return reinterpret_cast<uint8_t*>((reinterpret_cast<size_t>(addr) + alignedSize - 1) & ~(alignedSize - static_cast<size_t>(1))); |
| } |
| }; |
| |
| class Address : public Operand { |
| public: |
| enum Mode { |
| M_ModRM, |
| M_64bitDisp, |
| M_rip, |
| M_ripAddr |
| }; |
| XBYAK_CONSTEXPR Address(uint32_t sizeBit, bool broadcast, const RegExp& e) |
| : Operand(0, MEM, sizeBit), e_(e), label_(0), mode_(M_ModRM), broadcast_(broadcast), optimize_(true) |
| { |
| e_.verify(); |
| } |
| #ifdef XBYAK64 |
| explicit XBYAK_CONSTEXPR Address(size_t disp) |
| : Operand(0, MEM, 64), e_(disp), label_(0), mode_(M_64bitDisp), broadcast_(false), optimize_(true) { } |
| XBYAK_CONSTEXPR Address(uint32_t sizeBit, bool broadcast, const RegRip& addr) |
| : Operand(0, MEM, sizeBit), e_(addr.disp_), label_(addr.label_), mode_(addr.isAddr_ ? M_ripAddr : M_rip), broadcast_(broadcast), optimize_(true) { } |
| #endif |
| RegExp getRegExp() const |
| { |
| return optimize_ ? e_.optimize() : e_; |
| } |
| Address cloneNoOptimize() const { Address addr = *this; addr.optimize_ = false; return addr; } |
| Mode getMode() const { return mode_; } |
| bool is32bit() const { return e_.getBase().getBit() == 32 || e_.getIndex().getBit() == 32; } |
| bool isOnlyDisp() const { return !e_.getBase().getBit() && !e_.getIndex().getBit(); } // for mov eax |
| size_t getDisp() const { return e_.getDisp(); } |
| bool is64bitDisp() const { return mode_ == M_64bitDisp; } // for moffset |
| bool isBroadcast() const { return broadcast_; } |
| bool hasRex2() const { return e_.getBase().hasRex2() || e_.getIndex().hasRex2(); } |
| const Label* getLabel() const { return label_; } |
| bool operator==(const Address& rhs) const |
| { |
| return getBit() == rhs.getBit() && e_ == rhs.e_ && label_ == rhs.label_ && mode_ == rhs.mode_ && broadcast_ == rhs.broadcast_; |
| } |
| bool operator!=(const Address& rhs) const { return !operator==(rhs); } |
| bool isVsib() const { return e_.isVsib(); } |
| private: |
| RegExp e_; |
| const Label* label_; |
| Mode mode_; |
| bool broadcast_; |
| bool optimize_; |
| }; |
| |
| inline const Address& Operand::getAddress() const |
| { |
| assert(isMEM()); |
| return static_cast<const Address&>(*this); |
| } |
| |
| inline bool Operand::operator==(const Operand& rhs) const |
| { |
| if (isMEM() && rhs.isMEM()) return this->getAddress() == rhs.getAddress(); |
| return isEqualIfNotInherited(rhs); |
| } |
| |
| inline XBYAK_CONSTEXPR bool Operand::hasRex2() const |
| { |
| return (isREG() && isExtIdx2()) || (isMEM() && static_cast<const Address&>(*this).hasRex2()); |
| } |
| |
| class AddressFrame { |
| void operator=(const AddressFrame&); |
| AddressFrame(const AddressFrame&); |
| public: |
| const uint32_t bit_; |
| const bool broadcast_; |
| explicit XBYAK_CONSTEXPR AddressFrame(uint32_t bit, bool broadcast = false) : bit_(bit), broadcast_(broadcast) { } |
| Address operator[](const RegExp& e) const |
| { |
| return Address(bit_, broadcast_, e); |
| } |
| Address operator[](const void *disp) const |
| { |
| return Address(bit_, broadcast_, RegExp(reinterpret_cast<size_t>(disp))); |
| } |
| #ifdef XBYAK64 |
| Address operator[](uint64_t disp) const { return Address(disp); } |
| Address operator[](const RegRip& addr) const { return Address(bit_, broadcast_, addr); } |
| #endif |
| }; |
| |
| struct JmpLabel { |
| size_t endOfJmp; /* offset from top to the end address of jmp */ |
| int jmpSize; |
| inner::LabelMode mode; |
| size_t disp; // disp for [rip + disp] |
| explicit JmpLabel(size_t endOfJmp = 0, int jmpSize = 0, inner::LabelMode mode = inner::LasIs, size_t disp = 0) |
| : endOfJmp(endOfJmp), jmpSize(jmpSize), mode(mode), disp(disp) |
| { |
| } |
| }; |
| |
| class LabelManager; |
| |
| class Label { |
| mutable LabelManager *mgr; |
| mutable int id; |
| friend class LabelManager; |
| public: |
| Label() : mgr(0), id(0) {} |
| Label(const Label& rhs); |
| Label& operator=(const Label& rhs); |
| ~Label(); |
| void clear() { mgr = 0; id = 0; } |
| int getId() const { return id; } |
| const uint8_t *getAddress() const; |
| |
| // backward compatibility |
| static inline std::string toStr(int num) |
| { |
| char buf[16]; |
| #if defined(_MSC_VER) && (_MSC_VER < 1900) |
| _snprintf_s |
| #else |
| snprintf |
| #endif |
| (buf, sizeof(buf), ".%08x", num); |
| return buf; |
| } |
| }; |
| |
| class LabelManager { |
| // for string label |
| struct SlabelVal { |
| size_t offset; |
| SlabelVal(size_t offset) : offset(offset) {} |
| }; |
| typedef XBYAK_STD_UNORDERED_MAP<std::string, SlabelVal> SlabelDefList; |
| typedef XBYAK_STD_UNORDERED_MULTIMAP<std::string, const JmpLabel> SlabelUndefList; |
| struct SlabelState { |
| SlabelDefList defList; |
| SlabelUndefList undefList; |
| }; |
| typedef std::list<SlabelState> StateList; |
| // for Label class |
| struct ClabelVal { |
| ClabelVal(size_t offset = 0) : offset(offset), refCount(1) {} |
| size_t offset; |
| int refCount; |
| }; |
| typedef XBYAK_STD_UNORDERED_MAP<int, ClabelVal> ClabelDefList; |
| typedef XBYAK_STD_UNORDERED_MULTIMAP<int, const JmpLabel> ClabelUndefList; |
| typedef XBYAK_STD_UNORDERED_SET<Label*> LabelPtrList; |
| |
| CodeArray *base_; |
| // global : stateList_.front(), local : stateList_.back() |
| StateList stateList_; |
| mutable int labelId_; |
| ClabelDefList clabelDefList_; |
| ClabelUndefList clabelUndefList_; |
| LabelPtrList labelPtrList_; |
| |
| int getId(const Label& label) const |
| { |
| if (label.id == 0) label.id = labelId_++; |
| return label.id; |
| } |
| template<class DefList, class UndefList, class T> |
| void define_inner(DefList& defList, UndefList& undefList, const T& labelId, size_t addrOffset) |
| { |
| // add label |
| typename DefList::value_type item(labelId, addrOffset); |
| std::pair<typename DefList::iterator, bool> ret = defList.insert(item); |
| if (!ret.second) XBYAK_THROW(ERR_LABEL_IS_REDEFINED) |
| // search undefined label |
| for (;;) { |
| typename UndefList::iterator itr = undefList.find(labelId); |
| if (itr == undefList.end()) break; |
| const JmpLabel *jmp = &itr->second; |
| const size_t offset = jmp->endOfJmp - jmp->jmpSize; |
| size_t disp; |
| if (jmp->mode == inner::LaddTop) { |
| disp = addrOffset; |
| } else if (jmp->mode == inner::Labs) { |
| disp = size_t(base_->getCurr()); |
| } else { |
| disp = addrOffset - jmp->endOfJmp + jmp->disp; |
| #ifdef XBYAK64 |
| if (jmp->jmpSize <= 4 && !inner::IsInInt32(disp)) XBYAK_THROW(ERR_OFFSET_IS_TOO_BIG) |
| #endif |
| if (jmp->jmpSize == 1 && !inner::IsInDisp8((uint32_t)disp)) XBYAK_THROW(ERR_LABEL_IS_TOO_FAR) |
| } |
| if (base_->isAutoGrow()) { |
| base_->save(offset, disp, jmp->jmpSize, jmp->mode); |
| } else { |
| base_->rewrite(offset, disp, jmp->jmpSize); |
| } |
| undefList.erase(itr); |
| } |
| } |
| template<class DefList, class T> |
| bool getOffset_inner(const DefList& defList, size_t *offset, const T& label) const |
| { |
| typename DefList::const_iterator i = defList.find(label); |
| if (i == defList.end()) return false; |
| *offset = i->second.offset; |
| return true; |
| } |
| friend class Label; |
| void incRefCount(int id, Label *label) |
| { |
| clabelDefList_[id].refCount++; |
| labelPtrList_.insert(label); |
| } |
| void decRefCount(int id, Label *label) |
| { |
| labelPtrList_.erase(label); |
| ClabelDefList::iterator i = clabelDefList_.find(id); |
| if (i == clabelDefList_.end()) return; |
| if (i->second.refCount == 1) { |
| clabelDefList_.erase(id); |
| } else { |
| --i->second.refCount; |
| } |
| } |
| template<class T> |
| bool hasUndefinedLabel_inner(const T& list) const |
| { |
| #ifndef NDEBUG |
| for (typename T::const_iterator i = list.begin(); i != list.end(); ++i) { |
| std::cerr << "undefined label:" << i->first << std::endl; |
| } |
| #endif |
| return !list.empty(); |
| } |
| // detach all labels linked to LabelManager |
| void resetLabelPtrList() |
| { |
| for (LabelPtrList::iterator i = labelPtrList_.begin(), ie = labelPtrList_.end(); i != ie; ++i) { |
| (*i)->clear(); |
| } |
| labelPtrList_.clear(); |
| } |
| public: |
| LabelManager() |
| { |
| reset(); |
| } |
| ~LabelManager() |
| { |
| resetLabelPtrList(); |
| } |
| void reset() |
| { |
| base_ = 0; |
| labelId_ = 1; |
| stateList_.clear(); |
| stateList_.push_back(SlabelState()); |
| stateList_.push_back(SlabelState()); |
| clabelDefList_.clear(); |
| clabelUndefList_.clear(); |
| resetLabelPtrList(); |
| } |
| void enterLocal() |
| { |
| stateList_.push_back(SlabelState()); |
| } |
| void leaveLocal() |
| { |
| if (stateList_.size() <= 2) XBYAK_THROW(ERR_UNDER_LOCAL_LABEL) |
| if (hasUndefinedLabel_inner(stateList_.back().undefList)) XBYAK_THROW(ERR_LABEL_IS_NOT_FOUND) |
| stateList_.pop_back(); |
| } |
| void set(CodeArray *base) { base_ = base; } |
| void defineSlabel(std::string label) |
| { |
| if (label == "@b" || label == "@f") XBYAK_THROW(ERR_BAD_LABEL_STR) |
| if (label == "@@") { |
| SlabelDefList& defList = stateList_.front().defList; |
| SlabelDefList::iterator i = defList.find("@f"); |
| if (i != defList.end()) { |
| defList.erase(i); |
| label = "@b"; |
| } else { |
| i = defList.find("@b"); |
| if (i != defList.end()) { |
| defList.erase(i); |
| } |
| label = "@f"; |
| } |
| } |
| SlabelState& st = *label.c_str() == '.' ? stateList_.back() : stateList_.front(); |
| define_inner(st.defList, st.undefList, label, base_->getSize()); |
| } |
| void defineClabel(Label& label) |
| { |
| define_inner(clabelDefList_, clabelUndefList_, getId(label), base_->getSize()); |
| label.mgr = this; |
| labelPtrList_.insert(&label); |
| } |
| void assign(Label& dst, const Label& src) |
| { |
| ClabelDefList::const_iterator i = clabelDefList_.find(src.id); |
| if (i == clabelDefList_.end()) XBYAK_THROW(ERR_LABEL_ISNOT_SET_BY_L) |
| define_inner(clabelDefList_, clabelUndefList_, dst.id, i->second.offset); |
| dst.mgr = this; |
| labelPtrList_.insert(&dst); |
| } |
| bool getOffset(size_t *offset, std::string& label) const |
| { |
| const SlabelDefList& defList = stateList_.front().defList; |
| if (label == "@b") { |
| if (defList.find("@f") != defList.end()) { |
| label = "@f"; |
| } else if (defList.find("@b") == defList.end()) { |
| XBYAK_THROW_RET(ERR_LABEL_IS_NOT_FOUND, false) |
| } |
| } else if (label == "@f") { |
| if (defList.find("@f") != defList.end()) { |
| label = "@b"; |
| } |
| } |
| const SlabelState& st = *label.c_str() == '.' ? stateList_.back() : stateList_.front(); |
| return getOffset_inner(st.defList, offset, label); |
| } |
| bool getOffset(size_t *offset, const Label& label) const |
| { |
| return getOffset_inner(clabelDefList_, offset, getId(label)); |
| } |
| void addUndefinedLabel(const std::string& label, const JmpLabel& jmp) |
| { |
| SlabelState& st = *label.c_str() == '.' ? stateList_.back() : stateList_.front(); |
| st.undefList.insert(SlabelUndefList::value_type(label, jmp)); |
| } |
| void addUndefinedLabel(const Label& label, const JmpLabel& jmp) |
| { |
| clabelUndefList_.insert(ClabelUndefList::value_type(label.id, jmp)); |
| } |
| bool hasUndefSlabel() const |
| { |
| for (StateList::const_iterator i = stateList_.begin(), ie = stateList_.end(); i != ie; ++i) { |
| if (hasUndefinedLabel_inner(i->undefList)) return true; |
| } |
| return false; |
| } |
| bool hasUndefClabel() const { return hasUndefinedLabel_inner(clabelUndefList_); } |
| const uint8_t *getCode() const { return base_->getCode(); } |
| bool isReady() const { return !base_->isAutoGrow() || base_->isCalledCalcJmpAddress(); } |
| }; |
| |
| inline Label::Label(const Label& rhs) |
| { |
| id = rhs.id; |
| mgr = rhs.mgr; |
| if (mgr) mgr->incRefCount(id, this); |
| } |
| inline Label& Label::operator=(const Label& rhs) |
| { |
| if (id) XBYAK_THROW_RET(ERR_LABEL_IS_ALREADY_SET_BY_L, *this) |
| id = rhs.id; |
| mgr = rhs.mgr; |
| if (mgr) mgr->incRefCount(id, this); |
| return *this; |
| } |
| inline Label::~Label() |
| { |
| if (id && mgr) mgr->decRefCount(id, this); |
| } |
| inline const uint8_t* Label::getAddress() const |
| { |
| if (mgr == 0 || !mgr->isReady()) return 0; |
| size_t offset; |
| if (!mgr->getOffset(&offset, *this)) return 0; |
| return mgr->getCode() + offset; |
| } |
| |
| typedef enum { |
| DefaultEncoding, |
| VexEncoding, |
| EvexEncoding |
| } PreferredEncoding; |
| |
| class CodeGenerator : public CodeArray { |
| public: |
| enum LabelType { |
| T_SHORT, |
| T_NEAR, |
| T_FAR, // far jump |
| T_AUTO // T_SHORT if possible |
| }; |
| private: |
| CodeGenerator operator=(const CodeGenerator&); // don't call |
| #ifdef XBYAK64 |
| enum { i32e = 32 | 64, BIT = 64 }; |
| static const uint64_t dummyAddr = uint64_t(0x1122334455667788ull); |
| typedef Reg64 NativeReg; |
| #else |
| enum { i32e = 32, BIT = 32 }; |
| static const size_t dummyAddr = 0x12345678; |
| typedef Reg32 NativeReg; |
| #endif |
| // (XMM, XMM|MEM) |
| static inline bool isXMM_XMMorMEM(const Operand& op1, const Operand& op2) |
| { |
| return op1.isXMM() && (op2.isXMM() || op2.isMEM()); |
| } |
| // (MMX, MMX|MEM) or (XMM, XMM|MEM) |
| static inline bool isXMMorMMX_MEM(const Operand& op1, const Operand& op2) |
| { |
| return (op1.isMMX() && (op2.isMMX() || op2.isMEM())) || isXMM_XMMorMEM(op1, op2); |
| } |
| // (XMM, MMX|MEM) |
| static inline bool isXMM_MMXorMEM(const Operand& op1, const Operand& op2) |
| { |
| return op1.isXMM() && (op2.isMMX() || op2.isMEM()); |
| } |
| // (MMX, XMM|MEM) |
| static inline bool isMMX_XMMorMEM(const Operand& op1, const Operand& op2) |
| { |
| return op1.isMMX() && (op2.isXMM() || op2.isMEM()); |
| } |
| // (XMM, REG32|MEM) |
| static inline bool isXMM_REG32orMEM(const Operand& op1, const Operand& op2) |
| { |
| return op1.isXMM() && (op2.isREG(i32e) || op2.isMEM()); |
| } |
| // (REG32, XMM|MEM) |
| static inline bool isREG32_XMMorMEM(const Operand& op1, const Operand& op2) |
| { |
| return op1.isREG(i32e) && (op2.isXMM() || op2.isMEM()); |
| } |
| // (REG32, REG32|MEM) |
| static inline bool isREG32_REG32orMEM(const Operand& op1, const Operand& op2) |
| { |
| return op1.isREG(i32e) && ((op2.isREG(i32e) && op1.getBit() == op2.getBit()) || op2.isMEM()); |
| } |
| static inline bool isValidSSE(const Operand& op1) |
| { |
| // SSE instructions do not support XMM16 - XMM31 |
| return !(op1.isXMM() && op1.getIdx() >= 16); |
| } |
| static inline uint8_t rexRXB(int bit, int bit3, const Reg& r, const Reg& b, const Reg& x = Reg()) |
| { |
| int v = bit3 ? 8 : 0; |
| if (r.hasIdxBit(bit)) v |= 4; |
| if (x.hasIdxBit(bit)) v |= 2; |
| if (b.hasIdxBit(bit)) v |= 1; |
| return uint8_t(v); |
| } |
| void rex2(int bit3, int rex4bit, const Reg& r, const Reg& b, const Reg& x = Reg()) |
| { |
| db(0xD5); |
| db((rexRXB(4, bit3, r, b, x) << 4) | rex4bit); |
| } |
| void rex(const Operand& op1, const Operand& op2 = Operand(), uint64_t type = 0) |
| { |
| if (op1.getNF() | op2.getNF()) XBYAK_THROW(ERR_INVALID_NF) |
| if (op1.getZU() | op2.getZU()) XBYAK_THROW(ERR_INVALID_ZU) |
| uint8_t rex = 0; |
| const Operand *p1 = &op1, *p2 = &op2; |
| if (p1->isMEM()) std::swap(p1, p2); |
| if (p1->isMEM()) XBYAK_THROW(ERR_BAD_COMBINATION) |
| // except movsx(16bit, 32/64bit) |
| bool p66 = (op1.isBit(16) && !op2.isBit(i32e)) || (op2.isBit(16) && !op1.isBit(i32e)); |
| if ((type & T_66) || p66) db(0x66); |
| if (type & T_F2) { |
| db(0xF2); |
| } |
| if (type & T_F3) { |
| db(0xF3); |
| } |
| if (p2->isMEM()) { |
| const Reg& r = *static_cast<const Reg*>(p1); |
| const Address& addr = p2->getAddress(); |
| const RegExp e = addr.getRegExp(); |
| const Reg& base = e.getBase(); |
| const Reg& idx = e.getIndex(); |
| if (BIT == 64 && addr.is32bit()) db(0x67); |
| rex = rexRXB(3, r.isREG(64), r, base, idx); |
| if (r.hasRex2() || addr.hasRex2()) { |
| if (type & (T_0F|T_0F38|T_0F3A)) XBYAK_THROW(ERR_CANT_USE_REX2) |
| rex2(0, rex, r, base, idx); |
| return; |
| } |
| if (rex || r.isExt8bit()) rex |= 0x40; |
| } else { |
| const Reg& r1 = static_cast<const Reg&>(op1); |
| const Reg& r2 = static_cast<const Reg&>(op2); |
| // ModRM(reg, base); |
| rex = rexRXB(3, r1.isREG(64) || r2.isREG(64), r2, r1); |
| if (r1.hasRex2() || r2.hasRex2()) { |
| if (type & (T_0F|T_0F38|T_0F3A)) XBYAK_THROW(ERR_CANT_USE_REX2) |
| rex2(0, rex, r2, r1); |
| return; |
| } |
| if (rex || r1.isExt8bit() || r2.isExt8bit()) rex |= 0x40; |
| } |
| if (rex) db(rex); |
| } |
| // @@@begin of avx_type_def.h |
| static const uint64_t T_NONE = 0ull; |
| // low 3 bit |
| static const uint64_t T_N1 = 1ull; |
| static const uint64_t T_N2 = 2ull; |
| static const uint64_t T_N4 = 3ull; |
| static const uint64_t T_N8 = 4ull; |
| static const uint64_t T_N16 = 5ull; |
| static const uint64_t T_N32 = 6ull; |
| static const uint64_t T_NX_MASK = 7ull; |
| static const uint64_t T_DUP = T_NX_MASK;//1 << 4, // N = (8, 32, 64) |
| static const uint64_t T_N_VL = 1ull << 3; // N * (1, 2, 4) for VL |
| static const uint64_t T_APX = 1ull << 4; |
| static const uint64_t T_66 = 1ull << 5; // pp = 1 |
| static const uint64_t T_F3 = 1ull << 6; // pp = 2 |
| static const uint64_t T_ER_R = 1ull << 7; // reg{er} |
| static const uint64_t T_0F = 1ull << 8; |
| static const uint64_t T_0F38 = 1ull << 9; |
| static const uint64_t T_0F3A = 1ull << 10; |
| static const uint64_t T_L0 = 1ull << 11; |
| static const uint64_t T_L1 = 1ull << 12; |
| static const uint64_t T_W0 = 1ull << 13; |
| static const uint64_t T_W1 = 1ull << 14; |
| static const uint64_t T_EW0 = 1ull << 15; |
| static const uint64_t T_EW1 = 1ull << 16; |
| static const uint64_t T_YMM = 1ull << 17; // support YMM, ZMM |
| static const uint64_t T_EVEX = 1ull << 18; |
| static const uint64_t T_ER_X = 1ull << 19; // xmm{er} |
| static const uint64_t T_ER_Y = 1ull << 20; // ymm{er} |
| static const uint64_t T_ER_Z = 1ull << 21; // zmm{er} |
| static const uint64_t T_SAE_X = 1ull << 22; // xmm{sae} |
| static const uint64_t T_SAE_Y = 1ull << 23; // ymm{sae} |
| static const uint64_t T_SAE_Z = 1ull << 24; // zmm{sae} |
| static const uint64_t T_MUST_EVEX = 1ull << 25; // contains T_EVEX |
| static const uint64_t T_B32 = 1ull << 26; // m32bcst |
| static const uint64_t T_B64 = 1ull << 27; // m64bcst |
| static const uint64_t T_B16 = T_B32 | T_B64; // m16bcst (Be careful) |
| static const uint64_t T_M_K = 1ull << 28; // mem{k} |
| static const uint64_t T_VSIB = 1ull << 29; |
| static const uint64_t T_MEM_EVEX = 1ull << 30; // use evex if mem |
| static const uint64_t T_FP16 = 1ull << 31; // avx512-fp16 |
| static const uint64_t T_MAP5 = T_FP16 | T_0F; |
| static const uint64_t T_MAP6 = T_FP16 | T_0F38; |
| static const uint64_t T_NF = 1ull << 32; // T_nf |
| static const uint64_t T_CODE1_IF1 = 1ull << 33; // code|=1 if !r.isBit(8) |
| |
| static const uint64_t T_ND1 = 1ull << 35; // ND=1 |
| static const uint64_t T_ZU = 1ull << 36; // ND=ZU |
| static const uint64_t T_F2 = 1ull << 37; // pp = 3 |
| // T_66 = 1, T_F3 = 2, T_F2 = 3 |
| static inline uint32_t getPP(uint64_t type) { return (type & T_66) ? 1 : (type & T_F3) ? 2 : (type & T_F2) ? 3 : 0; } |
| // @@@end of avx_type_def.h |
| static inline uint32_t getMap(uint64_t type) { return (type & T_0F) ? 1 : (type & T_0F38) ? 2 : (type & T_0F3A) ? 3 : 0; } |
| void vex(const Reg& reg, const Reg& base, const Operand *v, uint64_t type, int code, bool x = false) |
| { |
| int w = (type & T_W1) ? 1 : 0; |
| bool is256 = (type & T_L1) ? true : (type & T_L0) ? false : reg.isYMM(); |
| bool r = reg.isExtIdx(); |
| bool b = base.isExtIdx(); |
| int idx = v ? v->getIdx() : 0; |
| if ((idx | reg.getIdx() | base.getIdx()) >= 16) XBYAK_THROW(ERR_BAD_COMBINATION) |
| uint32_t pp = getPP(type); |
| uint32_t vvvv = (((~idx) & 15) << 3) | (is256 ? 4 : 0) | pp; |
| if (!b && !x && !w && (type & T_0F)) { |
| db(0xC5); db((r ? 0 : 0x80) | vvvv); |
| } else { |
| uint32_t mmmm = getMap(type); |
| db(0xC4); db((r ? 0 : 0x80) | (x ? 0 : 0x40) | (b ? 0 : 0x20) | mmmm); db((w << 7) | vvvv); |
| } |
| db(code); |
| } |
| void verifySAE(const Reg& r, uint64_t type) const |
| { |
| if (((type & T_SAE_X) && r.isXMM()) || ((type & T_SAE_Y) && r.isYMM()) || ((type & T_SAE_Z) && r.isZMM())) return; |
| XBYAK_THROW(ERR_SAE_IS_INVALID) |
| } |
| void verifyER(const Reg& r, uint64_t type) const |
| { |
| if ((type & T_ER_R) && r.isREG(32|64)) return; |
| if (((type & T_ER_X) && r.isXMM()) || ((type & T_ER_Y) && r.isYMM()) || ((type & T_ER_Z) && r.isZMM())) return; |
| XBYAK_THROW(ERR_ER_IS_INVALID) |
| } |
| // (a, b, c) contains non zero two or three values then err |
| int verifyDuplicate(int a, int b, int c, int err) |
| { |
| int v = a | b | c; |
| if ((a > 0 && a != v) + (b > 0 && b != v) + (c > 0 && c != v) > 0) XBYAK_THROW_RET(err, 0) |
| return v; |
| } |
| int evex(const Reg& reg, const Reg& base, const Operand *v, uint64_t type, int code, const Reg *x = 0, bool b = false, int aaa = 0, uint32_t VL = 0, bool Hi16Vidx = false) |
| { |
| if (!(type & (T_EVEX | T_MUST_EVEX))) XBYAK_THROW_RET(ERR_EVEX_IS_INVALID, 0) |
| int w = (type & T_EW1) ? 1 : 0; |
| uint32_t mmm = getMap(type); |
| if (type & T_FP16) mmm |= 4; |
| uint32_t pp = getPP(type); |
| int idx = v ? v->getIdx() : 0; |
| uint32_t vvvv = ~idx; |
| |
| bool R = reg.isExtIdx(); |
| bool X3 = (x && x->isExtIdx()) || (base.isSIMD() && base.isExtIdx2()); |
| bool B4 = base.isREG() && base.isExtIdx2(); |
| bool X4 = x && (x->isREG() && x->isExtIdx2()); |
| bool B = base.isExtIdx(); |
| bool Rp = reg.isExtIdx2(); |
| int LL; |
| int rounding = verifyDuplicate(reg.getRounding(), base.getRounding(), v ? v->getRounding() : 0, ERR_ROUNDING_IS_ALREADY_SET); |
| int disp8N = 1; |
| if (rounding) { |
| if (rounding == EvexModifierRounding::T_SAE) { |
| verifySAE(base, type); LL = 0; |
| } else { |
| verifyER(base, type); LL = rounding - 1; |
| } |
| b = true; |
| } else { |
| if (v) VL = (std::max)(VL, v->getBit()); |
| VL = (std::max)((std::max)(reg.getBit(), base.getBit()), VL); |
| LL = (VL == 512) ? 2 : (VL == 256) ? 1 : 0; |
| if (b) { |
| disp8N = ((type & T_B16) == T_B16) ? 2 : (type & T_B32) ? 4 : 8; |
| } else if ((type & T_NX_MASK) == T_DUP) { |
| disp8N = VL == 128 ? 8 : VL == 256 ? 32 : 64; |
| } else { |
| if ((type & (T_NX_MASK | T_N_VL)) == 0) { |
| type |= T_N16 | T_N_VL; // default |
| } |
| int low = type & T_NX_MASK; |
| if (low > 0) { |
| disp8N = 1 << (low - 1); |
| if (type & T_N_VL) disp8N *= (VL == 512 ? 4 : VL == 256 ? 2 : 1); |
| } |
| } |
| } |
| bool V4 = ((v ? v->isExtIdx2() : 0) || Hi16Vidx); |
| bool z = reg.hasZero() || base.hasZero() || (v ? v->hasZero() : false); |
| if (aaa == 0) aaa = verifyDuplicate(base.getOpmaskIdx(), reg.getOpmaskIdx(), (v ? v->getOpmaskIdx() : 0), ERR_OPMASK_IS_ALREADY_SET); |
| if (aaa == 0) z = 0; // clear T_z if mask is not set |
| db(0x62); |
| db((R ? 0 : 0x80) | (X3 ? 0 : 0x40) | (B ? 0 : 0x20) | (Rp ? 0 : 0x10) | (B4 ? 8 : 0) | mmm); |
| db((w == 1 ? 0x80 : 0) | ((vvvv & 15) << 3) | (X4 ? 0 : 4) | (pp & 3)); |
| db((z ? 0x80 : 0) | ((LL & 3) << 5) | (b ? 0x10 : 0) | (V4 ? 0 : 8) | (aaa & 7)); |
| db(code); |
| return disp8N; |
| } |
| // evex of Legacy |
| void evexLeg(const Reg& r, const Reg& b, const Reg& x, const Reg& v, uint64_t type, int sc = NONE) |
| { |
| int M = getMap(type); if (M == 0) M = 4; // legacy |
| int R3 = !r.isExtIdx(); |
| int X3 = !x.isExtIdx(); |
| int B3 = b.isExtIdx() ? 0 : 0x20; |
| int R4 = r.isExtIdx2() ? 0 : 0x10; |
| int B4 = b.isExtIdx2() ? 0x08 : 0; |
| int w = (type & T_W0) ? 0 : (r.isBit(64) || v.isBit(64) || (type & T_W1)); |
| int V = (~v.getIdx() & 15) << 3; |
| int X4 = x.isExtIdx2() ? 0 : 0x04; |
| int pp = (type & (T_F2|T_F3|T_66)) ? getPP(type) : (r.isBit(16) || v.isBit(16)); |
| int V4 = !v.isExtIdx2(); |
| int ND = (type & T_ZU) ? (r.getZU() || b.getZU()) : (type & T_ND1) ? 1 : (type & T_APX) ? 0 : v.isREG(); |
| int NF = r.getNF() | b.getNF() | x.getNF() | v.getNF(); |
| int L = 0; |
| if ((type & T_NF) == 0 && NF) XBYAK_THROW(ERR_INVALID_NF) |
| if ((type & T_ZU) == 0 && r.getZU()) XBYAK_THROW(ERR_INVALID_ZU) |
| db(0x62); |
| db((R3<<7) | (X3<<6) | B3 | R4 | B4 | M); |
| db((w<<7) | V | X4 | pp); |
| if (sc != NONE) { |
| db((L<<5) | (ND<<4) | sc); |
| } else { |
| db((L<<5) | (ND<<4) | (V4<<3) | (NF<<2)); |
| } |
| } |
| void setModRM(int mod, int r1, int r2) |
| { |
| db(static_cast<uint8_t>((mod << 6) | ((r1 & 7) << 3) | (r2 & 7))); |
| } |
| void setSIB(const RegExp& e, int reg, int disp8N = 0) |
| { |
| uint64_t disp64 = e.getDisp(); |
| #if defined(XBYAK64) && !defined(__ILP32__) |
| #ifdef XBYAK_OLD_DISP_CHECK |
| // treat 0xffffffff as 0xffffffffffffffff |
| uint64_t high = disp64 >> 32; |
| if (high != 0 && high != 0xFFFFFFFF) XBYAK_THROW(ERR_OFFSET_IS_TOO_BIG) |
| #else |
| // displacement should be a signed 32-bit value, so also check sign bit |
| uint64_t high = disp64 >> 31; |
| if (high != 0 && high != 0x1FFFFFFFF) XBYAK_THROW(ERR_OFFSET_IS_TOO_BIG) |
| #endif |
| #endif |
| uint32_t disp = static_cast<uint32_t>(disp64); |
| const Reg& base = e.getBase(); |
| const Reg& index = e.getIndex(); |
| const int baseIdx = base.getIdx(); |
| const int baseBit = base.getBit(); |
| const int indexBit = index.getBit(); |
| enum { |
| mod00 = 0, mod01 = 1, mod10 = 2 |
| }; |
| int mod = mod10; // disp32 |
| if (!baseBit || ((baseIdx & 7) != Operand::EBP && disp == 0)) { |
| mod = mod00; |
| } else { |
| if (disp8N == 0) { |
| if (inner::IsInDisp8(disp)) { |
| mod = mod01; |
| } |
| } else { |
| // disp must be casted to signed |
| uint32_t t = static_cast<uint32_t>(static_cast<int>(disp) / disp8N); |
| if ((disp % disp8N) == 0 && inner::IsInDisp8(t)) { |
| disp = t; |
| mod = mod01; |
| } |
| } |
| } |
| const int newBaseIdx = baseBit ? (baseIdx & 7) : Operand::EBP; |
| /* ModR/M = [2:3:3] = [Mod:reg/code:R/M] */ |
| bool hasSIB = indexBit || (baseIdx & 7) == Operand::ESP; |
| #ifdef XBYAK64 |
| if (!baseBit && !indexBit) hasSIB = true; |
| #endif |
| if (hasSIB) { |
| setModRM(mod, reg, Operand::ESP); |
| /* SIB = [2:3:3] = [SS:index:base(=rm)] */ |
| const int idx = indexBit ? (index.getIdx() & 7) : Operand::ESP; |
| const int scale = e.getScale(); |
| const int SS = (scale == 8) ? 3 : (scale == 4) ? 2 : (scale == 2) ? 1 : 0; |
| setModRM(SS, idx, newBaseIdx); |
| } else { |
| setModRM(mod, reg, newBaseIdx); |
| } |
| if (mod == mod01) { |
| db(disp); |
| } else if (mod == mod10 || (mod == mod00 && !baseBit)) { |
| dd(disp); |
| } |
| } |
| LabelManager labelMgr_; |
| bool isInDisp16(uint32_t x) const { return 0xFFFF8000 <= x || x <= 0x7FFF; } |
| void writeCode(uint64_t type, const Reg& r, int code) |
| { |
| if (!(type & T_APX)) { |
| if (type & T_0F) { |
| db(0x0F); |
| } else if (type & T_0F38) { |
| db(0x0F); db(0x38); |
| } else if (type & T_0F3A) { |
| db(0x0F); db(0x3A); |
| } |
| } |
| db(code | ((type == 0 || (type & T_CODE1_IF1)) && !r.isBit(8))); |
| } |
| void opRR(const Reg& reg1, const Reg& reg2, uint64_t type, int code) |
| { |
| rex(reg2, reg1, type); |
| writeCode(type, reg1, code); |
| setModRM(3, reg1.getIdx(), reg2.getIdx()); |
| } |
| void opMR(const Address& addr, const Reg& r, uint64_t type, int code, int immSize = 0) |
| { |
| if (addr.is64bitDisp()) XBYAK_THROW(ERR_CANT_USE_64BIT_DISP) |
| rex(addr, r, type); |
| writeCode(type, r, code); |
| opAddr(addr, r.getIdx(), immSize); |
| } |
| void opLoadSeg(const Address& addr, const Reg& reg, uint64_t type, int code) |
| { |
| if (reg.isBit(8)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| if (addr.is64bitDisp()) XBYAK_THROW(ERR_CANT_USE_64BIT_DISP) |
| // can't use opMR |
| rex(addr, reg, type); |
| if (type & T_0F) db(0x0F); |
| db(code); |
| opAddr(addr, reg.getIdx()); |
| } |
| // for only MPX(bnd*) |
| void opMIB(const Address& addr, const Reg& reg, uint64_t type, int code) |
| { |
| if (addr.getMode() != Address::M_ModRM) XBYAK_THROW(ERR_INVALID_MIB_ADDRESS) |
| opMR(addr.cloneNoOptimize(), reg, type, code); |
| } |
| void makeJmp(uint32_t disp, LabelType type, uint8_t shortCode, uint8_t longCode, uint8_t longPref) |
| { |
| const int shortJmpSize = 2; |
| const int longHeaderSize = longPref ? 2 : 1; |
| const int longJmpSize = longHeaderSize + 4; |
| if (type != T_NEAR && inner::IsInDisp8(disp - shortJmpSize)) { |
| db(shortCode); db(disp - shortJmpSize); |
| } else { |
| if (type == T_SHORT) XBYAK_THROW(ERR_LABEL_IS_TOO_FAR) |
| if (longPref) db(longPref); |
| db(longCode); dd(disp - longJmpSize); |
| } |
| } |
| bool isNEAR(LabelType type) const { return type == T_NEAR || (type == T_AUTO && isDefaultJmpNEAR_); } |
| template<class T> |
| void opJmp(T& label, LabelType type, uint8_t shortCode, uint8_t longCode, uint8_t longPref) |
| { |
| if (type == T_FAR) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| if (isAutoGrow() && size_ + 16 >= maxSize_) growMemory(); /* avoid splitting code of jmp */ |
| size_t offset = 0; |
| if (labelMgr_.getOffset(&offset, label)) { /* label exists */ |
| makeJmp(inner::VerifyInInt32(offset - size_), type, shortCode, longCode, longPref); |
| } else { |
| int jmpSize = 0; |
| if (isNEAR(type)) { |
| jmpSize = 4; |
| if (longPref) db(longPref); |
| db(longCode); dd(0); |
| } else { |
| jmpSize = 1; |
| db(shortCode); db(0); |
| } |
| JmpLabel jmp(size_, jmpSize, inner::LasIs); |
| labelMgr_.addUndefinedLabel(label, jmp); |
| } |
| } |
| void opJmpAbs(const void *addr, LabelType type, uint8_t shortCode, uint8_t longCode, uint8_t longPref = 0) |
| { |
| if (type == T_FAR) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| if (isAutoGrow()) { |
| if (!isNEAR(type)) XBYAK_THROW(ERR_ONLY_T_NEAR_IS_SUPPORTED_IN_AUTO_GROW) |
| if (size_ + 16 >= maxSize_) growMemory(); |
| if (longPref) db(longPref); |
| db(longCode); |
| dd(0); |
| save(size_ - 4, size_t(addr) - size_, 4, inner::Labs); |
| } else { |
| makeJmp(inner::VerifyInInt32(reinterpret_cast<const uint8_t*>(addr) - getCurr()), type, shortCode, longCode, longPref); |
| } |
| |
| } |
| void opJmpOp(const Operand& op, LabelType type, int ext) |
| { |
| const int bit = 16|i32e; |
| if (type == T_FAR) { |
| if (!op.isMEM(bit)) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| opRext(op, bit, ext + 1, 0, 0xFF, false); |
| } else { |
| opRext(op, bit, ext, 0, 0xFF, true); |
| } |
| } |
| // reg is reg field of ModRM |
| // immSize is the size for immediate value |
| // disp8N = 0(normal), disp8N = 1(force disp32), disp8N = {2, 4, 8} ; compressed displacement |
| void opAddr(const Address &addr, int reg, int immSize = 0, int disp8N = 0, bool permitVisb = false) |
| { |
| if (!permitVisb && addr.isVsib()) XBYAK_THROW(ERR_BAD_VSIB_ADDRESSING) |
| if (addr.getMode() == Address::M_ModRM) { |
| setSIB(addr.getRegExp(), reg, disp8N); |
| } else if (addr.getMode() == Address::M_rip || addr.getMode() == Address::M_ripAddr) { |
| setModRM(0, reg, 5); |
| if (addr.getLabel()) { // [rip + Label] |
| putL_inner(*addr.getLabel(), true, addr.getDisp() - immSize); |
| } else { |
| size_t disp = addr.getDisp(); |
| if (addr.getMode() == Address::M_ripAddr) { |
| if (isAutoGrow()) XBYAK_THROW(ERR_INVALID_RIP_IN_AUTO_GROW) |
| disp -= (size_t)getCurr() + 4 + immSize; |
| } |
| dd(inner::VerifyInInt32(disp)); |
| } |
| } |
| } |
| void opSSE(const Reg& r, const Operand& op, uint64_t type, int code, bool isValid(const Operand&, const Operand&), int imm8 = NONE) |
| { |
| if (isValid && !isValid(r, op)) XBYAK_THROW(ERR_BAD_COMBINATION) |
| if (!isValidSSE(r) || !isValidSSE(op)) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| opRO(r, op, type, code, true, (imm8 != NONE) ? 1 : 0); |
| if (imm8 != NONE) db(imm8); |
| } |
| void opMMX_IMM(const Mmx& mmx, int imm8, int code, int ext) |
| { |
| if (!isValidSSE(mmx)) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| uint64_t type = T_0F; |
| if (mmx.isXMM()) type |= T_66; |
| opRR(Reg32(ext), mmx, type, code); |
| db(imm8); |
| } |
| void opMMX(const Mmx& mmx, const Operand& op, int code, uint64_t type = T_0F, uint64_t pref = T_66, int imm8 = NONE) |
| { |
| if (mmx.isXMM()) type |= pref; |
| opSSE(mmx, op, type, code, isXMMorMMX_MEM, imm8); |
| } |
| void opMovXMM(const Operand& op1, const Operand& op2, uint64_t type, int code) |
| { |
| if (!isValidSSE(op1) || !isValidSSE(op2)) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| if (op1.isXMM() && op2.isMEM()) { |
| opMR(op2.getAddress(), op1.getReg(), type, code); |
| } else if (op1.isMEM() && op2.isXMM()) { |
| opMR(op1.getAddress(), op2.getReg(), type, code | 1); |
| } else { |
| XBYAK_THROW(ERR_BAD_COMBINATION) |
| } |
| } |
| // pextr{w,b,d}, extractps |
| void opExt(const Operand& op, const Mmx& mmx, int code, int imm, bool hasMMX2 = false) |
| { |
| if (!isValidSSE(op) || !isValidSSE(mmx)) XBYAK_THROW(ERR_NOT_SUPPORTED) |
| if (hasMMX2 && op.isREG(i32e)) { /* pextrw is special */ |
| if (mmx.isXMM()) db(0x66); |
| opRR(op.getReg(), mmx, T_0F, 0xC5); db(imm); |
| } else { |
| opSSE(mmx, op, T_66 | T_0F3A, code, isXMM_REG32orMEM, imm); |
| } |
| } |
| // (r, r, m) or (r, m, r) |
| bool opROO(const Reg& d, const Operand& op1, const Operand& op2, uint64_t type, int code, int immSize = 0, int sc = NONE) |
| { |
| if (!(type & T_MUST_EVEX) && !d.isREG() && !(d.hasRex2NFZU() || op1.hasRex2NFZU() || op2.hasRex2NFZU())) return false; |
| const Operand *p1 = &op1, *p2 = &op2; |
| if (p1->isMEM()) { std::swap(p1, p2); } else { if (p2->isMEM()) code |= 2; } |
| if (p1->isMEM()) XBYAK_THROW_RET(ERR_BAD_COMBINATION, false) |
| if (p2->isMEM()) { |
| const Reg& r = *static_cast<const Reg*>(p1); |
| const Address& addr = p2->getAddress(); |
| const RegExp e = addr.getRegExp(); |
| evexLeg(r, e.getBase(), e.getIndex(), d, type, sc); |
| writeCode(type, d, code); |
| opAddr(addr, r.getIdx(), immSize); |
| } else { |
| evexLeg(static_cast<const Reg&>(op2), static_cast<const Reg&>(op1), Reg(), d, type, sc); |
| writeCode(type, d, code); |
| setModRM(3, op2.getIdx(), op1.getIdx()); |
| } |
| return true; |
| } |
| void opRext(const Operand& op, int bit, int ext, uint64_t type, int code, bool disableRex = false, int immSize = 0, const Reg *d = 0) |
| { |
| int opBit = op.getBit(); |
| if (disableRex && opBit == 64) opBit = 32; |
| const Reg r(ext, Operand::REG, opBit); |
| if ((type & T_APX) && op.hasRex2NFZU() && opROO(d ? *d : Reg(0, Operand::REG, opBit), op, r, type, code)) return; |
| if (op.isMEM()) { |
| opMR(op.getAddress(), r, type, code, immSize); |
| } else if (op.isREG(bit)) { |
| opRR(r, op.getReg().changeBit(opBit), type, code); |
| } else { |
| XBYAK_THROW(ERR_BAD_COMBINATION) |
| } |
| } |
| void opShift(const Operand& op, int imm, int ext, const Reg *d = 0) |
| { |
| if (d == 0) verifyMemHasSize(op); |
| if (d && op.getBit() != 0 && d->getBit() != op.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| uint64_t type = T_APX|T_CODE1_IF1; if (ext & 8) type |= T_NF; if (d) type |= T_ND1; |
| opRext(op, 0, ext&7, type, (0xC0 | ((imm == 1 ? 1 : 0) << 4)), false, (imm != 1) ? 1 : 0, d); |
| if (imm != 1) db(imm); |
| } |
| void opShift(const Operand& op, const Reg8& _cl, int ext, const Reg *d = 0) |
| { |
| if (_cl.getIdx() != Operand::CL) XBYAK_THROW(ERR_BAD_COMBINATION) |
| if (d && op.getBit() != 0 && d->getBit() != op.getBit()) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| uint64_t type = T_APX|T_CODE1_IF1; if (ext & 8) type |= T_NF; if (d) type |= T_ND1; |
| opRext(op, 0, ext&7, type, 0xD2, false, 0, d); |
| } |
| // condR assumes that op.isREG() is true |
| void opRO(const Reg& r, const Operand& op, uint64_t type, int code, bool condR = true, int immSize = 0) |
| { |
| if (op.isMEM()) { |
| opMR(op.getAddress(), r, type, code, immSize); |
| } else if (condR) { |
| opRR(r, op.getReg(), type, code); |
| } else { |
| XBYAK_THROW(ERR_BAD_COMBINATION) |
| } |
| } |
| void opShxd(const Reg& d, const Operand& op, const Reg& reg, uint8_t imm, int code, int code2, const Reg8 *_cl = 0) |
| { |
| if (_cl && _cl->getIdx() != Operand::CL) XBYAK_THROW(ERR_BAD_COMBINATION) |
| if (!reg.isREG(16|i32e)) XBYAK_THROW(ERR_BAD_SIZE_OF_REGISTER) |
| int immSize = _cl ? 0 : 1; |
| if (_cl) code |= 1; |
| uint64_t type = T_APX | T_NF; |
| if (d.isREG()) type |= T_ND1; |
| if (!opROO(d, op, reg, type, _cl ? code : code2, immSize)) { |
| opRO(reg, op, T_0F, code, true, immSize); |
| } |
| if (!_cl) db(imm); |
| } |
| // (REG, REG|MEM), (MEM, REG) |
| void opRO_MR(const Operand& op1, const Operand& op2, int code) |
| { |
| if (op2.isMEM()) { |
| if (!op1.isREG()) XBYAK_THROW(ERR_BAD_COMBINATION) |
| opMR(op2.getAddress(), op1.getReg(), 0, code | 2); |
| } else { |
| opRO(static_cast<const Reg&>(op2), op1, 0, code, op1.getKind() == op2.getKind()); |
| } |
| } |
| uint32_t getImmBit(const Operand& op, uint32_t imm) |
| { |
| verifyMemHasSize(op); |
| uint32_t immBit = inner::IsInDisp8(imm) ? 8 : isInDisp16(imm) ? 16 : 32; |
| if (op.isBit(8)) immBit = 8; |
| if (op.getBit() < immBit) XBYAK_THROW_RET(ERR_IMM_IS_TOO_BIG, 0) |
| if (op.isBit(32|64) && immBit == 16) immBit = 32; /* don't use MEM16 if 32/64bit mode */ |
| return immBit; |
| } |
| // (REG|MEM, IMM) |
| void opOI(const Operand& op, uint32_t imm, int code, int ext) |
| { |
| uint32_t immBit = getImmBit(op, imm); |
| if (op.isREG() && op.getIdx() == 0 && (op.getBit() == immBit || (op.isBit(64) && immBit == 32))) { // rax, eax, ax, al |
| rex(op); |
| db(code | 4 | (immBit == 8 ? 0 : 1)); |
| } else { |
| int tmp = immBit < (std::min)(op.getBit(), 32U) ? 2 : 0; |
| opRext(op, 0, ext, 0, 0x80 | tmp, false, immBit / 8); |
| } |
| db(imm, immBit / 8); |
| } |
| // (r, r/m, imm) |
| void opROI(const Reg& d, const Operand& op, uint32_t imm, uint64_t type, int ext) |
| { |
| uint32_t immBit = getImmBit(d, imm); |
| int code = immBit < (std::min)(d.getBit(), 32U) ? 2 : 0; |
| opROO(d, op, Reg(ext, Operand::REG, d.getBit()), type, 0x80 | code, immBit / 8); |
| db(imm, immBit / 8); |
| |