/*
 * Copyright 2019 Google LLC
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "include/core/SkString.h"
#include "include/private/SkSpinlock.h"
#include "include/private/SkThreadID.h"
#include "include/private/SkVx.h"
#include "src/core/SkCpu.h"
#include "src/core/SkOpts.h"
#include "src/core/SkVM.h"
#include <string.h>
#if defined(SKVM_JIT)
    #define XBYAK_NO_OP_NAMES
    #include "xbyak/xbyak.h"
#endif

namespace skvm {

    Program::~Program() = default;
    Program::Program(Program&&) = default;
    Program& Program::operator=(Program&&) = default;

    Program::Program(std::vector<Instruction> instructions, int regs, int loop)
        : fInstructions(std::move(instructions))
        , fRegs(regs)
        , fLoop(loop)
    {}

    Program Builder::done() {
        // Basic liveness analysis (and free dead code elimination).
        for (ID id = fProgram.size(); id --> 0; ) {
            Instruction& inst = fProgram[id];

            // All side-effect-only instructions (stores) are live.
            if (inst.op <= Op::store32) {
                inst.life = id;
            }
            // The arguments of a live instruction must live until that instruction.
            if (inst.life != NA) {
                // Notice how we're walking backward, storing the latest instruction in life.
                if (inst.x != NA && fProgram[inst.x].life == NA) { fProgram[inst.x].life = id; }
                if (inst.y != NA && fProgram[inst.y].life == NA) { fProgram[inst.y].life = id; }
                if (inst.z != NA && fProgram[inst.z].life == NA) { fProgram[inst.z].life = id; }
            }
        }

        // Look to see if there are any instructions that can be hoisted outside the program's loop.
        for (ID id = 0; id < (ID)fProgram.size(); id++) {
            Instruction& inst = fProgram[id];

            // Loads and stores cannot be hoisted out of the loop.
            if (inst.op <= Op::load32) {
                inst.hoist = false;
            }

            // If any of an instruction's arguments can't be hoisted, it can't be hoisted itself.
            if (inst.hoist) {
                if (inst.x != NA) { inst.hoist &= fProgram[inst.x].hoist; }
                if (inst.y != NA) { inst.hoist &= fProgram[inst.y].hoist; }
                if (inst.z != NA) { inst.hoist &= fProgram[inst.z].hoist; }
            }
        }

        // We'll need to map each live value to a register.
        std::unordered_map<ID, ID> val_to_reg;

        // Count the registers we've used so far.
        ID next_reg = 0;

        // Our first pass of register assignment assigns hoisted values to eternal registers.
        for (ID val = 0; val < (ID)fProgram.size(); val++) {
            Instruction& inst = fProgram[val];
            if (inst.life == NA || !inst.hoist) {
                continue;
            }

            // Hoisted values are needed forever, so they each get their own register.
            val_to_reg[val] = next_reg++;
        }

        // Now we'll assign registers to values that can't be hoisted out of the loop.  These
        // values have finite liftimes, so we track pre-owned registers that have become available
        // and a schedule of which registers become available as we reach a given instruction.
        std::vector<ID>                         avail;
        std::unordered_map<ID, std::vector<ID>> deaths;

        for (ID val = 0; val < (ID)fProgram.size(); val++) {
            Instruction& inst = fProgram[val];
            if (inst.life == NA || inst.hoist) {
                continue;
            }

            // All the values that are no longer needed after this instruction
            // can make their registers available to this and future values.
            const std::vector<ID>& dying = deaths[val];
            avail.insert(avail.end(),
                         dying.begin(), dying.end());

            // Allocate a register if we have to, but prefer to reuse one that's available.
            ID reg;
            if (avail.empty()) {
                reg = next_reg++;
            } else {
                reg = avail.back();
                avail.pop_back();
            }

            // Schedule this value's own death.  When we reach the instruction at inst.life,
            // this value is no longer needed and its register becomes available for reuse.
            deaths[inst.life].push_back(reg);

            val_to_reg[val] = reg;
        }

        // Add a dummy mapping for the N/A sentinel value to "register N/A",
        // so that the lookups don't have to know which arguments are used by which Ops.
        auto lookup_register = [&](ID val) {
            return val == NA ? NA
                             : val_to_reg[val];
        };

        // Finally translate Builder::Instructions to Program::Instructions by mapping values to
        // registers.  This will be two passes again, first outside the loop, then inside.

        // The loop begins at the loop'th Instruction.
        int loop = 0;
        std::vector<Program::Instruction> program;

        auto push_instruction = [&](ID id, const Builder::Instruction& inst) {
            Program::Instruction pinst{
                inst.op,
                lookup_register(id),
                lookup_register(inst.x),
               {lookup_register(inst.y)},
               {lookup_register(inst.z)},
            };
            if (inst.y == NA) { pinst.y.imm = inst.immy; }
            if (inst.z == NA) { pinst.z.imm = inst.immz; }
            program.push_back(pinst);
        };

        for (ID id = 0; id < (ID)fProgram.size(); id++) {
            Instruction& inst = fProgram[id];
            if (inst.life == NA || !inst.hoist) {
                continue;
            }

            push_instruction(id, inst);
            loop++;
        }
        for (ID id = 0; id < (ID)fProgram.size(); id++) {
            Instruction& inst = fProgram[id];
            if (inst.life == NA || inst.hoist) {
                continue;
            }

            push_instruction(id, inst);
        }

        return { std::move(program), /*register count = */next_reg, loop };
    }

    // Most instructions produce a value and return it by ID,
    // the value-producing instruction's own index in the program vector.

    ID Builder::push(Op op, ID x, ID y, ID z, int immy, int immz) {
        Instruction inst{op, /*hoist=*/true, /*life=*/NA, x, y, z, immy, immz};

        // Basic common subexpression elimination:
        // if we've already seen this exact Instruction, use it instead of creating a new one.
        auto lookup = fIndex.find(inst);
        if (lookup != fIndex.end()) {
            return lookup->second;
        }

        ID id = static_cast<ID>(fProgram.size());
        fProgram.push_back(inst);
        fIndex[inst] = id;
        return id;
    }

    bool Builder::isZero(ID id) const {
        return fProgram[id].op   == Op::splat
            && fProgram[id].immy == 0;
    }

    Arg Builder::arg(int ix) { return {ix}; }

    void Builder::store8 (Arg ptr, I32 val) { (void)this->push(Op::store8 , val.id,NA,NA, ptr.ix); }
    void Builder::store32(Arg ptr, I32 val) { (void)this->push(Op::store32, val.id,NA,NA, ptr.ix); }

    I32 Builder::load8 (Arg ptr) { return {this->push(Op::load8 , NA,NA,NA, ptr.ix) }; }
    I32 Builder::load32(Arg ptr) { return {this->push(Op::load32, NA,NA,NA, ptr.ix) }; }

    // The two splat() functions are just syntax sugar over splatting a 4-byte bit pattern.
    I32 Builder::splat(int   n) { return {this->push(Op::splat, NA,NA,NA, n) }; }
    F32 Builder::splat(float f) {
        int bits;
        memcpy(&bits, &f, 4);
        return {this->push(Op::splat, NA,NA,NA, bits)};
    }

    F32 Builder::add(F32 x, F32 y       ) { return {this->push(Op::add_f32, x.id, y.id)}; }
    F32 Builder::sub(F32 x, F32 y       ) { return {this->push(Op::sub_f32, x.id, y.id)}; }
    F32 Builder::mul(F32 x, F32 y       ) { return {this->push(Op::mul_f32, x.id, y.id)}; }
    F32 Builder::div(F32 x, F32 y       ) { return {this->push(Op::div_f32, x.id, y.id)}; }
    F32 Builder::mad(F32 x, F32 y, F32 z) {
        if (this->isZero(z.id)) {
            return this->mul(x,y);
        }
        return {this->push(Op::mad_f32, x.id, y.id, z.id)};
    }

    I32 Builder::add(I32 x, I32 y) { return {this->push(Op::add_i32, x.id, y.id)}; }
    I32 Builder::sub(I32 x, I32 y) { return {this->push(Op::sub_i32, x.id, y.id)}; }
    I32 Builder::mul(I32 x, I32 y) { return {this->push(Op::mul_i32, x.id, y.id)}; }

    I32 Builder::sub_16x2(I32 x, I32 y) { return {this->push(Op::sub_i16x2, x.id, y.id)}; }
    I32 Builder::mul_16x2(I32 x, I32 y) { return {this->push(Op::mul_i16x2, x.id, y.id)}; }
    I32 Builder::shr_16x2(I32 x, int bits) { return {this->push(Op::shr_i16x2, x.id,NA,NA, bits)}; }

    I32 Builder::bit_and(I32 x, I32 y) { return {this->push(Op::bit_and, x.id, y.id)}; }
    I32 Builder::bit_or (I32 x, I32 y) { return {this->push(Op::bit_or , x.id, y.id)}; }
    I32 Builder::bit_xor(I32 x, I32 y) { return {this->push(Op::bit_xor, x.id, y.id)}; }

    I32 Builder::shl(I32 x, int bits) { return {this->push(Op::shl, x.id,NA,NA, bits)}; }
    I32 Builder::shr(I32 x, int bits) { return {this->push(Op::shr, x.id,NA,NA, bits)}; }
    I32 Builder::sra(I32 x, int bits) { return {this->push(Op::sra, x.id,NA,NA, bits)}; }

    I32 Builder::extract(I32 x, int bits, I32 z) {
        return {this->push(Op::extract, x.id,NA,z.id, bits,0)};
    }

    I32 Builder::pack(I32 x, I32 y, int bits) {
        return {this->push(Op::pack, x.id,y.id,NA, 0,bits)};
    }

    F32 Builder::to_f32(I32 x) { return {this->push(Op::to_f32, x.id)}; }
    I32 Builder::to_i32(F32 x) { return {this->push(Op::to_i32, x.id)}; }

    // ~~~~ Program::dump() and co. ~~~~ //

    struct V { ID id; };
    struct R { ID id; };
    struct Shift { int bits; };
    struct Splat { int bits; };

    static void write(SkWStream* o, const char* s) {
        o->writeText(s);
    }

    static void write(SkWStream* o, Arg a) {
        write(o, "arg(");
        o->writeDecAsText(a.ix);
        write(o, ")");
    }
    static void write(SkWStream* o, V v) {
        write(o, "v");
        o->writeDecAsText(v.id);
    }
    static void write(SkWStream* o, R r) {
        write(o, "r");
        o->writeDecAsText(r.id);
    }
    static void write(SkWStream* o, Shift s) {
        o->writeDecAsText(s.bits);
    }
    static void write(SkWStream* o, Splat s) {
        float f;
        memcpy(&f, &s.bits, 4);
        o->writeHexAsText(s.bits);
        write(o, " (");
        o->writeScalarAsText(f);
        write(o, ")");
    }

    template <typename T, typename... Ts>
    static void write(SkWStream* o, T first, Ts... rest) {
        write(o, first);
        write(o, " ");
        write(o, rest...);
    }

    void Builder::dump(SkWStream* o) const {
        o->writeDecAsText(fProgram.size());
        o->writeText(" values:\n");
        for (ID id = 0; id < (ID)fProgram.size(); id++) {
            const Instruction& inst = fProgram[id];
            Op  op = inst.op;
            ID   x = inst.x,
                 y = inst.y,
                 z = inst.z;
            int immy = inst.immy,
                immz = inst.immz;
            write(o, inst.life == NA ? "☠ " :
                     inst.hoist      ? "⤴ " : "  ");
            switch (op) {
                case Op::store8:  write(o, "store8" , Arg{immy}, V{x}); break;
                case Op::store32: write(o, "store32", Arg{immy}, V{x}); break;

                case Op::load8:  write(o, V{id}, "= load8" , Arg{immy}); break;
                case Op::load32: write(o, V{id}, "= load32", Arg{immy}); break;

                case Op::splat:  write(o, V{id}, "= splat", Splat{immy}); break;

                case Op::add_f32: write(o, V{id}, "= add_f32", V{x}, V{y}      ); break;
                case Op::sub_f32: write(o, V{id}, "= sub_f32", V{x}, V{y}      ); break;
                case Op::mul_f32: write(o, V{id}, "= mul_f32", V{x}, V{y}      ); break;
                case Op::div_f32: write(o, V{id}, "= div_f32", V{x}, V{y}      ); break;
                case Op::mad_f32: write(o, V{id}, "= mad_f32", V{x}, V{y}, V{z}); break;

                case Op::add_i32: write(o, V{id}, "= add_i32", V{x}, V{y}); break;
                case Op::sub_i32: write(o, V{id}, "= sub_i32", V{x}, V{y}); break;
                case Op::mul_i32: write(o, V{id}, "= mul_i32", V{x}, V{y}); break;

                case Op::sub_i16x2: write(o, V{id}, "= sub_i16x2", V{x}, V{y}); break;
                case Op::mul_i16x2: write(o, V{id}, "= mul_i16x2", V{x}, V{y}); break;
                case Op::shr_i16x2: write(o, V{id}, "= shr_i16x2", V{x}, Shift{immy}); break;

                case Op::bit_and: write(o, V{id}, "= bit_and", V{x}, V{y}); break;
                case Op::bit_or : write(o, V{id}, "= bit_or" , V{x}, V{y}); break;
                case Op::bit_xor: write(o, V{id}, "= bit_xor", V{x}, V{y}); break;

                case Op::shl: write(o, V{id}, "= shl", V{x}, Shift{immy}); break;
                case Op::shr: write(o, V{id}, "= shr", V{x}, Shift{immy}); break;
                case Op::sra: write(o, V{id}, "= sra", V{x}, Shift{immy}); break;

                case Op::extract: write(o, V{id}, "= extract", V{x}, Shift{immy}, V{z}); break;
                case Op::pack:    write(o, V{id}, "= pack",    V{x}, V{y}, Shift{immz}); break;

                case Op::to_f32: write(o, V{id}, "= to_f32", V{x}); break;
                case Op::to_i32: write(o, V{id}, "= to_i32", V{x}); break;
            }

            write(o, "\n");
        }
    }

    void Program::dump(SkWStream* o) const {
        o->writeDecAsText(fRegs);
        o->writeText(" registers, ");
        o->writeDecAsText(fInstructions.size());
        o->writeText(" instructions:\n");
        for (int i = 0; i < (int)fInstructions.size(); i++) {
            if (i == fLoop) {
                write(o, "loop:\n");
            }
            const Instruction& inst = fInstructions[i];
            Op  op = inst.op;
            ID   d = inst.d,
                 x = inst.x;
            auto y = inst.y,
                 z = inst.z;
            switch (op) {
                case Op::store8:  write(o, "store8" , Arg{y.imm}, R{x}); break;
                case Op::store32: write(o, "store32", Arg{y.imm}, R{x}); break;

                case Op::load8:  write(o, R{d}, "= load8" , Arg{y.imm}); break;
                case Op::load32: write(o, R{d}, "= load32", Arg{y.imm}); break;

                case Op::splat:  write(o, R{d}, "= splat", Splat{y.imm}); break;

                case Op::add_f32: write(o, R{d}, "= add_f32", R{x}, R{y.id}           ); break;
                case Op::sub_f32: write(o, R{d}, "= sub_f32", R{x}, R{y.id}           ); break;
                case Op::mul_f32: write(o, R{d}, "= mul_f32", R{x}, R{y.id}           ); break;
                case Op::div_f32: write(o, R{d}, "= div_f32", R{x}, R{y.id}           ); break;
                case Op::mad_f32: write(o, R{d}, "= mad_f32", R{x}, R{y.id}, R{z.id}); break;

                case Op::add_i32: write(o, R{d}, "= add_i32", R{x}, R{y.id}); break;
                case Op::sub_i32: write(o, R{d}, "= sub_i32", R{x}, R{y.id}); break;
                case Op::mul_i32: write(o, R{d}, "= mul_i32", R{x}, R{y.id}); break;

                case Op::sub_i16x2: write(o, R{d}, "= sub_i16x2", R{x}, R{y.id}); break;
                case Op::mul_i16x2: write(o, R{d}, "= mul_i16x2", R{x}, R{y.id}); break;
                case Op::shr_i16x2: write(o, R{d}, "= shr_i16x2", R{x}, Shift{y.imm}); break;

                case Op::bit_and: write(o, R{d}, "= bit_and", R{x}, R{y.id}); break;
                case Op::bit_or : write(o, R{d}, "= bit_or" , R{x}, R{y.id}); break;
                case Op::bit_xor: write(o, R{d}, "= bit_xor", R{x}, R{y.id}); break;

                case Op::shl: write(o, R{d}, "= shl", R{x}, Shift{y.imm}); break;
                case Op::shr: write(o, R{d}, "= shr", R{x}, Shift{y.imm}); break;
                case Op::sra: write(o, R{d}, "= sra", R{x}, Shift{y.imm}); break;

                case Op::extract: write(o, R{d}, "= extract", R{x}, Shift{y.imm}, R{z.id}); break;
                case Op::pack:    write(o, R{d}, "= pack",    R{x}, R{y.id}, Shift{z.imm}); break;

                case Op::to_f32: write(o, R{d}, "= to_f32", R{x}); break;
                case Op::to_i32: write(o, R{d}, "= to_i32", R{x}); break;
            }
            write(o, "\n");
        }
    }

    // ~~~~ Program::eval() and co. ~~~~ //

    #if defined(SKVM_JIT)
        struct Program::JIT : Xbyak::CodeGenerator {
            // 8 float values in a ymm register.
            static constexpr int K = 8;

            static bool Supported(int regs) {
                return true
                    && SkCpu::Supports(SkCpu::HSW)
                    && regs < 16;
            }

            JIT(const std::vector<Program::Instruction>& instructions, int regs, int loop,
                size_t strides[], int nargs)
            {
                SkASSERT(Supported(regs));

            #if defined(SK_BUILD_FOR_WIN)
                // TODO  Windows ABI?
            #else
                // These registers are used to pass the first 6 arguments,
                // so if we stick to these we need not push, pop, spill, or move anything around.
                Xbyak::Reg N = rdi,
                       arg[] = { rsi, rdx, rcx, r8, r9 };

                // All 16 ymm registers are available as scratch.
                Xbyak::Ymm r[] = {
                    ymm0, ymm1, ymm2 , ymm3 , ymm4 , ymm5 , ymm6 , ymm7 ,
                    ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14,
                }, tmp = ymm15;

             #endif

                // Label / 4-byte values we need to write after ret.
                std::vector<std::pair<Xbyak::Label, int>> splats;

                for (int i = 0; i < (int)instructions.size(); i++) {
                    if (i == loop) {
                        L("loop");
                    }
                    const Instruction& inst = instructions[i];
                    Op  op = inst.op;

                    ID   d = inst.d,
                         x = inst.x;
                    auto y = inst.y,
                         z = inst.z;
                    switch (op) {
                        // Ops producing multiple AVX instructions should always
                        // use tmp as the result of all but the final instruction
                        // to avoid any possible dst/arg aliasing.  You don't want
                        // to overwrite your arguments before you're done using them!

                        case Op::store8:
                            if (SkCpu::Supports(SkCpu::SKX)) {
                                // One stop shop!  Pack 8x I32 -> U8 and store to ptr.
                                vpmovusdb(ptr[arg[y.imm]], r[x]);
                            } else {
                                vpackusdw(tmp, r[x], r[x]);      // pack 32-bit -> 16-bit
                                vpermq   (tmp, tmp, 0xd8);       // u64 tmp[0,1,2,3] = tmp[0,2,1,3]
                                vpackuswb(tmp, tmp, tmp);        // pack 16-bit -> 8-bit
                                vmovq(ptr[arg[y.imm]],           // store low 8 bytes
                                      Xbyak::Xmm{tmp.getIdx()}); // (arg must be an xmm register)
                            }
                            break;

                        case Op::store32: vmovups(ptr[arg[y.imm]], r[x]); break;

                        case Op::load8:  vpmovzxbd(r[d], ptr[arg[y.imm]]); break;
                        case Op::load32: vmovups  (r[d], ptr[arg[y.imm]]); break;

                        case Op::splat: splats.emplace_back(Xbyak::Label(), y.imm);
                                        vbroadcastss(r[d], ptr[rip + splats.back().first]);
                                        break;

                        case Op::add_f32: vaddps(r[d], r[x], r[y.id]); break;
                        case Op::sub_f32: vsubps(r[d], r[x], r[y.id]); break;
                        case Op::mul_f32: vmulps(r[d], r[x], r[y.id]); break;
                        case Op::div_f32: vdivps(r[d], r[x], r[y.id]); break;
                        case Op::mad_f32:
                            if (d == x   ) { vfmadd132ps(r[x   ], r[z.id], r[y.id]); } else
                            if (d == y.id) { vfmadd213ps(r[y.id], r[x   ], r[z.id]); } else
                            if (d == z.id) { vfmadd231ps(r[z.id], r[x   ], r[y.id]); } else
                                           { vmulps(tmp, r[x], r[y.id]);
                                             vaddps(r[d], tmp, r[z.id]); }
                            break;

                        case Op::add_i32: vpaddd (r[d], r[x], r[y.id]); break;
                        case Op::sub_i32: vpsubd (r[d], r[x], r[y.id]); break;
                        case Op::mul_i32: vpmulld(r[d], r[x], r[y.id]); break;

                        case Op::sub_i16x2: vpsubw (r[d], r[x], r[y.id]); break;
                        case Op::mul_i16x2: vpmullw(r[d], r[x], r[y.id]); break;
                        case Op::shr_i16x2: vpsrlw (r[d], r[x],   y.imm); break;

                        case Op::bit_and: vandps(r[d], r[x], r[y.id]); break;
                        case Op::bit_or : vorps (r[d], r[x], r[y.id]); break;
                        case Op::bit_xor: vxorps(r[d], r[x], r[y.id]); break;

                        case Op::shl: vpslld(r[d], r[x], y.imm); break;
                        case Op::shr: vpsrld(r[d], r[x], y.imm); break;
                        case Op::sra: vpsrad(r[d], r[x], y.imm); break;

                        case Op::extract: if (y.imm) {
                                              vpsrld(tmp, r[x], y.imm);
                                              vandps(r[d], tmp, r[z.id]);
                                          } else {
                                              vandps(r[d], r[x], r[z.id]);
                                          }
                                          break;

                        case Op::pack: vpslld(tmp, r[y.id], z.imm);
                                       vorps (r[d], tmp, r[x]);
                                       break;

                        case Op::to_f32: vcvtdq2ps (r[d], r[x]); break;
                        case Op::to_i32: vcvttps2dq(r[d], r[x]); break;
                    }
                }

                for (int i = 0; i < nargs; i++) {
                    add(arg[i], K*(int)strides[i]);
                }
                sub(N, K);
                jne("loop");

                vzeroupper();
                ret();

                for (auto splat : splats) {
                    align(4);
                    L(splat.first);
                    dd(splat.second);
                }
            }
        };
    #endif


    void Program::eval(int n, void* args[], size_t strides[], int nargs) const {
    #if defined(SKVM_JIT)
        if (!fJIT && JIT::Supported(fRegs)) {
            fJIT.reset(new JIT{fInstructions, fRegs, fLoop, strides, nargs});

        #if 1
            // We're doing some really stateful things below,
            // so one thread at a time please...
            static SkSpinlock dump_lock;
            SkAutoSpinlock lock(dump_lock);

            uint32_t hash = SkOpts::hash(fJIT->getCode(), fJIT->getSize());

            SkString name = SkStringPrintf("skvm-jit-%u", hash);

            // Create a jit-<pid>.dump file that we can `perf inject -j` into a
            // perf.data captured with `perf record -k 1`, letting us see each
            // JIT'd Program as if a function named skvm-jit-<hash>.   E.g.
            //
            //   ninja -C out nanobench
            //   perf record -k 1 out/nanobench -m SkVM_4096_I32\$
            //   perf inject -j -i perf.data -o perf.data.jit
            //   perf report -i perf.data.jit
            //
            // Running `perf inject -j` will also dump an .so for each JIT'd
            // program, named jitted-<pid>-<hash>.so.

            auto timestamp_ns = []() -> uint64_t {
                // It's important to use CLOCK_MONOTONIC here so that perf can
                // correlate our timestamps with those captured by `perf record
                // -k 1`.  That's also what `-k 1` does, by the way, tell perf
                // record to use CLOCK_MONOTONIC.
                struct timespec ts;
                clock_gettime(CLOCK_MONOTONIC, &ts);
                return ts.tv_sec * (uint64_t)1e9 + ts.tv_nsec;
            };

            // We'll open the jit-<pid>.dump file and write a small header once,
            // and just leave it open forever because we're lazy.
            static FILE* jitdump = [&]{
                // Must map as w+ for the mmap() call below to work.
                FILE* f = fopen(SkStringPrintf("jit-%d.dump", getpid()).c_str(), "w+");

                // Calling mmap() on the file adds a "hey they mmap()'d this" record to
                // the perf.data file that will point `perf inject -j` at this log file.
                // Kind of a strange way to tell `perf inject` where the file is...
                void* marker = mmap(nullptr,
                                    sysconf(_SC_PAGESIZE),
                                    PROT_READ|PROT_EXEC,
                                    MAP_PRIVATE,
                                    fileno(f),
                                    /*offset=*/0);
                SkASSERT_RELEASE(marker != MAP_FAILED);
                // Like never calling fclose(f), we'll also just always leave marker mmap()'d.

                struct Header {
                    uint32_t magic, version, header_size, elf_mach, reserved, pid;
                    uint64_t timestamp_us, flags;
                } header = {
                    0x4A695444, 1, sizeof(Header), 62/*x86-64*/, 0, (uint32_t)getpid(),
                    timestamp_ns() / 1000, 0,
                };
                fwrite(&header, sizeof(header), 1, f);

                return f;
            }();

            struct CodeLoad {
                uint32_t event_type, event_size;
                uint64_t timestamp_ns;

                uint32_t pid, tid;
                uint64_t vma/*???*/, code_addr, code_size, id;
            } load = {
                0/*code load*/, (uint32_t)(sizeof(CodeLoad) + name.size() + 1 + fJIT->getSize()),
                timestamp_ns(),

                (uint32_t)getpid(), (uint32_t)SkGetThreadID(),
                (uint64_t)fJIT->getCode(), (uint64_t)fJIT->getCode(), fJIT->getSize(), hash,
            };

            // Write the header, the JIT'd function name, and the JIT'd code itself.
            fwrite(&load, sizeof(load), 1, jitdump);
            fwrite(name.c_str(), 1, name.size(), jitdump);
            fwrite("\0", 1, 1, jitdump);
            fwrite(fJIT->getCode(), 1, fJIT->getSize(), jitdump);
        #endif
        }

        const int jitN = (n / JIT::K) * JIT::K;
        if (fJIT && jitN > 0) {
            bool ran = true;
            switch (nargs) {
                case 0: fJIT->getCode<void(*)(int              )>()(jitN                  ); break;
                case 1: fJIT->getCode<void(*)(int, void*       )>()(jitN, args[0]         ); break;
                case 2: fJIT->getCode<void(*)(int, void*, void*)>()(jitN, args[0], args[1]); break;
                default: ran = false; break;
            }
            if (ran) {
                // Step n and arguments forward to where the JIT stopped.
                n -= jitN;

                void**        arg    = args;
                const size_t* stride = strides;
                for (; *arg; arg++, stride++) {
                    *arg = (void*)( (char*)*arg + jitN * *stride );
                }
                SkASSERT(arg == args + nargs);
            }
        }
    #endif
        if (n) {
            SkOpts::eval(fInstructions.data(), (int)fInstructions.size(), fRegs, fLoop,
                         n, args, strides, nargs);
        }
    }
}

// TODO: argument strides (more generally types) should come earlier, the pointers themselves later.