12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628 |
- /*
- * Copyright 2019 Google LLC
- *
- * Use of this source code is governed by a BSD-style license that can be
- * found in the LICENSE file.
- */
- #include "include/private/SkSpinlock.h"
- #include "include/private/SkTFitsIn.h"
- #include "include/private/SkThreadID.h"
- #include "include/private/SkVx.h"
- #include "src/core/SkCpu.h"
- #include "src/core/SkVM.h"
- #include <string.h>
- #if defined(SKVM_JIT)
- #include <sys/mman.h>
- #endif
- namespace skvm {
- Program Builder::done(const char* debug_name) {
- // Basic liveness analysis:
- // an instruction is live until all live instructions that need its input have retired.
- for (Val id = fProgram.size(); id --> 0; ) {
- Instruction& inst = fProgram[id];
- // All side-effect-only instructions (stores) are live.
- if (inst.op <= Op::store32) {
- inst.death = id;
- }
- // The arguments of a live instruction must live until at least that instruction.
- if (inst.death != 0) {
- // Notice how we're walking backward, storing the latest instruction in death.
- if (inst.x != NA && fProgram[inst.x].death == 0) { fProgram[inst.x].death = id; }
- if (inst.y != NA && fProgram[inst.y].death == 0) { fProgram[inst.y].death = id; }
- if (inst.z != NA && fProgram[inst.z].death == 0) { fProgram[inst.z].death = id; }
- }
- }
- // Mark which values don't depend on the loop and can be hoisted.
- for (Val id = 0; id < (Val)fProgram.size(); id++) {
- Builder::Instruction& inst = fProgram[id];
- // Loads and stores cannot be hoisted out of the loop.
- if (inst.op <= Op::load32) {
- inst.hoist = false;
- }
- // If any of an instruction's inputs can't be hoisted, it can't be hoisted itself.
- if (inst.hoist) {
- if (inst.x != NA) { inst.hoist &= fProgram[inst.x].hoist; }
- if (inst.y != NA) { inst.hoist &= fProgram[inst.y].hoist; }
- if (inst.z != NA) { inst.hoist &= fProgram[inst.z].hoist; }
- }
- }
- return {fProgram, fStrides, debug_name};
- }
- static bool operator==(const Builder::Instruction& a, const Builder::Instruction& b) {
- return a.op == b.op
- && a.x == b.x
- && a.y == b.y
- && a.z == b.z
- && a.imm == b.imm
- && a.death == b.death
- && a.hoist == b.hoist;
- }
- // Most instructions produce a value and return it by ID,
- // the value-producing instruction's own index in the program vector.
- Val Builder::push(Op op, Val x, Val y, Val z, int imm) {
- Instruction inst{op, x, y, z, imm, /*death=*/0, /*hoist=*/true};
- // Basic common subexpression elimination:
- // if we've already seen this exact Instruction, use it instead of creating a new one.
- if (Val* id = fIndex.find(inst)) {
- return *id;
- }
- Val id = static_cast<Val>(fProgram.size());
- fProgram.push_back(inst);
- fIndex.set(inst, id);
- return id;
- }
- bool Builder::isZero(Val id) const {
- return fProgram[id].op == Op::splat
- && fProgram[id].imm == 0;
- }
- Arg Builder::arg(int stride) {
- int ix = (int)fStrides.size();
- fStrides.push_back(stride);
- return {ix};
- }
- void Builder::store8 (Arg ptr, I32 val) { (void)this->push(Op::store8 , val.id,NA,NA, ptr.ix); }
- void Builder::store32(Arg ptr, I32 val) { (void)this->push(Op::store32, val.id,NA,NA, ptr.ix); }
- I32 Builder::load8 (Arg ptr) { return {this->push(Op::load8 , NA,NA,NA, ptr.ix) }; }
- I32 Builder::load32(Arg ptr) { return {this->push(Op::load32, NA,NA,NA, ptr.ix) }; }
- // The two splat() functions are just syntax sugar over splatting a 4-byte bit pattern.
- I32 Builder::splat(int n) { return {this->push(Op::splat, NA,NA,NA, n) }; }
- F32 Builder::splat(float f) {
- int bits;
- memcpy(&bits, &f, 4);
- return {this->push(Op::splat, NA,NA,NA, bits)};
- }
- F32 Builder::add(F32 x, F32 y ) { return {this->push(Op::add_f32, x.id, y.id)}; }
- F32 Builder::sub(F32 x, F32 y ) { return {this->push(Op::sub_f32, x.id, y.id)}; }
- F32 Builder::mul(F32 x, F32 y ) { return {this->push(Op::mul_f32, x.id, y.id)}; }
- F32 Builder::div(F32 x, F32 y ) { return {this->push(Op::div_f32, x.id, y.id)}; }
- F32 Builder::mad(F32 x, F32 y, F32 z) {
- if (this->isZero(z.id)) {
- return this->mul(x,y);
- }
- return {this->push(Op::mad_f32, x.id, y.id, z.id)};
- }
- I32 Builder::add(I32 x, I32 y) { return {this->push(Op::add_i32, x.id, y.id)}; }
- I32 Builder::sub(I32 x, I32 y) { return {this->push(Op::sub_i32, x.id, y.id)}; }
- I32 Builder::mul(I32 x, I32 y) { return {this->push(Op::mul_i32, x.id, y.id)}; }
- I32 Builder::sub_16x2(I32 x, I32 y) { return {this->push(Op::sub_i16x2, x.id, y.id)}; }
- I32 Builder::mul_16x2(I32 x, I32 y) { return {this->push(Op::mul_i16x2, x.id, y.id)}; }
- I32 Builder::shr_16x2(I32 x, int bits) { return {this->push(Op::shr_i16x2, x.id,NA,NA, bits)}; }
- I32 Builder::bit_and (I32 x, I32 y) { return {this->push(Op::bit_and , x.id, y.id)}; }
- I32 Builder::bit_or (I32 x, I32 y) { return {this->push(Op::bit_or , x.id, y.id)}; }
- I32 Builder::bit_xor (I32 x, I32 y) { return {this->push(Op::bit_xor , x.id, y.id)}; }
- I32 Builder::bit_clear(I32 x, I32 y) { return {this->push(Op::bit_clear, x.id, y.id)}; }
- I32 Builder::shl(I32 x, int bits) { return {this->push(Op::shl, x.id,NA,NA, bits)}; }
- I32 Builder::shr(I32 x, int bits) { return {this->push(Op::shr, x.id,NA,NA, bits)}; }
- I32 Builder::sra(I32 x, int bits) { return {this->push(Op::sra, x.id,NA,NA, bits)}; }
- I32 Builder::extract(I32 x, int bits, I32 y) {
- return {this->push(Op::extract, x.id,y.id,NA, bits)};
- }
- I32 Builder::pack(I32 x, I32 y, int bits) {
- return {this->push(Op::pack, x.id,y.id,NA, bits)};
- }
- I32 Builder::bytes(I32 x, int control) {
- return {this->push(Op::bytes, x.id,NA,NA, control)};
- }
- F32 Builder::to_f32(I32 x) { return {this->push(Op::to_f32, x.id)}; }
- I32 Builder::to_i32(F32 x) { return {this->push(Op::to_i32, x.id)}; }
- // ~~~~ Program::eval() and co. ~~~~ //
- // Handy references for x86-64 instruction encoding:
- // https://wiki.osdev.org/X86-64_Instruction_Encoding
- // https://www-user.tu-chemnitz.de/~heha/viewchm.php/hs/x86.chm/x64.htm
- // https://www-user.tu-chemnitz.de/~heha/viewchm.php/hs/x86.chm/x86.htm
- // http://ref.x86asm.net/coder64.html
- // Used for ModRM / immediate instruction encoding.
- static uint8_t _233(int a, int b, int c) {
- return (a & 3) << 6
- | (b & 7) << 3
- | (c & 7) << 0;
- }
- // ModRM byte encodes the arguments of an opcode.
- enum class Mod { Indirect, OneByteImm, FourByteImm, Direct };
- static uint8_t mod_rm(Mod mod, int reg, int rm) {
- return _233((int)mod, reg, rm);
- }
- #if 0
- // SIB byte encodes a memory address, base + (index * scale).
- enum class Scale { One, Two, Four, Eight };
- static uint8_t sib(Scale scale, int index, int base) {
- return _233((int)scale, index, base);
- }
- #endif
- // The REX prefix is used to extend most old 32-bit instructions to 64-bit.
- static uint8_t rex(bool W, // If set, operation is 64-bit, otherwise default, usually 32-bit.
- bool R, // Extra top bit to select ModRM reg, registers 8-15.
- bool X, // Extra top bit for SIB index register.
- bool B) { // Extra top bit for SIB base or ModRM rm register.
- return 0b01000000 // Fixed 0100 for top four bits.
- | (W << 3)
- | (R << 2)
- | (X << 1)
- | (B << 0);
- }
- // The VEX prefix extends SSE operations to AVX. Used generally, even with XMM.
- struct VEX {
- int len;
- uint8_t bytes[3];
- };
- static VEX vex(bool WE, // Like REX W for int operations, or opcode extension for float?
- bool R, // Same as REX R. Pass high bit of dst register, dst>>3.
- bool X, // Same as REX X.
- bool B, // Same as REX B. Pass y>>3 for 3-arg ops, x>>3 for 2-arg.
- int map, // SSE opcode map selector: 0x0f, 0x380f, 0x3a0f.
- int vvvv, // 4-bit second operand register. Pass our x for 3-arg ops.
- bool L, // Set for 256-bit ymm operations, off for 128-bit xmm.
- int pp) { // SSE mandatory prefix: 0x66, 0xf3, 0xf2, else none.
- // Pack x86 opcode map selector to 5-bit VEX encoding.
- map = [map]{
- switch (map) {
- case 0x0f: return 0b00001;
- case 0x380f: return 0b00010;
- case 0x3a0f: return 0b00011;
- // Several more cases only used by XOP / TBM.
- }
- SkASSERT(false);
- return 0b00000;
- }();
- // Pack mandatory SSE opcode prefix byte to 2-bit VEX encoding.
- pp = [pp]{
- switch (pp) {
- case 0x66: return 0b01;
- case 0xf3: return 0b10;
- case 0xf2: return 0b11;
- }
- return 0b00;
- }();
- VEX vex = {0, {0,0,0}};
- if (X == 0 && B == 0 && WE == 0 && map == 0b00001) {
- // With these conditions met, we can optionally compress VEX to 2-byte.
- vex.len = 2;
- vex.bytes[0] = 0xc5;
- vex.bytes[1] = (pp & 3) << 0
- | (L & 1) << 2
- | (~vvvv & 15) << 3
- | (~(int)R & 1) << 7;
- } else {
- // We could use this 3-byte VEX prefix all the time if we like.
- vex.len = 3;
- vex.bytes[0] = 0xc4;
- vex.bytes[1] = (map & 31) << 0
- | (~(int)B & 1) << 5
- | (~(int)X & 1) << 6
- | (~(int)R & 1) << 7;
- vex.bytes[2] = (pp & 3) << 0
- | (L & 1) << 2
- | (~vvvv & 15) << 3
- | (WE & 1) << 7;
- }
- return vex;
- }
- Assembler::Assembler(void* buf) : fCode((uint8_t*)buf), fCurr(fCode), fSize(0) {}
- size_t Assembler::size() const { return fSize; }
- void Assembler::bytes(const void* p, int n) {
- if (fCurr) {
- memcpy(fCurr, p, n);
- fCurr += n;
- }
- fSize += n;
- }
- void Assembler::byte(uint8_t b) { this->bytes(&b, 1); }
- void Assembler::word(uint32_t w) { this->bytes(&w, 4); }
- void Assembler::align(int mod) {
- while (this->size() % mod) {
- this->byte(0x00);
- }
- }
- void Assembler::vzeroupper() {
- this->byte(0xc5);
- this->byte(0xf8);
- this->byte(0x77);
- }
- void Assembler::ret() { this->byte(0xc3); }
- // Common instruction building for 64-bit opcodes with an immediate argument.
- void Assembler::op(int opcode, int opcode_ext, GP64 dst, int imm) {
- opcode |= 0b0000'0001; // low bit set for 64-bit operands
- opcode |= 0b1000'0000; // top bit set for instructions with any immediate
- int imm_bytes = 4;
- if (SkTFitsIn<int8_t>(imm)) {
- imm_bytes = 1;
- opcode |= 0b0000'0010; // second bit set for 8-bit immediate, else 32-bit.
- }
- this->byte(rex(1,0,0,dst>>3));
- this->byte(opcode);
- this->byte(mod_rm(Mod::Direct, opcode_ext, dst&7));
- this->bytes(&imm, imm_bytes);
- }
- void Assembler::add(GP64 dst, int imm) { this->op(0,0b000, dst,imm); }
- void Assembler::sub(GP64 dst, int imm) { this->op(0,0b101, dst,imm); }
- void Assembler::cmp(GP64 reg, int imm) { this->op(0,0b111, reg,imm); }
- void Assembler::op(int prefix, int map, int opcode, Ymm dst, Ymm x, Ymm y, bool W/*=false*/) {
- VEX v = vex(W, dst>>3, 0, y>>3,
- map, x, 1/*ymm, not xmm*/, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Direct, dst&7, y&7));
- }
- void Assembler::vpaddd (Ymm dst, Ymm x, Ymm y) { this->op(0x66, 0x0f,0xfe, dst,x,y); }
- void Assembler::vpsubd (Ymm dst, Ymm x, Ymm y) { this->op(0x66, 0x0f,0xfa, dst,x,y); }
- void Assembler::vpmulld(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x380f,0x40, dst,x,y); }
- void Assembler::vpsubw (Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x0f,0xf9, dst,x,y); }
- void Assembler::vpmullw(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x0f,0xd5, dst,x,y); }
- void Assembler::vpand (Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x0f,0xdb, dst,x,y); }
- void Assembler::vpor (Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x0f,0xeb, dst,x,y); }
- void Assembler::vpxor (Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x0f,0xef, dst,x,y); }
- void Assembler::vpandn(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x0f,0xdf, dst,x,y); }
- void Assembler::vaddps(Ymm dst, Ymm x, Ymm y) { this->op(0,0x0f,0x58, dst,x,y); }
- void Assembler::vsubps(Ymm dst, Ymm x, Ymm y) { this->op(0,0x0f,0x5c, dst,x,y); }
- void Assembler::vmulps(Ymm dst, Ymm x, Ymm y) { this->op(0,0x0f,0x59, dst,x,y); }
- void Assembler::vdivps(Ymm dst, Ymm x, Ymm y) { this->op(0,0x0f,0x5e, dst,x,y); }
- void Assembler::vfmadd132ps(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x380f,0x98, dst,x,y); }
- void Assembler::vfmadd213ps(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x380f,0xa8, dst,x,y); }
- void Assembler::vfmadd231ps(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x380f,0xb8, dst,x,y); }
- void Assembler::vpackusdw(Ymm dst, Ymm x, Ymm y) { this->op(0x66,0x380f,0x2b, dst,x,y); }
- void Assembler::vpackuswb(Ymm dst, Ymm x, Ymm y) { this->op(0x66, 0x0f,0x67, dst,x,y); }
- // dst = x op /opcode_ext imm
- void Assembler::op(int prefix, int map, int opcode, int opcode_ext, Ymm dst, Ymm x, int imm) {
- // This is a little weird, but if we pass the opcode_ext as if it were the dst register,
- // the dst register as if x, and the x register as if y, all the bits end up where we want.
- this->op(prefix, map, opcode, (Ymm)opcode_ext,dst,x);
- this->byte(imm);
- }
- void Assembler::vpslld(Ymm dst, Ymm x, int imm) { this->op(0x66,0x0f,0x72,6, dst,x,imm); }
- void Assembler::vpsrld(Ymm dst, Ymm x, int imm) { this->op(0x66,0x0f,0x72,2, dst,x,imm); }
- void Assembler::vpsrad(Ymm dst, Ymm x, int imm) { this->op(0x66,0x0f,0x72,4, dst,x,imm); }
- void Assembler::vpsrlw(Ymm dst, Ymm x, int imm) { this->op(0x66,0x0f,0x71,2, dst,x,imm); }
- void Assembler::vpermq(Ymm dst, Ymm x, int imm) {
- // A bit unusual among the instructions we use, this is 64-bit operation, so we set W.
- bool W = true;
- this->op(0x66,0x3a0f,0x00, dst,x,W);
- this->byte(imm);
- }
- void Assembler::vcvtdq2ps (Ymm dst, Ymm x) { this->op(0, 0x0f,0x5b, dst,x); }
- void Assembler::vcvttps2dq(Ymm dst, Ymm x) { this->op(0xf3,0x0f,0x5b, dst,x); }
- Assembler::Label Assembler::here() {
- return { (int)this->size(), Label::None, {} };
- }
- int Assembler::disp19(Label* l) {
- SkASSERT(l->kind == Label::None ||
- l->kind == Label::ARMDisp19);
- l->kind = Label::ARMDisp19;
- l->references.push_back(here().offset);
- // ARM 19-bit instruction count, from the beginning of this instruction.
- return (l->offset - here().offset) / 4;
- }
- int Assembler::disp32(Label* l) {
- SkASSERT(l->kind == Label::None ||
- l->kind == Label::X86Disp32);
- l->kind = Label::X86Disp32;
- l->references.push_back(here().offset);
- // x86 32-bit byte count, from the end of this instruction.
- return l->offset - (here().offset + 4);
- }
- void Assembler::op(int prefix, int map, int opcode, Ymm dst, Ymm x, Label* l) {
- // IP-relative addressing uses Mod::Indirect with the R/M encoded as-if rbp or r13.
- const int rip = rbp;
- VEX v = vex(0, dst>>3, 0, rip>>3,
- map, x, /*ymm?*/1, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, dst&7, rip&7));
- this->word(this->disp32(l));
- }
- void Assembler::vbroadcastss(Ymm dst, Label* l) { this->op(0x66,0x380f,0x18, dst,l); }
- void Assembler::vpshufb(Ymm dst, Ymm x, Label* l) { this->op(0x66,0x380f,0x00, dst,x,l); }
- void Assembler::jump(uint8_t condition, Label* l) {
- // These conditional jumps can be either 2 bytes (short) or 6 bytes (near):
- // 7? one-byte-disp
- // 0F 8? four-byte-disp
- // We always use the near displacement to make updating labels simpler (no resizing).
- this->byte(0x0f);
- this->byte(condition);
- this->word(this->disp32(l));
- }
- void Assembler::je (Label* l) { this->jump(0x84, l); }
- void Assembler::jne(Label* l) { this->jump(0x85, l); }
- void Assembler::jl (Label* l) { this->jump(0x8c, l); }
- void Assembler::jmp(Label* l) {
- // Like above in jump(), we could use 8-bit displacement here, but always use 32-bit.
- this->byte(0xe9);
- this->word(this->disp32(l));
- }
- void Assembler::load_store(int prefix, int map, int opcode, Ymm ymm, GP64 ptr) {
- VEX v = vex(0, ymm>>3, 0, ptr>>3,
- map, 0, /*ymm?*/1, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, ymm&7, ptr&7));
- }
- void Assembler::vmovups (Ymm dst, GP64 src) { this->load_store(0 , 0x0f,0x10, dst,src); }
- void Assembler::vpmovzxbd(Ymm dst, GP64 src) { this->load_store(0x66,0x380f,0x31, dst,src); }
- void Assembler::vmovups (GP64 dst, Ymm src) { this->load_store(0 , 0x0f,0x11, src,dst); }
- void Assembler::vmovq(GP64 dst, Xmm src) {
- int prefix = 0x66,
- map = 0x0f,
- opcode = 0xd6;
- VEX v = vex(0, src>>3, 0, dst>>3,
- map, 0, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, src&7, dst&7));
- }
- void Assembler::vmovd(GP64 dst, Xmm src) {
- int prefix = 0x66,
- map = 0x0f,
- opcode = 0x7e;
- VEX v = vex(0, src>>3, 0, dst>>3,
- map, 0, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, src&7, dst&7));
- }
- void Assembler::vmovd_direct(GP64 dst, Xmm src) {
- int prefix = 0x66,
- map = 0x0f,
- opcode = 0x7e;
- VEX v = vex(0, src>>3, 0, dst>>3,
- map, 0, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Direct, src&7, dst&7));
- }
- void Assembler::vmovd(Xmm dst, GP64 src) {
- int prefix = 0x66,
- map = 0x0f,
- opcode = 0x6e;
- VEX v = vex(0, dst>>3, 0, src>>3,
- map, 0, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, dst&7, src&7));
- }
- void Assembler::vmovd_direct(Xmm dst, GP64 src) {
- int prefix = 0x66,
- map = 0x0f,
- opcode = 0x6e;
- VEX v = vex(0, dst>>3, 0, src>>3,
- map, 0, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Direct, dst&7, src&7));
- }
- void Assembler::movzbl(GP64 dst, GP64 src) {
- if ((dst>>3) || (src>>3)) {
- this->byte(rex(0,dst>>3,0,src>>3));
- }
- this->byte(0x0f);
- this->byte(0xb6);
- this->byte(mod_rm(Mod::Indirect, dst&7, src&7));
- }
- void Assembler::movb(GP64 dst, GP64 src) {
- if ((dst>>3) || (src>>3)) {
- this->byte(rex(0,src>>3,0,dst>>3));
- }
- this->byte(0x88);
- this->byte(mod_rm(Mod::Indirect, src&7, dst&7));
- }
- void Assembler::vpinsrb(Xmm dst, Xmm src, GP64 ptr, int imm) {
- int prefix = 0x66,
- map = 0x3a0f,
- opcode = 0x20;
- VEX v = vex(0, dst>>3, 0, ptr>>3,
- map, src, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, dst&7, ptr&7));
- this->byte(imm);
- }
- void Assembler::vpextrb(GP64 ptr, Xmm src, int imm) {
- int prefix = 0x66,
- map = 0x3a0f,
- opcode = 0x14;
- VEX v = vex(0, src>>3, 0, ptr>>3,
- map, 0, /*ymm?*/0, prefix);
- this->bytes(v.bytes, v.len);
- this->byte(opcode);
- this->byte(mod_rm(Mod::Indirect, src&7, ptr&7));
- this->byte(imm);
- }
- // https://static.docs.arm.com/ddi0596/a/DDI_0596_ARM_a64_instruction_set_architecture.pdf
- static int operator"" _mask(unsigned long long bits) { return (1<<(int)bits)-1; }
- void Assembler::op(uint32_t hi, V m, uint32_t lo, V n, V d) {
- this->word( (hi & 11_mask) << 21
- | (m & 5_mask) << 16
- | (lo & 6_mask) << 10
- | (n & 5_mask) << 5
- | (d & 5_mask) << 0);
- }
- void Assembler::and16b(V d, V n, V m) { this->op(0b0'1'0'01110'00'1, m, 0b00011'1, n, d); }
- void Assembler::orr16b(V d, V n, V m) { this->op(0b0'1'0'01110'10'1, m, 0b00011'1, n, d); }
- void Assembler::eor16b(V d, V n, V m) { this->op(0b0'1'1'01110'00'1, m, 0b00011'1, n, d); }
- void Assembler::bic16b(V d, V n, V m) { this->op(0b0'1'0'01110'01'1, m, 0b00011'1, n, d); }
- void Assembler::add4s(V d, V n, V m) { this->op(0b0'1'0'01110'10'1, m, 0b10000'1, n, d); }
- void Assembler::sub4s(V d, V n, V m) { this->op(0b0'1'1'01110'10'1, m, 0b10000'1, n, d); }
- void Assembler::mul4s(V d, V n, V m) { this->op(0b0'1'0'01110'10'1, m, 0b10011'1, n, d); }
- void Assembler::sub8h(V d, V n, V m) { this->op(0b0'1'1'01110'01'1, m, 0b10000'1, n, d); }
- void Assembler::mul8h(V d, V n, V m) { this->op(0b0'1'0'01110'01'1, m, 0b10011'1, n, d); }
- void Assembler::fadd4s(V d, V n, V m) { this->op(0b0'1'0'01110'0'0'1, m, 0b11010'1, n, d); }
- void Assembler::fsub4s(V d, V n, V m) { this->op(0b0'1'0'01110'1'0'1, m, 0b11010'1, n, d); }
- void Assembler::fmul4s(V d, V n, V m) { this->op(0b0'1'1'01110'0'0'1, m, 0b11011'1, n, d); }
- void Assembler::fdiv4s(V d, V n, V m) { this->op(0b0'1'1'01110'0'0'1, m, 0b11111'1, n, d); }
- void Assembler::fmla4s(V d, V n, V m) { this->op(0b0'1'0'01110'0'0'1, m, 0b11001'1, n, d); }
- void Assembler::tbl(V d, V n, V m) { this->op(0b0'1'001110'00'0, m, 0b0'00'0'00, n, d); }
- void Assembler::op(uint32_t op22, int imm, V n, V d) {
- this->word( (op22 & 22_mask) << 10
- | imm << 16 // imm is embedded inside op, bit size depends on op
- | (n & 5_mask) << 5
- | (d & 5_mask) << 0);
- }
- void Assembler::sli4s(V d, V n, int imm) {
- this->op(0b0'1'1'011110'0100'000'01010'1, ( imm&31), n, d);
- }
- void Assembler::shl4s(V d, V n, int imm) {
- this->op(0b0'1'0'011110'0100'000'01010'1, ( imm&31), n, d);
- }
- void Assembler::sshr4s(V d, V n, int imm) {
- this->op(0b0'1'0'011110'0100'000'00'0'0'0'1, (-imm&31), n, d);
- }
- void Assembler::ushr4s(V d, V n, int imm) {
- this->op(0b0'1'1'011110'0100'000'00'0'0'0'1, (-imm&31), n, d);
- }
- void Assembler::ushr8h(V d, V n, int imm) {
- this->op(0b0'1'1'011110'0010'000'00'0'0'0'1, (-imm&15), n, d);
- }
- void Assembler::scvtf4s (V d, V n) { this->op(0b0'1'0'01110'0'0'10000'11101'10, n,d); }
- void Assembler::fcvtzs4s(V d, V n) { this->op(0b0'1'0'01110'1'0'10000'1101'1'10, n,d); }
- void Assembler::xtns2h(V d, V n) { this->op(0b0'0'0'01110'01'10000'10010'10, n,d); }
- void Assembler::xtnh2b(V d, V n) { this->op(0b0'0'0'01110'00'10000'10010'10, n,d); }
- void Assembler::uxtlb2h(V d, V n) { this->op(0b0'0'1'011110'0001'000'10100'1, n,d); }
- void Assembler::uxtlh2s(V d, V n) { this->op(0b0'0'1'011110'0010'000'10100'1, n,d); }
- void Assembler::ret(X n) {
- this->word(0b1101011'0'0'10'11111'0000'0'0 << 10
- | (n & 5_mask) << 5);
- }
- void Assembler::add(X d, X n, int imm12) {
- this->word(0b1'0'0'10001'00 << 22
- | (imm12 & 12_mask) << 10
- | (n & 5_mask) << 5
- | (d & 5_mask) << 0);
- }
- void Assembler::sub(X d, X n, int imm12) {
- this->word( 0b1'1'0'10001'00 << 22
- | (imm12 & 12_mask) << 10
- | (n & 5_mask) << 5
- | (d & 5_mask) << 0);
- }
- void Assembler::subs(X d, X n, int imm12) {
- this->word( 0b1'1'1'10001'00 << 22
- | (imm12 & 12_mask) << 10
- | (n & 5_mask) << 5
- | (d & 5_mask) << 0);
- }
- void Assembler::b(Condition cond, Label* l) {
- const int imm19 = this->disp19(l);
- this->word( 0b0101010'0 << 24
- | (imm19 & 19_mask) << 5
- | ((int)cond & 4_mask) << 0);
- }
- void Assembler::cbz(X t, Label* l) {
- const int imm19 = this->disp19(l);
- this->word( 0b1'011010'0 << 24
- | (imm19 & 19_mask) << 5
- | (t & 5_mask) << 0);
- }
- void Assembler::cbnz(X t, Label* l) {
- const int imm19 = this->disp19(l);
- this->word( 0b1'011010'1 << 24
- | (imm19 & 19_mask) << 5
- | (t & 5_mask) << 0);
- }
- void Assembler::ldrq(V dst, X src) { this->op(0b00'111'1'01'11'000000000000, src, dst); }
- void Assembler::ldrs(V dst, X src) { this->op(0b10'111'1'01'01'000000000000, src, dst); }
- void Assembler::ldrb(V dst, X src) { this->op(0b00'111'1'01'01'000000000000, src, dst); }
- void Assembler::strq(V src, X dst) { this->op(0b00'111'1'01'10'000000000000, dst, src); }
- void Assembler::strs(V src, X dst) { this->op(0b10'111'1'01'00'000000000000, dst, src); }
- void Assembler::strb(V src, X dst) { this->op(0b00'111'1'01'00'000000000000, dst, src); }
- void Assembler::ldrq(V dst, Label* l) {
- const int imm19 = this->disp19(l);
- this->word( 0b10'011'1'00 << 24
- | (imm19 & 19_mask) << 5
- | (dst & 5_mask) << 0);
- }
- void Assembler::label(Label* l) {
- if (fCode) {
- // The instructions all currently point to l->offset.
- // We'll want to add a delta to point them to here().
- int delta = here().offset - l->offset;
- l->offset = here().offset;
- if (l->kind == Label::ARMDisp19) {
- for (int ref : l->references) {
- // ref points to a 32-bit instruction with 19-bit displacement in instructions.
- uint32_t inst;
- memcpy(&inst, fCode + ref, 4);
- // [ 8 bits to preserve] [ 19 bit signed displacement ] [ 5 bits to preserve ]
- int disp = (int)(inst << 8) >> 13;
- disp += delta/4; // delta is in bytes, we want instructions.
- // Put it all back together, preserving the high 8 bits and low 5.
- inst = ((disp << 5) & (19_mask << 5))
- | ((inst ) & ~(19_mask << 5));
- memcpy(fCode + ref, &inst, 4);
- }
- }
- if (l->kind == Label::X86Disp32) {
- for (int ref : l->references) {
- // ref points to a 32-bit displacement in bytes.
- int disp;
- memcpy(&disp, fCode + ref, 4);
- disp += delta;
- memcpy(fCode + ref, &disp, 4);
- }
- }
- }
- }
- void Program::eval(int n, void* args[]) const {
- const int nargs = (int)fStrides.size();
- if (fJITBuf) {
- switch (nargs) {
- case 0: return ((void(*)(int ))fJITBuf)(n );
- case 1: return ((void(*)(int, void* ))fJITBuf)(n, args[0] );
- case 2: return ((void(*)(int, void*, void*))fJITBuf)(n, args[0], args[1]);
- default: SkUNREACHABLE; // TODO
- }
- }
- // We'll operate in SIMT style, knocking off K-size chunks from n while possible.
- constexpr int K = 16;
- using I32 = skvx::Vec<K, int>;
- using F32 = skvx::Vec<K, float>;
- using U32 = skvx::Vec<K, uint32_t>;
- using U8 = skvx::Vec<K, uint8_t>;
- using I16x2 = skvx::Vec<2*K, int16_t>;
- using U16x2 = skvx::Vec<2*K, uint16_t>;
- union Slot {
- I32 i32;
- U32 u32;
- F32 f32;
- };
- Slot few_regs[16];
- std::unique_ptr<char[]> many_regs;
- Slot* regs = few_regs;
- if (fRegs > (int)SK_ARRAY_COUNT(few_regs)) {
- // Annoyingly we can't trust that malloc() or new will work with Slot because
- // the skvx::Vec types may have alignment greater than what they provide.
- // We'll overallocate one extra register so we can align manually.
- many_regs.reset(new char[ sizeof(Slot) * (fRegs + 1) ]);
- uintptr_t addr = (uintptr_t)many_regs.get();
- addr += alignof(Slot) -
- (addr & (alignof(Slot) - 1));
- SkASSERT((addr & (alignof(Slot) - 1)) == 0);
- regs = (Slot*)addr;
- }
- auto r = [&](Reg id) -> Slot& {
- SkASSERT(0 <= id && id < fRegs);
- return regs[id];
- };
- auto arg = [&](int ix) {
- SkASSERT(0 <= ix && ix < nargs);
- return args[ix];
- };
- // Step each argument pointer ahead by its stride a number of times.
- auto step_args = [&](int times) {
- // Looping by marching pointers until *arg == nullptr helps the
- // compiler to keep this loop scalar. Otherwise it'd create a
- // rather large and useless autovectorized version.
- void** arg = args;
- const int* stride = fStrides.data();
- for (; *arg; arg++, stride++) {
- *arg = (void*)( (char*)*arg + times * *stride );
- }
- SkASSERT(arg == args + nargs);
- };
- int start = 0,
- stride;
- for ( ; n > 0; start = fLoop, n -= stride, step_args(stride)) {
- stride = n >= K ? K : 1;
- for (int i = start; i < (int)fInstructions.size(); i++) {
- Instruction inst = fInstructions[i];
- // d = op(x,y,z/imm)
- Reg d = inst.d,
- x = inst.x,
- y = inst.y,
- z = inst.z;
- int imm = inst.imm;
- // Ops that interact with memory need to know whether we're stride=1 or K,
- // but all non-memory ops can run the same code no matter the stride.
- switch (2*(int)inst.op + (stride == K ? 1 : 0)) {
- #define STRIDE_1(op) case 2*(int)op
- #define STRIDE_K(op) case 2*(int)op + 1
- STRIDE_1(Op::store8 ): memcpy(arg(imm), &r(x).i32, 1); break;
- STRIDE_1(Op::store32): memcpy(arg(imm), &r(x).i32, 4); break;
- STRIDE_K(Op::store8 ): skvx::cast<uint8_t>(r(x).i32).store(arg(imm)); break;
- STRIDE_K(Op::store32): (r(x).i32).store(arg(imm)); break;
- STRIDE_1(Op::load8 ): r(d).i32 = 0; memcpy(&r(d).i32, arg(imm), 1); break;
- STRIDE_1(Op::load32): r(d).i32 = 0; memcpy(&r(d).i32, arg(imm), 4); break;
- STRIDE_K(Op::load8 ): r(d).i32= skvx::cast<int>(U8 ::Load(arg(imm))); break;
- STRIDE_K(Op::load32): r(d).i32= I32::Load(arg(imm)) ; break;
- #undef STRIDE_1
- #undef STRIDE_K
- // Ops that don't interact with memory should never care about the stride.
- #define CASE(op) case 2*(int)op: /*fallthrough*/ case 2*(int)op+1
- CASE(Op::splat): r(d).i32 = imm; break;
- CASE(Op::add_f32): r(d).f32 = r(x).f32 + r(y).f32; break;
- CASE(Op::sub_f32): r(d).f32 = r(x).f32 - r(y).f32; break;
- CASE(Op::mul_f32): r(d).f32 = r(x).f32 * r(y).f32; break;
- CASE(Op::div_f32): r(d).f32 = r(x).f32 / r(y).f32; break;
- CASE(Op::mad_f32): r(d).f32 = r(x).f32 * r(y).f32 + r(z).f32; break;
- CASE(Op::add_i32): r(d).i32 = r(x).i32 + r(y).i32; break;
- CASE(Op::sub_i32): r(d).i32 = r(x).i32 - r(y).i32; break;
- CASE(Op::mul_i32): r(d).i32 = r(x).i32 * r(y).i32; break;
- CASE(Op::sub_i16x2):
- r(d).i32 = skvx::bit_pun<I32>(skvx::bit_pun<I16x2>(r(x).i32) -
- skvx::bit_pun<I16x2>(r(y).i32) ); break;
- CASE(Op::mul_i16x2):
- r(d).i32 = skvx::bit_pun<I32>(skvx::bit_pun<I16x2>(r(x).i32) *
- skvx::bit_pun<I16x2>(r(y).i32) ); break;
- CASE(Op::shr_i16x2):
- r(d).i32 = skvx::bit_pun<I32>(skvx::bit_pun<U16x2>(r(x).i32) >> imm);
- break;
- CASE(Op::bit_and): r(d).i32 = r(x).i32 & r(y).i32; break;
- CASE(Op::bit_or ): r(d).i32 = r(x).i32 | r(y).i32; break;
- CASE(Op::bit_xor): r(d).i32 = r(x).i32 ^ r(y).i32; break;
- CASE(Op::bit_clear): r(d).i32 = r(x).i32 & ~r(y).i32; break;
- CASE(Op::shl): r(d).i32 = r(x).i32 << imm; break;
- CASE(Op::sra): r(d).i32 = r(x).i32 >> imm; break;
- CASE(Op::shr): r(d).u32 = r(x).u32 >> imm; break;
- CASE(Op::extract): r(d).u32 = (r(x).u32 >> imm) & r(y).u32; break;
- CASE(Op::pack): r(d).u32 = r(x).u32 | (r(y).u32 << imm); break;
- CASE(Op::bytes): {
- const U32 table[] = {
- 0,
- (r(x).u32 ) & 0xff,
- (r(x).u32 >> 8) & 0xff,
- (r(x).u32 >> 16) & 0xff,
- (r(x).u32 >> 24) & 0xff,
- };
- r(d).u32 = table[(imm >> 0) & 0xf] << 0
- | table[(imm >> 4) & 0xf] << 8
- | table[(imm >> 8) & 0xf] << 16
- | table[(imm >> 12) & 0xf] << 24;
- } break;
- CASE(Op::to_f32): r(d).f32 = skvx::cast<float>(r(x).i32); break;
- CASE(Op::to_i32): r(d).i32 = skvx::cast<int> (r(x).f32); break;
- #undef CASE
- }
- }
- }
- }
- void Program::dropJIT() {
- #if defined(SKVM_JIT)
- if (fJITBuf) {
- munmap(fJITBuf, fJITSize);
- }
- #else
- SkASSERT(fJITBuf == nullptr);
- #endif
- fJITBuf = nullptr;
- fJITSize = 0;
- }
- Program::~Program() { this->dropJIT(); }
- Program::Program(Program&& other) {
- fInstructions = std::move(other.fInstructions);
- fRegs = other.fRegs;
- fLoop = other.fLoop;
- fStrides = std::move(other.fStrides);
- std::swap(fJITBuf , other.fJITBuf);
- std::swap(fJITSize , other.fJITSize);
- }
- Program& Program::operator=(Program&& other) {
- fInstructions = std::move(other.fInstructions);
- fRegs = other.fRegs;
- fLoop = other.fLoop;
- fStrides = std::move(other.fStrides);
- std::swap(fJITBuf , other.fJITBuf);
- std::swap(fJITSize , other.fJITSize);
- return *this;
- }
- Program::Program(const std::vector<Builder::Instruction>& instructions,
- const std::vector<int>& strides,
- const char* debug_name) : fStrides(strides) {
- this->setupInterpreter(instructions);
- #if defined(SKVM_JIT)
- this->setupJIT(instructions, debug_name);
- #endif
- }
- // Translate Builder::Instructions to Program::Instructions used by the interpreter.
- void Program::setupInterpreter(const std::vector<Builder::Instruction>& instructions) {
- // Register each instruction is assigned to.
- std::vector<Reg> reg(instructions.size());
- // This next bit is a bit more complicated than strictly necessary;
- // we could just assign every live instruction to its own register.
- //
- // But recycling registers in the loop is fairly cheap, and good practice
- // for the JITs where minimizing register pressure really is important.
- // (Also helps minimize unit test diffs.)
- // Assign a register to each live hoisted instruction. We'll never recycle these.
- fRegs = 0;
- int live_instructions = 0;
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- const Builder::Instruction& inst = instructions[id];
- if (inst.death != 0 && inst.hoist) {
- live_instructions++;
- reg[id] = fRegs++;
- }
- }
- // Assign registers to each live loop instruction, recycling them when we can.
- std::vector<Reg> avail;
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- const Builder::Instruction& inst = instructions[id];
- if (inst.death != 0 && !inst.hoist) {
- live_instructions++;
- /// If an instruction's input is no longer live, we can recycle its register.
- auto maybe_recycle_register = [&](Val input) {
- // If this is a real input and it's lifetime ends at this instruction,
- // we can recycle the register it's occupying.
- if (input != NA
- && !instructions[input].hoist
- && instructions[input].death == id) {
- avail.push_back(reg[input]);
- }
- };
- // Take care to not recycle the same register twice.
- if (true ) { maybe_recycle_register(inst.x); }
- if (inst.y != inst.x ) { maybe_recycle_register(inst.y); }
- if (inst.z != inst.x && inst.z != inst.y) { maybe_recycle_register(inst.z); }
- // Allocate a register if we have to, preferring to reuse anything available.
- if (avail.empty()) {
- reg[id] = fRegs++;
- } else {
- reg[id] = avail.back();
- avail.pop_back();
- }
- }
- }
- // Translate Builder::Instructions to Program::Instructions by mapping values to
- // registers. This will be two passes, first hoisted instructions, then inside the loop.
- // The loop begins at the fLoop'th Instruction.
- fLoop = 0;
- fInstructions.reserve(live_instructions);
- // Add a dummy mapping for the N/A sentinel Val to any arbitrary register
- // so lookups don't have to know which arguments are used by which Ops.
- auto lookup_register = [&](Val id) {
- return id == NA ? (Reg)0
- : reg[id];
- };
- auto push_instruction = [&](Val id, const Builder::Instruction& inst) {
- Program::Instruction pinst{
- inst.op,
- lookup_register(id),
- lookup_register(inst.x),
- lookup_register(inst.y),
- {lookup_register(inst.z)},
- };
- if (inst.z == NA) { pinst.imm = inst.imm; }
- fInstructions.push_back(pinst);
- };
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- const Builder::Instruction& inst = instructions[id];
- if (inst.death != 0 && inst.hoist) {
- push_instruction(id, inst);
- fLoop++;
- }
- }
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- const Builder::Instruction& inst = instructions[id];
- if (inst.death != 0 && !inst.hoist) {
- push_instruction(id, inst);
- }
- }
- }
- #if defined(SKVM_JIT)
- // Just so happens that we can translate the immediate control for our bytes() op
- // to a single 128-bit mask that can be consumed by both AVX2 vpshufb and NEON tbl!
- static void bytes_control(int imm, int mask[4]) {
- auto nibble_to_vpshufb = [](uint8_t n) -> uint8_t {
- // 0 -> 0xff, Fill with zero
- // 1 -> 0x00, Select byte 0
- // 2 -> 0x01, " 1
- // 3 -> 0x02, " 2
- // 4 -> 0x03, " 3
- return n - 1;
- };
- uint8_t control[] = {
- nibble_to_vpshufb( (imm >> 0) & 0xf ),
- nibble_to_vpshufb( (imm >> 4) & 0xf ),
- nibble_to_vpshufb( (imm >> 8) & 0xf ),
- nibble_to_vpshufb( (imm >> 12) & 0xf ),
- };
- for (int i = 0; i < 4; i++) {
- mask[i] = (int)control[0] << 0
- | (int)control[1] << 8
- | (int)control[2] << 16
- | (int)control[3] << 24;
- // Update each byte that refers to a byte index by 4 to
- // point into the next 32-bit lane, but leave any 0xff
- // that fills with zero alone.
- control[0] += control[0] == 0xff ? 0 : 4;
- control[1] += control[1] == 0xff ? 0 : 4;
- control[2] += control[2] == 0xff ? 0 : 4;
- control[3] += control[3] == 0xff ? 0 : 4;
- }
- }
- bool Program::jit(const std::vector<Builder::Instruction>& instructions,
- const bool hoist,
- Assembler* a) const {
- using A = Assembler;
- #if defined(__x86_64__)
- if (!SkCpu::Supports(SkCpu::HSW)) {
- return false;
- }
- A::GP64 N = A::rdi,
- arg[] = { A::rsi, A::rdx, A::rcx, A::r8, A::r9 };
- // All 16 ymm registers are available to use.
- using Reg = A::Ymm;
- uint32_t avail = 0xffff;
- #elif defined(__aarch64__)
- A::X N = A::x0,
- arg[] = { A::x1, A::x2, A::x3, A::x4, A::x5, A::x6, A::x7 };
- // We can use v0-v7 and v16-v31 freely; we'd need to preseve v8-v15.
- using Reg = A::V;
- uint32_t avail = 0xffff00ff;
- #endif
- if (SK_ARRAY_COUNT(arg) < fStrides.size()) {
- return false;
- }
- auto hoisted = [&](Val id) { return hoist && instructions[id].hoist; };
- std::vector<Reg> r(instructions.size());
- struct LabelAndReg {
- A::Label label;
- Reg reg;
- };
- SkTHashMap<int, LabelAndReg> splats,
- bytes_masks;
- auto warmup = [&](Val id) {
- const Builder::Instruction& inst = instructions[id];
- if (inst.death == 0) {
- return true;
- }
- Op op = inst.op;
- int imm = inst.imm;
- switch (op) {
- default: break;
- case Op::splat: if (!splats.find(imm)) { splats.set(imm, {}); }
- break;
- case Op::bytes: if (!bytes_masks.find(imm)) {
- bytes_masks.set(imm, {});
- if (hoist) {
- // vpshufb can always work with the mask from memory,
- // but it helps to hoist the mask to a register for tbl.
- #if defined(__aarch64__)
- LabelAndReg* entry = bytes_masks.find(imm);
- if (int found = __builtin_ffs(avail)) {
- entry->reg = (Reg)(found-1);
- avail ^= 1 << entry->reg;
- a->ldrq(entry->reg, &entry->label);
- } else {
- return false;
- }
- #endif
- }
- }
- break;
- }
- return true;
- };
- auto emit = [&](Val id, bool scalar) {
- const Builder::Instruction& inst = instructions[id];
- // No need to emit dead code instructions that produce values that are never used.
- if (inst.death == 0) {
- return true;
- }
- Op op = inst.op;
- Val x = inst.x,
- y = inst.y,
- z = inst.z;
- int imm = inst.imm;
- // Most (but not all) ops create an output value and need a register to hold it, dst.
- // We track each instruction's dst in r[] so we can thread it through as an input
- // to any future instructions needing that value.
- //
- // And some ops may need a temporary scratch register, tmp. Some need both tmp and dst.
- //
- // tmp and dst are very similar and can and will often be assigned the same register,
- // but tmp may never alias any of the instructions's inputs, while dst may when this
- // instruction consumes that input, i.e. if the input reaches its end of life here.
- //
- // We'll assign both registers lazily to keep register pressure as low as possible.
- bool tmp_is_set = false,
- dst_is_set = false;
- Reg tmp_reg = (Reg)0; // This initial value won't matter... anything legal is fine.
- bool ok = true; // Set to false if we need to assign a register and none's available.
- // First lock in how to choose tmp if we need to based on the registers
- // available before this instruction, not including any of its input registers.
- auto tmp = [&,avail/*important, closing over avail's current value*/]{
- if (!tmp_is_set) {
- tmp_is_set = true;
- if (int found = __builtin_ffs(avail)) {
- // This is a scratch register just for this op,
- // so we leave it marked available for future ops.
- tmp_reg = (Reg)(found - 1);
- } else {
- // We needed a tmp register but couldn't find one available. :'(
- // This will cause emit() to return false, in turn causing jit() to fail.
- ok = false;
- }
- }
- return tmp_reg;
- };
- // Now make available any registers that are consumed by this instruction.
- // (The register pool we can pick dst from is >= the pool for tmp, adding any of these.)
- if (x != NA && instructions[x].death == id && !hoisted(x)) { avail |= 1 << r[x]; }
- if (y != NA && instructions[y].death == id && !hoisted(y)) { avail |= 1 << r[y]; }
- if (z != NA && instructions[z].death == id && !hoisted(z)) { avail |= 1 << r[z]; }
- // set_dst() and dst() will work read/write with this perhaps-just-updated avail.
- // Some ops may decide dst on their own to best fit the instruction (see Op::mad_f32).
- auto set_dst = [&](Reg reg){
- SkASSERT(dst_is_set == false);
- dst_is_set = true;
- SkASSERT(avail & (1<<reg));
- avail ^= 1<<reg;
- r[id] = reg;
- };
- // Thanks to AVX and NEON's 3-argument instruction sets,
- // most ops can use any register as dst.
- auto dst = [&]{
- if (!dst_is_set) {
- if (int found = __builtin_ffs(avail)) {
- set_dst((Reg)(found-1));
- } else {
- // Same deal as with tmp... all the registers are occupied. Time to fail!
- ok = false;
- }
- }
- return r[id];
- };
- // Because we use the same logic to pick an arbitrary dst and to pick tmp,
- // and we know that tmp will never overlap any of the inputs, `dst() == tmp()`
- // is a simple idiom to check that the destination does not overlap any of the inputs.
- // Sometimes we can use this knowledge to do better instruction selection.
- // Ok! Keep in mind that we haven't assigned tmp or dst yet,
- // just laid out hooks for how to do so if we need them, depending on the instruction.
- //
- // Now let's actually assemble the instruction!
- switch (op) {
- #if defined(__x86_64__)
- case Op::store8: if (scalar) { a->vpextrb (arg[imm], (A::Xmm)r[x], 0); }
- else { a->vpackusdw(tmp(), r[x], r[x]);
- a->vpermq (tmp(), tmp(), 0xd8);
- a->vpackuswb(tmp(), tmp(), tmp());
- a->vmovq (arg[imm], (A::Xmm)tmp()); }
- break;
- // TODO: the else case is a situation where we could use r[x]
- // as tmp if it's available... we don't need it after the
- // first instruction.
- case Op::store32: if (scalar) { a->vmovd (arg[imm], (A::Xmm)r[x]); }
- else { a->vmovups(arg[imm], r[x]); }
- break;
- case Op::load8: if (scalar) {
- a->vpxor (dst(), dst(), dst());
- a->vpinsrb((A::Xmm)dst(), (A::Xmm)dst(), arg[imm], 0);
- } else {
- a->vpmovzxbd(dst(), arg[imm]);
- } break;
- case Op::load32: if (scalar) { a->vmovd ((A::Xmm)dst(), arg[imm]); }
- else { a->vmovups( dst(), arg[imm]); }
- break;
- case Op::splat: a->vbroadcastss(dst(), &splats.find(imm)->label);
- break;
- // TODO: many of these instructions have variants that
- // can read one of their arugments from 32-byte memory
- // instead of a register. Find a way to avoid needing
- // to splat most* constants out at all?
- // (*Might work for x - 255 but not 255 - x, so will
- // always need to be able to splat to a register.)
- case Op::add_f32: a->vaddps(dst(), r[x], r[y]); break;
- case Op::sub_f32: a->vsubps(dst(), r[x], r[y]); break;
- case Op::mul_f32: a->vmulps(dst(), r[x], r[y]); break;
- case Op::div_f32: a->vdivps(dst(), r[x], r[y]); break;
- case Op::mad_f32:
- if (avail & (1<<r[x])) { set_dst(r[x]); a->vfmadd132ps(r[x], r[z], r[y]); }
- else if (avail & (1<<r[y])) { set_dst(r[y]); a->vfmadd213ps(r[y], r[x], r[z]); }
- else if (avail & (1<<r[z])) { set_dst(r[z]); a->vfmadd231ps(r[z], r[x], r[y]); }
- else { SkASSERT(dst() == tmp());
- // TODO: vpor -> vmovdqa here?
- a->vpor (dst(),r[x], r[x]);
- a->vfmadd132ps(dst(),r[z], r[y]); }
- break;
- case Op::add_i32: a->vpaddd (dst(), r[x], r[y]); break;
- case Op::sub_i32: a->vpsubd (dst(), r[x], r[y]); break;
- case Op::mul_i32: a->vpmulld(dst(), r[x], r[y]); break;
- case Op::sub_i16x2: a->vpsubw (dst(), r[x], r[y]); break;
- case Op::mul_i16x2: a->vpmullw(dst(), r[x], r[y]); break;
- case Op::shr_i16x2: a->vpsrlw (dst(), r[x], imm); break;
- case Op::bit_and: a->vpand (dst(), r[x], r[y]); break;
- case Op::bit_or : a->vpor (dst(), r[x], r[y]); break;
- case Op::bit_xor: a->vpxor (dst(), r[x], r[y]); break;
- case Op::bit_clear: a->vpandn(dst(), r[y], r[x]); break; // N.B. Y then X.
- case Op::shl: a->vpslld(dst(), r[x], imm); break;
- case Op::shr: a->vpsrld(dst(), r[x], imm); break;
- case Op::sra: a->vpsrad(dst(), r[x], imm); break;
- case Op::extract: if (imm == 0) { a->vpand (dst(), r[x], r[y]); }
- else { a->vpsrld(tmp(), r[x], imm);
- a->vpand (dst(), tmp(), r[y]); }
- break;
- case Op::pack: a->vpslld(tmp(), r[y], imm);
- a->vpor (dst(), tmp(), r[x]);
- break;
- case Op::to_f32: a->vcvtdq2ps (dst(), r[x]); break;
- case Op::to_i32: a->vcvttps2dq(dst(), r[x]); break;
- case Op::bytes: a->vpshufb(dst(), r[x], &bytes_masks.find(imm)->label);
- break;
- #elif defined(__aarch64__)
- case Op::store8: a->xtns2h(tmp(), r[x]);
- a->xtnh2b(tmp(), tmp());
- if (scalar) { a->strb (tmp(), arg[imm]); }
- else { a->strs (tmp(), arg[imm]); }
- break;
- // TODO: another case where it'd be okay to alias r[x] and tmp if r[x] dies here.
- case Op::store32: if (scalar) { a->strs(r[x], arg[imm]); }
- else { a->strq(r[x], arg[imm]); }
- break;
- case Op::load8: if (scalar) { a->ldrb(tmp(), arg[imm]); }
- else { a->ldrs(tmp(), arg[imm]); }
- a->uxtlb2h(tmp(), tmp());
- a->uxtlh2s(dst(), tmp());
- break;
- case Op::load32: if (scalar) { a->ldrs(dst(), arg[imm]); }
- else { a->ldrq(dst(), arg[imm]); }
- break;
- case Op::splat: a->ldrq(dst(), &splats.find(imm)->label);
- break;
- // TODO: If we hoist these, pack 4 values in each register
- // and use vector/lane operations, cutting the register
- // pressure cost of hoisting by 4?
- case Op::add_f32: a->fadd4s(dst(), r[x], r[y]); break;
- case Op::sub_f32: a->fsub4s(dst(), r[x], r[y]); break;
- case Op::mul_f32: a->fmul4s(dst(), r[x], r[y]); break;
- case Op::div_f32: a->fdiv4s(dst(), r[x], r[y]); break;
- case Op::mad_f32:
- if (avail & (1<<r[z])) { set_dst(r[z]); a->fmla4s( r[z], r[x], r[y]); }
- else { a->orr16b(tmp(), r[z], r[z]);
- a->fmla4s(tmp(), r[x], r[y]);
- if(dst() != tmp()) { a->orr16b(dst(), tmp(), tmp()); } }
- break;
- case Op::add_i32: a->add4s(dst(), r[x], r[y]); break;
- case Op::sub_i32: a->sub4s(dst(), r[x], r[y]); break;
- case Op::mul_i32: a->mul4s(dst(), r[x], r[y]); break;
- case Op::sub_i16x2: a->sub8h (dst(), r[x], r[y]); break;
- case Op::mul_i16x2: a->mul8h (dst(), r[x], r[y]); break;
- case Op::shr_i16x2: a->ushr8h(dst(), r[x], imm); break;
- case Op::bit_and: a->and16b(dst(), r[x], r[y]); break;
- case Op::bit_or : a->orr16b(dst(), r[x], r[y]); break;
- case Op::bit_xor: a->eor16b(dst(), r[x], r[y]); break;
- case Op::bit_clear: a->bic16b(dst(), r[x], r[y]); break;
- case Op::shl: a-> shl4s(dst(), r[x], imm); break;
- case Op::shr: a->ushr4s(dst(), r[x], imm); break;
- case Op::sra: a->sshr4s(dst(), r[x], imm); break;
- case Op::extract: if (imm) { a->ushr4s(tmp(), r[x], imm);
- a->and16b(dst(), tmp(), r[y]); }
- else { a->and16b(dst(), r[x], r[y]); }
- break;
- case Op::pack:
- if (avail & (1<<r[x])) { set_dst(r[x]); a->sli4s ( r[x], r[y], imm); }
- else { a->shl4s (tmp(), r[y], imm);
- a->orr16b(dst(), tmp(), r[x]); }
- break;
- case Op::to_f32: a->scvtf4s (dst(), r[x]); break;
- case Op::to_i32: a->fcvtzs4s(dst(), r[x]); break;
- case Op::bytes: if (hoist) { a->tbl (dst(), r[x], bytes_masks.find(imm)->reg); }
- else { a->ldrq(tmp(), &bytes_masks.find(imm)->label);
- a->tbl (dst(), r[x], tmp()); }
- break;
- #endif
- }
- // Calls to tmp() or dst() might have flipped this false from its default true state.
- return ok;
- };
- #if defined(__x86_64__)
- const int K = 8;
- auto jump_if_less = [&](A::Label* l) { a->jl (l); };
- auto jump = [&](A::Label* l) { a->jmp(l); };
- auto add = [&](A::GP64 gp, int imm) { a->add(gp, imm); };
- auto sub = [&](A::GP64 gp, int imm) { a->sub(gp, imm); };
- auto exit = [&]{ a->vzeroupper(); a->ret(); };
- #elif defined(__aarch64__)
- const int K = 4;
- auto jump_if_less = [&](A::Label* l) { a->blt(l); };
- auto jump = [&](A::Label* l) { a->b (l); };
- auto add = [&](A::X gp, int imm) { a->add(gp, gp, imm); };
- auto sub = [&](A::X gp, int imm) { a->sub(gp, gp, imm); };
- auto exit = [&]{ a->ret(A::x30); };
- #endif
- A::Label body,
- tail,
- done;
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- if (!warmup(id)) {
- return false;
- }
- if (hoisted(id) && !emit(id, /*scalar=*/false)) {
- return false;
- }
- }
- a->label(&body);
- {
- a->cmp(N, K);
- jump_if_less(&tail);
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- if (!hoisted(id) && !emit(id, /*scalar=*/false)) {
- return false;
- }
- }
- for (int i = 0; i < (int)fStrides.size(); i++) {
- add(arg[i], K*fStrides[i]);
- }
- sub(N, K);
- jump(&body);
- }
- a->label(&tail);
- {
- a->cmp(N, 1);
- jump_if_less(&done);
- for (Val id = 0; id < (Val)instructions.size(); id++) {
- if (!hoisted(id) && !emit(id, /*scalar=*/true)) {
- return false;
- }
- }
- for (int i = 0; i < (int)fStrides.size(); i++) {
- add(arg[i], 1*fStrides[i]);
- }
- sub(N, 1);
- jump(&tail);
- }
- a->label(&done);
- {
- exit();
- }
- bytes_masks.foreach([&](int imm, LabelAndReg* entry) {
- // One 16-byte pattern for ARM tbl, that same pattern twice for x86-64 vpshufb.
- #if defined(__x86_64__)
- a->align(32);
- #elif defined(__aarch64__)
- a->align(4);
- #endif
- a->label(&entry->label);
- int mask[4];
- bytes_control(imm, mask);
- a->bytes(mask, sizeof(mask));
- #if defined(__x86_64__)
- a->bytes(mask, sizeof(mask));
- #endif
- });
- splats.foreach([&](int imm, LabelAndReg* entry) {
- // vbroadcastss 4 bytes on x86-64, or simply load 16-bytes on aarch64.
- a->align(4);
- a->label(&entry->label);
- a->word(imm);
- #if defined(__aarch64__)
- a->word(imm);
- a->word(imm);
- a->word(imm);
- #endif
- });
- return true;
- }
- void Program::setupJIT(const std::vector<Builder::Instruction>& instructions,
- const char* debug_name) {
- // Assemble with no buffer to determine a.size(), the number of bytes we'll assemble.
- Assembler a{nullptr};
- // First try allowing code hoisting (faster code)
- // then again without if that fails (lower register pressure).
- bool hoist = true;
- if (!this->jit(instructions, hoist, &a)) {
- hoist = false;
- if (!this->jit(instructions, hoist, &a)) {
- return;
- }
- }
- // Allocate space that we can remap as executable.
- const size_t page = sysconf(_SC_PAGESIZE);
- fJITSize = ((a.size() + page - 1) / page) * page; // mprotect works at page granularity.
- fJITBuf = mmap(nullptr,fJITSize, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1,0);
- // Assemble the program for real.
- a = Assembler{fJITBuf};
- SkAssertResult(this->jit(instructions, hoist, &a));
- SkASSERT(a.size() <= fJITSize);
- // Remap as executable, and flush caches on platforms that need that.
- mprotect(fJITBuf, fJITSize, PROT_READ|PROT_EXEC);
- __builtin___clear_cache((char*)fJITBuf,
- (char*)fJITBuf + fJITSize);
- #if defined(SKVM_PERF_DUMPS)
- this->dumpJIT(debug_name, a.size());
- #endif
- }
- #endif
- #if defined(SKVM_PERF_DUMPS)
- void Program::dumpJIT(const char* debug_name, size_t size) const {
- #if 0 && defined(__aarch64__)
- if (debug_name) {
- SkDebugf("\n%s:", debug_name);
- }
- // cat | llvm-mc -arch aarch64 -disassemble
- auto cur = (const uint8_t*)fJITBuf;
- for (int i = 0; i < (int)size; i++) {
- if (i % 4 == 0) {
- SkDebugf("\n");
- }
- SkDebugf("0x%02x ", *cur++);
- }
- SkDebugf("\n");
- #endif
- // We're doing some really stateful things below so one thread at a time please...
- static SkSpinlock dump_lock;
- SkAutoSpinlock lock(dump_lock);
- auto fnv1a = [](const void* vbuf, size_t n) {
- uint32_t hash = 2166136261;
- for (auto buf = (const uint8_t*)vbuf; n --> 0; buf++) {
- hash ^= *buf;
- hash *= 16777619;
- }
- return hash;
- };
- char name[64];
- uint32_t hash = fnv1a(fJITBuf, size);
- if (debug_name) {
- sprintf(name, "skvm-jit-%s", debug_name);
- } else {
- sprintf(name, "skvm-jit-%u", hash);
- }
- // Create a jit-<pid>.dump file that we can `perf inject -j` into a
- // perf.data captured with `perf record -k 1`, letting us see each
- // JIT'd Program as if a function named skvm-jit-<hash>. E.g.
- //
- // ninja -C out nanobench
- // perf record -k 1 out/nanobench -m SkVM_4096_I32\$
- // perf inject -j -i perf.data -o perf.data.jit
- // perf report -i perf.data.jit
- //
- // Running `perf inject -j` will also dump an .so for each JIT'd
- // program, named jitted-<pid>-<hash>.so.
- //
- // https://lwn.net/Articles/638566/
- // https://v8.dev/docs/linux-perf
- // https://cs.chromium.org/chromium/src/v8/src/diagnostics/perf-jit.cc
- // https://lore.kernel.org/patchwork/patch/622240/
- auto timestamp_ns = []() -> uint64_t {
- // It's important to use CLOCK_MONOTONIC here so that perf can
- // correlate our timestamps with those captured by `perf record
- // -k 1`. That's also what `-k 1` does, by the way, tell perf
- // record to use CLOCK_MONOTONIC.
- struct timespec ts;
- clock_gettime(CLOCK_MONOTONIC, &ts);
- return ts.tv_sec * (uint64_t)1e9 + ts.tv_nsec;
- };
- // We'll open the jit-<pid>.dump file and write a small header once,
- // and just leave it open forever because we're lazy.
- static FILE* jitdump = [&]{
- // Must map as w+ for the mmap() call below to work.
- char path[64];
- sprintf(path, "jit-%d.dump", getpid());
- FILE* f = fopen(path, "w+");
- // Calling mmap() on the file adds a "hey they mmap()'d this" record to
- // the perf.data file that will point `perf inject -j` at this log file.
- // Kind of a strange way to tell `perf inject` where the file is...
- void* marker = mmap(nullptr, sysconf(_SC_PAGESIZE),
- PROT_READ|PROT_EXEC, MAP_PRIVATE,
- fileno(f), /*offset=*/0);
- SkASSERT_RELEASE(marker != MAP_FAILED);
- // Like never calling fclose(f), we'll also just always leave marker mmap()'d.
- #if defined(__x86_64__)
- const uint32_t elf_mach = 62;
- #elif defined(__aarch64__)
- const uint32_t elf_mach = 183;
- #endif
- struct Header {
- uint32_t magic, version, header_size, elf_mach, reserved, pid;
- uint64_t timestamp_us, flags;
- } header = {
- 0x4A695444, 1, sizeof(Header), elf_mach, 0, (uint32_t)getpid(),
- timestamp_ns() / 1000, 0,
- };
- fwrite(&header, sizeof(header), 1, f);
- return f;
- }();
- struct CodeLoad {
- uint32_t event_type, event_size;
- uint64_t timestamp_ns;
- uint32_t pid, tid;
- uint64_t vma/*???*/, code_addr, code_size, id;
- } load = {
- 0/*code load*/, (uint32_t)(sizeof(CodeLoad) + strlen(name) + 1 + size),
- timestamp_ns(),
- (uint32_t)getpid(), (uint32_t)SkGetThreadID(),
- (uint64_t)fJITBuf, (uint64_t)fJITBuf, size, hash,
- };
- // Write the header, the JIT'd function name, and the JIT'd code itself.
- fwrite(&load, sizeof(load), 1, jitdump);
- fwrite(name, 1, strlen(name), jitdump);
- fwrite("\0", 1, 1, jitdump);
- fwrite(fJITBuf, 1, size, jitdump);
- }
- #endif
- } // namespace skvm
|