bpf_jit_comp_64.c 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/moduleloader.h>
  3. #include <linux/workqueue.h>
  4. #include <linux/netdevice.h>
  5. #include <linux/filter.h>
  6. #include <linux/bpf.h>
  7. #include <linux/cache.h>
  8. #include <linux/if_vlan.h>
  9. #include <asm/cacheflush.h>
  10. #include <asm/ptrace.h>
  11. #include "bpf_jit_64.h"
  12. static inline bool is_simm13(unsigned int value)
  13. {
  14. return value + 0x1000 < 0x2000;
  15. }
  16. static inline bool is_simm10(unsigned int value)
  17. {
  18. return value + 0x200 < 0x400;
  19. }
  20. static inline bool is_simm5(unsigned int value)
  21. {
  22. return value + 0x10 < 0x20;
  23. }
  24. static inline bool is_sethi(unsigned int value)
  25. {
  26. return (value & ~0x3fffff) == 0;
  27. }
  28. static void bpf_flush_icache(void *start_, void *end_)
  29. {
  30. /* Cheetah's I-cache is fully coherent. */
  31. if (tlb_type == spitfire) {
  32. unsigned long start = (unsigned long) start_;
  33. unsigned long end = (unsigned long) end_;
  34. start &= ~7UL;
  35. end = (end + 7UL) & ~7UL;
  36. while (start < end) {
  37. flushi(start);
  38. start += 32;
  39. }
  40. }
  41. }
  42. #define S13(X) ((X) & 0x1fff)
  43. #define S5(X) ((X) & 0x1f)
  44. #define IMMED 0x00002000
  45. #define RD(X) ((X) << 25)
  46. #define RS1(X) ((X) << 14)
  47. #define RS2(X) ((X))
  48. #define OP(X) ((X) << 30)
  49. #define OP2(X) ((X) << 22)
  50. #define OP3(X) ((X) << 19)
  51. #define COND(X) (((X) & 0xf) << 25)
  52. #define CBCOND(X) (((X) & 0x1f) << 25)
  53. #define F1(X) OP(X)
  54. #define F2(X, Y) (OP(X) | OP2(Y))
  55. #define F3(X, Y) (OP(X) | OP3(Y))
  56. #define ASI(X) (((X) & 0xff) << 5)
  57. #define CONDN COND(0x0)
  58. #define CONDE COND(0x1)
  59. #define CONDLE COND(0x2)
  60. #define CONDL COND(0x3)
  61. #define CONDLEU COND(0x4)
  62. #define CONDCS COND(0x5)
  63. #define CONDNEG COND(0x6)
  64. #define CONDVC COND(0x7)
  65. #define CONDA COND(0x8)
  66. #define CONDNE COND(0x9)
  67. #define CONDG COND(0xa)
  68. #define CONDGE COND(0xb)
  69. #define CONDGU COND(0xc)
  70. #define CONDCC COND(0xd)
  71. #define CONDPOS COND(0xe)
  72. #define CONDVS COND(0xf)
  73. #define CONDGEU CONDCC
  74. #define CONDLU CONDCS
  75. #define WDISP22(X) (((X) >> 2) & 0x3fffff)
  76. #define WDISP19(X) (((X) >> 2) & 0x7ffff)
  77. /* The 10-bit branch displacement for CBCOND is split into two fields */
  78. static u32 WDISP10(u32 off)
  79. {
  80. u32 ret = ((off >> 2) & 0xff) << 5;
  81. ret |= ((off >> (2 + 8)) & 0x03) << 19;
  82. return ret;
  83. }
  84. #define CBCONDE CBCOND(0x09)
  85. #define CBCONDLE CBCOND(0x0a)
  86. #define CBCONDL CBCOND(0x0b)
  87. #define CBCONDLEU CBCOND(0x0c)
  88. #define CBCONDCS CBCOND(0x0d)
  89. #define CBCONDN CBCOND(0x0e)
  90. #define CBCONDVS CBCOND(0x0f)
  91. #define CBCONDNE CBCOND(0x19)
  92. #define CBCONDG CBCOND(0x1a)
  93. #define CBCONDGE CBCOND(0x1b)
  94. #define CBCONDGU CBCOND(0x1c)
  95. #define CBCONDCC CBCOND(0x1d)
  96. #define CBCONDPOS CBCOND(0x1e)
  97. #define CBCONDVC CBCOND(0x1f)
  98. #define CBCONDGEU CBCONDCC
  99. #define CBCONDLU CBCONDCS
  100. #define ANNUL (1 << 29)
  101. #define XCC (1 << 21)
  102. #define BRANCH (F2(0, 1) | XCC)
  103. #define CBCOND_OP (F2(0, 3) | XCC)
  104. #define BA (BRANCH | CONDA)
  105. #define BG (BRANCH | CONDG)
  106. #define BL (BRANCH | CONDL)
  107. #define BLE (BRANCH | CONDLE)
  108. #define BGU (BRANCH | CONDGU)
  109. #define BLEU (BRANCH | CONDLEU)
  110. #define BGE (BRANCH | CONDGE)
  111. #define BGEU (BRANCH | CONDGEU)
  112. #define BLU (BRANCH | CONDLU)
  113. #define BE (BRANCH | CONDE)
  114. #define BNE (BRANCH | CONDNE)
  115. #define SETHI(K, REG) \
  116. (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
  117. #define OR_LO(K, REG) \
  118. (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG))
  119. #define ADD F3(2, 0x00)
  120. #define AND F3(2, 0x01)
  121. #define ANDCC F3(2, 0x11)
  122. #define OR F3(2, 0x02)
  123. #define XOR F3(2, 0x03)
  124. #define SUB F3(2, 0x04)
  125. #define SUBCC F3(2, 0x14)
  126. #define MUL F3(2, 0x0a)
  127. #define MULX F3(2, 0x09)
  128. #define UDIVX F3(2, 0x0d)
  129. #define DIV F3(2, 0x0e)
  130. #define SLL F3(2, 0x25)
  131. #define SLLX (F3(2, 0x25)|(1<<12))
  132. #define SRA F3(2, 0x27)
  133. #define SRAX (F3(2, 0x27)|(1<<12))
  134. #define SRL F3(2, 0x26)
  135. #define SRLX (F3(2, 0x26)|(1<<12))
  136. #define JMPL F3(2, 0x38)
  137. #define SAVE F3(2, 0x3c)
  138. #define RESTORE F3(2, 0x3d)
  139. #define CALL F1(1)
  140. #define BR F2(0, 0x01)
  141. #define RD_Y F3(2, 0x28)
  142. #define WR_Y F3(2, 0x30)
  143. #define LD32 F3(3, 0x00)
  144. #define LD8 F3(3, 0x01)
  145. #define LD16 F3(3, 0x02)
  146. #define LD64 F3(3, 0x0b)
  147. #define LD64A F3(3, 0x1b)
  148. #define ST8 F3(3, 0x05)
  149. #define ST16 F3(3, 0x06)
  150. #define ST32 F3(3, 0x04)
  151. #define ST64 F3(3, 0x0e)
  152. #define CAS F3(3, 0x3c)
  153. #define CASX F3(3, 0x3e)
  154. #define LDPTR LD64
  155. #define BASE_STACKFRAME 176
  156. #define LD32I (LD32 | IMMED)
  157. #define LD8I (LD8 | IMMED)
  158. #define LD16I (LD16 | IMMED)
  159. #define LD64I (LD64 | IMMED)
  160. #define LDPTRI (LDPTR | IMMED)
  161. #define ST32I (ST32 | IMMED)
  162. struct jit_ctx {
  163. struct bpf_prog *prog;
  164. unsigned int *offset;
  165. int idx;
  166. int epilogue_offset;
  167. bool tmp_1_used;
  168. bool tmp_2_used;
  169. bool tmp_3_used;
  170. bool saw_frame_pointer;
  171. bool saw_call;
  172. bool saw_tail_call;
  173. u32 *image;
  174. };
  175. #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
  176. #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
  177. #define TMP_REG_3 (MAX_BPF_JIT_REG + 2)
  178. /* Map BPF registers to SPARC registers */
  179. static const int bpf2sparc[] = {
  180. /* return value from in-kernel function, and exit value from eBPF */
  181. [BPF_REG_0] = O5,
  182. /* arguments from eBPF program to in-kernel function */
  183. [BPF_REG_1] = O0,
  184. [BPF_REG_2] = O1,
  185. [BPF_REG_3] = O2,
  186. [BPF_REG_4] = O3,
  187. [BPF_REG_5] = O4,
  188. /* callee saved registers that in-kernel function will preserve */
  189. [BPF_REG_6] = L0,
  190. [BPF_REG_7] = L1,
  191. [BPF_REG_8] = L2,
  192. [BPF_REG_9] = L3,
  193. /* read-only frame pointer to access stack */
  194. [BPF_REG_FP] = L6,
  195. [BPF_REG_AX] = G7,
  196. /* temporary register for internal BPF JIT */
  197. [TMP_REG_1] = G1,
  198. [TMP_REG_2] = G2,
  199. [TMP_REG_3] = G3,
  200. };
  201. static void emit(const u32 insn, struct jit_ctx *ctx)
  202. {
  203. if (ctx->image != NULL)
  204. ctx->image[ctx->idx] = insn;
  205. ctx->idx++;
  206. }
  207. static void emit_call(u32 *func, struct jit_ctx *ctx)
  208. {
  209. if (ctx->image != NULL) {
  210. void *here = &ctx->image[ctx->idx];
  211. unsigned int off;
  212. off = (void *)func - here;
  213. ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff);
  214. }
  215. ctx->idx++;
  216. }
  217. static void emit_nop(struct jit_ctx *ctx)
  218. {
  219. emit(SETHI(0, G0), ctx);
  220. }
  221. static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx)
  222. {
  223. emit(OR | RS1(G0) | RS2(from) | RD(to), ctx);
  224. }
  225. /* Emit 32-bit constant, zero extended. */
  226. static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx)
  227. {
  228. emit(SETHI(K, reg), ctx);
  229. emit(OR_LO(K, reg), ctx);
  230. }
  231. /* Emit 32-bit constant, sign extended. */
  232. static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx)
  233. {
  234. if (K >= 0) {
  235. emit(SETHI(K, reg), ctx);
  236. emit(OR_LO(K, reg), ctx);
  237. } else {
  238. u32 hbits = ~(u32) K;
  239. u32 lbits = -0x400 | (u32) K;
  240. emit(SETHI(hbits, reg), ctx);
  241. emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx);
  242. }
  243. }
  244. static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx)
  245. {
  246. emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx);
  247. }
  248. static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx)
  249. {
  250. emit(opcode | RS1(a) | RS2(b) | RD(c), ctx);
  251. }
  252. static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm,
  253. struct jit_ctx *ctx)
  254. {
  255. bool small_immed = is_simm13(imm);
  256. unsigned int insn = opcode;
  257. insn |= RS1(dst) | RD(dst);
  258. if (small_immed) {
  259. emit(insn | IMMED | S13(imm), ctx);
  260. } else {
  261. unsigned int tmp = bpf2sparc[TMP_REG_1];
  262. ctx->tmp_1_used = true;
  263. emit_set_const_sext(imm, tmp, ctx);
  264. emit(insn | RS2(tmp), ctx);
  265. }
  266. }
  267. static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm,
  268. unsigned int dst, struct jit_ctx *ctx)
  269. {
  270. bool small_immed = is_simm13(imm);
  271. unsigned int insn = opcode;
  272. insn |= RS1(src) | RD(dst);
  273. if (small_immed) {
  274. emit(insn | IMMED | S13(imm), ctx);
  275. } else {
  276. unsigned int tmp = bpf2sparc[TMP_REG_1];
  277. ctx->tmp_1_used = true;
  278. emit_set_const_sext(imm, tmp, ctx);
  279. emit(insn | RS2(tmp), ctx);
  280. }
  281. }
  282. static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx)
  283. {
  284. if (K >= 0 && is_simm13(K)) {
  285. /* or %g0, K, DEST */
  286. emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
  287. } else {
  288. emit_set_const(K, dest, ctx);
  289. }
  290. }
  291. static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx)
  292. {
  293. if (is_simm13(K)) {
  294. /* or %g0, K, DEST */
  295. emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
  296. } else {
  297. emit_set_const(K, dest, ctx);
  298. }
  299. }
  300. static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx)
  301. {
  302. if (is_simm13(K)) {
  303. /* or %g0, K, DEST */
  304. emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx);
  305. } else {
  306. emit_set_const_sext(K, dest, ctx);
  307. }
  308. }
  309. static void analyze_64bit_constant(u32 high_bits, u32 low_bits,
  310. int *hbsp, int *lbsp, int *abbasp)
  311. {
  312. int lowest_bit_set, highest_bit_set, all_bits_between_are_set;
  313. int i;
  314. lowest_bit_set = highest_bit_set = -1;
  315. i = 0;
  316. do {
  317. if ((lowest_bit_set == -1) && ((low_bits >> i) & 1))
  318. lowest_bit_set = i;
  319. if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1))
  320. highest_bit_set = (64 - i - 1);
  321. } while (++i < 32 && (highest_bit_set == -1 ||
  322. lowest_bit_set == -1));
  323. if (i == 32) {
  324. i = 0;
  325. do {
  326. if (lowest_bit_set == -1 && ((high_bits >> i) & 1))
  327. lowest_bit_set = i + 32;
  328. if (highest_bit_set == -1 &&
  329. ((low_bits >> (32 - i - 1)) & 1))
  330. highest_bit_set = 32 - i - 1;
  331. } while (++i < 32 && (highest_bit_set == -1 ||
  332. lowest_bit_set == -1));
  333. }
  334. all_bits_between_are_set = 1;
  335. for (i = lowest_bit_set; i <= highest_bit_set; i++) {
  336. if (i < 32) {
  337. if ((low_bits & (1 << i)) != 0)
  338. continue;
  339. } else {
  340. if ((high_bits & (1 << (i - 32))) != 0)
  341. continue;
  342. }
  343. all_bits_between_are_set = 0;
  344. break;
  345. }
  346. *hbsp = highest_bit_set;
  347. *lbsp = lowest_bit_set;
  348. *abbasp = all_bits_between_are_set;
  349. }
  350. static unsigned long create_simple_focus_bits(unsigned long high_bits,
  351. unsigned long low_bits,
  352. int lowest_bit_set, int shift)
  353. {
  354. long hi, lo;
  355. if (lowest_bit_set < 32) {
  356. lo = (low_bits >> lowest_bit_set) << shift;
  357. hi = ((high_bits << (32 - lowest_bit_set)) << shift);
  358. } else {
  359. lo = 0;
  360. hi = ((high_bits >> (lowest_bit_set - 32)) << shift);
  361. }
  362. return hi | lo;
  363. }
  364. static bool const64_is_2insns(unsigned long high_bits,
  365. unsigned long low_bits)
  366. {
  367. int highest_bit_set, lowest_bit_set, all_bits_between_are_set;
  368. if (high_bits == 0 || high_bits == 0xffffffff)
  369. return true;
  370. analyze_64bit_constant(high_bits, low_bits,
  371. &highest_bit_set, &lowest_bit_set,
  372. &all_bits_between_are_set);
  373. if ((highest_bit_set == 63 || lowest_bit_set == 0) &&
  374. all_bits_between_are_set != 0)
  375. return true;
  376. if (highest_bit_set - lowest_bit_set < 21)
  377. return true;
  378. return false;
  379. }
  380. static void sparc_emit_set_const64_quick2(unsigned long high_bits,
  381. unsigned long low_imm,
  382. unsigned int dest,
  383. int shift_count, struct jit_ctx *ctx)
  384. {
  385. emit_loadimm32(high_bits, dest, ctx);
  386. /* Now shift it up into place. */
  387. emit_alu_K(SLLX, dest, shift_count, ctx);
  388. /* If there is a low immediate part piece, finish up by
  389. * putting that in as well.
  390. */
  391. if (low_imm != 0)
  392. emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx);
  393. }
  394. static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx)
  395. {
  396. int all_bits_between_are_set, lowest_bit_set, highest_bit_set;
  397. unsigned int tmp = bpf2sparc[TMP_REG_1];
  398. u32 low_bits = (K & 0xffffffff);
  399. u32 high_bits = (K >> 32);
  400. /* These two tests also take care of all of the one
  401. * instruction cases.
  402. */
  403. if (high_bits == 0xffffffff && (low_bits & 0x80000000))
  404. return emit_loadimm_sext(K, dest, ctx);
  405. if (high_bits == 0x00000000)
  406. return emit_loadimm32(K, dest, ctx);
  407. analyze_64bit_constant(high_bits, low_bits, &highest_bit_set,
  408. &lowest_bit_set, &all_bits_between_are_set);
  409. /* 1) mov -1, %reg
  410. * sllx %reg, shift, %reg
  411. * 2) mov -1, %reg
  412. * srlx %reg, shift, %reg
  413. * 3) mov some_small_const, %reg
  414. * sllx %reg, shift, %reg
  415. */
  416. if (((highest_bit_set == 63 || lowest_bit_set == 0) &&
  417. all_bits_between_are_set != 0) ||
  418. ((highest_bit_set - lowest_bit_set) < 12)) {
  419. int shift = lowest_bit_set;
  420. long the_const = -1;
  421. if ((highest_bit_set != 63 && lowest_bit_set != 0) ||
  422. all_bits_between_are_set == 0) {
  423. the_const =
  424. create_simple_focus_bits(high_bits, low_bits,
  425. lowest_bit_set, 0);
  426. } else if (lowest_bit_set == 0)
  427. shift = -(63 - highest_bit_set);
  428. emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx);
  429. if (shift > 0)
  430. emit_alu_K(SLLX, dest, shift, ctx);
  431. else if (shift < 0)
  432. emit_alu_K(SRLX, dest, -shift, ctx);
  433. return;
  434. }
  435. /* Now a range of 22 or less bits set somewhere.
  436. * 1) sethi %hi(focus_bits), %reg
  437. * sllx %reg, shift, %reg
  438. * 2) sethi %hi(focus_bits), %reg
  439. * srlx %reg, shift, %reg
  440. */
  441. if ((highest_bit_set - lowest_bit_set) < 21) {
  442. unsigned long focus_bits =
  443. create_simple_focus_bits(high_bits, low_bits,
  444. lowest_bit_set, 10);
  445. emit(SETHI(focus_bits, dest), ctx);
  446. /* If lowest_bit_set == 10 then a sethi alone could
  447. * have done it.
  448. */
  449. if (lowest_bit_set < 10)
  450. emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx);
  451. else if (lowest_bit_set > 10)
  452. emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx);
  453. return;
  454. }
  455. /* Ok, now 3 instruction sequences. */
  456. if (low_bits == 0) {
  457. emit_loadimm32(high_bits, dest, ctx);
  458. emit_alu_K(SLLX, dest, 32, ctx);
  459. return;
  460. }
  461. /* We may be able to do something quick
  462. * when the constant is negated, so try that.
  463. */
  464. if (const64_is_2insns((~high_bits) & 0xffffffff,
  465. (~low_bits) & 0xfffffc00)) {
  466. /* NOTE: The trailing bits get XOR'd so we need the
  467. * non-negated bits, not the negated ones.
  468. */
  469. unsigned long trailing_bits = low_bits & 0x3ff;
  470. if ((((~high_bits) & 0xffffffff) == 0 &&
  471. ((~low_bits) & 0x80000000) == 0) ||
  472. (((~high_bits) & 0xffffffff) == 0xffffffff &&
  473. ((~low_bits) & 0x80000000) != 0)) {
  474. unsigned long fast_int = (~low_bits & 0xffffffff);
  475. if ((is_sethi(fast_int) &&
  476. (~high_bits & 0xffffffff) == 0)) {
  477. emit(SETHI(fast_int, dest), ctx);
  478. } else if (is_simm13(fast_int)) {
  479. emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx);
  480. } else {
  481. emit_loadimm64(fast_int, dest, ctx);
  482. }
  483. } else {
  484. u64 n = ((~low_bits) & 0xfffffc00) |
  485. (((unsigned long)((~high_bits) & 0xffffffff))<<32);
  486. emit_loadimm64(n, dest, ctx);
  487. }
  488. low_bits = -0x400 | trailing_bits;
  489. emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx);
  490. return;
  491. }
  492. /* 1) sethi %hi(xxx), %reg
  493. * or %reg, %lo(xxx), %reg
  494. * sllx %reg, yyy, %reg
  495. */
  496. if ((highest_bit_set - lowest_bit_set) < 32) {
  497. unsigned long focus_bits =
  498. create_simple_focus_bits(high_bits, low_bits,
  499. lowest_bit_set, 0);
  500. /* So what we know is that the set bits straddle the
  501. * middle of the 64-bit word.
  502. */
  503. sparc_emit_set_const64_quick2(focus_bits, 0, dest,
  504. lowest_bit_set, ctx);
  505. return;
  506. }
  507. /* 1) sethi %hi(high_bits), %reg
  508. * or %reg, %lo(high_bits), %reg
  509. * sllx %reg, 32, %reg
  510. * or %reg, low_bits, %reg
  511. */
  512. if (is_simm13(low_bits) && ((int)low_bits > 0)) {
  513. sparc_emit_set_const64_quick2(high_bits, low_bits,
  514. dest, 32, ctx);
  515. return;
  516. }
  517. /* Oh well, we tried... Do a full 64-bit decomposition. */
  518. ctx->tmp_1_used = true;
  519. emit_loadimm32(high_bits, tmp, ctx);
  520. emit_loadimm32(low_bits, dest, ctx);
  521. emit_alu_K(SLLX, tmp, 32, ctx);
  522. emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx);
  523. }
  524. static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx,
  525. struct jit_ctx *ctx)
  526. {
  527. unsigned int off = to_idx - from_idx;
  528. if (br_opc & XCC)
  529. emit(br_opc | WDISP19(off << 2), ctx);
  530. else
  531. emit(br_opc | WDISP22(off << 2), ctx);
  532. }
  533. static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
  534. const u8 dst, const u8 src, struct jit_ctx *ctx)
  535. {
  536. unsigned int off = to_idx - from_idx;
  537. emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx);
  538. }
  539. static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx,
  540. const u8 dst, s32 imm, struct jit_ctx *ctx)
  541. {
  542. unsigned int off = to_idx - from_idx;
  543. emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx);
  544. }
  545. #define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX)
  546. #define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX)
  547. #define emit_cmp(R1, R2, CTX) \
  548. emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
  549. #define emit_cmpi(R1, IMM, CTX) \
  550. emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
  551. #define emit_btst(R1, R2, CTX) \
  552. emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX)
  553. #define emit_btsti(R1, IMM, CTX) \
  554. emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX)
  555. static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src,
  556. const s32 imm, bool is_imm, int branch_dst,
  557. struct jit_ctx *ctx)
  558. {
  559. bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0;
  560. const u8 tmp = bpf2sparc[TMP_REG_1];
  561. branch_dst = ctx->offset[branch_dst];
  562. if (!is_simm10(branch_dst - ctx->idx) ||
  563. BPF_OP(code) == BPF_JSET)
  564. use_cbcond = false;
  565. if (is_imm) {
  566. bool fits = true;
  567. if (use_cbcond) {
  568. if (!is_simm5(imm))
  569. fits = false;
  570. } else if (!is_simm13(imm)) {
  571. fits = false;
  572. }
  573. if (!fits) {
  574. ctx->tmp_1_used = true;
  575. emit_loadimm_sext(imm, tmp, ctx);
  576. src = tmp;
  577. is_imm = false;
  578. }
  579. }
  580. if (!use_cbcond) {
  581. u32 br_opcode;
  582. if (BPF_OP(code) == BPF_JSET) {
  583. if (is_imm)
  584. emit_btsti(dst, imm, ctx);
  585. else
  586. emit_btst(dst, src, ctx);
  587. } else {
  588. if (is_imm)
  589. emit_cmpi(dst, imm, ctx);
  590. else
  591. emit_cmp(dst, src, ctx);
  592. }
  593. switch (BPF_OP(code)) {
  594. case BPF_JEQ:
  595. br_opcode = BE;
  596. break;
  597. case BPF_JGT:
  598. br_opcode = BGU;
  599. break;
  600. case BPF_JLT:
  601. br_opcode = BLU;
  602. break;
  603. case BPF_JGE:
  604. br_opcode = BGEU;
  605. break;
  606. case BPF_JLE:
  607. br_opcode = BLEU;
  608. break;
  609. case BPF_JSET:
  610. case BPF_JNE:
  611. br_opcode = BNE;
  612. break;
  613. case BPF_JSGT:
  614. br_opcode = BG;
  615. break;
  616. case BPF_JSLT:
  617. br_opcode = BL;
  618. break;
  619. case BPF_JSGE:
  620. br_opcode = BGE;
  621. break;
  622. case BPF_JSLE:
  623. br_opcode = BLE;
  624. break;
  625. default:
  626. /* Make sure we dont leak kernel information to the
  627. * user.
  628. */
  629. return -EFAULT;
  630. }
  631. emit_branch(br_opcode, ctx->idx, branch_dst, ctx);
  632. emit_nop(ctx);
  633. } else {
  634. u32 cbcond_opcode;
  635. switch (BPF_OP(code)) {
  636. case BPF_JEQ:
  637. cbcond_opcode = CBCONDE;
  638. break;
  639. case BPF_JGT:
  640. cbcond_opcode = CBCONDGU;
  641. break;
  642. case BPF_JLT:
  643. cbcond_opcode = CBCONDLU;
  644. break;
  645. case BPF_JGE:
  646. cbcond_opcode = CBCONDGEU;
  647. break;
  648. case BPF_JLE:
  649. cbcond_opcode = CBCONDLEU;
  650. break;
  651. case BPF_JNE:
  652. cbcond_opcode = CBCONDNE;
  653. break;
  654. case BPF_JSGT:
  655. cbcond_opcode = CBCONDG;
  656. break;
  657. case BPF_JSLT:
  658. cbcond_opcode = CBCONDL;
  659. break;
  660. case BPF_JSGE:
  661. cbcond_opcode = CBCONDGE;
  662. break;
  663. case BPF_JSLE:
  664. cbcond_opcode = CBCONDLE;
  665. break;
  666. default:
  667. /* Make sure we dont leak kernel information to the
  668. * user.
  669. */
  670. return -EFAULT;
  671. }
  672. cbcond_opcode |= CBCOND_OP;
  673. if (is_imm)
  674. emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst,
  675. dst, imm, ctx);
  676. else
  677. emit_cbcond(cbcond_opcode, ctx->idx, branch_dst,
  678. dst, src, ctx);
  679. }
  680. return 0;
  681. }
  682. /* Just skip the save instruction and the ctx register move. */
  683. #define BPF_TAILCALL_PROLOGUE_SKIP 32
  684. #define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128)
  685. static void build_prologue(struct jit_ctx *ctx)
  686. {
  687. s32 stack_needed = BASE_STACKFRAME;
  688. if (ctx->saw_frame_pointer || ctx->saw_tail_call) {
  689. struct bpf_prog *prog = ctx->prog;
  690. u32 stack_depth;
  691. stack_depth = prog->aux->stack_depth;
  692. stack_needed += round_up(stack_depth, 16);
  693. }
  694. if (ctx->saw_tail_call)
  695. stack_needed += 8;
  696. /* save %sp, -176, %sp */
  697. emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx);
  698. /* tail_call_cnt = 0 */
  699. if (ctx->saw_tail_call) {
  700. u32 off = BPF_TAILCALL_CNT_SP_OFF;
  701. emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx);
  702. } else {
  703. emit_nop(ctx);
  704. }
  705. if (ctx->saw_frame_pointer) {
  706. const u8 vfp = bpf2sparc[BPF_REG_FP];
  707. emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx);
  708. } else {
  709. emit_nop(ctx);
  710. }
  711. emit_reg_move(I0, O0, ctx);
  712. emit_reg_move(I1, O1, ctx);
  713. emit_reg_move(I2, O2, ctx);
  714. emit_reg_move(I3, O3, ctx);
  715. emit_reg_move(I4, O4, ctx);
  716. /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */
  717. }
  718. static void build_epilogue(struct jit_ctx *ctx)
  719. {
  720. ctx->epilogue_offset = ctx->idx;
  721. /* ret (jmpl %i7 + 8, %g0) */
  722. emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx);
  723. /* restore %i5, %g0, %o0 */
  724. emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx);
  725. }
  726. static void emit_tail_call(struct jit_ctx *ctx)
  727. {
  728. const u8 bpf_array = bpf2sparc[BPF_REG_2];
  729. const u8 bpf_index = bpf2sparc[BPF_REG_3];
  730. const u8 tmp = bpf2sparc[TMP_REG_1];
  731. u32 off;
  732. ctx->saw_tail_call = true;
  733. off = offsetof(struct bpf_array, map.max_entries);
  734. emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx);
  735. emit_cmp(bpf_index, tmp, ctx);
  736. #define OFFSET1 17
  737. emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx);
  738. emit_nop(ctx);
  739. off = BPF_TAILCALL_CNT_SP_OFF;
  740. emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
  741. emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx);
  742. #define OFFSET2 13
  743. emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx);
  744. emit_nop(ctx);
  745. emit_alu_K(ADD, tmp, 1, ctx);
  746. off = BPF_TAILCALL_CNT_SP_OFF;
  747. emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx);
  748. emit_alu3_K(SLL, bpf_index, 3, tmp, ctx);
  749. emit_alu(ADD, bpf_array, tmp, ctx);
  750. off = offsetof(struct bpf_array, ptrs);
  751. emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
  752. emit_cmpi(tmp, 0, ctx);
  753. #define OFFSET3 5
  754. emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx);
  755. emit_nop(ctx);
  756. off = offsetof(struct bpf_prog, bpf_func);
  757. emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx);
  758. off = BPF_TAILCALL_PROLOGUE_SKIP;
  759. emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx);
  760. emit_nop(ctx);
  761. }
  762. static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
  763. {
  764. const u8 code = insn->code;
  765. const u8 dst = bpf2sparc[insn->dst_reg];
  766. const u8 src = bpf2sparc[insn->src_reg];
  767. const int i = insn - ctx->prog->insnsi;
  768. const s16 off = insn->off;
  769. const s32 imm = insn->imm;
  770. if (insn->src_reg == BPF_REG_FP)
  771. ctx->saw_frame_pointer = true;
  772. switch (code) {
  773. /* dst = src */
  774. case BPF_ALU | BPF_MOV | BPF_X:
  775. emit_alu3_K(SRL, src, 0, dst, ctx);
  776. if (insn_is_zext(&insn[1]))
  777. return 1;
  778. break;
  779. case BPF_ALU64 | BPF_MOV | BPF_X:
  780. emit_reg_move(src, dst, ctx);
  781. break;
  782. /* dst = dst OP src */
  783. case BPF_ALU | BPF_ADD | BPF_X:
  784. case BPF_ALU64 | BPF_ADD | BPF_X:
  785. emit_alu(ADD, src, dst, ctx);
  786. goto do_alu32_trunc;
  787. case BPF_ALU | BPF_SUB | BPF_X:
  788. case BPF_ALU64 | BPF_SUB | BPF_X:
  789. emit_alu(SUB, src, dst, ctx);
  790. goto do_alu32_trunc;
  791. case BPF_ALU | BPF_AND | BPF_X:
  792. case BPF_ALU64 | BPF_AND | BPF_X:
  793. emit_alu(AND, src, dst, ctx);
  794. goto do_alu32_trunc;
  795. case BPF_ALU | BPF_OR | BPF_X:
  796. case BPF_ALU64 | BPF_OR | BPF_X:
  797. emit_alu(OR, src, dst, ctx);
  798. goto do_alu32_trunc;
  799. case BPF_ALU | BPF_XOR | BPF_X:
  800. case BPF_ALU64 | BPF_XOR | BPF_X:
  801. emit_alu(XOR, src, dst, ctx);
  802. goto do_alu32_trunc;
  803. case BPF_ALU | BPF_MUL | BPF_X:
  804. emit_alu(MUL, src, dst, ctx);
  805. goto do_alu32_trunc;
  806. case BPF_ALU64 | BPF_MUL | BPF_X:
  807. emit_alu(MULX, src, dst, ctx);
  808. break;
  809. case BPF_ALU | BPF_DIV | BPF_X:
  810. emit_write_y(G0, ctx);
  811. emit_alu(DIV, src, dst, ctx);
  812. if (insn_is_zext(&insn[1]))
  813. return 1;
  814. break;
  815. case BPF_ALU64 | BPF_DIV | BPF_X:
  816. emit_alu(UDIVX, src, dst, ctx);
  817. break;
  818. case BPF_ALU | BPF_MOD | BPF_X: {
  819. const u8 tmp = bpf2sparc[TMP_REG_1];
  820. ctx->tmp_1_used = true;
  821. emit_write_y(G0, ctx);
  822. emit_alu3(DIV, dst, src, tmp, ctx);
  823. emit_alu3(MULX, tmp, src, tmp, ctx);
  824. emit_alu3(SUB, dst, tmp, dst, ctx);
  825. goto do_alu32_trunc;
  826. }
  827. case BPF_ALU64 | BPF_MOD | BPF_X: {
  828. const u8 tmp = bpf2sparc[TMP_REG_1];
  829. ctx->tmp_1_used = true;
  830. emit_alu3(UDIVX, dst, src, tmp, ctx);
  831. emit_alu3(MULX, tmp, src, tmp, ctx);
  832. emit_alu3(SUB, dst, tmp, dst, ctx);
  833. break;
  834. }
  835. case BPF_ALU | BPF_LSH | BPF_X:
  836. emit_alu(SLL, src, dst, ctx);
  837. goto do_alu32_trunc;
  838. case BPF_ALU64 | BPF_LSH | BPF_X:
  839. emit_alu(SLLX, src, dst, ctx);
  840. break;
  841. case BPF_ALU | BPF_RSH | BPF_X:
  842. emit_alu(SRL, src, dst, ctx);
  843. if (insn_is_zext(&insn[1]))
  844. return 1;
  845. break;
  846. case BPF_ALU64 | BPF_RSH | BPF_X:
  847. emit_alu(SRLX, src, dst, ctx);
  848. break;
  849. case BPF_ALU | BPF_ARSH | BPF_X:
  850. emit_alu(SRA, src, dst, ctx);
  851. goto do_alu32_trunc;
  852. case BPF_ALU64 | BPF_ARSH | BPF_X:
  853. emit_alu(SRAX, src, dst, ctx);
  854. break;
  855. /* dst = -dst */
  856. case BPF_ALU | BPF_NEG:
  857. case BPF_ALU64 | BPF_NEG:
  858. emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx);
  859. goto do_alu32_trunc;
  860. case BPF_ALU | BPF_END | BPF_FROM_BE:
  861. switch (imm) {
  862. case 16:
  863. emit_alu_K(SLL, dst, 16, ctx);
  864. emit_alu_K(SRL, dst, 16, ctx);
  865. if (insn_is_zext(&insn[1]))
  866. return 1;
  867. break;
  868. case 32:
  869. if (!ctx->prog->aux->verifier_zext)
  870. emit_alu_K(SRL, dst, 0, ctx);
  871. break;
  872. case 64:
  873. /* nop */
  874. break;
  875. }
  876. break;
  877. /* dst = BSWAP##imm(dst) */
  878. case BPF_ALU | BPF_END | BPF_FROM_LE: {
  879. const u8 tmp = bpf2sparc[TMP_REG_1];
  880. const u8 tmp2 = bpf2sparc[TMP_REG_2];
  881. ctx->tmp_1_used = true;
  882. switch (imm) {
  883. case 16:
  884. emit_alu3_K(AND, dst, 0xff, tmp, ctx);
  885. emit_alu3_K(SRL, dst, 8, dst, ctx);
  886. emit_alu3_K(AND, dst, 0xff, dst, ctx);
  887. emit_alu3_K(SLL, tmp, 8, tmp, ctx);
  888. emit_alu(OR, tmp, dst, ctx);
  889. if (insn_is_zext(&insn[1]))
  890. return 1;
  891. break;
  892. case 32:
  893. ctx->tmp_2_used = true;
  894. emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */
  895. emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */
  896. emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
  897. emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */
  898. emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
  899. emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */
  900. emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */
  901. emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */
  902. emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */
  903. emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */
  904. emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */
  905. emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */
  906. if (insn_is_zext(&insn[1]))
  907. return 1;
  908. break;
  909. case 64:
  910. emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx);
  911. emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx);
  912. emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx);
  913. break;
  914. }
  915. break;
  916. }
  917. /* dst = imm */
  918. case BPF_ALU | BPF_MOV | BPF_K:
  919. emit_loadimm32(imm, dst, ctx);
  920. if (insn_is_zext(&insn[1]))
  921. return 1;
  922. break;
  923. case BPF_ALU64 | BPF_MOV | BPF_K:
  924. emit_loadimm_sext(imm, dst, ctx);
  925. break;
  926. /* dst = dst OP imm */
  927. case BPF_ALU | BPF_ADD | BPF_K:
  928. case BPF_ALU64 | BPF_ADD | BPF_K:
  929. emit_alu_K(ADD, dst, imm, ctx);
  930. goto do_alu32_trunc;
  931. case BPF_ALU | BPF_SUB | BPF_K:
  932. case BPF_ALU64 | BPF_SUB | BPF_K:
  933. emit_alu_K(SUB, dst, imm, ctx);
  934. goto do_alu32_trunc;
  935. case BPF_ALU | BPF_AND | BPF_K:
  936. case BPF_ALU64 | BPF_AND | BPF_K:
  937. emit_alu_K(AND, dst, imm, ctx);
  938. goto do_alu32_trunc;
  939. case BPF_ALU | BPF_OR | BPF_K:
  940. case BPF_ALU64 | BPF_OR | BPF_K:
  941. emit_alu_K(OR, dst, imm, ctx);
  942. goto do_alu32_trunc;
  943. case BPF_ALU | BPF_XOR | BPF_K:
  944. case BPF_ALU64 | BPF_XOR | BPF_K:
  945. emit_alu_K(XOR, dst, imm, ctx);
  946. goto do_alu32_trunc;
  947. case BPF_ALU | BPF_MUL | BPF_K:
  948. emit_alu_K(MUL, dst, imm, ctx);
  949. goto do_alu32_trunc;
  950. case BPF_ALU64 | BPF_MUL | BPF_K:
  951. emit_alu_K(MULX, dst, imm, ctx);
  952. break;
  953. case BPF_ALU | BPF_DIV | BPF_K:
  954. if (imm == 0)
  955. return -EINVAL;
  956. emit_write_y(G0, ctx);
  957. emit_alu_K(DIV, dst, imm, ctx);
  958. goto do_alu32_trunc;
  959. case BPF_ALU64 | BPF_DIV | BPF_K:
  960. if (imm == 0)
  961. return -EINVAL;
  962. emit_alu_K(UDIVX, dst, imm, ctx);
  963. break;
  964. case BPF_ALU64 | BPF_MOD | BPF_K:
  965. case BPF_ALU | BPF_MOD | BPF_K: {
  966. const u8 tmp = bpf2sparc[TMP_REG_2];
  967. unsigned int div;
  968. if (imm == 0)
  969. return -EINVAL;
  970. div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV;
  971. ctx->tmp_2_used = true;
  972. if (BPF_CLASS(code) != BPF_ALU64)
  973. emit_write_y(G0, ctx);
  974. if (is_simm13(imm)) {
  975. emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx);
  976. emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx);
  977. emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
  978. } else {
  979. const u8 tmp1 = bpf2sparc[TMP_REG_1];
  980. ctx->tmp_1_used = true;
  981. emit_set_const_sext(imm, tmp1, ctx);
  982. emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx);
  983. emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx);
  984. emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx);
  985. }
  986. goto do_alu32_trunc;
  987. }
  988. case BPF_ALU | BPF_LSH | BPF_K:
  989. emit_alu_K(SLL, dst, imm, ctx);
  990. goto do_alu32_trunc;
  991. case BPF_ALU64 | BPF_LSH | BPF_K:
  992. emit_alu_K(SLLX, dst, imm, ctx);
  993. break;
  994. case BPF_ALU | BPF_RSH | BPF_K:
  995. emit_alu_K(SRL, dst, imm, ctx);
  996. if (insn_is_zext(&insn[1]))
  997. return 1;
  998. break;
  999. case BPF_ALU64 | BPF_RSH | BPF_K:
  1000. emit_alu_K(SRLX, dst, imm, ctx);
  1001. break;
  1002. case BPF_ALU | BPF_ARSH | BPF_K:
  1003. emit_alu_K(SRA, dst, imm, ctx);
  1004. goto do_alu32_trunc;
  1005. case BPF_ALU64 | BPF_ARSH | BPF_K:
  1006. emit_alu_K(SRAX, dst, imm, ctx);
  1007. break;
  1008. do_alu32_trunc:
  1009. if (BPF_CLASS(code) == BPF_ALU &&
  1010. !ctx->prog->aux->verifier_zext)
  1011. emit_alu_K(SRL, dst, 0, ctx);
  1012. break;
  1013. /* JUMP off */
  1014. case BPF_JMP | BPF_JA:
  1015. emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx);
  1016. emit_nop(ctx);
  1017. break;
  1018. /* IF (dst COND src) JUMP off */
  1019. case BPF_JMP | BPF_JEQ | BPF_X:
  1020. case BPF_JMP | BPF_JGT | BPF_X:
  1021. case BPF_JMP | BPF_JLT | BPF_X:
  1022. case BPF_JMP | BPF_JGE | BPF_X:
  1023. case BPF_JMP | BPF_JLE | BPF_X:
  1024. case BPF_JMP | BPF_JNE | BPF_X:
  1025. case BPF_JMP | BPF_JSGT | BPF_X:
  1026. case BPF_JMP | BPF_JSLT | BPF_X:
  1027. case BPF_JMP | BPF_JSGE | BPF_X:
  1028. case BPF_JMP | BPF_JSLE | BPF_X:
  1029. case BPF_JMP | BPF_JSET | BPF_X: {
  1030. int err;
  1031. err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx);
  1032. if (err)
  1033. return err;
  1034. break;
  1035. }
  1036. /* IF (dst COND imm) JUMP off */
  1037. case BPF_JMP | BPF_JEQ | BPF_K:
  1038. case BPF_JMP | BPF_JGT | BPF_K:
  1039. case BPF_JMP | BPF_JLT | BPF_K:
  1040. case BPF_JMP | BPF_JGE | BPF_K:
  1041. case BPF_JMP | BPF_JLE | BPF_K:
  1042. case BPF_JMP | BPF_JNE | BPF_K:
  1043. case BPF_JMP | BPF_JSGT | BPF_K:
  1044. case BPF_JMP | BPF_JSLT | BPF_K:
  1045. case BPF_JMP | BPF_JSGE | BPF_K:
  1046. case BPF_JMP | BPF_JSLE | BPF_K:
  1047. case BPF_JMP | BPF_JSET | BPF_K: {
  1048. int err;
  1049. err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx);
  1050. if (err)
  1051. return err;
  1052. break;
  1053. }
  1054. /* function call */
  1055. case BPF_JMP | BPF_CALL:
  1056. {
  1057. u8 *func = ((u8 *)__bpf_call_base) + imm;
  1058. ctx->saw_call = true;
  1059. emit_call((u32 *)func, ctx);
  1060. emit_nop(ctx);
  1061. emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx);
  1062. break;
  1063. }
  1064. /* tail call */
  1065. case BPF_JMP | BPF_TAIL_CALL:
  1066. emit_tail_call(ctx);
  1067. break;
  1068. /* function return */
  1069. case BPF_JMP | BPF_EXIT:
  1070. /* Optimization: when last instruction is EXIT,
  1071. simply fallthrough to epilogue. */
  1072. if (i == ctx->prog->len - 1)
  1073. break;
  1074. emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx);
  1075. emit_nop(ctx);
  1076. break;
  1077. /* dst = imm64 */
  1078. case BPF_LD | BPF_IMM | BPF_DW:
  1079. {
  1080. const struct bpf_insn insn1 = insn[1];
  1081. u64 imm64;
  1082. imm64 = (u64)insn1.imm << 32 | (u32)imm;
  1083. emit_loadimm64(imm64, dst, ctx);
  1084. return 1;
  1085. }
  1086. /* LDX: dst = *(size *)(src + off) */
  1087. case BPF_LDX | BPF_MEM | BPF_W:
  1088. case BPF_LDX | BPF_MEM | BPF_H:
  1089. case BPF_LDX | BPF_MEM | BPF_B:
  1090. case BPF_LDX | BPF_MEM | BPF_DW: {
  1091. const u8 tmp = bpf2sparc[TMP_REG_1];
  1092. u32 opcode = 0, rs2;
  1093. ctx->tmp_1_used = true;
  1094. switch (BPF_SIZE(code)) {
  1095. case BPF_W:
  1096. opcode = LD32;
  1097. break;
  1098. case BPF_H:
  1099. opcode = LD16;
  1100. break;
  1101. case BPF_B:
  1102. opcode = LD8;
  1103. break;
  1104. case BPF_DW:
  1105. opcode = LD64;
  1106. break;
  1107. }
  1108. if (is_simm13(off)) {
  1109. opcode |= IMMED;
  1110. rs2 = S13(off);
  1111. } else {
  1112. emit_loadimm(off, tmp, ctx);
  1113. rs2 = RS2(tmp);
  1114. }
  1115. emit(opcode | RS1(src) | rs2 | RD(dst), ctx);
  1116. if (opcode != LD64 && insn_is_zext(&insn[1]))
  1117. return 1;
  1118. break;
  1119. }
  1120. /* speculation barrier */
  1121. case BPF_ST | BPF_NOSPEC:
  1122. break;
  1123. /* ST: *(size *)(dst + off) = imm */
  1124. case BPF_ST | BPF_MEM | BPF_W:
  1125. case BPF_ST | BPF_MEM | BPF_H:
  1126. case BPF_ST | BPF_MEM | BPF_B:
  1127. case BPF_ST | BPF_MEM | BPF_DW: {
  1128. const u8 tmp = bpf2sparc[TMP_REG_1];
  1129. const u8 tmp2 = bpf2sparc[TMP_REG_2];
  1130. u32 opcode = 0, rs2;
  1131. if (insn->dst_reg == BPF_REG_FP)
  1132. ctx->saw_frame_pointer = true;
  1133. ctx->tmp_2_used = true;
  1134. emit_loadimm(imm, tmp2, ctx);
  1135. switch (BPF_SIZE(code)) {
  1136. case BPF_W:
  1137. opcode = ST32;
  1138. break;
  1139. case BPF_H:
  1140. opcode = ST16;
  1141. break;
  1142. case BPF_B:
  1143. opcode = ST8;
  1144. break;
  1145. case BPF_DW:
  1146. opcode = ST64;
  1147. break;
  1148. }
  1149. if (is_simm13(off)) {
  1150. opcode |= IMMED;
  1151. rs2 = S13(off);
  1152. } else {
  1153. ctx->tmp_1_used = true;
  1154. emit_loadimm(off, tmp, ctx);
  1155. rs2 = RS2(tmp);
  1156. }
  1157. emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx);
  1158. break;
  1159. }
  1160. /* STX: *(size *)(dst + off) = src */
  1161. case BPF_STX | BPF_MEM | BPF_W:
  1162. case BPF_STX | BPF_MEM | BPF_H:
  1163. case BPF_STX | BPF_MEM | BPF_B:
  1164. case BPF_STX | BPF_MEM | BPF_DW: {
  1165. const u8 tmp = bpf2sparc[TMP_REG_1];
  1166. u32 opcode = 0, rs2;
  1167. if (insn->dst_reg == BPF_REG_FP)
  1168. ctx->saw_frame_pointer = true;
  1169. switch (BPF_SIZE(code)) {
  1170. case BPF_W:
  1171. opcode = ST32;
  1172. break;
  1173. case BPF_H:
  1174. opcode = ST16;
  1175. break;
  1176. case BPF_B:
  1177. opcode = ST8;
  1178. break;
  1179. case BPF_DW:
  1180. opcode = ST64;
  1181. break;
  1182. }
  1183. if (is_simm13(off)) {
  1184. opcode |= IMMED;
  1185. rs2 = S13(off);
  1186. } else {
  1187. ctx->tmp_1_used = true;
  1188. emit_loadimm(off, tmp, ctx);
  1189. rs2 = RS2(tmp);
  1190. }
  1191. emit(opcode | RS1(dst) | rs2 | RD(src), ctx);
  1192. break;
  1193. }
  1194. /* STX XADD: lock *(u32 *)(dst + off) += src */
  1195. case BPF_STX | BPF_XADD | BPF_W: {
  1196. const u8 tmp = bpf2sparc[TMP_REG_1];
  1197. const u8 tmp2 = bpf2sparc[TMP_REG_2];
  1198. const u8 tmp3 = bpf2sparc[TMP_REG_3];
  1199. if (insn->dst_reg == BPF_REG_FP)
  1200. ctx->saw_frame_pointer = true;
  1201. ctx->tmp_1_used = true;
  1202. ctx->tmp_2_used = true;
  1203. ctx->tmp_3_used = true;
  1204. emit_loadimm(off, tmp, ctx);
  1205. emit_alu3(ADD, dst, tmp, tmp, ctx);
  1206. emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
  1207. emit_alu3(ADD, tmp2, src, tmp3, ctx);
  1208. emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
  1209. emit_cmp(tmp2, tmp3, ctx);
  1210. emit_branch(BNE, 4, 0, ctx);
  1211. emit_nop(ctx);
  1212. break;
  1213. }
  1214. /* STX XADD: lock *(u64 *)(dst + off) += src */
  1215. case BPF_STX | BPF_XADD | BPF_DW: {
  1216. const u8 tmp = bpf2sparc[TMP_REG_1];
  1217. const u8 tmp2 = bpf2sparc[TMP_REG_2];
  1218. const u8 tmp3 = bpf2sparc[TMP_REG_3];
  1219. if (insn->dst_reg == BPF_REG_FP)
  1220. ctx->saw_frame_pointer = true;
  1221. ctx->tmp_1_used = true;
  1222. ctx->tmp_2_used = true;
  1223. ctx->tmp_3_used = true;
  1224. emit_loadimm(off, tmp, ctx);
  1225. emit_alu3(ADD, dst, tmp, tmp, ctx);
  1226. emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx);
  1227. emit_alu3(ADD, tmp2, src, tmp3, ctx);
  1228. emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx);
  1229. emit_cmp(tmp2, tmp3, ctx);
  1230. emit_branch(BNE, 4, 0, ctx);
  1231. emit_nop(ctx);
  1232. break;
  1233. }
  1234. default:
  1235. pr_err_once("unknown opcode %02x\n", code);
  1236. return -EINVAL;
  1237. }
  1238. return 0;
  1239. }
  1240. static int build_body(struct jit_ctx *ctx)
  1241. {
  1242. const struct bpf_prog *prog = ctx->prog;
  1243. int i;
  1244. for (i = 0; i < prog->len; i++) {
  1245. const struct bpf_insn *insn = &prog->insnsi[i];
  1246. int ret;
  1247. ret = build_insn(insn, ctx);
  1248. if (ret > 0) {
  1249. i++;
  1250. ctx->offset[i] = ctx->idx;
  1251. continue;
  1252. }
  1253. ctx->offset[i] = ctx->idx;
  1254. if (ret)
  1255. return ret;
  1256. }
  1257. return 0;
  1258. }
  1259. static void jit_fill_hole(void *area, unsigned int size)
  1260. {
  1261. u32 *ptr;
  1262. /* We are guaranteed to have aligned memory. */
  1263. for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
  1264. *ptr++ = 0x91d02005; /* ta 5 */
  1265. }
  1266. bool bpf_jit_needs_zext(void)
  1267. {
  1268. return true;
  1269. }
  1270. struct sparc64_jit_data {
  1271. struct bpf_binary_header *header;
  1272. u8 *image;
  1273. struct jit_ctx ctx;
  1274. };
  1275. struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  1276. {
  1277. struct bpf_prog *tmp, *orig_prog = prog;
  1278. struct sparc64_jit_data *jit_data;
  1279. struct bpf_binary_header *header;
  1280. u32 prev_image_size, image_size;
  1281. bool tmp_blinded = false;
  1282. bool extra_pass = false;
  1283. struct jit_ctx ctx;
  1284. u8 *image_ptr;
  1285. int pass, i;
  1286. if (!prog->jit_requested)
  1287. return orig_prog;
  1288. tmp = bpf_jit_blind_constants(prog);
  1289. /* If blinding was requested and we failed during blinding,
  1290. * we must fall back to the interpreter.
  1291. */
  1292. if (IS_ERR(tmp))
  1293. return orig_prog;
  1294. if (tmp != prog) {
  1295. tmp_blinded = true;
  1296. prog = tmp;
  1297. }
  1298. jit_data = prog->aux->jit_data;
  1299. if (!jit_data) {
  1300. jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  1301. if (!jit_data) {
  1302. prog = orig_prog;
  1303. goto out;
  1304. }
  1305. prog->aux->jit_data = jit_data;
  1306. }
  1307. if (jit_data->ctx.offset) {
  1308. ctx = jit_data->ctx;
  1309. image_ptr = jit_data->image;
  1310. header = jit_data->header;
  1311. extra_pass = true;
  1312. image_size = sizeof(u32) * ctx.idx;
  1313. prev_image_size = image_size;
  1314. pass = 1;
  1315. goto skip_init_ctx;
  1316. }
  1317. memset(&ctx, 0, sizeof(ctx));
  1318. ctx.prog = prog;
  1319. ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
  1320. if (ctx.offset == NULL) {
  1321. prog = orig_prog;
  1322. goto out_off;
  1323. }
  1324. /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook
  1325. * the offset array so that we converge faster.
  1326. */
  1327. for (i = 0; i < prog->len; i++)
  1328. ctx.offset[i] = i * (12 * 4);
  1329. prev_image_size = ~0U;
  1330. for (pass = 1; pass < 40; pass++) {
  1331. ctx.idx = 0;
  1332. build_prologue(&ctx);
  1333. if (build_body(&ctx)) {
  1334. prog = orig_prog;
  1335. goto out_off;
  1336. }
  1337. build_epilogue(&ctx);
  1338. if (bpf_jit_enable > 1)
  1339. pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
  1340. ctx.idx * 4,
  1341. ctx.tmp_1_used ? '1' : ' ',
  1342. ctx.tmp_2_used ? '2' : ' ',
  1343. ctx.tmp_3_used ? '3' : ' ',
  1344. ctx.saw_frame_pointer ? 'F' : ' ',
  1345. ctx.saw_call ? 'C' : ' ',
  1346. ctx.saw_tail_call ? 'T' : ' ');
  1347. if (ctx.idx * 4 == prev_image_size)
  1348. break;
  1349. prev_image_size = ctx.idx * 4;
  1350. cond_resched();
  1351. }
  1352. /* Now we know the actual image size. */
  1353. image_size = sizeof(u32) * ctx.idx;
  1354. header = bpf_jit_binary_alloc(image_size, &image_ptr,
  1355. sizeof(u32), jit_fill_hole);
  1356. if (header == NULL) {
  1357. prog = orig_prog;
  1358. goto out_off;
  1359. }
  1360. ctx.image = (u32 *)image_ptr;
  1361. skip_init_ctx:
  1362. ctx.idx = 0;
  1363. build_prologue(&ctx);
  1364. if (build_body(&ctx)) {
  1365. bpf_jit_binary_free(header);
  1366. prog = orig_prog;
  1367. goto out_off;
  1368. }
  1369. build_epilogue(&ctx);
  1370. if (ctx.idx * 4 != prev_image_size) {
  1371. pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
  1372. prev_image_size, ctx.idx * 4);
  1373. bpf_jit_binary_free(header);
  1374. prog = orig_prog;
  1375. goto out_off;
  1376. }
  1377. if (bpf_jit_enable > 1)
  1378. bpf_jit_dump(prog->len, image_size, pass, ctx.image);
  1379. bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
  1380. if (!prog->is_func || extra_pass) {
  1381. bpf_jit_binary_lock_ro(header);
  1382. } else {
  1383. jit_data->ctx = ctx;
  1384. jit_data->image = image_ptr;
  1385. jit_data->header = header;
  1386. }
  1387. prog->bpf_func = (void *)ctx.image;
  1388. prog->jited = 1;
  1389. prog->jited_len = image_size;
  1390. if (!prog->is_func || extra_pass) {
  1391. bpf_prog_fill_jited_linfo(prog, ctx.offset);
  1392. out_off:
  1393. kfree(ctx.offset);
  1394. kfree(jit_data);
  1395. prog->aux->jit_data = NULL;
  1396. }
  1397. out:
  1398. if (tmp_blinded)
  1399. bpf_jit_prog_release_other(prog, prog == orig_prog ?
  1400. tmp : orig_prog);
  1401. return prog;
  1402. }