uprobes.c 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * User-space Probes (UProbes) for x86
  4. *
  5. * Copyright (C) IBM Corporation, 2008-2011
  6. * Authors:
  7. * Srikar Dronamraju
  8. * Jim Keniston
  9. */
  10. #include <linux/kernel.h>
  11. #include <linux/sched.h>
  12. #include <linux/ptrace.h>
  13. #include <linux/uprobes.h>
  14. #include <linux/uaccess.h>
  15. #include <linux/kdebug.h>
  16. #include <asm/processor.h>
  17. #include <asm/insn.h>
  18. #include <asm/mmu_context.h>
  19. /* Post-execution fixups. */
  20. /* Adjust IP back to vicinity of actual insn */
  21. #define UPROBE_FIX_IP 0x01
  22. /* Adjust the return address of a call insn */
  23. #define UPROBE_FIX_CALL 0x02
  24. /* Instruction will modify TF, don't change it */
  25. #define UPROBE_FIX_SETF 0x04
  26. #define UPROBE_FIX_RIP_SI 0x08
  27. #define UPROBE_FIX_RIP_DI 0x10
  28. #define UPROBE_FIX_RIP_BX 0x20
  29. #define UPROBE_FIX_RIP_MASK \
  30. (UPROBE_FIX_RIP_SI | UPROBE_FIX_RIP_DI | UPROBE_FIX_RIP_BX)
  31. #define UPROBE_TRAP_NR UINT_MAX
  32. /* Adaptations for mhiramat x86 decoder v14. */
  33. #define OPCODE1(insn) ((insn)->opcode.bytes[0])
  34. #define OPCODE2(insn) ((insn)->opcode.bytes[1])
  35. #define OPCODE3(insn) ((insn)->opcode.bytes[2])
  36. #define MODRM_REG(insn) X86_MODRM_REG((insn)->modrm.value)
  37. #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
  38. (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
  39. (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
  40. (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
  41. (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
  42. << (row % 32))
  43. /*
  44. * Good-instruction tables for 32-bit apps. This is non-const and volatile
  45. * to keep gcc from statically optimizing it out, as variable_test_bit makes
  46. * some versions of gcc to think only *(unsigned long*) is used.
  47. *
  48. * Opcodes we'll probably never support:
  49. * 6c-6f - ins,outs. SEGVs if used in userspace
  50. * e4-e7 - in,out imm. SEGVs if used in userspace
  51. * ec-ef - in,out acc. SEGVs if used in userspace
  52. * cc - int3. SIGTRAP if used in userspace
  53. * ce - into. Not used in userspace - no kernel support to make it useful. SEGVs
  54. * (why we support bound (62) then? it's similar, and similarly unused...)
  55. * f1 - int1. SIGTRAP if used in userspace
  56. * f4 - hlt. SEGVs if used in userspace
  57. * fa - cli. SEGVs if used in userspace
  58. * fb - sti. SEGVs if used in userspace
  59. *
  60. * Opcodes which need some work to be supported:
  61. * 07,17,1f - pop es/ss/ds
  62. * Normally not used in userspace, but would execute if used.
  63. * Can cause GP or stack exception if tries to load wrong segment descriptor.
  64. * We hesitate to run them under single step since kernel's handling
  65. * of userspace single-stepping (TF flag) is fragile.
  66. * We can easily refuse to support push es/cs/ss/ds (06/0e/16/1e)
  67. * on the same grounds that they are never used.
  68. * cd - int N.
  69. * Used by userspace for "int 80" syscall entry. (Other "int N"
  70. * cause GP -> SEGV since their IDT gates don't allow calls from CPL 3).
  71. * Not supported since kernel's handling of userspace single-stepping
  72. * (TF flag) is fragile.
  73. * cf - iret. Normally not used in userspace. Doesn't SEGV unless arguments are bad
  74. */
  75. #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
  76. static volatile u32 good_insns_32[256 / 32] = {
  77. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  78. /* ---------------------------------------------- */
  79. W(0x00, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 00 */
  80. W(0x10, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 10 */
  81. W(0x20, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
  82. W(0x30, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 30 */
  83. W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
  84. W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
  85. W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
  86. W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
  87. W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
  88. W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
  89. W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
  90. W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
  91. W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
  92. W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
  93. W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* e0 */
  94. W(0xf0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1) /* f0 */
  95. /* ---------------------------------------------- */
  96. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  97. };
  98. #else
  99. #define good_insns_32 NULL
  100. #endif
  101. /* Good-instruction tables for 64-bit apps.
  102. *
  103. * Genuinely invalid opcodes:
  104. * 06,07 - formerly push/pop es
  105. * 0e - formerly push cs
  106. * 16,17 - formerly push/pop ss
  107. * 1e,1f - formerly push/pop ds
  108. * 27,2f,37,3f - formerly daa/das/aaa/aas
  109. * 60,61 - formerly pusha/popa
  110. * 62 - formerly bound. EVEX prefix for AVX512 (not yet supported)
  111. * 82 - formerly redundant encoding of Group1
  112. * 9a - formerly call seg:ofs
  113. * ce - formerly into
  114. * d4,d5 - formerly aam/aad
  115. * d6 - formerly undocumented salc
  116. * ea - formerly jmp seg:ofs
  117. *
  118. * Opcodes we'll probably never support:
  119. * 6c-6f - ins,outs. SEGVs if used in userspace
  120. * e4-e7 - in,out imm. SEGVs if used in userspace
  121. * ec-ef - in,out acc. SEGVs if used in userspace
  122. * cc - int3. SIGTRAP if used in userspace
  123. * f1 - int1. SIGTRAP if used in userspace
  124. * f4 - hlt. SEGVs if used in userspace
  125. * fa - cli. SEGVs if used in userspace
  126. * fb - sti. SEGVs if used in userspace
  127. *
  128. * Opcodes which need some work to be supported:
  129. * cd - int N.
  130. * Used by userspace for "int 80" syscall entry. (Other "int N"
  131. * cause GP -> SEGV since their IDT gates don't allow calls from CPL 3).
  132. * Not supported since kernel's handling of userspace single-stepping
  133. * (TF flag) is fragile.
  134. * cf - iret. Normally not used in userspace. Doesn't SEGV unless arguments are bad
  135. */
  136. #if defined(CONFIG_X86_64)
  137. static volatile u32 good_insns_64[256 / 32] = {
  138. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  139. /* ---------------------------------------------- */
  140. W(0x00, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1) | /* 00 */
  141. W(0x10, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) , /* 10 */
  142. W(0x20, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) | /* 20 */
  143. W(0x30, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 30 */
  144. W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
  145. W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
  146. W(0x60, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
  147. W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
  148. W(0x80, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
  149. W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1) , /* 90 */
  150. W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
  151. W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
  152. W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
  153. W(0xd0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
  154. W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0) | /* e0 */
  155. W(0xf0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1) /* f0 */
  156. /* ---------------------------------------------- */
  157. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  158. };
  159. #else
  160. #define good_insns_64 NULL
  161. #endif
  162. /* Using this for both 64-bit and 32-bit apps.
  163. * Opcodes we don't support:
  164. * 0f 00 - SLDT/STR/LLDT/LTR/VERR/VERW/-/- group. System insns
  165. * 0f 01 - SGDT/SIDT/LGDT/LIDT/SMSW/-/LMSW/INVLPG group.
  166. * Also encodes tons of other system insns if mod=11.
  167. * Some are in fact non-system: xend, xtest, rdtscp, maybe more
  168. * 0f 05 - syscall
  169. * 0f 06 - clts (CPL0 insn)
  170. * 0f 07 - sysret
  171. * 0f 08 - invd (CPL0 insn)
  172. * 0f 09 - wbinvd (CPL0 insn)
  173. * 0f 0b - ud2
  174. * 0f 30 - wrmsr (CPL0 insn) (then why rdmsr is allowed, it's also CPL0 insn?)
  175. * 0f 34 - sysenter
  176. * 0f 35 - sysexit
  177. * 0f 37 - getsec
  178. * 0f 78 - vmread (Intel VMX. CPL0 insn)
  179. * 0f 79 - vmwrite (Intel VMX. CPL0 insn)
  180. * Note: with prefixes, these two opcodes are
  181. * extrq/insertq/AVX512 convert vector ops.
  182. * 0f ae - group15: [f]xsave,[f]xrstor,[v]{ld,st}mxcsr,clflush[opt],
  183. * {rd,wr}{fs,gs}base,{s,l,m}fence.
  184. * Why? They are all user-executable.
  185. */
  186. static volatile u32 good_2byte_insns[256 / 32] = {
  187. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  188. /* ---------------------------------------------- */
  189. W(0x00, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1) | /* 00 */
  190. W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 10 */
  191. W(0x20, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
  192. W(0x30, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) , /* 30 */
  193. W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
  194. W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
  195. W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 60 */
  196. W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1) , /* 70 */
  197. W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
  198. W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
  199. W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1) | /* a0 */
  200. W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
  201. W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
  202. W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
  203. W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* e0 */
  204. W(0xf0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) /* f0 */
  205. /* ---------------------------------------------- */
  206. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  207. };
  208. #undef W
  209. /*
  210. * opcodes we may need to refine support for:
  211. *
  212. * 0f - 2-byte instructions: For many of these instructions, the validity
  213. * depends on the prefix and/or the reg field. On such instructions, we
  214. * just consider the opcode combination valid if it corresponds to any
  215. * valid instruction.
  216. *
  217. * 8f - Group 1 - only reg = 0 is OK
  218. * c6-c7 - Group 11 - only reg = 0 is OK
  219. * d9-df - fpu insns with some illegal encodings
  220. * f2, f3 - repnz, repz prefixes. These are also the first byte for
  221. * certain floating-point instructions, such as addsd.
  222. *
  223. * fe - Group 4 - only reg = 0 or 1 is OK
  224. * ff - Group 5 - only reg = 0-6 is OK
  225. *
  226. * others -- Do we need to support these?
  227. *
  228. * 0f - (floating-point?) prefetch instructions
  229. * 07, 17, 1f - pop es, pop ss, pop ds
  230. * 26, 2e, 36, 3e - es:, cs:, ss:, ds: segment prefixes --
  231. * but 64 and 65 (fs: and gs:) seem to be used, so we support them
  232. * 67 - addr16 prefix
  233. * ce - into
  234. * f0 - lock prefix
  235. */
  236. /*
  237. * TODO:
  238. * - Where necessary, examine the modrm byte and allow only valid instructions
  239. * in the different Groups and fpu instructions.
  240. */
  241. static bool is_prefix_bad(struct insn *insn)
  242. {
  243. insn_byte_t p;
  244. int i;
  245. for_each_insn_prefix(insn, i, p) {
  246. insn_attr_t attr;
  247. attr = inat_get_opcode_attribute(p);
  248. switch (attr) {
  249. case INAT_MAKE_PREFIX(INAT_PFX_ES):
  250. case INAT_MAKE_PREFIX(INAT_PFX_CS):
  251. case INAT_MAKE_PREFIX(INAT_PFX_DS):
  252. case INAT_MAKE_PREFIX(INAT_PFX_SS):
  253. case INAT_MAKE_PREFIX(INAT_PFX_LOCK):
  254. return true;
  255. }
  256. }
  257. return false;
  258. }
  259. static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
  260. {
  261. u32 volatile *good_insns;
  262. insn_init(insn, auprobe->insn, sizeof(auprobe->insn), x86_64);
  263. /* has the side-effect of processing the entire instruction */
  264. insn_get_length(insn);
  265. if (!insn_complete(insn))
  266. return -ENOEXEC;
  267. if (is_prefix_bad(insn))
  268. return -ENOTSUPP;
  269. /* We should not singlestep on the exception masking instructions */
  270. if (insn_masking_exception(insn))
  271. return -ENOTSUPP;
  272. if (x86_64)
  273. good_insns = good_insns_64;
  274. else
  275. good_insns = good_insns_32;
  276. if (test_bit(OPCODE1(insn), (unsigned long *)good_insns))
  277. return 0;
  278. if (insn->opcode.nbytes == 2) {
  279. if (test_bit(OPCODE2(insn), (unsigned long *)good_2byte_insns))
  280. return 0;
  281. }
  282. return -ENOTSUPP;
  283. }
  284. #ifdef CONFIG_X86_64
  285. /*
  286. * If arch_uprobe->insn doesn't use rip-relative addressing, return
  287. * immediately. Otherwise, rewrite the instruction so that it accesses
  288. * its memory operand indirectly through a scratch register. Set
  289. * defparam->fixups accordingly. (The contents of the scratch register
  290. * will be saved before we single-step the modified instruction,
  291. * and restored afterward).
  292. *
  293. * We do this because a rip-relative instruction can access only a
  294. * relatively small area (+/- 2 GB from the instruction), and the XOL
  295. * area typically lies beyond that area. At least for instructions
  296. * that store to memory, we can't execute the original instruction
  297. * and "fix things up" later, because the misdirected store could be
  298. * disastrous.
  299. *
  300. * Some useful facts about rip-relative instructions:
  301. *
  302. * - There's always a modrm byte with bit layout "00 reg 101".
  303. * - There's never a SIB byte.
  304. * - The displacement is always 4 bytes.
  305. * - REX.B=1 bit in REX prefix, which normally extends r/m field,
  306. * has no effect on rip-relative mode. It doesn't make modrm byte
  307. * with r/m=101 refer to register 1101 = R13.
  308. */
  309. static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
  310. {
  311. u8 *cursor;
  312. u8 reg;
  313. u8 reg2;
  314. if (!insn_rip_relative(insn))
  315. return;
  316. /*
  317. * insn_rip_relative() would have decoded rex_prefix, vex_prefix, modrm.
  318. * Clear REX.b bit (extension of MODRM.rm field):
  319. * we want to encode low numbered reg, not r8+.
  320. */
  321. if (insn->rex_prefix.nbytes) {
  322. cursor = auprobe->insn + insn_offset_rex_prefix(insn);
  323. /* REX byte has 0100wrxb layout, clearing REX.b bit */
  324. *cursor &= 0xfe;
  325. }
  326. /*
  327. * Similar treatment for VEX3/EVEX prefix.
  328. * TODO: add XOP treatment when insn decoder supports them
  329. */
  330. if (insn->vex_prefix.nbytes >= 3) {
  331. /*
  332. * vex2: c5 rvvvvLpp (has no b bit)
  333. * vex3/xop: c4/8f rxbmmmmm wvvvvLpp
  334. * evex: 62 rxbR00mm wvvvv1pp zllBVaaa
  335. * Setting VEX3.b (setting because it has inverted meaning).
  336. * Setting EVEX.x since (in non-SIB encoding) EVEX.x
  337. * is the 4th bit of MODRM.rm, and needs the same treatment.
  338. * For VEX3-encoded insns, VEX3.x value has no effect in
  339. * non-SIB encoding, the change is superfluous but harmless.
  340. */
  341. cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;
  342. *cursor |= 0x60;
  343. }
  344. /*
  345. * Convert from rip-relative addressing to register-relative addressing
  346. * via a scratch register.
  347. *
  348. * This is tricky since there are insns with modrm byte
  349. * which also use registers not encoded in modrm byte:
  350. * [i]div/[i]mul: implicitly use dx:ax
  351. * shift ops: implicitly use cx
  352. * cmpxchg: implicitly uses ax
  353. * cmpxchg8/16b: implicitly uses dx:ax and bx:cx
  354. * Encoding: 0f c7/1 modrm
  355. * The code below thinks that reg=1 (cx), chooses si as scratch.
  356. * mulx: implicitly uses dx: mulx r/m,r1,r2 does r1:r2 = dx * r/m.
  357. * First appeared in Haswell (BMI2 insn). It is vex-encoded.
  358. * Example where none of bx,cx,dx can be used as scratch reg:
  359. * c4 e2 63 f6 0d disp32 mulx disp32(%rip),%ebx,%ecx
  360. * [v]pcmpistri: implicitly uses cx, xmm0
  361. * [v]pcmpistrm: implicitly uses xmm0
  362. * [v]pcmpestri: implicitly uses ax, dx, cx, xmm0
  363. * [v]pcmpestrm: implicitly uses ax, dx, xmm0
  364. * Evil SSE4.2 string comparison ops from hell.
  365. * maskmovq/[v]maskmovdqu: implicitly uses (ds:rdi) as destination.
  366. * Encoding: 0f f7 modrm, 66 0f f7 modrm, vex-encoded: c5 f9 f7 modrm.
  367. * Store op1, byte-masked by op2 msb's in each byte, to (ds:rdi).
  368. * AMD says it has no 3-operand form (vex.vvvv must be 1111)
  369. * and that it can have only register operands, not mem
  370. * (its modrm byte must have mode=11).
  371. * If these restrictions will ever be lifted,
  372. * we'll need code to prevent selection of di as scratch reg!
  373. *
  374. * Summary: I don't know any insns with modrm byte which
  375. * use SI register implicitly. DI register is used only
  376. * by one insn (maskmovq) and BX register is used
  377. * only by one too (cmpxchg8b).
  378. * BP is stack-segment based (may be a problem?).
  379. * AX, DX, CX are off-limits (many implicit users).
  380. * SP is unusable (it's stack pointer - think about "pop mem";
  381. * also, rsp+disp32 needs sib encoding -> insn length change).
  382. */
  383. reg = MODRM_REG(insn); /* Fetch modrm.reg */
  384. reg2 = 0xff; /* Fetch vex.vvvv */
  385. if (insn->vex_prefix.nbytes)
  386. reg2 = insn->vex_prefix.bytes[2];
  387. /*
  388. * TODO: add XOP vvvv reading.
  389. *
  390. * vex.vvvv field is in bits 6-3, bits are inverted.
  391. * But in 32-bit mode, high-order bit may be ignored.
  392. * Therefore, let's consider only 3 low-order bits.
  393. */
  394. reg2 = ((reg2 >> 3) & 0x7) ^ 0x7;
  395. /*
  396. * Register numbering is ax,cx,dx,bx, sp,bp,si,di, r8..r15.
  397. *
  398. * Choose scratch reg. Order is important: must not select bx
  399. * if we can use si (cmpxchg8b case!)
  400. */
  401. if (reg != 6 && reg2 != 6) {
  402. reg2 = 6;
  403. auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;
  404. } else if (reg != 7 && reg2 != 7) {
  405. reg2 = 7;
  406. auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;
  407. /* TODO (paranoia): force maskmovq to not use di */
  408. } else {
  409. reg2 = 3;
  410. auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;
  411. }
  412. /*
  413. * Point cursor at the modrm byte. The next 4 bytes are the
  414. * displacement. Beyond the displacement, for some instructions,
  415. * is the immediate operand.
  416. */
  417. cursor = auprobe->insn + insn_offset_modrm(insn);
  418. /*
  419. * Change modrm from "00 reg 101" to "10 reg reg2". Example:
  420. * 89 05 disp32 mov %eax,disp32(%rip) becomes
  421. * 89 86 disp32 mov %eax,disp32(%rsi)
  422. */
  423. *cursor = 0x80 | (reg << 3) | reg2;
  424. }
  425. static inline unsigned long *
  426. scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)
  427. {
  428. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)
  429. return &regs->si;
  430. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)
  431. return &regs->di;
  432. return &regs->bx;
  433. }
  434. /*
  435. * If we're emulating a rip-relative instruction, save the contents
  436. * of the scratch register and store the target address in that register.
  437. */
  438. static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  439. {
  440. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
  441. struct uprobe_task *utask = current->utask;
  442. unsigned long *sr = scratch_reg(auprobe, regs);
  443. utask->autask.saved_scratch_register = *sr;
  444. *sr = utask->vaddr + auprobe->defparam.ilen;
  445. }
  446. }
  447. static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  448. {
  449. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
  450. struct uprobe_task *utask = current->utask;
  451. unsigned long *sr = scratch_reg(auprobe, regs);
  452. *sr = utask->autask.saved_scratch_register;
  453. }
  454. }
  455. #else /* 32-bit: */
  456. /*
  457. * No RIP-relative addressing on 32-bit
  458. */
  459. static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
  460. {
  461. }
  462. static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  463. {
  464. }
  465. static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  466. {
  467. }
  468. #endif /* CONFIG_X86_64 */
  469. struct uprobe_xol_ops {
  470. bool (*emulate)(struct arch_uprobe *, struct pt_regs *);
  471. int (*pre_xol)(struct arch_uprobe *, struct pt_regs *);
  472. int (*post_xol)(struct arch_uprobe *, struct pt_regs *);
  473. void (*abort)(struct arch_uprobe *, struct pt_regs *);
  474. };
  475. static inline int sizeof_long(struct pt_regs *regs)
  476. {
  477. /*
  478. * Check registers for mode as in_xxx_syscall() does not apply here.
  479. */
  480. return user_64bit_mode(regs) ? 8 : 4;
  481. }
  482. static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  483. {
  484. riprel_pre_xol(auprobe, regs);
  485. return 0;
  486. }
  487. static int emulate_push_stack(struct pt_regs *regs, unsigned long val)
  488. {
  489. unsigned long new_sp = regs->sp - sizeof_long(regs);
  490. if (copy_to_user((void __user *)new_sp, &val, sizeof_long(regs)))
  491. return -EFAULT;
  492. regs->sp = new_sp;
  493. return 0;
  494. }
  495. /*
  496. * We have to fix things up as follows:
  497. *
  498. * Typically, the new ip is relative to the copied instruction. We need
  499. * to make it relative to the original instruction (FIX_IP). Exceptions
  500. * are return instructions and absolute or indirect jump or call instructions.
  501. *
  502. * If the single-stepped instruction was a call, the return address that
  503. * is atop the stack is the address following the copied instruction. We
  504. * need to make it the address following the original instruction (FIX_CALL).
  505. *
  506. * If the original instruction was a rip-relative instruction such as
  507. * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
  508. * instruction using a scratch register -- e.g., "movl %edx,0xnnnn(%rsi)".
  509. * We need to restore the contents of the scratch register
  510. * (FIX_RIP_reg).
  511. */
  512. static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  513. {
  514. struct uprobe_task *utask = current->utask;
  515. riprel_post_xol(auprobe, regs);
  516. if (auprobe->defparam.fixups & UPROBE_FIX_IP) {
  517. long correction = utask->vaddr - utask->xol_vaddr;
  518. regs->ip += correction;
  519. } else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {
  520. regs->sp += sizeof_long(regs); /* Pop incorrect return address */
  521. if (emulate_push_stack(regs, utask->vaddr + auprobe->defparam.ilen))
  522. return -ERESTART;
  523. }
  524. /* popf; tell the caller to not touch TF */
  525. if (auprobe->defparam.fixups & UPROBE_FIX_SETF)
  526. utask->autask.saved_tf = true;
  527. return 0;
  528. }
  529. static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  530. {
  531. riprel_post_xol(auprobe, regs);
  532. }
  533. static const struct uprobe_xol_ops default_xol_ops = {
  534. .pre_xol = default_pre_xol_op,
  535. .post_xol = default_post_xol_op,
  536. .abort = default_abort_op,
  537. };
  538. static bool branch_is_call(struct arch_uprobe *auprobe)
  539. {
  540. return auprobe->branch.opc1 == 0xe8;
  541. }
  542. #define CASE_COND \
  543. COND(70, 71, XF(OF)) \
  544. COND(72, 73, XF(CF)) \
  545. COND(74, 75, XF(ZF)) \
  546. COND(78, 79, XF(SF)) \
  547. COND(7a, 7b, XF(PF)) \
  548. COND(76, 77, XF(CF) || XF(ZF)) \
  549. COND(7c, 7d, XF(SF) != XF(OF)) \
  550. COND(7e, 7f, XF(ZF) || XF(SF) != XF(OF))
  551. #define COND(op_y, op_n, expr) \
  552. case 0x ## op_y: DO((expr) != 0) \
  553. case 0x ## op_n: DO((expr) == 0)
  554. #define XF(xf) (!!(flags & X86_EFLAGS_ ## xf))
  555. static bool is_cond_jmp_opcode(u8 opcode)
  556. {
  557. switch (opcode) {
  558. #define DO(expr) \
  559. return true;
  560. CASE_COND
  561. #undef DO
  562. default:
  563. return false;
  564. }
  565. }
  566. static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)
  567. {
  568. unsigned long flags = regs->flags;
  569. switch (auprobe->branch.opc1) {
  570. #define DO(expr) \
  571. return expr;
  572. CASE_COND
  573. #undef DO
  574. default: /* not a conditional jmp */
  575. return true;
  576. }
  577. }
  578. #undef XF
  579. #undef COND
  580. #undef CASE_COND
  581. static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  582. {
  583. unsigned long new_ip = regs->ip += auprobe->branch.ilen;
  584. unsigned long offs = (long)auprobe->branch.offs;
  585. if (branch_is_call(auprobe)) {
  586. /*
  587. * If it fails we execute this (mangled, see the comment in
  588. * branch_clear_offset) insn out-of-line. In the likely case
  589. * this should trigger the trap, and the probed application
  590. * should die or restart the same insn after it handles the
  591. * signal, arch_uprobe_post_xol() won't be even called.
  592. *
  593. * But there is corner case, see the comment in ->post_xol().
  594. */
  595. if (emulate_push_stack(regs, new_ip))
  596. return false;
  597. } else if (!check_jmp_cond(auprobe, regs)) {
  598. offs = 0;
  599. }
  600. regs->ip = new_ip + offs;
  601. return true;
  602. }
  603. static bool push_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  604. {
  605. unsigned long *src_ptr = (void *)regs + auprobe->push.reg_offset;
  606. if (emulate_push_stack(regs, *src_ptr))
  607. return false;
  608. regs->ip += auprobe->push.ilen;
  609. return true;
  610. }
  611. static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  612. {
  613. BUG_ON(!branch_is_call(auprobe));
  614. /*
  615. * We can only get here if branch_emulate_op() failed to push the ret
  616. * address _and_ another thread expanded our stack before the (mangled)
  617. * "call" insn was executed out-of-line. Just restore ->sp and restart.
  618. * We could also restore ->ip and try to call branch_emulate_op() again.
  619. */
  620. regs->sp += sizeof_long(regs);
  621. return -ERESTART;
  622. }
  623. static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)
  624. {
  625. /*
  626. * Turn this insn into "call 1f; 1:", this is what we will execute
  627. * out-of-line if ->emulate() fails. We only need this to generate
  628. * a trap, so that the probed task receives the correct signal with
  629. * the properly filled siginfo.
  630. *
  631. * But see the comment in ->post_xol(), in the unlikely case it can
  632. * succeed. So we need to ensure that the new ->ip can not fall into
  633. * the non-canonical area and trigger #GP.
  634. *
  635. * We could turn it into (say) "pushf", but then we would need to
  636. * divorce ->insn[] and ->ixol[]. We need to preserve the 1st byte
  637. * of ->insn[] for set_orig_insn().
  638. */
  639. memset(auprobe->insn + insn_offset_immediate(insn),
  640. 0, insn->immediate.nbytes);
  641. }
  642. static const struct uprobe_xol_ops branch_xol_ops = {
  643. .emulate = branch_emulate_op,
  644. .post_xol = branch_post_xol_op,
  645. };
  646. static const struct uprobe_xol_ops push_xol_ops = {
  647. .emulate = push_emulate_op,
  648. };
  649. /* Returns -ENOSYS if branch_xol_ops doesn't handle this insn */
  650. static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
  651. {
  652. u8 opc1 = OPCODE1(insn);
  653. insn_byte_t p;
  654. int i;
  655. switch (opc1) {
  656. case 0xeb: /* jmp 8 */
  657. case 0xe9: /* jmp 32 */
  658. case 0x90: /* prefix* + nop; same as jmp with .offs = 0 */
  659. break;
  660. case 0xe8: /* call relative */
  661. branch_clear_offset(auprobe, insn);
  662. break;
  663. case 0x0f:
  664. if (insn->opcode.nbytes != 2)
  665. return -ENOSYS;
  666. /*
  667. * If it is a "near" conditional jmp, OPCODE2() - 0x10 matches
  668. * OPCODE1() of the "short" jmp which checks the same condition.
  669. */
  670. opc1 = OPCODE2(insn) - 0x10;
  671. fallthrough;
  672. default:
  673. if (!is_cond_jmp_opcode(opc1))
  674. return -ENOSYS;
  675. }
  676. /*
  677. * 16-bit overrides such as CALLW (66 e8 nn nn) are not supported.
  678. * Intel and AMD behavior differ in 64-bit mode: Intel ignores 66 prefix.
  679. * No one uses these insns, reject any branch insns with such prefix.
  680. */
  681. for_each_insn_prefix(insn, i, p) {
  682. if (p == 0x66)
  683. return -ENOTSUPP;
  684. }
  685. auprobe->branch.opc1 = opc1;
  686. auprobe->branch.ilen = insn->length;
  687. auprobe->branch.offs = insn->immediate.value;
  688. auprobe->ops = &branch_xol_ops;
  689. return 0;
  690. }
  691. /* Returns -ENOSYS if push_xol_ops doesn't handle this insn */
  692. static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
  693. {
  694. u8 opc1 = OPCODE1(insn), reg_offset = 0;
  695. if (opc1 < 0x50 || opc1 > 0x57)
  696. return -ENOSYS;
  697. if (insn->length > 2)
  698. return -ENOSYS;
  699. if (insn->length == 2) {
  700. /* only support rex_prefix 0x41 (x64 only) */
  701. #ifdef CONFIG_X86_64
  702. if (insn->rex_prefix.nbytes != 1 ||
  703. insn->rex_prefix.bytes[0] != 0x41)
  704. return -ENOSYS;
  705. switch (opc1) {
  706. case 0x50:
  707. reg_offset = offsetof(struct pt_regs, r8);
  708. break;
  709. case 0x51:
  710. reg_offset = offsetof(struct pt_regs, r9);
  711. break;
  712. case 0x52:
  713. reg_offset = offsetof(struct pt_regs, r10);
  714. break;
  715. case 0x53:
  716. reg_offset = offsetof(struct pt_regs, r11);
  717. break;
  718. case 0x54:
  719. reg_offset = offsetof(struct pt_regs, r12);
  720. break;
  721. case 0x55:
  722. reg_offset = offsetof(struct pt_regs, r13);
  723. break;
  724. case 0x56:
  725. reg_offset = offsetof(struct pt_regs, r14);
  726. break;
  727. case 0x57:
  728. reg_offset = offsetof(struct pt_regs, r15);
  729. break;
  730. }
  731. #else
  732. return -ENOSYS;
  733. #endif
  734. } else {
  735. switch (opc1) {
  736. case 0x50:
  737. reg_offset = offsetof(struct pt_regs, ax);
  738. break;
  739. case 0x51:
  740. reg_offset = offsetof(struct pt_regs, cx);
  741. break;
  742. case 0x52:
  743. reg_offset = offsetof(struct pt_regs, dx);
  744. break;
  745. case 0x53:
  746. reg_offset = offsetof(struct pt_regs, bx);
  747. break;
  748. case 0x54:
  749. reg_offset = offsetof(struct pt_regs, sp);
  750. break;
  751. case 0x55:
  752. reg_offset = offsetof(struct pt_regs, bp);
  753. break;
  754. case 0x56:
  755. reg_offset = offsetof(struct pt_regs, si);
  756. break;
  757. case 0x57:
  758. reg_offset = offsetof(struct pt_regs, di);
  759. break;
  760. }
  761. }
  762. auprobe->push.reg_offset = reg_offset;
  763. auprobe->push.ilen = insn->length;
  764. auprobe->ops = &push_xol_ops;
  765. return 0;
  766. }
  767. /**
  768. * arch_uprobe_analyze_insn - instruction analysis including validity and fixups.
  769. * @auprobe: the probepoint information.
  770. * @mm: the probed address space.
  771. * @addr: virtual address at which to install the probepoint
  772. * Return 0 on success or a -ve number on error.
  773. */
  774. int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)
  775. {
  776. struct insn insn;
  777. u8 fix_ip_or_call = UPROBE_FIX_IP;
  778. int ret;
  779. ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
  780. if (ret)
  781. return ret;
  782. ret = branch_setup_xol_ops(auprobe, &insn);
  783. if (ret != -ENOSYS)
  784. return ret;
  785. ret = push_setup_xol_ops(auprobe, &insn);
  786. if (ret != -ENOSYS)
  787. return ret;
  788. /*
  789. * Figure out which fixups default_post_xol_op() will need to perform,
  790. * and annotate defparam->fixups accordingly.
  791. */
  792. switch (OPCODE1(&insn)) {
  793. case 0x9d: /* popf */
  794. auprobe->defparam.fixups |= UPROBE_FIX_SETF;
  795. break;
  796. case 0xc3: /* ret or lret -- ip is correct */
  797. case 0xcb:
  798. case 0xc2:
  799. case 0xca:
  800. case 0xea: /* jmp absolute -- ip is correct */
  801. fix_ip_or_call = 0;
  802. break;
  803. case 0x9a: /* call absolute - Fix return addr, not ip */
  804. fix_ip_or_call = UPROBE_FIX_CALL;
  805. break;
  806. case 0xff:
  807. switch (MODRM_REG(&insn)) {
  808. case 2: case 3: /* call or lcall, indirect */
  809. fix_ip_or_call = UPROBE_FIX_CALL;
  810. break;
  811. case 4: case 5: /* jmp or ljmp, indirect */
  812. fix_ip_or_call = 0;
  813. break;
  814. }
  815. fallthrough;
  816. default:
  817. riprel_analyze(auprobe, &insn);
  818. }
  819. auprobe->defparam.ilen = insn.length;
  820. auprobe->defparam.fixups |= fix_ip_or_call;
  821. auprobe->ops = &default_xol_ops;
  822. return 0;
  823. }
  824. /*
  825. * arch_uprobe_pre_xol - prepare to execute out of line.
  826. * @auprobe: the probepoint information.
  827. * @regs: reflects the saved user state of current task.
  828. */
  829. int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  830. {
  831. struct uprobe_task *utask = current->utask;
  832. if (auprobe->ops->pre_xol) {
  833. int err = auprobe->ops->pre_xol(auprobe, regs);
  834. if (err)
  835. return err;
  836. }
  837. regs->ip = utask->xol_vaddr;
  838. utask->autask.saved_trap_nr = current->thread.trap_nr;
  839. current->thread.trap_nr = UPROBE_TRAP_NR;
  840. utask->autask.saved_tf = !!(regs->flags & X86_EFLAGS_TF);
  841. regs->flags |= X86_EFLAGS_TF;
  842. if (test_tsk_thread_flag(current, TIF_BLOCKSTEP))
  843. set_task_blockstep(current, false);
  844. return 0;
  845. }
  846. /*
  847. * If xol insn itself traps and generates a signal(Say,
  848. * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped
  849. * instruction jumps back to its own address. It is assumed that anything
  850. * like do_page_fault/do_trap/etc sets thread.trap_nr != -1.
  851. *
  852. * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr,
  853. * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to
  854. * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol().
  855. */
  856. bool arch_uprobe_xol_was_trapped(struct task_struct *t)
  857. {
  858. if (t->thread.trap_nr != UPROBE_TRAP_NR)
  859. return true;
  860. return false;
  861. }
  862. /*
  863. * Called after single-stepping. To avoid the SMP problems that can
  864. * occur when we temporarily put back the original opcode to
  865. * single-step, we single-stepped a copy of the instruction.
  866. *
  867. * This function prepares to resume execution after the single-step.
  868. */
  869. int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  870. {
  871. struct uprobe_task *utask = current->utask;
  872. bool send_sigtrap = utask->autask.saved_tf;
  873. int err = 0;
  874. WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
  875. current->thread.trap_nr = utask->autask.saved_trap_nr;
  876. if (auprobe->ops->post_xol) {
  877. err = auprobe->ops->post_xol(auprobe, regs);
  878. if (err) {
  879. /*
  880. * Restore ->ip for restart or post mortem analysis.
  881. * ->post_xol() must not return -ERESTART unless this
  882. * is really possible.
  883. */
  884. regs->ip = utask->vaddr;
  885. if (err == -ERESTART)
  886. err = 0;
  887. send_sigtrap = false;
  888. }
  889. }
  890. /*
  891. * arch_uprobe_pre_xol() doesn't save the state of TIF_BLOCKSTEP
  892. * so we can get an extra SIGTRAP if we do not clear TF. We need
  893. * to examine the opcode to make it right.
  894. */
  895. if (send_sigtrap)
  896. send_sig(SIGTRAP, current, 0);
  897. if (!utask->autask.saved_tf)
  898. regs->flags &= ~X86_EFLAGS_TF;
  899. return err;
  900. }
  901. /* callback routine for handling exceptions. */
  902. int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data)
  903. {
  904. struct die_args *args = data;
  905. struct pt_regs *regs = args->regs;
  906. int ret = NOTIFY_DONE;
  907. /* We are only interested in userspace traps */
  908. if (regs && !user_mode(regs))
  909. return NOTIFY_DONE;
  910. switch (val) {
  911. case DIE_INT3:
  912. if (uprobe_pre_sstep_notifier(regs))
  913. ret = NOTIFY_STOP;
  914. break;
  915. case DIE_DEBUG:
  916. if (uprobe_post_sstep_notifier(regs))
  917. ret = NOTIFY_STOP;
  918. default:
  919. break;
  920. }
  921. return ret;
  922. }
  923. /*
  924. * This function gets called when XOL instruction either gets trapped or
  925. * the thread has a fatal signal. Reset the instruction pointer to its
  926. * probed address for the potential restart or for post mortem analysis.
  927. */
  928. void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  929. {
  930. struct uprobe_task *utask = current->utask;
  931. if (auprobe->ops->abort)
  932. auprobe->ops->abort(auprobe, regs);
  933. current->thread.trap_nr = utask->autask.saved_trap_nr;
  934. regs->ip = utask->vaddr;
  935. /* clear TF if it was set by us in arch_uprobe_pre_xol() */
  936. if (!utask->autask.saved_tf)
  937. regs->flags &= ~X86_EFLAGS_TF;
  938. }
  939. static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
  940. {
  941. if (auprobe->ops->emulate)
  942. return auprobe->ops->emulate(auprobe, regs);
  943. return false;
  944. }
  945. bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
  946. {
  947. bool ret = __skip_sstep(auprobe, regs);
  948. if (ret && (regs->flags & X86_EFLAGS_TF))
  949. send_sig(SIGTRAP, current, 0);
  950. return ret;
  951. }
  952. unsigned long
  953. arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs)
  954. {
  955. int rasize = sizeof_long(regs), nleft;
  956. unsigned long orig_ret_vaddr = 0; /* clear high bits for 32-bit apps */
  957. if (copy_from_user(&orig_ret_vaddr, (void __user *)regs->sp, rasize))
  958. return -1;
  959. /* check whether address has been already hijacked */
  960. if (orig_ret_vaddr == trampoline_vaddr)
  961. return orig_ret_vaddr;
  962. nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);
  963. if (likely(!nleft))
  964. return orig_ret_vaddr;
  965. if (nleft != rasize) {
  966. pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n",
  967. current->pid, regs->sp, regs->ip);
  968. force_sig(SIGSEGV);
  969. }
  970. return -1;
  971. }
  972. bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
  973. struct pt_regs *regs)
  974. {
  975. if (ctx == RP_CHECK_CALL) /* sp was just decremented by "call" insn */
  976. return regs->sp < ret->stack;
  977. else
  978. return regs->sp <= ret->stack;
  979. }