uprobes.c 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * User-space Probes (UProbes)
  4. *
  5. * Copyright (C) IBM Corporation, 2008-2012
  6. * Authors:
  7. * Srikar Dronamraju
  8. * Jim Keniston
  9. * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
  10. */
  11. #include <linux/kernel.h>
  12. #include <linux/highmem.h>
  13. #include <linux/pagemap.h> /* read_mapping_page */
  14. #include <linux/slab.h>
  15. #include <linux/sched.h>
  16. #include <linux/sched/mm.h>
  17. #include <linux/sched/coredump.h>
  18. #include <linux/export.h>
  19. #include <linux/rmap.h> /* anon_vma_prepare */
  20. #include <linux/mmu_notifier.h> /* set_pte_at_notify */
  21. #include <linux/swap.h> /* try_to_free_swap */
  22. #include <linux/ptrace.h> /* user_enable_single_step */
  23. #include <linux/kdebug.h> /* notifier mechanism */
  24. #include "../../mm/internal.h" /* munlock_vma_page */
  25. #include <linux/percpu-rwsem.h>
  26. #include <linux/task_work.h>
  27. #include <linux/shmem_fs.h>
  28. #include <linux/khugepaged.h>
  29. #include <linux/uprobes.h>
  30. #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
  31. #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
  32. static struct rb_root uprobes_tree = RB_ROOT;
  33. /*
  34. * allows us to skip the uprobe_mmap if there are no uprobe events active
  35. * at this time. Probably a fine grained per inode count is better?
  36. */
  37. #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
  38. static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
  39. #define UPROBES_HASH_SZ 13
  40. /* serialize uprobe->pending_list */
  41. static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
  42. #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
  43. DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
  44. /* Have a copy of original instruction */
  45. #define UPROBE_COPY_INSN 0
  46. struct uprobe {
  47. struct rb_node rb_node; /* node in the rb tree */
  48. refcount_t ref;
  49. struct rw_semaphore register_rwsem;
  50. struct rw_semaphore consumer_rwsem;
  51. struct list_head pending_list;
  52. struct uprobe_consumer *consumers;
  53. struct inode *inode; /* Also hold a ref to inode */
  54. loff_t offset;
  55. loff_t ref_ctr_offset;
  56. unsigned long flags;
  57. /*
  58. * The generic code assumes that it has two members of unknown type
  59. * owned by the arch-specific code:
  60. *
  61. * insn - copy_insn() saves the original instruction here for
  62. * arch_uprobe_analyze_insn().
  63. *
  64. * ixol - potentially modified instruction to execute out of
  65. * line, copied to xol_area by xol_get_insn_slot().
  66. */
  67. struct arch_uprobe arch;
  68. };
  69. struct delayed_uprobe {
  70. struct list_head list;
  71. struct uprobe *uprobe;
  72. struct mm_struct *mm;
  73. };
  74. static DEFINE_MUTEX(delayed_uprobe_lock);
  75. static LIST_HEAD(delayed_uprobe_list);
  76. /*
  77. * Execute out of line area: anonymous executable mapping installed
  78. * by the probed task to execute the copy of the original instruction
  79. * mangled by set_swbp().
  80. *
  81. * On a breakpoint hit, thread contests for a slot. It frees the
  82. * slot after singlestep. Currently a fixed number of slots are
  83. * allocated.
  84. */
  85. struct xol_area {
  86. wait_queue_head_t wq; /* if all slots are busy */
  87. atomic_t slot_count; /* number of in-use slots */
  88. unsigned long *bitmap; /* 0 = free slot */
  89. struct vm_special_mapping xol_mapping;
  90. struct page *pages[2];
  91. /*
  92. * We keep the vma's vm_start rather than a pointer to the vma
  93. * itself. The probed process or a naughty kernel module could make
  94. * the vma go away, and we must handle that reasonably gracefully.
  95. */
  96. unsigned long vaddr; /* Page(s) of instruction slots */
  97. };
  98. /*
  99. * valid_vma: Verify if the specified vma is an executable vma
  100. * Relax restrictions while unregistering: vm_flags might have
  101. * changed after breakpoint was inserted.
  102. * - is_register: indicates if we are in register context.
  103. * - Return 1 if the specified virtual address is in an
  104. * executable vma.
  105. */
  106. static bool valid_vma(struct vm_area_struct *vma, bool is_register)
  107. {
  108. vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
  109. if (is_register)
  110. flags |= VM_WRITE;
  111. return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
  112. }
  113. static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
  114. {
  115. return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
  116. }
  117. static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
  118. {
  119. return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
  120. }
  121. /**
  122. * __replace_page - replace page in vma by new page.
  123. * based on replace_page in mm/ksm.c
  124. *
  125. * @vma: vma that holds the pte pointing to page
  126. * @addr: address the old @page is mapped at
  127. * @old_page: the page we are replacing by new_page
  128. * @new_page: the modified page we replace page by
  129. *
  130. * If @new_page is NULL, only unmap @old_page.
  131. *
  132. * Returns 0 on success, negative error code otherwise.
  133. */
  134. static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
  135. struct page *old_page, struct page *new_page)
  136. {
  137. struct mm_struct *mm = vma->vm_mm;
  138. struct page_vma_mapped_walk pvmw = {
  139. .page = compound_head(old_page),
  140. .vma = vma,
  141. .address = addr,
  142. };
  143. int err;
  144. struct mmu_notifier_range range;
  145. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
  146. addr + PAGE_SIZE);
  147. if (new_page) {
  148. err = mem_cgroup_charge(new_page, vma->vm_mm, GFP_KERNEL);
  149. if (err)
  150. return err;
  151. }
  152. /* For try_to_free_swap() and munlock_vma_page() below */
  153. lock_page(old_page);
  154. mmu_notifier_invalidate_range_start(&range);
  155. err = -EAGAIN;
  156. if (!page_vma_mapped_walk(&pvmw))
  157. goto unlock;
  158. VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
  159. if (new_page) {
  160. get_page(new_page);
  161. page_add_new_anon_rmap(new_page, vma, addr, false);
  162. lru_cache_add_inactive_or_unevictable(new_page, vma);
  163. } else
  164. /* no new page, just dec_mm_counter for old_page */
  165. dec_mm_counter(mm, MM_ANONPAGES);
  166. if (!PageAnon(old_page)) {
  167. dec_mm_counter(mm, mm_counter_file(old_page));
  168. inc_mm_counter(mm, MM_ANONPAGES);
  169. }
  170. flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
  171. ptep_clear_flush_notify(vma, addr, pvmw.pte);
  172. if (new_page)
  173. set_pte_at_notify(mm, addr, pvmw.pte,
  174. mk_pte(new_page, vma->vm_page_prot));
  175. page_remove_rmap(old_page, false);
  176. if (!page_mapped(old_page))
  177. try_to_free_swap(old_page);
  178. page_vma_mapped_walk_done(&pvmw);
  179. if ((vma->vm_flags & VM_LOCKED) && !PageCompound(old_page))
  180. munlock_vma_page(old_page);
  181. put_page(old_page);
  182. err = 0;
  183. unlock:
  184. mmu_notifier_invalidate_range_end(&range);
  185. unlock_page(old_page);
  186. return err;
  187. }
  188. /**
  189. * is_swbp_insn - check if instruction is breakpoint instruction.
  190. * @insn: instruction to be checked.
  191. * Default implementation of is_swbp_insn
  192. * Returns true if @insn is a breakpoint instruction.
  193. */
  194. bool __weak is_swbp_insn(uprobe_opcode_t *insn)
  195. {
  196. return *insn == UPROBE_SWBP_INSN;
  197. }
  198. /**
  199. * is_trap_insn - check if instruction is breakpoint instruction.
  200. * @insn: instruction to be checked.
  201. * Default implementation of is_trap_insn
  202. * Returns true if @insn is a breakpoint instruction.
  203. *
  204. * This function is needed for the case where an architecture has multiple
  205. * trap instructions (like powerpc).
  206. */
  207. bool __weak is_trap_insn(uprobe_opcode_t *insn)
  208. {
  209. return is_swbp_insn(insn);
  210. }
  211. static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
  212. {
  213. void *kaddr = kmap_atomic(page);
  214. memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
  215. kunmap_atomic(kaddr);
  216. }
  217. static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
  218. {
  219. void *kaddr = kmap_atomic(page);
  220. memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
  221. kunmap_atomic(kaddr);
  222. }
  223. static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
  224. {
  225. uprobe_opcode_t old_opcode;
  226. bool is_swbp;
  227. /*
  228. * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
  229. * We do not check if it is any other 'trap variant' which could
  230. * be conditional trap instruction such as the one powerpc supports.
  231. *
  232. * The logic is that we do not care if the underlying instruction
  233. * is a trap variant; uprobes always wins over any other (gdb)
  234. * breakpoint.
  235. */
  236. copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
  237. is_swbp = is_swbp_insn(&old_opcode);
  238. if (is_swbp_insn(new_opcode)) {
  239. if (is_swbp) /* register: already installed? */
  240. return 0;
  241. } else {
  242. if (!is_swbp) /* unregister: was it changed by us? */
  243. return 0;
  244. }
  245. return 1;
  246. }
  247. static struct delayed_uprobe *
  248. delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
  249. {
  250. struct delayed_uprobe *du;
  251. list_for_each_entry(du, &delayed_uprobe_list, list)
  252. if (du->uprobe == uprobe && du->mm == mm)
  253. return du;
  254. return NULL;
  255. }
  256. static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
  257. {
  258. struct delayed_uprobe *du;
  259. if (delayed_uprobe_check(uprobe, mm))
  260. return 0;
  261. du = kzalloc(sizeof(*du), GFP_KERNEL);
  262. if (!du)
  263. return -ENOMEM;
  264. du->uprobe = uprobe;
  265. du->mm = mm;
  266. list_add(&du->list, &delayed_uprobe_list);
  267. return 0;
  268. }
  269. static void delayed_uprobe_delete(struct delayed_uprobe *du)
  270. {
  271. if (WARN_ON(!du))
  272. return;
  273. list_del(&du->list);
  274. kfree(du);
  275. }
  276. static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
  277. {
  278. struct list_head *pos, *q;
  279. struct delayed_uprobe *du;
  280. if (!uprobe && !mm)
  281. return;
  282. list_for_each_safe(pos, q, &delayed_uprobe_list) {
  283. du = list_entry(pos, struct delayed_uprobe, list);
  284. if (uprobe && du->uprobe != uprobe)
  285. continue;
  286. if (mm && du->mm != mm)
  287. continue;
  288. delayed_uprobe_delete(du);
  289. }
  290. }
  291. static bool valid_ref_ctr_vma(struct uprobe *uprobe,
  292. struct vm_area_struct *vma)
  293. {
  294. unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
  295. return uprobe->ref_ctr_offset &&
  296. vma->vm_file &&
  297. file_inode(vma->vm_file) == uprobe->inode &&
  298. (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
  299. vma->vm_start <= vaddr &&
  300. vma->vm_end > vaddr;
  301. }
  302. static struct vm_area_struct *
  303. find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
  304. {
  305. struct vm_area_struct *tmp;
  306. for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
  307. if (valid_ref_ctr_vma(uprobe, tmp))
  308. return tmp;
  309. return NULL;
  310. }
  311. static int
  312. __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
  313. {
  314. void *kaddr;
  315. struct page *page;
  316. struct vm_area_struct *vma;
  317. int ret;
  318. short *ptr;
  319. if (!vaddr || !d)
  320. return -EINVAL;
  321. ret = get_user_pages_remote(mm, vaddr, 1,
  322. FOLL_WRITE, &page, &vma, NULL);
  323. if (unlikely(ret <= 0)) {
  324. /*
  325. * We are asking for 1 page. If get_user_pages_remote() fails,
  326. * it may return 0, in that case we have to return error.
  327. */
  328. return ret == 0 ? -EBUSY : ret;
  329. }
  330. kaddr = kmap_atomic(page);
  331. ptr = kaddr + (vaddr & ~PAGE_MASK);
  332. if (unlikely(*ptr + d < 0)) {
  333. pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
  334. "curr val: %d, delta: %d\n", vaddr, *ptr, d);
  335. ret = -EINVAL;
  336. goto out;
  337. }
  338. *ptr += d;
  339. ret = 0;
  340. out:
  341. kunmap_atomic(kaddr);
  342. put_page(page);
  343. return ret;
  344. }
  345. static void update_ref_ctr_warn(struct uprobe *uprobe,
  346. struct mm_struct *mm, short d)
  347. {
  348. pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
  349. "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
  350. d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
  351. (unsigned long long) uprobe->offset,
  352. (unsigned long long) uprobe->ref_ctr_offset, mm);
  353. }
  354. static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
  355. short d)
  356. {
  357. struct vm_area_struct *rc_vma;
  358. unsigned long rc_vaddr;
  359. int ret = 0;
  360. rc_vma = find_ref_ctr_vma(uprobe, mm);
  361. if (rc_vma) {
  362. rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
  363. ret = __update_ref_ctr(mm, rc_vaddr, d);
  364. if (ret)
  365. update_ref_ctr_warn(uprobe, mm, d);
  366. if (d > 0)
  367. return ret;
  368. }
  369. mutex_lock(&delayed_uprobe_lock);
  370. if (d > 0)
  371. ret = delayed_uprobe_add(uprobe, mm);
  372. else
  373. delayed_uprobe_remove(uprobe, mm);
  374. mutex_unlock(&delayed_uprobe_lock);
  375. return ret;
  376. }
  377. /*
  378. * NOTE:
  379. * Expect the breakpoint instruction to be the smallest size instruction for
  380. * the architecture. If an arch has variable length instruction and the
  381. * breakpoint instruction is not of the smallest length instruction
  382. * supported by that architecture then we need to modify is_trap_at_addr and
  383. * uprobe_write_opcode accordingly. This would never be a problem for archs
  384. * that have fixed length instructions.
  385. *
  386. * uprobe_write_opcode - write the opcode at a given virtual address.
  387. * @mm: the probed process address space.
  388. * @vaddr: the virtual address to store the opcode.
  389. * @opcode: opcode to be written at @vaddr.
  390. *
  391. * Called with mm->mmap_lock held for write.
  392. * Return 0 (success) or a negative errno.
  393. */
  394. int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
  395. unsigned long vaddr, uprobe_opcode_t opcode)
  396. {
  397. struct uprobe *uprobe;
  398. struct page *old_page, *new_page;
  399. struct vm_area_struct *vma;
  400. int ret, is_register, ref_ctr_updated = 0;
  401. bool orig_page_huge = false;
  402. unsigned int gup_flags = FOLL_FORCE;
  403. is_register = is_swbp_insn(&opcode);
  404. uprobe = container_of(auprobe, struct uprobe, arch);
  405. retry:
  406. if (is_register)
  407. gup_flags |= FOLL_SPLIT_PMD;
  408. /* Read the page with vaddr into memory */
  409. ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
  410. &old_page, &vma, NULL);
  411. if (ret <= 0)
  412. return ret;
  413. ret = verify_opcode(old_page, vaddr, &opcode);
  414. if (ret <= 0)
  415. goto put_old;
  416. if (WARN(!is_register && PageCompound(old_page),
  417. "uprobe unregister should never work on compound page\n")) {
  418. ret = -EINVAL;
  419. goto put_old;
  420. }
  421. /* We are going to replace instruction, update ref_ctr. */
  422. if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
  423. ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
  424. if (ret)
  425. goto put_old;
  426. ref_ctr_updated = 1;
  427. }
  428. ret = 0;
  429. if (!is_register && !PageAnon(old_page))
  430. goto put_old;
  431. ret = anon_vma_prepare(vma);
  432. if (ret)
  433. goto put_old;
  434. ret = -ENOMEM;
  435. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
  436. if (!new_page)
  437. goto put_old;
  438. __SetPageUptodate(new_page);
  439. copy_highpage(new_page, old_page);
  440. copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
  441. if (!is_register) {
  442. struct page *orig_page;
  443. pgoff_t index;
  444. VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
  445. index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
  446. orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
  447. index);
  448. if (orig_page) {
  449. if (PageUptodate(orig_page) &&
  450. pages_identical(new_page, orig_page)) {
  451. /* let go new_page */
  452. put_page(new_page);
  453. new_page = NULL;
  454. if (PageCompound(orig_page))
  455. orig_page_huge = true;
  456. }
  457. put_page(orig_page);
  458. }
  459. }
  460. ret = __replace_page(vma, vaddr, old_page, new_page);
  461. if (new_page)
  462. put_page(new_page);
  463. put_old:
  464. put_page(old_page);
  465. if (unlikely(ret == -EAGAIN))
  466. goto retry;
  467. /* Revert back reference counter if instruction update failed. */
  468. if (ret && is_register && ref_ctr_updated)
  469. update_ref_ctr(uprobe, mm, -1);
  470. /* try collapse pmd for compound page */
  471. if (!ret && orig_page_huge)
  472. collapse_pte_mapped_thp(mm, vaddr);
  473. return ret;
  474. }
  475. /**
  476. * set_swbp - store breakpoint at a given address.
  477. * @auprobe: arch specific probepoint information.
  478. * @mm: the probed process address space.
  479. * @vaddr: the virtual address to insert the opcode.
  480. *
  481. * For mm @mm, store the breakpoint instruction at @vaddr.
  482. * Return 0 (success) or a negative errno.
  483. */
  484. int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
  485. {
  486. return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
  487. }
  488. /**
  489. * set_orig_insn - Restore the original instruction.
  490. * @mm: the probed process address space.
  491. * @auprobe: arch specific probepoint information.
  492. * @vaddr: the virtual address to insert the opcode.
  493. *
  494. * For mm @mm, restore the original opcode (opcode) at @vaddr.
  495. * Return 0 (success) or a negative errno.
  496. */
  497. int __weak
  498. set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
  499. {
  500. return uprobe_write_opcode(auprobe, mm, vaddr,
  501. *(uprobe_opcode_t *)&auprobe->insn);
  502. }
  503. static struct uprobe *get_uprobe(struct uprobe *uprobe)
  504. {
  505. refcount_inc(&uprobe->ref);
  506. return uprobe;
  507. }
  508. static void put_uprobe(struct uprobe *uprobe)
  509. {
  510. if (refcount_dec_and_test(&uprobe->ref)) {
  511. /*
  512. * If application munmap(exec_vma) before uprobe_unregister()
  513. * gets called, we don't get a chance to remove uprobe from
  514. * delayed_uprobe_list from remove_breakpoint(). Do it here.
  515. */
  516. mutex_lock(&delayed_uprobe_lock);
  517. delayed_uprobe_remove(uprobe, NULL);
  518. mutex_unlock(&delayed_uprobe_lock);
  519. kfree(uprobe);
  520. }
  521. }
  522. static int match_uprobe(struct uprobe *l, struct uprobe *r)
  523. {
  524. if (l->inode < r->inode)
  525. return -1;
  526. if (l->inode > r->inode)
  527. return 1;
  528. if (l->offset < r->offset)
  529. return -1;
  530. if (l->offset > r->offset)
  531. return 1;
  532. return 0;
  533. }
  534. static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
  535. {
  536. struct uprobe u = { .inode = inode, .offset = offset };
  537. struct rb_node *n = uprobes_tree.rb_node;
  538. struct uprobe *uprobe;
  539. int match;
  540. while (n) {
  541. uprobe = rb_entry(n, struct uprobe, rb_node);
  542. match = match_uprobe(&u, uprobe);
  543. if (!match)
  544. return get_uprobe(uprobe);
  545. if (match < 0)
  546. n = n->rb_left;
  547. else
  548. n = n->rb_right;
  549. }
  550. return NULL;
  551. }
  552. /*
  553. * Find a uprobe corresponding to a given inode:offset
  554. * Acquires uprobes_treelock
  555. */
  556. static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
  557. {
  558. struct uprobe *uprobe;
  559. spin_lock(&uprobes_treelock);
  560. uprobe = __find_uprobe(inode, offset);
  561. spin_unlock(&uprobes_treelock);
  562. return uprobe;
  563. }
  564. static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
  565. {
  566. struct rb_node **p = &uprobes_tree.rb_node;
  567. struct rb_node *parent = NULL;
  568. struct uprobe *u;
  569. int match;
  570. while (*p) {
  571. parent = *p;
  572. u = rb_entry(parent, struct uprobe, rb_node);
  573. match = match_uprobe(uprobe, u);
  574. if (!match)
  575. return get_uprobe(u);
  576. if (match < 0)
  577. p = &parent->rb_left;
  578. else
  579. p = &parent->rb_right;
  580. }
  581. u = NULL;
  582. rb_link_node(&uprobe->rb_node, parent, p);
  583. rb_insert_color(&uprobe->rb_node, &uprobes_tree);
  584. /* get access + creation ref */
  585. refcount_set(&uprobe->ref, 2);
  586. return u;
  587. }
  588. /*
  589. * Acquire uprobes_treelock.
  590. * Matching uprobe already exists in rbtree;
  591. * increment (access refcount) and return the matching uprobe.
  592. *
  593. * No matching uprobe; insert the uprobe in rb_tree;
  594. * get a double refcount (access + creation) and return NULL.
  595. */
  596. static struct uprobe *insert_uprobe(struct uprobe *uprobe)
  597. {
  598. struct uprobe *u;
  599. spin_lock(&uprobes_treelock);
  600. u = __insert_uprobe(uprobe);
  601. spin_unlock(&uprobes_treelock);
  602. return u;
  603. }
  604. static void
  605. ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
  606. {
  607. pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
  608. "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
  609. uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
  610. (unsigned long long) cur_uprobe->ref_ctr_offset,
  611. (unsigned long long) uprobe->ref_ctr_offset);
  612. }
  613. static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
  614. loff_t ref_ctr_offset)
  615. {
  616. struct uprobe *uprobe, *cur_uprobe;
  617. uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
  618. if (!uprobe)
  619. return NULL;
  620. uprobe->inode = inode;
  621. uprobe->offset = offset;
  622. uprobe->ref_ctr_offset = ref_ctr_offset;
  623. init_rwsem(&uprobe->register_rwsem);
  624. init_rwsem(&uprobe->consumer_rwsem);
  625. /* add to uprobes_tree, sorted on inode:offset */
  626. cur_uprobe = insert_uprobe(uprobe);
  627. /* a uprobe exists for this inode:offset combination */
  628. if (cur_uprobe) {
  629. if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
  630. ref_ctr_mismatch_warn(cur_uprobe, uprobe);
  631. put_uprobe(cur_uprobe);
  632. kfree(uprobe);
  633. return ERR_PTR(-EINVAL);
  634. }
  635. kfree(uprobe);
  636. uprobe = cur_uprobe;
  637. }
  638. return uprobe;
  639. }
  640. static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
  641. {
  642. down_write(&uprobe->consumer_rwsem);
  643. uc->next = uprobe->consumers;
  644. uprobe->consumers = uc;
  645. up_write(&uprobe->consumer_rwsem);
  646. }
  647. /*
  648. * For uprobe @uprobe, delete the consumer @uc.
  649. * Return true if the @uc is deleted successfully
  650. * or return false.
  651. */
  652. static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
  653. {
  654. struct uprobe_consumer **con;
  655. bool ret = false;
  656. down_write(&uprobe->consumer_rwsem);
  657. for (con = &uprobe->consumers; *con; con = &(*con)->next) {
  658. if (*con == uc) {
  659. *con = uc->next;
  660. ret = true;
  661. break;
  662. }
  663. }
  664. up_write(&uprobe->consumer_rwsem);
  665. return ret;
  666. }
  667. static int __copy_insn(struct address_space *mapping, struct file *filp,
  668. void *insn, int nbytes, loff_t offset)
  669. {
  670. struct page *page;
  671. /*
  672. * Ensure that the page that has the original instruction is populated
  673. * and in page-cache. If ->readpage == NULL it must be shmem_mapping(),
  674. * see uprobe_register().
  675. */
  676. if (mapping->a_ops->readpage)
  677. page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
  678. else
  679. page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  680. if (IS_ERR(page))
  681. return PTR_ERR(page);
  682. copy_from_page(page, offset, insn, nbytes);
  683. put_page(page);
  684. return 0;
  685. }
  686. static int copy_insn(struct uprobe *uprobe, struct file *filp)
  687. {
  688. struct address_space *mapping = uprobe->inode->i_mapping;
  689. loff_t offs = uprobe->offset;
  690. void *insn = &uprobe->arch.insn;
  691. int size = sizeof(uprobe->arch.insn);
  692. int len, err = -EIO;
  693. /* Copy only available bytes, -EIO if nothing was read */
  694. do {
  695. if (offs >= i_size_read(uprobe->inode))
  696. break;
  697. len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
  698. err = __copy_insn(mapping, filp, insn, len, offs);
  699. if (err)
  700. break;
  701. insn += len;
  702. offs += len;
  703. size -= len;
  704. } while (size);
  705. return err;
  706. }
  707. static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
  708. struct mm_struct *mm, unsigned long vaddr)
  709. {
  710. int ret = 0;
  711. if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
  712. return ret;
  713. /* TODO: move this into _register, until then we abuse this sem. */
  714. down_write(&uprobe->consumer_rwsem);
  715. if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
  716. goto out;
  717. ret = copy_insn(uprobe, file);
  718. if (ret)
  719. goto out;
  720. ret = -ENOTSUPP;
  721. if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
  722. goto out;
  723. ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
  724. if (ret)
  725. goto out;
  726. smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
  727. set_bit(UPROBE_COPY_INSN, &uprobe->flags);
  728. out:
  729. up_write(&uprobe->consumer_rwsem);
  730. return ret;
  731. }
  732. static inline bool consumer_filter(struct uprobe_consumer *uc,
  733. enum uprobe_filter_ctx ctx, struct mm_struct *mm)
  734. {
  735. return !uc->filter || uc->filter(uc, ctx, mm);
  736. }
  737. static bool filter_chain(struct uprobe *uprobe,
  738. enum uprobe_filter_ctx ctx, struct mm_struct *mm)
  739. {
  740. struct uprobe_consumer *uc;
  741. bool ret = false;
  742. down_read(&uprobe->consumer_rwsem);
  743. for (uc = uprobe->consumers; uc; uc = uc->next) {
  744. ret = consumer_filter(uc, ctx, mm);
  745. if (ret)
  746. break;
  747. }
  748. up_read(&uprobe->consumer_rwsem);
  749. return ret;
  750. }
  751. static int
  752. install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
  753. struct vm_area_struct *vma, unsigned long vaddr)
  754. {
  755. bool first_uprobe;
  756. int ret;
  757. ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
  758. if (ret)
  759. return ret;
  760. /*
  761. * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
  762. * the task can hit this breakpoint right after __replace_page().
  763. */
  764. first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
  765. if (first_uprobe)
  766. set_bit(MMF_HAS_UPROBES, &mm->flags);
  767. ret = set_swbp(&uprobe->arch, mm, vaddr);
  768. if (!ret)
  769. clear_bit(MMF_RECALC_UPROBES, &mm->flags);
  770. else if (first_uprobe)
  771. clear_bit(MMF_HAS_UPROBES, &mm->flags);
  772. return ret;
  773. }
  774. static int
  775. remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
  776. {
  777. set_bit(MMF_RECALC_UPROBES, &mm->flags);
  778. return set_orig_insn(&uprobe->arch, mm, vaddr);
  779. }
  780. static inline bool uprobe_is_active(struct uprobe *uprobe)
  781. {
  782. return !RB_EMPTY_NODE(&uprobe->rb_node);
  783. }
  784. /*
  785. * There could be threads that have already hit the breakpoint. They
  786. * will recheck the current insn and restart if find_uprobe() fails.
  787. * See find_active_uprobe().
  788. */
  789. static void delete_uprobe(struct uprobe *uprobe)
  790. {
  791. if (WARN_ON(!uprobe_is_active(uprobe)))
  792. return;
  793. spin_lock(&uprobes_treelock);
  794. rb_erase(&uprobe->rb_node, &uprobes_tree);
  795. spin_unlock(&uprobes_treelock);
  796. RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
  797. put_uprobe(uprobe);
  798. }
  799. struct map_info {
  800. struct map_info *next;
  801. struct mm_struct *mm;
  802. unsigned long vaddr;
  803. };
  804. static inline struct map_info *free_map_info(struct map_info *info)
  805. {
  806. struct map_info *next = info->next;
  807. kfree(info);
  808. return next;
  809. }
  810. static struct map_info *
  811. build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
  812. {
  813. unsigned long pgoff = offset >> PAGE_SHIFT;
  814. struct vm_area_struct *vma;
  815. struct map_info *curr = NULL;
  816. struct map_info *prev = NULL;
  817. struct map_info *info;
  818. int more = 0;
  819. again:
  820. i_mmap_lock_read(mapping);
  821. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  822. if (!valid_vma(vma, is_register))
  823. continue;
  824. if (!prev && !more) {
  825. /*
  826. * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
  827. * reclaim. This is optimistic, no harm done if it fails.
  828. */
  829. prev = kmalloc(sizeof(struct map_info),
  830. GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
  831. if (prev)
  832. prev->next = NULL;
  833. }
  834. if (!prev) {
  835. more++;
  836. continue;
  837. }
  838. if (!mmget_not_zero(vma->vm_mm))
  839. continue;
  840. info = prev;
  841. prev = prev->next;
  842. info->next = curr;
  843. curr = info;
  844. info->mm = vma->vm_mm;
  845. info->vaddr = offset_to_vaddr(vma, offset);
  846. }
  847. i_mmap_unlock_read(mapping);
  848. if (!more)
  849. goto out;
  850. prev = curr;
  851. while (curr) {
  852. mmput(curr->mm);
  853. curr = curr->next;
  854. }
  855. do {
  856. info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
  857. if (!info) {
  858. curr = ERR_PTR(-ENOMEM);
  859. goto out;
  860. }
  861. info->next = prev;
  862. prev = info;
  863. } while (--more);
  864. goto again;
  865. out:
  866. while (prev)
  867. prev = free_map_info(prev);
  868. return curr;
  869. }
  870. static int
  871. register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
  872. {
  873. bool is_register = !!new;
  874. struct map_info *info;
  875. int err = 0;
  876. percpu_down_write(&dup_mmap_sem);
  877. info = build_map_info(uprobe->inode->i_mapping,
  878. uprobe->offset, is_register);
  879. if (IS_ERR(info)) {
  880. err = PTR_ERR(info);
  881. goto out;
  882. }
  883. while (info) {
  884. struct mm_struct *mm = info->mm;
  885. struct vm_area_struct *vma;
  886. if (err && is_register)
  887. goto free;
  888. mmap_write_lock(mm);
  889. vma = find_vma(mm, info->vaddr);
  890. if (!vma || !valid_vma(vma, is_register) ||
  891. file_inode(vma->vm_file) != uprobe->inode)
  892. goto unlock;
  893. if (vma->vm_start > info->vaddr ||
  894. vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
  895. goto unlock;
  896. if (is_register) {
  897. /* consult only the "caller", new consumer. */
  898. if (consumer_filter(new,
  899. UPROBE_FILTER_REGISTER, mm))
  900. err = install_breakpoint(uprobe, mm, vma, info->vaddr);
  901. } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
  902. if (!filter_chain(uprobe,
  903. UPROBE_FILTER_UNREGISTER, mm))
  904. err |= remove_breakpoint(uprobe, mm, info->vaddr);
  905. }
  906. unlock:
  907. mmap_write_unlock(mm);
  908. free:
  909. mmput(mm);
  910. info = free_map_info(info);
  911. }
  912. out:
  913. percpu_up_write(&dup_mmap_sem);
  914. return err;
  915. }
  916. static void
  917. __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
  918. {
  919. int err;
  920. if (WARN_ON(!consumer_del(uprobe, uc)))
  921. return;
  922. err = register_for_each_vma(uprobe, NULL);
  923. /* TODO : cant unregister? schedule a worker thread */
  924. if (!uprobe->consumers && !err)
  925. delete_uprobe(uprobe);
  926. }
  927. /*
  928. * uprobe_unregister - unregister an already registered probe.
  929. * @inode: the file in which the probe has to be removed.
  930. * @offset: offset from the start of the file.
  931. * @uc: identify which probe if multiple probes are colocated.
  932. */
  933. void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
  934. {
  935. struct uprobe *uprobe;
  936. uprobe = find_uprobe(inode, offset);
  937. if (WARN_ON(!uprobe))
  938. return;
  939. down_write(&uprobe->register_rwsem);
  940. __uprobe_unregister(uprobe, uc);
  941. up_write(&uprobe->register_rwsem);
  942. put_uprobe(uprobe);
  943. }
  944. EXPORT_SYMBOL_GPL(uprobe_unregister);
  945. /*
  946. * __uprobe_register - register a probe
  947. * @inode: the file in which the probe has to be placed.
  948. * @offset: offset from the start of the file.
  949. * @uc: information on howto handle the probe..
  950. *
  951. * Apart from the access refcount, __uprobe_register() takes a creation
  952. * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
  953. * inserted into the rbtree (i.e first consumer for a @inode:@offset
  954. * tuple). Creation refcount stops uprobe_unregister from freeing the
  955. * @uprobe even before the register operation is complete. Creation
  956. * refcount is released when the last @uc for the @uprobe
  957. * unregisters. Caller of __uprobe_register() is required to keep @inode
  958. * (and the containing mount) referenced.
  959. *
  960. * Return errno if it cannot successully install probes
  961. * else return 0 (success)
  962. */
  963. static int __uprobe_register(struct inode *inode, loff_t offset,
  964. loff_t ref_ctr_offset, struct uprobe_consumer *uc)
  965. {
  966. struct uprobe *uprobe;
  967. int ret;
  968. /* Uprobe must have at least one set consumer */
  969. if (!uc->handler && !uc->ret_handler)
  970. return -EINVAL;
  971. /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
  972. if (!inode->i_mapping->a_ops->readpage && !shmem_mapping(inode->i_mapping))
  973. return -EIO;
  974. /* Racy, just to catch the obvious mistakes */
  975. if (offset > i_size_read(inode))
  976. return -EINVAL;
  977. /*
  978. * This ensures that copy_from_page(), copy_to_page() and
  979. * __update_ref_ctr() can't cross page boundary.
  980. */
  981. if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
  982. return -EINVAL;
  983. if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
  984. return -EINVAL;
  985. retry:
  986. uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
  987. if (!uprobe)
  988. return -ENOMEM;
  989. if (IS_ERR(uprobe))
  990. return PTR_ERR(uprobe);
  991. /*
  992. * We can race with uprobe_unregister()->delete_uprobe().
  993. * Check uprobe_is_active() and retry if it is false.
  994. */
  995. down_write(&uprobe->register_rwsem);
  996. ret = -EAGAIN;
  997. if (likely(uprobe_is_active(uprobe))) {
  998. consumer_add(uprobe, uc);
  999. ret = register_for_each_vma(uprobe, uc);
  1000. if (ret)
  1001. __uprobe_unregister(uprobe, uc);
  1002. }
  1003. up_write(&uprobe->register_rwsem);
  1004. put_uprobe(uprobe);
  1005. if (unlikely(ret == -EAGAIN))
  1006. goto retry;
  1007. return ret;
  1008. }
  1009. int uprobe_register(struct inode *inode, loff_t offset,
  1010. struct uprobe_consumer *uc)
  1011. {
  1012. return __uprobe_register(inode, offset, 0, uc);
  1013. }
  1014. EXPORT_SYMBOL_GPL(uprobe_register);
  1015. int uprobe_register_refctr(struct inode *inode, loff_t offset,
  1016. loff_t ref_ctr_offset, struct uprobe_consumer *uc)
  1017. {
  1018. return __uprobe_register(inode, offset, ref_ctr_offset, uc);
  1019. }
  1020. EXPORT_SYMBOL_GPL(uprobe_register_refctr);
  1021. /*
  1022. * uprobe_apply - unregister an already registered probe.
  1023. * @inode: the file in which the probe has to be removed.
  1024. * @offset: offset from the start of the file.
  1025. * @uc: consumer which wants to add more or remove some breakpoints
  1026. * @add: add or remove the breakpoints
  1027. */
  1028. int uprobe_apply(struct inode *inode, loff_t offset,
  1029. struct uprobe_consumer *uc, bool add)
  1030. {
  1031. struct uprobe *uprobe;
  1032. struct uprobe_consumer *con;
  1033. int ret = -ENOENT;
  1034. uprobe = find_uprobe(inode, offset);
  1035. if (WARN_ON(!uprobe))
  1036. return ret;
  1037. down_write(&uprobe->register_rwsem);
  1038. for (con = uprobe->consumers; con && con != uc ; con = con->next)
  1039. ;
  1040. if (con)
  1041. ret = register_for_each_vma(uprobe, add ? uc : NULL);
  1042. up_write(&uprobe->register_rwsem);
  1043. put_uprobe(uprobe);
  1044. return ret;
  1045. }
  1046. static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
  1047. {
  1048. struct vm_area_struct *vma;
  1049. int err = 0;
  1050. mmap_read_lock(mm);
  1051. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  1052. unsigned long vaddr;
  1053. loff_t offset;
  1054. if (!valid_vma(vma, false) ||
  1055. file_inode(vma->vm_file) != uprobe->inode)
  1056. continue;
  1057. offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
  1058. if (uprobe->offset < offset ||
  1059. uprobe->offset >= offset + vma->vm_end - vma->vm_start)
  1060. continue;
  1061. vaddr = offset_to_vaddr(vma, uprobe->offset);
  1062. err |= remove_breakpoint(uprobe, mm, vaddr);
  1063. }
  1064. mmap_read_unlock(mm);
  1065. return err;
  1066. }
  1067. static struct rb_node *
  1068. find_node_in_range(struct inode *inode, loff_t min, loff_t max)
  1069. {
  1070. struct rb_node *n = uprobes_tree.rb_node;
  1071. while (n) {
  1072. struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
  1073. if (inode < u->inode) {
  1074. n = n->rb_left;
  1075. } else if (inode > u->inode) {
  1076. n = n->rb_right;
  1077. } else {
  1078. if (max < u->offset)
  1079. n = n->rb_left;
  1080. else if (min > u->offset)
  1081. n = n->rb_right;
  1082. else
  1083. break;
  1084. }
  1085. }
  1086. return n;
  1087. }
  1088. /*
  1089. * For a given range in vma, build a list of probes that need to be inserted.
  1090. */
  1091. static void build_probe_list(struct inode *inode,
  1092. struct vm_area_struct *vma,
  1093. unsigned long start, unsigned long end,
  1094. struct list_head *head)
  1095. {
  1096. loff_t min, max;
  1097. struct rb_node *n, *t;
  1098. struct uprobe *u;
  1099. INIT_LIST_HEAD(head);
  1100. min = vaddr_to_offset(vma, start);
  1101. max = min + (end - start) - 1;
  1102. spin_lock(&uprobes_treelock);
  1103. n = find_node_in_range(inode, min, max);
  1104. if (n) {
  1105. for (t = n; t; t = rb_prev(t)) {
  1106. u = rb_entry(t, struct uprobe, rb_node);
  1107. if (u->inode != inode || u->offset < min)
  1108. break;
  1109. list_add(&u->pending_list, head);
  1110. get_uprobe(u);
  1111. }
  1112. for (t = n; (t = rb_next(t)); ) {
  1113. u = rb_entry(t, struct uprobe, rb_node);
  1114. if (u->inode != inode || u->offset > max)
  1115. break;
  1116. list_add(&u->pending_list, head);
  1117. get_uprobe(u);
  1118. }
  1119. }
  1120. spin_unlock(&uprobes_treelock);
  1121. }
  1122. /* @vma contains reference counter, not the probed instruction. */
  1123. static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
  1124. {
  1125. struct list_head *pos, *q;
  1126. struct delayed_uprobe *du;
  1127. unsigned long vaddr;
  1128. int ret = 0, err = 0;
  1129. mutex_lock(&delayed_uprobe_lock);
  1130. list_for_each_safe(pos, q, &delayed_uprobe_list) {
  1131. du = list_entry(pos, struct delayed_uprobe, list);
  1132. if (du->mm != vma->vm_mm ||
  1133. !valid_ref_ctr_vma(du->uprobe, vma))
  1134. continue;
  1135. vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
  1136. ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
  1137. if (ret) {
  1138. update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
  1139. if (!err)
  1140. err = ret;
  1141. }
  1142. delayed_uprobe_delete(du);
  1143. }
  1144. mutex_unlock(&delayed_uprobe_lock);
  1145. return err;
  1146. }
  1147. /*
  1148. * Called from mmap_region/vma_adjust with mm->mmap_lock acquired.
  1149. *
  1150. * Currently we ignore all errors and always return 0, the callers
  1151. * can't handle the failure anyway.
  1152. */
  1153. int uprobe_mmap(struct vm_area_struct *vma)
  1154. {
  1155. struct list_head tmp_list;
  1156. struct uprobe *uprobe, *u;
  1157. struct inode *inode;
  1158. if (no_uprobe_events())
  1159. return 0;
  1160. if (vma->vm_file &&
  1161. (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
  1162. test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
  1163. delayed_ref_ctr_inc(vma);
  1164. if (!valid_vma(vma, true))
  1165. return 0;
  1166. inode = file_inode(vma->vm_file);
  1167. if (!inode)
  1168. return 0;
  1169. mutex_lock(uprobes_mmap_hash(inode));
  1170. build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
  1171. /*
  1172. * We can race with uprobe_unregister(), this uprobe can be already
  1173. * removed. But in this case filter_chain() must return false, all
  1174. * consumers have gone away.
  1175. */
  1176. list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
  1177. if (!fatal_signal_pending(current) &&
  1178. filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
  1179. unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
  1180. install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
  1181. }
  1182. put_uprobe(uprobe);
  1183. }
  1184. mutex_unlock(uprobes_mmap_hash(inode));
  1185. return 0;
  1186. }
  1187. static bool
  1188. vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  1189. {
  1190. loff_t min, max;
  1191. struct inode *inode;
  1192. struct rb_node *n;
  1193. inode = file_inode(vma->vm_file);
  1194. min = vaddr_to_offset(vma, start);
  1195. max = min + (end - start) - 1;
  1196. spin_lock(&uprobes_treelock);
  1197. n = find_node_in_range(inode, min, max);
  1198. spin_unlock(&uprobes_treelock);
  1199. return !!n;
  1200. }
  1201. /*
  1202. * Called in context of a munmap of a vma.
  1203. */
  1204. void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  1205. {
  1206. if (no_uprobe_events() || !valid_vma(vma, false))
  1207. return;
  1208. if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
  1209. return;
  1210. if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
  1211. test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
  1212. return;
  1213. if (vma_has_uprobes(vma, start, end))
  1214. set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
  1215. }
  1216. /* Slot allocation for XOL */
  1217. static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
  1218. {
  1219. struct vm_area_struct *vma;
  1220. int ret;
  1221. if (mmap_write_lock_killable(mm))
  1222. return -EINTR;
  1223. if (mm->uprobes_state.xol_area) {
  1224. ret = -EALREADY;
  1225. goto fail;
  1226. }
  1227. if (!area->vaddr) {
  1228. /* Try to map as high as possible, this is only a hint. */
  1229. area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
  1230. PAGE_SIZE, 0, 0);
  1231. if (IS_ERR_VALUE(area->vaddr)) {
  1232. ret = area->vaddr;
  1233. goto fail;
  1234. }
  1235. }
  1236. vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
  1237. VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
  1238. &area->xol_mapping);
  1239. if (IS_ERR(vma)) {
  1240. ret = PTR_ERR(vma);
  1241. goto fail;
  1242. }
  1243. ret = 0;
  1244. /* pairs with get_xol_area() */
  1245. smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
  1246. fail:
  1247. mmap_write_unlock(mm);
  1248. return ret;
  1249. }
  1250. static struct xol_area *__create_xol_area(unsigned long vaddr)
  1251. {
  1252. struct mm_struct *mm = current->mm;
  1253. uprobe_opcode_t insn = UPROBE_SWBP_INSN;
  1254. struct xol_area *area;
  1255. area = kmalloc(sizeof(*area), GFP_KERNEL);
  1256. if (unlikely(!area))
  1257. goto out;
  1258. area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
  1259. GFP_KERNEL);
  1260. if (!area->bitmap)
  1261. goto free_area;
  1262. area->xol_mapping.name = "[uprobes]";
  1263. area->xol_mapping.fault = NULL;
  1264. area->xol_mapping.pages = area->pages;
  1265. area->pages[0] = alloc_page(GFP_HIGHUSER);
  1266. if (!area->pages[0])
  1267. goto free_bitmap;
  1268. area->pages[1] = NULL;
  1269. area->vaddr = vaddr;
  1270. init_waitqueue_head(&area->wq);
  1271. /* Reserve the 1st slot for get_trampoline_vaddr() */
  1272. set_bit(0, area->bitmap);
  1273. atomic_set(&area->slot_count, 1);
  1274. arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
  1275. if (!xol_add_vma(mm, area))
  1276. return area;
  1277. __free_page(area->pages[0]);
  1278. free_bitmap:
  1279. kfree(area->bitmap);
  1280. free_area:
  1281. kfree(area);
  1282. out:
  1283. return NULL;
  1284. }
  1285. /*
  1286. * get_xol_area - Allocate process's xol_area if necessary.
  1287. * This area will be used for storing instructions for execution out of line.
  1288. *
  1289. * Returns the allocated area or NULL.
  1290. */
  1291. static struct xol_area *get_xol_area(void)
  1292. {
  1293. struct mm_struct *mm = current->mm;
  1294. struct xol_area *area;
  1295. if (!mm->uprobes_state.xol_area)
  1296. __create_xol_area(0);
  1297. /* Pairs with xol_add_vma() smp_store_release() */
  1298. area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
  1299. return area;
  1300. }
  1301. /*
  1302. * uprobe_clear_state - Free the area allocated for slots.
  1303. */
  1304. void uprobe_clear_state(struct mm_struct *mm)
  1305. {
  1306. struct xol_area *area = mm->uprobes_state.xol_area;
  1307. mutex_lock(&delayed_uprobe_lock);
  1308. delayed_uprobe_remove(NULL, mm);
  1309. mutex_unlock(&delayed_uprobe_lock);
  1310. if (!area)
  1311. return;
  1312. put_page(area->pages[0]);
  1313. kfree(area->bitmap);
  1314. kfree(area);
  1315. }
  1316. void uprobe_start_dup_mmap(void)
  1317. {
  1318. percpu_down_read(&dup_mmap_sem);
  1319. }
  1320. void uprobe_end_dup_mmap(void)
  1321. {
  1322. percpu_up_read(&dup_mmap_sem);
  1323. }
  1324. void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
  1325. {
  1326. if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
  1327. set_bit(MMF_HAS_UPROBES, &newmm->flags);
  1328. /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
  1329. set_bit(MMF_RECALC_UPROBES, &newmm->flags);
  1330. }
  1331. }
  1332. /*
  1333. * - search for a free slot.
  1334. */
  1335. static unsigned long xol_take_insn_slot(struct xol_area *area)
  1336. {
  1337. unsigned long slot_addr;
  1338. int slot_nr;
  1339. do {
  1340. slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
  1341. if (slot_nr < UINSNS_PER_PAGE) {
  1342. if (!test_and_set_bit(slot_nr, area->bitmap))
  1343. break;
  1344. slot_nr = UINSNS_PER_PAGE;
  1345. continue;
  1346. }
  1347. wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
  1348. } while (slot_nr >= UINSNS_PER_PAGE);
  1349. slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
  1350. atomic_inc(&area->slot_count);
  1351. return slot_addr;
  1352. }
  1353. /*
  1354. * xol_get_insn_slot - allocate a slot for xol.
  1355. * Returns the allocated slot address or 0.
  1356. */
  1357. static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
  1358. {
  1359. struct xol_area *area;
  1360. unsigned long xol_vaddr;
  1361. area = get_xol_area();
  1362. if (!area)
  1363. return 0;
  1364. xol_vaddr = xol_take_insn_slot(area);
  1365. if (unlikely(!xol_vaddr))
  1366. return 0;
  1367. arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
  1368. &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
  1369. return xol_vaddr;
  1370. }
  1371. /*
  1372. * xol_free_insn_slot - If slot was earlier allocated by
  1373. * @xol_get_insn_slot(), make the slot available for
  1374. * subsequent requests.
  1375. */
  1376. static void xol_free_insn_slot(struct task_struct *tsk)
  1377. {
  1378. struct xol_area *area;
  1379. unsigned long vma_end;
  1380. unsigned long slot_addr;
  1381. if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
  1382. return;
  1383. slot_addr = tsk->utask->xol_vaddr;
  1384. if (unlikely(!slot_addr))
  1385. return;
  1386. area = tsk->mm->uprobes_state.xol_area;
  1387. vma_end = area->vaddr + PAGE_SIZE;
  1388. if (area->vaddr <= slot_addr && slot_addr < vma_end) {
  1389. unsigned long offset;
  1390. int slot_nr;
  1391. offset = slot_addr - area->vaddr;
  1392. slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
  1393. if (slot_nr >= UINSNS_PER_PAGE)
  1394. return;
  1395. clear_bit(slot_nr, area->bitmap);
  1396. atomic_dec(&area->slot_count);
  1397. smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
  1398. if (waitqueue_active(&area->wq))
  1399. wake_up(&area->wq);
  1400. tsk->utask->xol_vaddr = 0;
  1401. }
  1402. }
  1403. void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
  1404. void *src, unsigned long len)
  1405. {
  1406. /* Initialize the slot */
  1407. copy_to_page(page, vaddr, src, len);
  1408. /*
  1409. * We probably need flush_icache_user_page() but it needs vma.
  1410. * This should work on most of architectures by default. If
  1411. * architecture needs to do something different it can define
  1412. * its own version of the function.
  1413. */
  1414. flush_dcache_page(page);
  1415. }
  1416. /**
  1417. * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
  1418. * @regs: Reflects the saved state of the task after it has hit a breakpoint
  1419. * instruction.
  1420. * Return the address of the breakpoint instruction.
  1421. */
  1422. unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
  1423. {
  1424. return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
  1425. }
  1426. unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
  1427. {
  1428. struct uprobe_task *utask = current->utask;
  1429. if (unlikely(utask && utask->active_uprobe))
  1430. return utask->vaddr;
  1431. return instruction_pointer(regs);
  1432. }
  1433. static struct return_instance *free_ret_instance(struct return_instance *ri)
  1434. {
  1435. struct return_instance *next = ri->next;
  1436. put_uprobe(ri->uprobe);
  1437. kfree(ri);
  1438. return next;
  1439. }
  1440. /*
  1441. * Called with no locks held.
  1442. * Called in context of an exiting or an exec-ing thread.
  1443. */
  1444. void uprobe_free_utask(struct task_struct *t)
  1445. {
  1446. struct uprobe_task *utask = t->utask;
  1447. struct return_instance *ri;
  1448. if (!utask)
  1449. return;
  1450. if (utask->active_uprobe)
  1451. put_uprobe(utask->active_uprobe);
  1452. ri = utask->return_instances;
  1453. while (ri)
  1454. ri = free_ret_instance(ri);
  1455. xol_free_insn_slot(t);
  1456. kfree(utask);
  1457. t->utask = NULL;
  1458. }
  1459. /*
  1460. * Allocate a uprobe_task object for the task if necessary.
  1461. * Called when the thread hits a breakpoint.
  1462. *
  1463. * Returns:
  1464. * - pointer to new uprobe_task on success
  1465. * - NULL otherwise
  1466. */
  1467. static struct uprobe_task *get_utask(void)
  1468. {
  1469. if (!current->utask)
  1470. current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
  1471. return current->utask;
  1472. }
  1473. static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
  1474. {
  1475. struct uprobe_task *n_utask;
  1476. struct return_instance **p, *o, *n;
  1477. n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
  1478. if (!n_utask)
  1479. return -ENOMEM;
  1480. t->utask = n_utask;
  1481. p = &n_utask->return_instances;
  1482. for (o = o_utask->return_instances; o; o = o->next) {
  1483. n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
  1484. if (!n)
  1485. return -ENOMEM;
  1486. *n = *o;
  1487. get_uprobe(n->uprobe);
  1488. n->next = NULL;
  1489. *p = n;
  1490. p = &n->next;
  1491. n_utask->depth++;
  1492. }
  1493. return 0;
  1494. }
  1495. static void uprobe_warn(struct task_struct *t, const char *msg)
  1496. {
  1497. pr_warn("uprobe: %s:%d failed to %s\n",
  1498. current->comm, current->pid, msg);
  1499. }
  1500. static void dup_xol_work(struct callback_head *work)
  1501. {
  1502. if (current->flags & PF_EXITING)
  1503. return;
  1504. if (!__create_xol_area(current->utask->dup_xol_addr) &&
  1505. !fatal_signal_pending(current))
  1506. uprobe_warn(current, "dup xol area");
  1507. }
  1508. /*
  1509. * Called in context of a new clone/fork from copy_process.
  1510. */
  1511. void uprobe_copy_process(struct task_struct *t, unsigned long flags)
  1512. {
  1513. struct uprobe_task *utask = current->utask;
  1514. struct mm_struct *mm = current->mm;
  1515. struct xol_area *area;
  1516. t->utask = NULL;
  1517. if (!utask || !utask->return_instances)
  1518. return;
  1519. if (mm == t->mm && !(flags & CLONE_VFORK))
  1520. return;
  1521. if (dup_utask(t, utask))
  1522. return uprobe_warn(t, "dup ret instances");
  1523. /* The task can fork() after dup_xol_work() fails */
  1524. area = mm->uprobes_state.xol_area;
  1525. if (!area)
  1526. return uprobe_warn(t, "dup xol area");
  1527. if (mm == t->mm)
  1528. return;
  1529. t->utask->dup_xol_addr = area->vaddr;
  1530. init_task_work(&t->utask->dup_xol_work, dup_xol_work);
  1531. task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
  1532. }
  1533. /*
  1534. * Current area->vaddr notion assume the trampoline address is always
  1535. * equal area->vaddr.
  1536. *
  1537. * Returns -1 in case the xol_area is not allocated.
  1538. */
  1539. static unsigned long get_trampoline_vaddr(void)
  1540. {
  1541. struct xol_area *area;
  1542. unsigned long trampoline_vaddr = -1;
  1543. /* Pairs with xol_add_vma() smp_store_release() */
  1544. area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
  1545. if (area)
  1546. trampoline_vaddr = area->vaddr;
  1547. return trampoline_vaddr;
  1548. }
  1549. static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
  1550. struct pt_regs *regs)
  1551. {
  1552. struct return_instance *ri = utask->return_instances;
  1553. enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
  1554. while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
  1555. ri = free_ret_instance(ri);
  1556. utask->depth--;
  1557. }
  1558. utask->return_instances = ri;
  1559. }
  1560. static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
  1561. {
  1562. struct return_instance *ri;
  1563. struct uprobe_task *utask;
  1564. unsigned long orig_ret_vaddr, trampoline_vaddr;
  1565. bool chained;
  1566. if (!get_xol_area())
  1567. return;
  1568. utask = get_utask();
  1569. if (!utask)
  1570. return;
  1571. if (utask->depth >= MAX_URETPROBE_DEPTH) {
  1572. printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
  1573. " nestedness limit pid/tgid=%d/%d\n",
  1574. current->pid, current->tgid);
  1575. return;
  1576. }
  1577. ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
  1578. if (!ri)
  1579. return;
  1580. trampoline_vaddr = get_trampoline_vaddr();
  1581. orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
  1582. if (orig_ret_vaddr == -1)
  1583. goto fail;
  1584. /* drop the entries invalidated by longjmp() */
  1585. chained = (orig_ret_vaddr == trampoline_vaddr);
  1586. cleanup_return_instances(utask, chained, regs);
  1587. /*
  1588. * We don't want to keep trampoline address in stack, rather keep the
  1589. * original return address of first caller thru all the consequent
  1590. * instances. This also makes breakpoint unwrapping easier.
  1591. */
  1592. if (chained) {
  1593. if (!utask->return_instances) {
  1594. /*
  1595. * This situation is not possible. Likely we have an
  1596. * attack from user-space.
  1597. */
  1598. uprobe_warn(current, "handle tail call");
  1599. goto fail;
  1600. }
  1601. orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
  1602. }
  1603. ri->uprobe = get_uprobe(uprobe);
  1604. ri->func = instruction_pointer(regs);
  1605. ri->stack = user_stack_pointer(regs);
  1606. ri->orig_ret_vaddr = orig_ret_vaddr;
  1607. ri->chained = chained;
  1608. utask->depth++;
  1609. ri->next = utask->return_instances;
  1610. utask->return_instances = ri;
  1611. return;
  1612. fail:
  1613. kfree(ri);
  1614. }
  1615. /* Prepare to single-step probed instruction out of line. */
  1616. static int
  1617. pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
  1618. {
  1619. struct uprobe_task *utask;
  1620. unsigned long xol_vaddr;
  1621. int err;
  1622. utask = get_utask();
  1623. if (!utask)
  1624. return -ENOMEM;
  1625. xol_vaddr = xol_get_insn_slot(uprobe);
  1626. if (!xol_vaddr)
  1627. return -ENOMEM;
  1628. utask->xol_vaddr = xol_vaddr;
  1629. utask->vaddr = bp_vaddr;
  1630. err = arch_uprobe_pre_xol(&uprobe->arch, regs);
  1631. if (unlikely(err)) {
  1632. xol_free_insn_slot(current);
  1633. return err;
  1634. }
  1635. utask->active_uprobe = uprobe;
  1636. utask->state = UTASK_SSTEP;
  1637. return 0;
  1638. }
  1639. /*
  1640. * If we are singlestepping, then ensure this thread is not connected to
  1641. * non-fatal signals until completion of singlestep. When xol insn itself
  1642. * triggers the signal, restart the original insn even if the task is
  1643. * already SIGKILL'ed (since coredump should report the correct ip). This
  1644. * is even more important if the task has a handler for SIGSEGV/etc, The
  1645. * _same_ instruction should be repeated again after return from the signal
  1646. * handler, and SSTEP can never finish in this case.
  1647. */
  1648. bool uprobe_deny_signal(void)
  1649. {
  1650. struct task_struct *t = current;
  1651. struct uprobe_task *utask = t->utask;
  1652. if (likely(!utask || !utask->active_uprobe))
  1653. return false;
  1654. WARN_ON_ONCE(utask->state != UTASK_SSTEP);
  1655. if (signal_pending(t)) {
  1656. spin_lock_irq(&t->sighand->siglock);
  1657. clear_tsk_thread_flag(t, TIF_SIGPENDING);
  1658. spin_unlock_irq(&t->sighand->siglock);
  1659. if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
  1660. utask->state = UTASK_SSTEP_TRAPPED;
  1661. set_tsk_thread_flag(t, TIF_UPROBE);
  1662. }
  1663. }
  1664. return true;
  1665. }
  1666. static void mmf_recalc_uprobes(struct mm_struct *mm)
  1667. {
  1668. struct vm_area_struct *vma;
  1669. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  1670. if (!valid_vma(vma, false))
  1671. continue;
  1672. /*
  1673. * This is not strictly accurate, we can race with
  1674. * uprobe_unregister() and see the already removed
  1675. * uprobe if delete_uprobe() was not yet called.
  1676. * Or this uprobe can be filtered out.
  1677. */
  1678. if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
  1679. return;
  1680. }
  1681. clear_bit(MMF_HAS_UPROBES, &mm->flags);
  1682. }
  1683. static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
  1684. {
  1685. struct page *page;
  1686. uprobe_opcode_t opcode;
  1687. int result;
  1688. if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
  1689. return -EINVAL;
  1690. pagefault_disable();
  1691. result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
  1692. pagefault_enable();
  1693. if (likely(result == 0))
  1694. goto out;
  1695. /*
  1696. * The NULL 'tsk' here ensures that any faults that occur here
  1697. * will not be accounted to the task. 'mm' *is* current->mm,
  1698. * but we treat this as a 'remote' access since it is
  1699. * essentially a kernel access to the memory.
  1700. */
  1701. result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
  1702. NULL, NULL);
  1703. if (result < 0)
  1704. return result;
  1705. copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
  1706. put_page(page);
  1707. out:
  1708. /* This needs to return true for any variant of the trap insn */
  1709. return is_trap_insn(&opcode);
  1710. }
  1711. static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
  1712. {
  1713. struct mm_struct *mm = current->mm;
  1714. struct uprobe *uprobe = NULL;
  1715. struct vm_area_struct *vma;
  1716. mmap_read_lock(mm);
  1717. vma = find_vma(mm, bp_vaddr);
  1718. if (vma && vma->vm_start <= bp_vaddr) {
  1719. if (valid_vma(vma, false)) {
  1720. struct inode *inode = file_inode(vma->vm_file);
  1721. loff_t offset = vaddr_to_offset(vma, bp_vaddr);
  1722. uprobe = find_uprobe(inode, offset);
  1723. }
  1724. if (!uprobe)
  1725. *is_swbp = is_trap_at_addr(mm, bp_vaddr);
  1726. } else {
  1727. *is_swbp = -EFAULT;
  1728. }
  1729. if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
  1730. mmf_recalc_uprobes(mm);
  1731. mmap_read_unlock(mm);
  1732. return uprobe;
  1733. }
  1734. static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
  1735. {
  1736. struct uprobe_consumer *uc;
  1737. int remove = UPROBE_HANDLER_REMOVE;
  1738. bool need_prep = false; /* prepare return uprobe, when needed */
  1739. down_read(&uprobe->register_rwsem);
  1740. for (uc = uprobe->consumers; uc; uc = uc->next) {
  1741. int rc = 0;
  1742. if (uc->handler) {
  1743. rc = uc->handler(uc, regs);
  1744. WARN(rc & ~UPROBE_HANDLER_MASK,
  1745. "bad rc=0x%x from %ps()\n", rc, uc->handler);
  1746. }
  1747. if (uc->ret_handler)
  1748. need_prep = true;
  1749. remove &= rc;
  1750. }
  1751. if (need_prep && !remove)
  1752. prepare_uretprobe(uprobe, regs); /* put bp at return */
  1753. if (remove && uprobe->consumers) {
  1754. WARN_ON(!uprobe_is_active(uprobe));
  1755. unapply_uprobe(uprobe, current->mm);
  1756. }
  1757. up_read(&uprobe->register_rwsem);
  1758. }
  1759. static void
  1760. handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
  1761. {
  1762. struct uprobe *uprobe = ri->uprobe;
  1763. struct uprobe_consumer *uc;
  1764. down_read(&uprobe->register_rwsem);
  1765. for (uc = uprobe->consumers; uc; uc = uc->next) {
  1766. if (uc->ret_handler)
  1767. uc->ret_handler(uc, ri->func, regs);
  1768. }
  1769. up_read(&uprobe->register_rwsem);
  1770. }
  1771. static struct return_instance *find_next_ret_chain(struct return_instance *ri)
  1772. {
  1773. bool chained;
  1774. do {
  1775. chained = ri->chained;
  1776. ri = ri->next; /* can't be NULL if chained */
  1777. } while (chained);
  1778. return ri;
  1779. }
  1780. static void handle_trampoline(struct pt_regs *regs)
  1781. {
  1782. struct uprobe_task *utask;
  1783. struct return_instance *ri, *next;
  1784. bool valid;
  1785. utask = current->utask;
  1786. if (!utask)
  1787. goto sigill;
  1788. ri = utask->return_instances;
  1789. if (!ri)
  1790. goto sigill;
  1791. do {
  1792. /*
  1793. * We should throw out the frames invalidated by longjmp().
  1794. * If this chain is valid, then the next one should be alive
  1795. * or NULL; the latter case means that nobody but ri->func
  1796. * could hit this trampoline on return. TODO: sigaltstack().
  1797. */
  1798. next = find_next_ret_chain(ri);
  1799. valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
  1800. instruction_pointer_set(regs, ri->orig_ret_vaddr);
  1801. do {
  1802. if (valid)
  1803. handle_uretprobe_chain(ri, regs);
  1804. ri = free_ret_instance(ri);
  1805. utask->depth--;
  1806. } while (ri != next);
  1807. } while (!valid);
  1808. utask->return_instances = ri;
  1809. return;
  1810. sigill:
  1811. uprobe_warn(current, "handle uretprobe, sending SIGILL.");
  1812. force_sig(SIGILL);
  1813. }
  1814. bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
  1815. {
  1816. return false;
  1817. }
  1818. bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
  1819. struct pt_regs *regs)
  1820. {
  1821. return true;
  1822. }
  1823. /*
  1824. * Run handler and ask thread to singlestep.
  1825. * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
  1826. */
  1827. static void handle_swbp(struct pt_regs *regs)
  1828. {
  1829. struct uprobe *uprobe;
  1830. unsigned long bp_vaddr;
  1831. int is_swbp;
  1832. bp_vaddr = uprobe_get_swbp_addr(regs);
  1833. if (bp_vaddr == get_trampoline_vaddr())
  1834. return handle_trampoline(regs);
  1835. uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
  1836. if (!uprobe) {
  1837. if (is_swbp > 0) {
  1838. /* No matching uprobe; signal SIGTRAP. */
  1839. force_sig(SIGTRAP);
  1840. } else {
  1841. /*
  1842. * Either we raced with uprobe_unregister() or we can't
  1843. * access this memory. The latter is only possible if
  1844. * another thread plays with our ->mm. In both cases
  1845. * we can simply restart. If this vma was unmapped we
  1846. * can pretend this insn was not executed yet and get
  1847. * the (correct) SIGSEGV after restart.
  1848. */
  1849. instruction_pointer_set(regs, bp_vaddr);
  1850. }
  1851. return;
  1852. }
  1853. /* change it in advance for ->handler() and restart */
  1854. instruction_pointer_set(regs, bp_vaddr);
  1855. /*
  1856. * TODO: move copy_insn/etc into _register and remove this hack.
  1857. * After we hit the bp, _unregister + _register can install the
  1858. * new and not-yet-analyzed uprobe at the same address, restart.
  1859. */
  1860. if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
  1861. goto out;
  1862. /*
  1863. * Pairs with the smp_wmb() in prepare_uprobe().
  1864. *
  1865. * Guarantees that if we see the UPROBE_COPY_INSN bit set, then
  1866. * we must also see the stores to &uprobe->arch performed by the
  1867. * prepare_uprobe() call.
  1868. */
  1869. smp_rmb();
  1870. /* Tracing handlers use ->utask to communicate with fetch methods */
  1871. if (!get_utask())
  1872. goto out;
  1873. if (arch_uprobe_ignore(&uprobe->arch, regs))
  1874. goto out;
  1875. handler_chain(uprobe, regs);
  1876. if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
  1877. goto out;
  1878. if (!pre_ssout(uprobe, regs, bp_vaddr))
  1879. return;
  1880. /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
  1881. out:
  1882. put_uprobe(uprobe);
  1883. }
  1884. /*
  1885. * Perform required fix-ups and disable singlestep.
  1886. * Allow pending signals to take effect.
  1887. */
  1888. static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
  1889. {
  1890. struct uprobe *uprobe;
  1891. int err = 0;
  1892. uprobe = utask->active_uprobe;
  1893. if (utask->state == UTASK_SSTEP_ACK)
  1894. err = arch_uprobe_post_xol(&uprobe->arch, regs);
  1895. else if (utask->state == UTASK_SSTEP_TRAPPED)
  1896. arch_uprobe_abort_xol(&uprobe->arch, regs);
  1897. else
  1898. WARN_ON_ONCE(1);
  1899. put_uprobe(uprobe);
  1900. utask->active_uprobe = NULL;
  1901. utask->state = UTASK_RUNNING;
  1902. xol_free_insn_slot(current);
  1903. spin_lock_irq(&current->sighand->siglock);
  1904. recalc_sigpending(); /* see uprobe_deny_signal() */
  1905. spin_unlock_irq(&current->sighand->siglock);
  1906. if (unlikely(err)) {
  1907. uprobe_warn(current, "execute the probed insn, sending SIGILL.");
  1908. force_sig(SIGILL);
  1909. }
  1910. }
  1911. /*
  1912. * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
  1913. * allows the thread to return from interrupt. After that handle_swbp()
  1914. * sets utask->active_uprobe.
  1915. *
  1916. * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
  1917. * and allows the thread to return from interrupt.
  1918. *
  1919. * While returning to userspace, thread notices the TIF_UPROBE flag and calls
  1920. * uprobe_notify_resume().
  1921. */
  1922. void uprobe_notify_resume(struct pt_regs *regs)
  1923. {
  1924. struct uprobe_task *utask;
  1925. clear_thread_flag(TIF_UPROBE);
  1926. utask = current->utask;
  1927. if (utask && utask->active_uprobe)
  1928. handle_singlestep(utask, regs);
  1929. else
  1930. handle_swbp(regs);
  1931. }
  1932. /*
  1933. * uprobe_pre_sstep_notifier gets called from interrupt context as part of
  1934. * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
  1935. */
  1936. int uprobe_pre_sstep_notifier(struct pt_regs *regs)
  1937. {
  1938. if (!current->mm)
  1939. return 0;
  1940. if (!test_bit(MMF_HAS_UPROBES, &current->mm->flags) &&
  1941. (!current->utask || !current->utask->return_instances))
  1942. return 0;
  1943. set_thread_flag(TIF_UPROBE);
  1944. return 1;
  1945. }
  1946. /*
  1947. * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
  1948. * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
  1949. */
  1950. int uprobe_post_sstep_notifier(struct pt_regs *regs)
  1951. {
  1952. struct uprobe_task *utask = current->utask;
  1953. if (!current->mm || !utask || !utask->active_uprobe)
  1954. /* task is currently not uprobed */
  1955. return 0;
  1956. utask->state = UTASK_SSTEP_ACK;
  1957. set_thread_flag(TIF_UPROBE);
  1958. return 1;
  1959. }
  1960. static struct notifier_block uprobe_exception_nb = {
  1961. .notifier_call = arch_uprobe_exception_notify,
  1962. .priority = INT_MAX-1, /* notified after kprobes, kgdb */
  1963. };
  1964. void __init uprobes_init(void)
  1965. {
  1966. int i;
  1967. for (i = 0; i < UPROBES_HASH_SZ; i++)
  1968. mutex_init(&uprobes_mmap_mutex[i]);
  1969. BUG_ON(register_die_notifier(&uprobe_exception_nb));
  1970. }