tracepoint.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2008-2014 Mathieu Desnoyers
  4. */
  5. #include <linux/module.h>
  6. #include <linux/mutex.h>
  7. #include <linux/types.h>
  8. #include <linux/jhash.h>
  9. #include <linux/list.h>
  10. #include <linux/rcupdate.h>
  11. #include <linux/tracepoint.h>
  12. #include <linux/err.h>
  13. #include <linux/slab.h>
  14. #include <linux/sched/signal.h>
  15. #include <linux/sched/task.h>
  16. #include <linux/static_key.h>
  17. enum tp_func_state {
  18. TP_FUNC_0,
  19. TP_FUNC_1,
  20. TP_FUNC_2,
  21. TP_FUNC_N,
  22. };
  23. extern tracepoint_ptr_t __start___tracepoints_ptrs[];
  24. extern tracepoint_ptr_t __stop___tracepoints_ptrs[];
  25. DEFINE_SRCU(tracepoint_srcu);
  26. EXPORT_SYMBOL_GPL(tracepoint_srcu);
  27. enum tp_transition_sync {
  28. TP_TRANSITION_SYNC_1_0_1,
  29. TP_TRANSITION_SYNC_N_2_1,
  30. _NR_TP_TRANSITION_SYNC,
  31. };
  32. struct tp_transition_snapshot {
  33. unsigned long rcu;
  34. unsigned long srcu;
  35. bool ongoing;
  36. };
  37. /* Protected by tracepoints_mutex */
  38. static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];
  39. static void tp_rcu_get_state(enum tp_transition_sync sync)
  40. {
  41. struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
  42. /* Keep the latest get_state snapshot. */
  43. snapshot->rcu = get_state_synchronize_rcu();
  44. snapshot->srcu = start_poll_synchronize_srcu(&tracepoint_srcu);
  45. snapshot->ongoing = true;
  46. }
  47. static void tp_rcu_cond_sync(enum tp_transition_sync sync)
  48. {
  49. struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
  50. if (!snapshot->ongoing)
  51. return;
  52. cond_synchronize_rcu(snapshot->rcu);
  53. if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu))
  54. synchronize_srcu(&tracepoint_srcu);
  55. snapshot->ongoing = false;
  56. }
  57. /* Set to 1 to enable tracepoint debug output */
  58. static const int tracepoint_debug;
  59. #ifdef CONFIG_MODULES
  60. /*
  61. * Tracepoint module list mutex protects the local module list.
  62. */
  63. static DEFINE_MUTEX(tracepoint_module_list_mutex);
  64. /* Local list of struct tp_module */
  65. static LIST_HEAD(tracepoint_module_list);
  66. #endif /* CONFIG_MODULES */
  67. /*
  68. * tracepoints_mutex protects the builtin and module tracepoints.
  69. * tracepoints_mutex nests inside tracepoint_module_list_mutex.
  70. */
  71. static DEFINE_MUTEX(tracepoints_mutex);
  72. static struct rcu_head *early_probes;
  73. static bool ok_to_free_tracepoints;
  74. /*
  75. * Note about RCU :
  76. * It is used to delay the free of multiple probes array until a quiescent
  77. * state is reached.
  78. */
  79. struct tp_probes {
  80. struct rcu_head rcu;
  81. struct tracepoint_func probes[];
  82. };
  83. /* Called in removal of a func but failed to allocate a new tp_funcs */
  84. static void tp_stub_func(void)
  85. {
  86. return;
  87. }
  88. static inline void *allocate_probes(int count)
  89. {
  90. struct tp_probes *p = kmalloc(struct_size(p, probes, count),
  91. GFP_KERNEL);
  92. return p == NULL ? NULL : p->probes;
  93. }
  94. static void srcu_free_old_probes(struct rcu_head *head)
  95. {
  96. kfree(container_of(head, struct tp_probes, rcu));
  97. }
  98. static void rcu_free_old_probes(struct rcu_head *head)
  99. {
  100. call_srcu(&tracepoint_srcu, head, srcu_free_old_probes);
  101. }
  102. static __init int release_early_probes(void)
  103. {
  104. struct rcu_head *tmp;
  105. ok_to_free_tracepoints = true;
  106. while (early_probes) {
  107. tmp = early_probes;
  108. early_probes = tmp->next;
  109. call_rcu(tmp, rcu_free_old_probes);
  110. }
  111. return 0;
  112. }
  113. /* SRCU is initialized at core_initcall */
  114. postcore_initcall(release_early_probes);
  115. static inline void release_probes(struct tracepoint_func *old)
  116. {
  117. if (old) {
  118. struct tp_probes *tp_probes = container_of(old,
  119. struct tp_probes, probes[0]);
  120. /*
  121. * We can't free probes if SRCU is not initialized yet.
  122. * Postpone the freeing till after SRCU is initialized.
  123. */
  124. if (unlikely(!ok_to_free_tracepoints)) {
  125. tp_probes->rcu.next = early_probes;
  126. early_probes = &tp_probes->rcu;
  127. return;
  128. }
  129. /*
  130. * Tracepoint probes are protected by both sched RCU and SRCU,
  131. * by calling the SRCU callback in the sched RCU callback we
  132. * cover both cases. So let us chain the SRCU and sched RCU
  133. * callbacks to wait for both grace periods.
  134. */
  135. call_rcu(&tp_probes->rcu, rcu_free_old_probes);
  136. }
  137. }
  138. static void debug_print_probes(struct tracepoint_func *funcs)
  139. {
  140. int i;
  141. if (!tracepoint_debug || !funcs)
  142. return;
  143. for (i = 0; funcs[i].func; i++)
  144. printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func);
  145. }
  146. static struct tracepoint_func *
  147. func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
  148. int prio)
  149. {
  150. struct tracepoint_func *old, *new;
  151. int nr_probes = 0;
  152. int stub_funcs = 0;
  153. int pos = -1;
  154. if (WARN_ON(!tp_func->func))
  155. return ERR_PTR(-EINVAL);
  156. debug_print_probes(*funcs);
  157. old = *funcs;
  158. if (old) {
  159. /* (N -> N+1), (N != 0, 1) probes */
  160. for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
  161. /* Insert before probes of lower priority */
  162. if (pos < 0 && old[nr_probes].prio < prio)
  163. pos = nr_probes;
  164. if (old[nr_probes].func == tp_func->func &&
  165. old[nr_probes].data == tp_func->data)
  166. return ERR_PTR(-EEXIST);
  167. if (old[nr_probes].func == tp_stub_func)
  168. stub_funcs++;
  169. }
  170. }
  171. /* + 2 : one for new probe, one for NULL func - stub functions */
  172. new = allocate_probes(nr_probes + 2 - stub_funcs);
  173. if (new == NULL)
  174. return ERR_PTR(-ENOMEM);
  175. if (old) {
  176. if (stub_funcs) {
  177. /* Need to copy one at a time to remove stubs */
  178. int probes = 0;
  179. pos = -1;
  180. for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
  181. if (old[nr_probes].func == tp_stub_func)
  182. continue;
  183. if (pos < 0 && old[nr_probes].prio < prio)
  184. pos = probes++;
  185. new[probes++] = old[nr_probes];
  186. }
  187. nr_probes = probes;
  188. if (pos < 0)
  189. pos = probes;
  190. else
  191. nr_probes--; /* Account for insertion */
  192. } else if (pos < 0) {
  193. pos = nr_probes;
  194. memcpy(new, old, nr_probes * sizeof(struct tracepoint_func));
  195. } else {
  196. /* Copy higher priority probes ahead of the new probe */
  197. memcpy(new, old, pos * sizeof(struct tracepoint_func));
  198. /* Copy the rest after it. */
  199. memcpy(new + pos + 1, old + pos,
  200. (nr_probes - pos) * sizeof(struct tracepoint_func));
  201. }
  202. } else
  203. pos = 0;
  204. new[pos] = *tp_func;
  205. new[nr_probes + 1].func = NULL;
  206. *funcs = new;
  207. debug_print_probes(*funcs);
  208. return old;
  209. }
  210. static void *func_remove(struct tracepoint_func **funcs,
  211. struct tracepoint_func *tp_func)
  212. {
  213. int nr_probes = 0, nr_del = 0, i;
  214. struct tracepoint_func *old, *new;
  215. old = *funcs;
  216. if (!old)
  217. return ERR_PTR(-ENOENT);
  218. debug_print_probes(*funcs);
  219. /* (N -> M), (N > 1, M >= 0) probes */
  220. if (tp_func->func) {
  221. for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
  222. if ((old[nr_probes].func == tp_func->func &&
  223. old[nr_probes].data == tp_func->data) ||
  224. old[nr_probes].func == tp_stub_func)
  225. nr_del++;
  226. }
  227. }
  228. /*
  229. * If probe is NULL, then nr_probes = nr_del = 0, and then the
  230. * entire entry will be removed.
  231. */
  232. if (nr_probes - nr_del == 0) {
  233. /* N -> 0, (N > 1) */
  234. *funcs = NULL;
  235. debug_print_probes(*funcs);
  236. return old;
  237. } else {
  238. int j = 0;
  239. /* N -> M, (N > 1, M > 0) */
  240. /* + 1 for NULL */
  241. new = allocate_probes(nr_probes - nr_del + 1);
  242. if (new) {
  243. for (i = 0; old[i].func; i++)
  244. if ((old[i].func != tp_func->func
  245. || old[i].data != tp_func->data)
  246. && old[i].func != tp_stub_func)
  247. new[j++] = old[i];
  248. new[nr_probes - nr_del].func = NULL;
  249. *funcs = new;
  250. } else {
  251. /*
  252. * Failed to allocate, replace the old function
  253. * with calls to tp_stub_func.
  254. */
  255. for (i = 0; old[i].func; i++)
  256. if (old[i].func == tp_func->func &&
  257. old[i].data == tp_func->data) {
  258. old[i].func = tp_stub_func;
  259. /* Set the prio to the next event. */
  260. if (old[i + 1].func)
  261. old[i].prio =
  262. old[i + 1].prio;
  263. else
  264. old[i].prio = -1;
  265. }
  266. *funcs = old;
  267. }
  268. }
  269. debug_print_probes(*funcs);
  270. return old;
  271. }
  272. /*
  273. * Count the number of functions (enum tp_func_state) in a tp_funcs array.
  274. */
  275. static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs)
  276. {
  277. if (!tp_funcs)
  278. return TP_FUNC_0;
  279. if (!tp_funcs[1].func)
  280. return TP_FUNC_1;
  281. if (!tp_funcs[2].func)
  282. return TP_FUNC_2;
  283. return TP_FUNC_N; /* 3 or more */
  284. }
  285. static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
  286. {
  287. void *func = tp->iterator;
  288. /* Synthetic events do not have static call sites */
  289. if (!tp->static_call_key)
  290. return;
  291. if (nr_func_state(tp_funcs) == TP_FUNC_1)
  292. func = tp_funcs[0].func;
  293. __static_call_update(tp->static_call_key, tp->static_call_tramp, func);
  294. }
  295. /*
  296. * Add the probe function to a tracepoint.
  297. */
  298. static int tracepoint_add_func(struct tracepoint *tp,
  299. struct tracepoint_func *func, int prio,
  300. bool warn)
  301. {
  302. struct tracepoint_func *old, *tp_funcs;
  303. int ret;
  304. if (tp->regfunc && !static_key_enabled(&tp->key)) {
  305. ret = tp->regfunc();
  306. if (ret < 0)
  307. return ret;
  308. }
  309. tp_funcs = rcu_dereference_protected(tp->funcs,
  310. lockdep_is_held(&tracepoints_mutex));
  311. old = func_add(&tp_funcs, func, prio);
  312. if (IS_ERR(old)) {
  313. WARN_ON_ONCE(warn && PTR_ERR(old) != -ENOMEM);
  314. return PTR_ERR(old);
  315. }
  316. /*
  317. * rcu_assign_pointer has as smp_store_release() which makes sure
  318. * that the new probe callbacks array is consistent before setting
  319. * a pointer to it. This array is referenced by __DO_TRACE from
  320. * include/linux/tracepoint.h using rcu_dereference_sched().
  321. */
  322. switch (nr_func_state(tp_funcs)) {
  323. case TP_FUNC_1: /* 0->1 */
  324. /*
  325. * Make sure new static func never uses old data after a
  326. * 1->0->1 transition sequence.
  327. */
  328. tp_rcu_cond_sync(TP_TRANSITION_SYNC_1_0_1);
  329. /* Set static call to first function */
  330. tracepoint_update_call(tp, tp_funcs);
  331. /* Both iterator and static call handle NULL tp->funcs */
  332. rcu_assign_pointer(tp->funcs, tp_funcs);
  333. static_key_enable(&tp->key);
  334. break;
  335. case TP_FUNC_2: /* 1->2 */
  336. /* Set iterator static call */
  337. tracepoint_update_call(tp, tp_funcs);
  338. /*
  339. * Iterator callback installed before updating tp->funcs.
  340. * Requires ordering between RCU assign/dereference and
  341. * static call update/call.
  342. */
  343. fallthrough;
  344. case TP_FUNC_N: /* N->N+1 (N>1) */
  345. rcu_assign_pointer(tp->funcs, tp_funcs);
  346. /*
  347. * Make sure static func never uses incorrect data after a
  348. * N->...->2->1 (N>1) transition sequence.
  349. */
  350. if (tp_funcs[0].data != old[0].data)
  351. tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
  352. break;
  353. default:
  354. WARN_ON_ONCE(1);
  355. break;
  356. }
  357. release_probes(old);
  358. return 0;
  359. }
  360. /*
  361. * Remove a probe function from a tracepoint.
  362. * Note: only waiting an RCU period after setting elem->call to the empty
  363. * function insures that the original callback is not used anymore. This insured
  364. * by preempt_disable around the call site.
  365. */
  366. static int tracepoint_remove_func(struct tracepoint *tp,
  367. struct tracepoint_func *func)
  368. {
  369. struct tracepoint_func *old, *tp_funcs;
  370. tp_funcs = rcu_dereference_protected(tp->funcs,
  371. lockdep_is_held(&tracepoints_mutex));
  372. old = func_remove(&tp_funcs, func);
  373. if (WARN_ON_ONCE(IS_ERR(old)))
  374. return PTR_ERR(old);
  375. if (tp_funcs == old)
  376. /* Failed allocating new tp_funcs, replaced func with stub */
  377. return 0;
  378. switch (nr_func_state(tp_funcs)) {
  379. case TP_FUNC_0: /* 1->0 */
  380. /* Removed last function */
  381. if (tp->unregfunc && static_key_enabled(&tp->key))
  382. tp->unregfunc();
  383. static_key_disable(&tp->key);
  384. /* Set iterator static call */
  385. tracepoint_update_call(tp, tp_funcs);
  386. /* Both iterator and static call handle NULL tp->funcs */
  387. rcu_assign_pointer(tp->funcs, NULL);
  388. /*
  389. * Make sure new static func never uses old data after a
  390. * 1->0->1 transition sequence.
  391. */
  392. tp_rcu_get_state(TP_TRANSITION_SYNC_1_0_1);
  393. break;
  394. case TP_FUNC_1: /* 2->1 */
  395. rcu_assign_pointer(tp->funcs, tp_funcs);
  396. /*
  397. * Make sure static func never uses incorrect data after a
  398. * N->...->2->1 (N>2) transition sequence. If the first
  399. * element's data has changed, then force the synchronization
  400. * to prevent current readers that have loaded the old data
  401. * from calling the new function.
  402. */
  403. if (tp_funcs[0].data != old[0].data)
  404. tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
  405. tp_rcu_cond_sync(TP_TRANSITION_SYNC_N_2_1);
  406. /* Set static call to first function */
  407. tracepoint_update_call(tp, tp_funcs);
  408. break;
  409. case TP_FUNC_2: /* N->N-1 (N>2) */
  410. fallthrough;
  411. case TP_FUNC_N:
  412. rcu_assign_pointer(tp->funcs, tp_funcs);
  413. /*
  414. * Make sure static func never uses incorrect data after a
  415. * N->...->2->1 (N>2) transition sequence.
  416. */
  417. if (tp_funcs[0].data != old[0].data)
  418. tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
  419. break;
  420. default:
  421. WARN_ON_ONCE(1);
  422. break;
  423. }
  424. release_probes(old);
  425. return 0;
  426. }
  427. /**
  428. * tracepoint_probe_register_prio_may_exist - Connect a probe to a tracepoint with priority
  429. * @tp: tracepoint
  430. * @probe: probe handler
  431. * @data: tracepoint data
  432. * @prio: priority of this function over other registered functions
  433. *
  434. * Same as tracepoint_probe_register_prio() except that it will not warn
  435. * if the tracepoint is already registered.
  436. */
  437. int tracepoint_probe_register_prio_may_exist(struct tracepoint *tp, void *probe,
  438. void *data, int prio)
  439. {
  440. struct tracepoint_func tp_func;
  441. int ret;
  442. mutex_lock(&tracepoints_mutex);
  443. tp_func.func = probe;
  444. tp_func.data = data;
  445. tp_func.prio = prio;
  446. ret = tracepoint_add_func(tp, &tp_func, prio, false);
  447. mutex_unlock(&tracepoints_mutex);
  448. return ret;
  449. }
  450. EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio_may_exist);
  451. /**
  452. * tracepoint_probe_register_prio - Connect a probe to a tracepoint with priority
  453. * @tp: tracepoint
  454. * @probe: probe handler
  455. * @data: tracepoint data
  456. * @prio: priority of this function over other registered functions
  457. *
  458. * Returns 0 if ok, error value on error.
  459. * Note: if @tp is within a module, the caller is responsible for
  460. * unregistering the probe before the module is gone. This can be
  461. * performed either with a tracepoint module going notifier, or from
  462. * within module exit functions.
  463. */
  464. int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe,
  465. void *data, int prio)
  466. {
  467. struct tracepoint_func tp_func;
  468. int ret;
  469. mutex_lock(&tracepoints_mutex);
  470. tp_func.func = probe;
  471. tp_func.data = data;
  472. tp_func.prio = prio;
  473. ret = tracepoint_add_func(tp, &tp_func, prio, true);
  474. mutex_unlock(&tracepoints_mutex);
  475. return ret;
  476. }
  477. EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio);
  478. /**
  479. * tracepoint_probe_register - Connect a probe to a tracepoint
  480. * @tp: tracepoint
  481. * @probe: probe handler
  482. * @data: tracepoint data
  483. *
  484. * Returns 0 if ok, error value on error.
  485. * Note: if @tp is within a module, the caller is responsible for
  486. * unregistering the probe before the module is gone. This can be
  487. * performed either with a tracepoint module going notifier, or from
  488. * within module exit functions.
  489. */
  490. int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data)
  491. {
  492. return tracepoint_probe_register_prio(tp, probe, data, TRACEPOINT_DEFAULT_PRIO);
  493. }
  494. EXPORT_SYMBOL_GPL(tracepoint_probe_register);
  495. /**
  496. * tracepoint_probe_unregister - Disconnect a probe from a tracepoint
  497. * @tp: tracepoint
  498. * @probe: probe function pointer
  499. * @data: tracepoint data
  500. *
  501. * Returns 0 if ok, error value on error.
  502. */
  503. int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data)
  504. {
  505. struct tracepoint_func tp_func;
  506. int ret;
  507. mutex_lock(&tracepoints_mutex);
  508. tp_func.func = probe;
  509. tp_func.data = data;
  510. ret = tracepoint_remove_func(tp, &tp_func);
  511. mutex_unlock(&tracepoints_mutex);
  512. return ret;
  513. }
  514. EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
  515. static void for_each_tracepoint_range(
  516. tracepoint_ptr_t *begin, tracepoint_ptr_t *end,
  517. void (*fct)(struct tracepoint *tp, void *priv),
  518. void *priv)
  519. {
  520. tracepoint_ptr_t *iter;
  521. if (!begin)
  522. return;
  523. for (iter = begin; iter < end; iter++)
  524. fct(tracepoint_ptr_deref(iter), priv);
  525. }
  526. #ifdef CONFIG_MODULES
  527. bool trace_module_has_bad_taint(struct module *mod)
  528. {
  529. return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) |
  530. (1 << TAINT_UNSIGNED_MODULE));
  531. }
  532. static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list);
  533. /**
  534. * register_tracepoint_notifier - register tracepoint coming/going notifier
  535. * @nb: notifier block
  536. *
  537. * Notifiers registered with this function are called on module
  538. * coming/going with the tracepoint_module_list_mutex held.
  539. * The notifier block callback should expect a "struct tp_module" data
  540. * pointer.
  541. */
  542. int register_tracepoint_module_notifier(struct notifier_block *nb)
  543. {
  544. struct tp_module *tp_mod;
  545. int ret;
  546. mutex_lock(&tracepoint_module_list_mutex);
  547. ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb);
  548. if (ret)
  549. goto end;
  550. list_for_each_entry(tp_mod, &tracepoint_module_list, list)
  551. (void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod);
  552. end:
  553. mutex_unlock(&tracepoint_module_list_mutex);
  554. return ret;
  555. }
  556. EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier);
  557. /**
  558. * unregister_tracepoint_notifier - unregister tracepoint coming/going notifier
  559. * @nb: notifier block
  560. *
  561. * The notifier block callback should expect a "struct tp_module" data
  562. * pointer.
  563. */
  564. int unregister_tracepoint_module_notifier(struct notifier_block *nb)
  565. {
  566. struct tp_module *tp_mod;
  567. int ret;
  568. mutex_lock(&tracepoint_module_list_mutex);
  569. ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb);
  570. if (ret)
  571. goto end;
  572. list_for_each_entry(tp_mod, &tracepoint_module_list, list)
  573. (void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod);
  574. end:
  575. mutex_unlock(&tracepoint_module_list_mutex);
  576. return ret;
  577. }
  578. EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier);
  579. /*
  580. * Ensure the tracer unregistered the module's probes before the module
  581. * teardown is performed. Prevents leaks of probe and data pointers.
  582. */
  583. static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv)
  584. {
  585. WARN_ON_ONCE(tp->funcs);
  586. }
  587. static int tracepoint_module_coming(struct module *mod)
  588. {
  589. struct tp_module *tp_mod;
  590. int ret = 0;
  591. if (!mod->num_tracepoints)
  592. return 0;
  593. /*
  594. * We skip modules that taint the kernel, especially those with different
  595. * module headers (for forced load), to make sure we don't cause a crash.
  596. * Staging, out-of-tree, and unsigned GPL modules are fine.
  597. */
  598. if (trace_module_has_bad_taint(mod))
  599. return 0;
  600. mutex_lock(&tracepoint_module_list_mutex);
  601. tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
  602. if (!tp_mod) {
  603. ret = -ENOMEM;
  604. goto end;
  605. }
  606. tp_mod->mod = mod;
  607. list_add_tail(&tp_mod->list, &tracepoint_module_list);
  608. blocking_notifier_call_chain(&tracepoint_notify_list,
  609. MODULE_STATE_COMING, tp_mod);
  610. end:
  611. mutex_unlock(&tracepoint_module_list_mutex);
  612. return ret;
  613. }
  614. static void tracepoint_module_going(struct module *mod)
  615. {
  616. struct tp_module *tp_mod;
  617. if (!mod->num_tracepoints)
  618. return;
  619. mutex_lock(&tracepoint_module_list_mutex);
  620. list_for_each_entry(tp_mod, &tracepoint_module_list, list) {
  621. if (tp_mod->mod == mod) {
  622. blocking_notifier_call_chain(&tracepoint_notify_list,
  623. MODULE_STATE_GOING, tp_mod);
  624. list_del(&tp_mod->list);
  625. kfree(tp_mod);
  626. /*
  627. * Called the going notifier before checking for
  628. * quiescence.
  629. */
  630. for_each_tracepoint_range(mod->tracepoints_ptrs,
  631. mod->tracepoints_ptrs + mod->num_tracepoints,
  632. tp_module_going_check_quiescent, NULL);
  633. break;
  634. }
  635. }
  636. /*
  637. * In the case of modules that were tainted at "coming", we'll simply
  638. * walk through the list without finding it. We cannot use the "tainted"
  639. * flag on "going", in case a module taints the kernel only after being
  640. * loaded.
  641. */
  642. mutex_unlock(&tracepoint_module_list_mutex);
  643. }
  644. static int tracepoint_module_notify(struct notifier_block *self,
  645. unsigned long val, void *data)
  646. {
  647. struct module *mod = data;
  648. int ret = 0;
  649. switch (val) {
  650. case MODULE_STATE_COMING:
  651. ret = tracepoint_module_coming(mod);
  652. break;
  653. case MODULE_STATE_LIVE:
  654. break;
  655. case MODULE_STATE_GOING:
  656. tracepoint_module_going(mod);
  657. break;
  658. case MODULE_STATE_UNFORMED:
  659. break;
  660. }
  661. return notifier_from_errno(ret);
  662. }
  663. static struct notifier_block tracepoint_module_nb = {
  664. .notifier_call = tracepoint_module_notify,
  665. .priority = 0,
  666. };
  667. static __init int init_tracepoints(void)
  668. {
  669. int ret;
  670. ret = register_module_notifier(&tracepoint_module_nb);
  671. if (ret)
  672. pr_warn("Failed to register tracepoint module enter notifier\n");
  673. return ret;
  674. }
  675. __initcall(init_tracepoints);
  676. #endif /* CONFIG_MODULES */
  677. /**
  678. * for_each_kernel_tracepoint - iteration on all kernel tracepoints
  679. * @fct: callback
  680. * @priv: private data
  681. */
  682. void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
  683. void *priv)
  684. {
  685. for_each_tracepoint_range(__start___tracepoints_ptrs,
  686. __stop___tracepoints_ptrs, fct, priv);
  687. }
  688. EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
  689. #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
  690. /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
  691. static int sys_tracepoint_refcount;
  692. int syscall_regfunc(void)
  693. {
  694. struct task_struct *p, *t;
  695. if (!sys_tracepoint_refcount) {
  696. read_lock(&tasklist_lock);
  697. for_each_process_thread(p, t) {
  698. set_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
  699. }
  700. read_unlock(&tasklist_lock);
  701. }
  702. sys_tracepoint_refcount++;
  703. return 0;
  704. }
  705. void syscall_unregfunc(void)
  706. {
  707. struct task_struct *p, *t;
  708. sys_tracepoint_refcount--;
  709. if (!sys_tracepoint_refcount) {
  710. read_lock(&tasklist_lock);
  711. for_each_process_thread(p, t) {
  712. clear_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
  713. }
  714. read_unlock(&tasklist_lock);
  715. }
  716. }
  717. #endif
  718. #ifdef CONFIG_ANDROID_VENDOR_HOOKS
  719. static void *rvh_zalloc_funcs(int count)
  720. {
  721. return kzalloc(sizeof(struct tracepoint_func) * count, GFP_KERNEL);
  722. }
  723. #define ANDROID_RVH_NR_PROBES_MAX 2
  724. static int rvh_func_add(struct tracepoint *tp, struct tracepoint_func *func)
  725. {
  726. int i;
  727. if (!static_key_enabled(&tp->key)) {
  728. /* '+ 1' for the last NULL element */
  729. tp->funcs = rvh_zalloc_funcs(ANDROID_RVH_NR_PROBES_MAX + 1);
  730. if (!tp->funcs)
  731. return ENOMEM;
  732. }
  733. for (i = 0; i < ANDROID_RVH_NR_PROBES_MAX; i++) {
  734. if (!tp->funcs[i].func) {
  735. if (!static_key_enabled(&tp->key))
  736. tp->funcs[i].data = func->data;
  737. WRITE_ONCE(tp->funcs[i].func, func->func);
  738. return 0;
  739. }
  740. }
  741. return -EBUSY;
  742. }
  743. static int android_rvh_add_func(struct tracepoint *tp, struct tracepoint_func *func)
  744. {
  745. int ret;
  746. if (tp->regfunc && !static_key_enabled(&tp->key)) {
  747. ret = tp->regfunc();
  748. if (ret < 0)
  749. return ret;
  750. }
  751. ret = rvh_func_add(tp, func);
  752. if (ret)
  753. return ret;
  754. tracepoint_update_call(tp, tp->funcs);
  755. static_key_enable(&tp->key);
  756. return 0;
  757. }
  758. int android_rvh_probe_register(struct tracepoint *tp, void *probe, void *data)
  759. {
  760. struct tracepoint_func tp_func;
  761. int ret;
  762. /*
  763. * Once the static key has been flipped, the array may be read
  764. * concurrently. Although __traceiter_*() always checks .func first,
  765. * it doesn't enforce read->read dependencies, and we can't strongly
  766. * guarantee it will see the correct .data for the second element
  767. * without adding smp_load_acquire() in the fast path. But this is a
  768. * corner case which is unlikely to be needed by anybody in practice,
  769. * so let's just forbid it and keep the fast path clean.
  770. */
  771. if (WARN_ON(static_key_enabled(&tp->key) && data))
  772. return -EINVAL;
  773. mutex_lock(&tracepoints_mutex);
  774. tp_func.func = probe;
  775. tp_func.data = data;
  776. ret = android_rvh_add_func(tp, &tp_func);
  777. mutex_unlock(&tracepoints_mutex);
  778. return ret;
  779. }
  780. EXPORT_SYMBOL_GPL(android_rvh_probe_register);
  781. #endif