notifier.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/kdebug.h>
  3. #include <linux/kprobes.h>
  4. #include <linux/export.h>
  5. #include <linux/notifier.h>
  6. #include <linux/rcupdate.h>
  7. #include <linux/vmalloc.h>
  8. #include <linux/reboot.h>
  9. /*
  10. * Notifier list for kernel code which wants to be called
  11. * at shutdown. This is used to stop any idling DMA operations
  12. * and the like.
  13. */
  14. BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
  15. /*
  16. * Notifier chain core routines. The exported routines below
  17. * are layered on top of these, with appropriate locking added.
  18. */
  19. static int notifier_chain_register(struct notifier_block **nl,
  20. struct notifier_block *n)
  21. {
  22. while ((*nl) != NULL) {
  23. if (unlikely((*nl) == n)) {
  24. WARN(1, "double register detected");
  25. return 0;
  26. }
  27. if (n->priority > (*nl)->priority)
  28. break;
  29. nl = &((*nl)->next);
  30. }
  31. n->next = *nl;
  32. rcu_assign_pointer(*nl, n);
  33. return 0;
  34. }
  35. static int notifier_chain_unregister(struct notifier_block **nl,
  36. struct notifier_block *n)
  37. {
  38. while ((*nl) != NULL) {
  39. if ((*nl) == n) {
  40. rcu_assign_pointer(*nl, n->next);
  41. return 0;
  42. }
  43. nl = &((*nl)->next);
  44. }
  45. return -ENOENT;
  46. }
  47. /**
  48. * notifier_call_chain - Informs the registered notifiers about an event.
  49. * @nl: Pointer to head of the blocking notifier chain
  50. * @val: Value passed unmodified to notifier function
  51. * @v: Pointer passed unmodified to notifier function
  52. * @nr_to_call: Number of notifier functions to be called. Don't care
  53. * value of this parameter is -1.
  54. * @nr_calls: Records the number of notifications sent. Don't care
  55. * value of this field is NULL.
  56. * @returns: notifier_call_chain returns the value returned by the
  57. * last notifier function called.
  58. */
  59. static int notifier_call_chain(struct notifier_block **nl,
  60. unsigned long val, void *v,
  61. int nr_to_call, int *nr_calls)
  62. {
  63. int ret = NOTIFY_DONE;
  64. struct notifier_block *nb, *next_nb;
  65. nb = rcu_dereference_raw(*nl);
  66. while (nb && nr_to_call) {
  67. next_nb = rcu_dereference_raw(nb->next);
  68. #ifdef CONFIG_DEBUG_NOTIFIERS
  69. if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
  70. WARN(1, "Invalid notifier called!");
  71. nb = next_nb;
  72. continue;
  73. }
  74. #endif
  75. ret = nb->notifier_call(nb, val, v);
  76. if (nr_calls)
  77. (*nr_calls)++;
  78. if (ret & NOTIFY_STOP_MASK)
  79. break;
  80. nb = next_nb;
  81. nr_to_call--;
  82. }
  83. return ret;
  84. }
  85. NOKPROBE_SYMBOL(notifier_call_chain);
  86. /**
  87. * notifier_call_chain_robust - Inform the registered notifiers about an event
  88. * and rollback on error.
  89. * @nl: Pointer to head of the blocking notifier chain
  90. * @val_up: Value passed unmodified to the notifier function
  91. * @val_down: Value passed unmodified to the notifier function when recovering
  92. * from an error on @val_up
  93. * @v Pointer passed unmodified to the notifier function
  94. *
  95. * NOTE: It is important the @nl chain doesn't change between the two
  96. * invocations of notifier_call_chain() such that we visit the
  97. * exact same notifier callbacks; this rules out any RCU usage.
  98. *
  99. * Returns: the return value of the @val_up call.
  100. */
  101. static int notifier_call_chain_robust(struct notifier_block **nl,
  102. unsigned long val_up, unsigned long val_down,
  103. void *v)
  104. {
  105. int ret, nr = 0;
  106. ret = notifier_call_chain(nl, val_up, v, -1, &nr);
  107. if (ret & NOTIFY_STOP_MASK)
  108. notifier_call_chain(nl, val_down, v, nr-1, NULL);
  109. return ret;
  110. }
  111. /*
  112. * Atomic notifier chain routines. Registration and unregistration
  113. * use a spinlock, and call_chain is synchronized by RCU (no locks).
  114. */
  115. /**
  116. * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
  117. * @nh: Pointer to head of the atomic notifier chain
  118. * @n: New entry in notifier chain
  119. *
  120. * Adds a notifier to an atomic notifier chain.
  121. *
  122. * Currently always returns zero.
  123. */
  124. int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
  125. struct notifier_block *n)
  126. {
  127. unsigned long flags;
  128. int ret;
  129. spin_lock_irqsave(&nh->lock, flags);
  130. ret = notifier_chain_register(&nh->head, n);
  131. spin_unlock_irqrestore(&nh->lock, flags);
  132. return ret;
  133. }
  134. EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
  135. /**
  136. * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
  137. * @nh: Pointer to head of the atomic notifier chain
  138. * @n: Entry to remove from notifier chain
  139. *
  140. * Removes a notifier from an atomic notifier chain.
  141. *
  142. * Returns zero on success or %-ENOENT on failure.
  143. */
  144. int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
  145. struct notifier_block *n)
  146. {
  147. unsigned long flags;
  148. int ret;
  149. spin_lock_irqsave(&nh->lock, flags);
  150. ret = notifier_chain_unregister(&nh->head, n);
  151. spin_unlock_irqrestore(&nh->lock, flags);
  152. synchronize_rcu();
  153. return ret;
  154. }
  155. EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
  156. int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
  157. unsigned long val_up, unsigned long val_down, void *v)
  158. {
  159. unsigned long flags;
  160. int ret;
  161. /*
  162. * Musn't use RCU; because then the notifier list can
  163. * change between the up and down traversal.
  164. */
  165. spin_lock_irqsave(&nh->lock, flags);
  166. ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
  167. spin_unlock_irqrestore(&nh->lock, flags);
  168. return ret;
  169. }
  170. EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust);
  171. NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust);
  172. /**
  173. * atomic_notifier_call_chain - Call functions in an atomic notifier chain
  174. * @nh: Pointer to head of the atomic notifier chain
  175. * @val: Value passed unmodified to notifier function
  176. * @v: Pointer passed unmodified to notifier function
  177. *
  178. * Calls each function in a notifier chain in turn. The functions
  179. * run in an atomic context, so they must not block.
  180. * This routine uses RCU to synchronize with changes to the chain.
  181. *
  182. * If the return value of the notifier can be and'ed
  183. * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
  184. * will return immediately, with the return value of
  185. * the notifier function which halted execution.
  186. * Otherwise the return value is the return value
  187. * of the last notifier function called.
  188. */
  189. int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
  190. unsigned long val, void *v)
  191. {
  192. int ret;
  193. rcu_read_lock();
  194. ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
  195. rcu_read_unlock();
  196. return ret;
  197. }
  198. EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
  199. NOKPROBE_SYMBOL(atomic_notifier_call_chain);
  200. /*
  201. * Blocking notifier chain routines. All access to the chain is
  202. * synchronized by an rwsem.
  203. */
  204. /**
  205. * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
  206. * @nh: Pointer to head of the blocking notifier chain
  207. * @n: New entry in notifier chain
  208. *
  209. * Adds a notifier to a blocking notifier chain.
  210. * Must be called in process context.
  211. *
  212. * Currently always returns zero.
  213. */
  214. int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
  215. struct notifier_block *n)
  216. {
  217. int ret;
  218. /*
  219. * This code gets used during boot-up, when task switching is
  220. * not yet working and interrupts must remain disabled. At
  221. * such times we must not call down_write().
  222. */
  223. if (unlikely(system_state == SYSTEM_BOOTING))
  224. return notifier_chain_register(&nh->head, n);
  225. down_write(&nh->rwsem);
  226. ret = notifier_chain_register(&nh->head, n);
  227. up_write(&nh->rwsem);
  228. return ret;
  229. }
  230. EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
  231. /**
  232. * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
  233. * @nh: Pointer to head of the blocking notifier chain
  234. * @n: Entry to remove from notifier chain
  235. *
  236. * Removes a notifier from a blocking notifier chain.
  237. * Must be called from process context.
  238. *
  239. * Returns zero on success or %-ENOENT on failure.
  240. */
  241. int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
  242. struct notifier_block *n)
  243. {
  244. int ret;
  245. /*
  246. * This code gets used during boot-up, when task switching is
  247. * not yet working and interrupts must remain disabled. At
  248. * such times we must not call down_write().
  249. */
  250. if (unlikely(system_state == SYSTEM_BOOTING))
  251. return notifier_chain_unregister(&nh->head, n);
  252. down_write(&nh->rwsem);
  253. ret = notifier_chain_unregister(&nh->head, n);
  254. up_write(&nh->rwsem);
  255. return ret;
  256. }
  257. EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
  258. int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
  259. unsigned long val_up, unsigned long val_down, void *v)
  260. {
  261. int ret = NOTIFY_DONE;
  262. /*
  263. * We check the head outside the lock, but if this access is
  264. * racy then it does not matter what the result of the test
  265. * is, we re-check the list after having taken the lock anyway:
  266. */
  267. if (rcu_access_pointer(nh->head)) {
  268. down_read(&nh->rwsem);
  269. ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
  270. up_read(&nh->rwsem);
  271. }
  272. return ret;
  273. }
  274. EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
  275. /**
  276. * blocking_notifier_call_chain - Call functions in a blocking notifier chain
  277. * @nh: Pointer to head of the blocking notifier chain
  278. * @val: Value passed unmodified to notifier function
  279. * @v: Pointer passed unmodified to notifier function
  280. *
  281. * Calls each function in a notifier chain in turn. The functions
  282. * run in a process context, so they are allowed to block.
  283. *
  284. * If the return value of the notifier can be and'ed
  285. * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
  286. * will return immediately, with the return value of
  287. * the notifier function which halted execution.
  288. * Otherwise the return value is the return value
  289. * of the last notifier function called.
  290. */
  291. int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
  292. unsigned long val, void *v)
  293. {
  294. int ret = NOTIFY_DONE;
  295. /*
  296. * We check the head outside the lock, but if this access is
  297. * racy then it does not matter what the result of the test
  298. * is, we re-check the list after having taken the lock anyway:
  299. */
  300. if (rcu_access_pointer(nh->head)) {
  301. down_read(&nh->rwsem);
  302. ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
  303. up_read(&nh->rwsem);
  304. }
  305. return ret;
  306. }
  307. EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
  308. /*
  309. * Raw notifier chain routines. There is no protection;
  310. * the caller must provide it. Use at your own risk!
  311. */
  312. /**
  313. * raw_notifier_chain_register - Add notifier to a raw notifier chain
  314. * @nh: Pointer to head of the raw notifier chain
  315. * @n: New entry in notifier chain
  316. *
  317. * Adds a notifier to a raw notifier chain.
  318. * All locking must be provided by the caller.
  319. *
  320. * Currently always returns zero.
  321. */
  322. int raw_notifier_chain_register(struct raw_notifier_head *nh,
  323. struct notifier_block *n)
  324. {
  325. return notifier_chain_register(&nh->head, n);
  326. }
  327. EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
  328. /**
  329. * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
  330. * @nh: Pointer to head of the raw notifier chain
  331. * @n: Entry to remove from notifier chain
  332. *
  333. * Removes a notifier from a raw notifier chain.
  334. * All locking must be provided by the caller.
  335. *
  336. * Returns zero on success or %-ENOENT on failure.
  337. */
  338. int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
  339. struct notifier_block *n)
  340. {
  341. return notifier_chain_unregister(&nh->head, n);
  342. }
  343. EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
  344. int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
  345. unsigned long val_up, unsigned long val_down, void *v)
  346. {
  347. return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
  348. }
  349. EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
  350. /**
  351. * raw_notifier_call_chain - Call functions in a raw notifier chain
  352. * @nh: Pointer to head of the raw notifier chain
  353. * @val: Value passed unmodified to notifier function
  354. * @v: Pointer passed unmodified to notifier function
  355. *
  356. * Calls each function in a notifier chain in turn. The functions
  357. * run in an undefined context.
  358. * All locking must be provided by the caller.
  359. *
  360. * If the return value of the notifier can be and'ed
  361. * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
  362. * will return immediately, with the return value of
  363. * the notifier function which halted execution.
  364. * Otherwise the return value is the return value
  365. * of the last notifier function called.
  366. */
  367. int raw_notifier_call_chain(struct raw_notifier_head *nh,
  368. unsigned long val, void *v)
  369. {
  370. return notifier_call_chain(&nh->head, val, v, -1, NULL);
  371. }
  372. EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
  373. #ifdef CONFIG_SRCU
  374. /*
  375. * SRCU notifier chain routines. Registration and unregistration
  376. * use a mutex, and call_chain is synchronized by SRCU (no locks).
  377. */
  378. /**
  379. * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
  380. * @nh: Pointer to head of the SRCU notifier chain
  381. * @n: New entry in notifier chain
  382. *
  383. * Adds a notifier to an SRCU notifier chain.
  384. * Must be called in process context.
  385. *
  386. * Currently always returns zero.
  387. */
  388. int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
  389. struct notifier_block *n)
  390. {
  391. int ret;
  392. /*
  393. * This code gets used during boot-up, when task switching is
  394. * not yet working and interrupts must remain disabled. At
  395. * such times we must not call mutex_lock().
  396. */
  397. if (unlikely(system_state == SYSTEM_BOOTING))
  398. return notifier_chain_register(&nh->head, n);
  399. mutex_lock(&nh->mutex);
  400. ret = notifier_chain_register(&nh->head, n);
  401. mutex_unlock(&nh->mutex);
  402. return ret;
  403. }
  404. EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
  405. /**
  406. * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
  407. * @nh: Pointer to head of the SRCU notifier chain
  408. * @n: Entry to remove from notifier chain
  409. *
  410. * Removes a notifier from an SRCU notifier chain.
  411. * Must be called from process context.
  412. *
  413. * Returns zero on success or %-ENOENT on failure.
  414. */
  415. int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
  416. struct notifier_block *n)
  417. {
  418. int ret;
  419. /*
  420. * This code gets used during boot-up, when task switching is
  421. * not yet working and interrupts must remain disabled. At
  422. * such times we must not call mutex_lock().
  423. */
  424. if (unlikely(system_state == SYSTEM_BOOTING))
  425. return notifier_chain_unregister(&nh->head, n);
  426. mutex_lock(&nh->mutex);
  427. ret = notifier_chain_unregister(&nh->head, n);
  428. mutex_unlock(&nh->mutex);
  429. synchronize_srcu(&nh->srcu);
  430. return ret;
  431. }
  432. EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
  433. /**
  434. * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
  435. * @nh: Pointer to head of the SRCU notifier chain
  436. * @val: Value passed unmodified to notifier function
  437. * @v: Pointer passed unmodified to notifier function
  438. *
  439. * Calls each function in a notifier chain in turn. The functions
  440. * run in a process context, so they are allowed to block.
  441. *
  442. * If the return value of the notifier can be and'ed
  443. * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
  444. * will return immediately, with the return value of
  445. * the notifier function which halted execution.
  446. * Otherwise the return value is the return value
  447. * of the last notifier function called.
  448. */
  449. int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
  450. unsigned long val, void *v)
  451. {
  452. int ret;
  453. int idx;
  454. idx = srcu_read_lock(&nh->srcu);
  455. ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
  456. srcu_read_unlock(&nh->srcu, idx);
  457. return ret;
  458. }
  459. EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
  460. /**
  461. * srcu_init_notifier_head - Initialize an SRCU notifier head
  462. * @nh: Pointer to head of the srcu notifier chain
  463. *
  464. * Unlike other sorts of notifier heads, SRCU notifier heads require
  465. * dynamic initialization. Be sure to call this routine before
  466. * calling any of the other SRCU notifier routines for this head.
  467. *
  468. * If an SRCU notifier head is deallocated, it must first be cleaned
  469. * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
  470. * per-cpu data (used by the SRCU mechanism) will leak.
  471. */
  472. void srcu_init_notifier_head(struct srcu_notifier_head *nh)
  473. {
  474. mutex_init(&nh->mutex);
  475. if (init_srcu_struct(&nh->srcu) < 0)
  476. BUG();
  477. nh->head = NULL;
  478. }
  479. EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
  480. #endif /* CONFIG_SRCU */
  481. static ATOMIC_NOTIFIER_HEAD(die_chain);
  482. int notrace notify_die(enum die_val val, const char *str,
  483. struct pt_regs *regs, long err, int trap, int sig)
  484. {
  485. struct die_args args = {
  486. .regs = regs,
  487. .str = str,
  488. .err = err,
  489. .trapnr = trap,
  490. .signr = sig,
  491. };
  492. RCU_LOCKDEP_WARN(!rcu_is_watching(),
  493. "notify_die called but RCU thinks we're quiescent");
  494. return atomic_notifier_call_chain(&die_chain, val, &args);
  495. }
  496. NOKPROBE_SYMBOL(notify_die);
  497. int register_die_notifier(struct notifier_block *nb)
  498. {
  499. return atomic_notifier_chain_register(&die_chain, nb);
  500. }
  501. EXPORT_SYMBOL_GPL(register_die_notifier);
  502. int unregister_die_notifier(struct notifier_block *nb)
  503. {
  504. return atomic_notifier_chain_unregister(&die_chain, nb);
  505. }
  506. EXPORT_SYMBOL_GPL(unregister_die_notifier);