static_call.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/init.h>
  3. #include <linux/static_call.h>
  4. #include <linux/bug.h>
  5. #include <linux/smp.h>
  6. #include <linux/sort.h>
  7. #include <linux/slab.h>
  8. #include <linux/module.h>
  9. #include <linux/cpu.h>
  10. #include <linux/processor.h>
  11. #include <asm/sections.h>
  12. extern struct static_call_site __start_static_call_sites[],
  13. __stop_static_call_sites[];
  14. extern struct static_call_tramp_key __start_static_call_tramp_key[],
  15. __stop_static_call_tramp_key[];
  16. static bool static_call_initialized;
  17. /* mutex to protect key modules/sites */
  18. static DEFINE_MUTEX(static_call_mutex);
  19. static void static_call_lock(void)
  20. {
  21. mutex_lock(&static_call_mutex);
  22. }
  23. static void static_call_unlock(void)
  24. {
  25. mutex_unlock(&static_call_mutex);
  26. }
  27. static inline void *static_call_addr(struct static_call_site *site)
  28. {
  29. return (void *)((long)site->addr + (long)&site->addr);
  30. }
  31. static inline unsigned long __static_call_key(const struct static_call_site *site)
  32. {
  33. return (long)site->key + (long)&site->key;
  34. }
  35. static inline struct static_call_key *static_call_key(const struct static_call_site *site)
  36. {
  37. return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
  38. }
  39. /* These assume the key is word-aligned. */
  40. static inline bool static_call_is_init(struct static_call_site *site)
  41. {
  42. return __static_call_key(site) & STATIC_CALL_SITE_INIT;
  43. }
  44. static inline bool static_call_is_tail(struct static_call_site *site)
  45. {
  46. return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
  47. }
  48. static inline void static_call_set_init(struct static_call_site *site)
  49. {
  50. site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
  51. (long)&site->key;
  52. }
  53. static int static_call_site_cmp(const void *_a, const void *_b)
  54. {
  55. const struct static_call_site *a = _a;
  56. const struct static_call_site *b = _b;
  57. const struct static_call_key *key_a = static_call_key(a);
  58. const struct static_call_key *key_b = static_call_key(b);
  59. if (key_a < key_b)
  60. return -1;
  61. if (key_a > key_b)
  62. return 1;
  63. return 0;
  64. }
  65. static void static_call_site_swap(void *_a, void *_b, int size)
  66. {
  67. long delta = (unsigned long)_a - (unsigned long)_b;
  68. struct static_call_site *a = _a;
  69. struct static_call_site *b = _b;
  70. struct static_call_site tmp = *a;
  71. a->addr = b->addr - delta;
  72. a->key = b->key - delta;
  73. b->addr = tmp.addr + delta;
  74. b->key = tmp.key + delta;
  75. }
  76. static inline void static_call_sort_entries(struct static_call_site *start,
  77. struct static_call_site *stop)
  78. {
  79. sort(start, stop - start, sizeof(struct static_call_site),
  80. static_call_site_cmp, static_call_site_swap);
  81. }
  82. static inline bool static_call_key_has_mods(struct static_call_key *key)
  83. {
  84. return !(key->type & 1);
  85. }
  86. static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
  87. {
  88. if (!static_call_key_has_mods(key))
  89. return NULL;
  90. return key->mods;
  91. }
  92. static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
  93. {
  94. if (static_call_key_has_mods(key))
  95. return NULL;
  96. return (struct static_call_site *)(key->type & ~1);
  97. }
  98. void __static_call_update(struct static_call_key *key, void *tramp, void *func)
  99. {
  100. struct static_call_site *site, *stop;
  101. struct static_call_mod *site_mod, first;
  102. cpus_read_lock();
  103. static_call_lock();
  104. if (key->func == func)
  105. goto done;
  106. key->func = func;
  107. arch_static_call_transform(NULL, tramp, func, false);
  108. /*
  109. * If uninitialized, we'll not update the callsites, but they still
  110. * point to the trampoline and we just patched that.
  111. */
  112. if (WARN_ON_ONCE(!static_call_initialized))
  113. goto done;
  114. first = (struct static_call_mod){
  115. .next = static_call_key_next(key),
  116. .mod = NULL,
  117. .sites = static_call_key_sites(key),
  118. };
  119. for (site_mod = &first; site_mod; site_mod = site_mod->next) {
  120. bool init = system_state < SYSTEM_RUNNING;
  121. struct module *mod = site_mod->mod;
  122. if (!site_mod->sites) {
  123. /*
  124. * This can happen if the static call key is defined in
  125. * a module which doesn't use it.
  126. *
  127. * It also happens in the has_mods case, where the
  128. * 'first' entry has no sites associated with it.
  129. */
  130. continue;
  131. }
  132. stop = __stop_static_call_sites;
  133. if (mod) {
  134. #ifdef CONFIG_MODULES
  135. stop = mod->static_call_sites +
  136. mod->num_static_call_sites;
  137. init = mod->state == MODULE_STATE_COMING;
  138. #endif
  139. }
  140. for (site = site_mod->sites;
  141. site < stop && static_call_key(site) == key; site++) {
  142. void *site_addr = static_call_addr(site);
  143. if (!init && static_call_is_init(site))
  144. continue;
  145. if (!kernel_text_address((unsigned long)site_addr)) {
  146. /*
  147. * This skips patching built-in __exit, which
  148. * is part of init_section_contains() but is
  149. * not part of kernel_text_address().
  150. *
  151. * Skipping built-in __exit is fine since it
  152. * will never be executed.
  153. */
  154. WARN_ONCE(!static_call_is_init(site),
  155. "can't patch static call site at %pS",
  156. site_addr);
  157. continue;
  158. }
  159. arch_static_call_transform(site_addr, NULL, func,
  160. static_call_is_tail(site));
  161. }
  162. }
  163. done:
  164. static_call_unlock();
  165. cpus_read_unlock();
  166. }
  167. EXPORT_SYMBOL_GPL(__static_call_update);
  168. static int __static_call_init(struct module *mod,
  169. struct static_call_site *start,
  170. struct static_call_site *stop)
  171. {
  172. struct static_call_site *site;
  173. struct static_call_key *key, *prev_key = NULL;
  174. struct static_call_mod *site_mod;
  175. if (start == stop)
  176. return 0;
  177. static_call_sort_entries(start, stop);
  178. for (site = start; site < stop; site++) {
  179. void *site_addr = static_call_addr(site);
  180. if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
  181. (!mod && init_section_contains(site_addr, 1)))
  182. static_call_set_init(site);
  183. key = static_call_key(site);
  184. if (key != prev_key) {
  185. prev_key = key;
  186. /*
  187. * For vmlinux (!mod) avoid the allocation by storing
  188. * the sites pointer in the key itself. Also see
  189. * __static_call_update()'s @first.
  190. *
  191. * This allows architectures (eg. x86) to call
  192. * static_call_init() before memory allocation works.
  193. */
  194. if (!mod) {
  195. key->sites = site;
  196. key->type |= 1;
  197. goto do_transform;
  198. }
  199. site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
  200. if (!site_mod)
  201. return -ENOMEM;
  202. /*
  203. * When the key has a direct sites pointer, extract
  204. * that into an explicit struct static_call_mod, so we
  205. * can have a list of modules.
  206. */
  207. if (static_call_key_sites(key)) {
  208. site_mod->mod = NULL;
  209. site_mod->next = NULL;
  210. site_mod->sites = static_call_key_sites(key);
  211. key->mods = site_mod;
  212. site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
  213. if (!site_mod)
  214. return -ENOMEM;
  215. }
  216. site_mod->mod = mod;
  217. site_mod->sites = site;
  218. site_mod->next = static_call_key_next(key);
  219. key->mods = site_mod;
  220. }
  221. do_transform:
  222. arch_static_call_transform(site_addr, NULL, key->func,
  223. static_call_is_tail(site));
  224. }
  225. return 0;
  226. }
  227. static int addr_conflict(struct static_call_site *site, void *start, void *end)
  228. {
  229. unsigned long addr = (unsigned long)static_call_addr(site);
  230. if (addr <= (unsigned long)end &&
  231. addr + CALL_INSN_SIZE > (unsigned long)start)
  232. return 1;
  233. return 0;
  234. }
  235. static int __static_call_text_reserved(struct static_call_site *iter_start,
  236. struct static_call_site *iter_stop,
  237. void *start, void *end, bool init)
  238. {
  239. struct static_call_site *iter = iter_start;
  240. while (iter < iter_stop) {
  241. if (init || !static_call_is_init(iter)) {
  242. if (addr_conflict(iter, start, end))
  243. return 1;
  244. }
  245. iter++;
  246. }
  247. return 0;
  248. }
  249. #ifdef CONFIG_MODULES
  250. static int __static_call_mod_text_reserved(void *start, void *end)
  251. {
  252. struct module *mod;
  253. int ret;
  254. preempt_disable();
  255. mod = __module_text_address((unsigned long)start);
  256. WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
  257. if (!try_module_get(mod))
  258. mod = NULL;
  259. preempt_enable();
  260. if (!mod)
  261. return 0;
  262. ret = __static_call_text_reserved(mod->static_call_sites,
  263. mod->static_call_sites + mod->num_static_call_sites,
  264. start, end, mod->state == MODULE_STATE_COMING);
  265. module_put(mod);
  266. return ret;
  267. }
  268. static unsigned long tramp_key_lookup(unsigned long addr)
  269. {
  270. struct static_call_tramp_key *start = __start_static_call_tramp_key;
  271. struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
  272. struct static_call_tramp_key *tramp_key;
  273. for (tramp_key = start; tramp_key != stop; tramp_key++) {
  274. unsigned long tramp;
  275. tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
  276. if (tramp == addr)
  277. return (long)tramp_key->key + (long)&tramp_key->key;
  278. }
  279. return 0;
  280. }
  281. static int static_call_add_module(struct module *mod)
  282. {
  283. struct static_call_site *start = mod->static_call_sites;
  284. struct static_call_site *stop = start + mod->num_static_call_sites;
  285. struct static_call_site *site;
  286. for (site = start; site != stop; site++) {
  287. unsigned long s_key = __static_call_key(site);
  288. unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
  289. unsigned long key;
  290. /*
  291. * Is the key is exported, 'addr' points to the key, which
  292. * means modules are allowed to call static_call_update() on
  293. * it.
  294. *
  295. * Otherwise, the key isn't exported, and 'addr' points to the
  296. * trampoline so we need to lookup the key.
  297. *
  298. * We go through this dance to prevent crazy modules from
  299. * abusing sensitive static calls.
  300. */
  301. if (!kernel_text_address(addr))
  302. continue;
  303. key = tramp_key_lookup(addr);
  304. if (!key) {
  305. pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
  306. static_call_addr(site));
  307. return -EINVAL;
  308. }
  309. key |= s_key & STATIC_CALL_SITE_FLAGS;
  310. site->key = key - (long)&site->key;
  311. }
  312. return __static_call_init(mod, start, stop);
  313. }
  314. static void static_call_del_module(struct module *mod)
  315. {
  316. struct static_call_site *start = mod->static_call_sites;
  317. struct static_call_site *stop = mod->static_call_sites +
  318. mod->num_static_call_sites;
  319. struct static_call_key *key, *prev_key = NULL;
  320. struct static_call_mod *site_mod, **prev;
  321. struct static_call_site *site;
  322. for (site = start; site < stop; site++) {
  323. key = static_call_key(site);
  324. if (key == prev_key)
  325. continue;
  326. prev_key = key;
  327. for (prev = &key->mods, site_mod = key->mods;
  328. site_mod && site_mod->mod != mod;
  329. prev = &site_mod->next, site_mod = site_mod->next)
  330. ;
  331. if (!site_mod)
  332. continue;
  333. *prev = site_mod->next;
  334. kfree(site_mod);
  335. }
  336. }
  337. static int static_call_module_notify(struct notifier_block *nb,
  338. unsigned long val, void *data)
  339. {
  340. struct module *mod = data;
  341. int ret = 0;
  342. cpus_read_lock();
  343. static_call_lock();
  344. switch (val) {
  345. case MODULE_STATE_COMING:
  346. ret = static_call_add_module(mod);
  347. if (ret) {
  348. WARN(1, "Failed to allocate memory for static calls");
  349. static_call_del_module(mod);
  350. }
  351. break;
  352. case MODULE_STATE_GOING:
  353. static_call_del_module(mod);
  354. break;
  355. }
  356. static_call_unlock();
  357. cpus_read_unlock();
  358. return notifier_from_errno(ret);
  359. }
  360. static struct notifier_block static_call_module_nb = {
  361. .notifier_call = static_call_module_notify,
  362. };
  363. #else
  364. static inline int __static_call_mod_text_reserved(void *start, void *end)
  365. {
  366. return 0;
  367. }
  368. #endif /* CONFIG_MODULES */
  369. int static_call_text_reserved(void *start, void *end)
  370. {
  371. bool init = system_state < SYSTEM_RUNNING;
  372. int ret = __static_call_text_reserved(__start_static_call_sites,
  373. __stop_static_call_sites, start, end, init);
  374. if (ret)
  375. return ret;
  376. return __static_call_mod_text_reserved(start, end);
  377. }
  378. int __init static_call_init(void)
  379. {
  380. int ret;
  381. if (static_call_initialized)
  382. return 0;
  383. cpus_read_lock();
  384. static_call_lock();
  385. ret = __static_call_init(NULL, __start_static_call_sites,
  386. __stop_static_call_sites);
  387. static_call_unlock();
  388. cpus_read_unlock();
  389. if (ret) {
  390. pr_err("Failed to allocate memory for static_call!\n");
  391. BUG();
  392. }
  393. static_call_initialized = true;
  394. #ifdef CONFIG_MODULES
  395. register_module_notifier(&static_call_module_nb);
  396. #endif
  397. return 0;
  398. }
  399. early_initcall(static_call_init);
  400. #ifdef CONFIG_STATIC_CALL_SELFTEST
  401. static int func_a(int x)
  402. {
  403. return x+1;
  404. }
  405. static int func_b(int x)
  406. {
  407. return x+2;
  408. }
  409. DEFINE_STATIC_CALL(sc_selftest, func_a);
  410. static struct static_call_data {
  411. int (*func)(int);
  412. int val;
  413. int expect;
  414. } static_call_data [] __initdata = {
  415. { NULL, 2, 3 },
  416. { func_b, 2, 4 },
  417. { func_a, 2, 3 }
  418. };
  419. static int __init test_static_call_init(void)
  420. {
  421. int i;
  422. for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
  423. struct static_call_data *scd = &static_call_data[i];
  424. if (scd->func)
  425. static_call_update(sc_selftest, scd->func);
  426. WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
  427. }
  428. return 0;
  429. }
  430. early_initcall(test_static_call_init);
  431. #endif /* CONFIG_STATIC_CALL_SELFTEST */