transition.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * transition.c - Kernel Live Patching transition functions
  4. *
  5. * Copyright (C) 2015-2016 Josh Poimboeuf <jpoimboe@redhat.com>
  6. */
  7. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  8. #include <linux/cpu.h>
  9. #include <linux/stacktrace.h>
  10. #include "core.h"
  11. #include "patch.h"
  12. #include "transition.h"
  13. #include "../sched/sched.h"
  14. #define MAX_STACK_ENTRIES 100
  15. #define STACK_ERR_BUF_SIZE 128
  16. #define SIGNALS_TIMEOUT 15
  17. struct klp_patch *klp_transition_patch;
  18. static int klp_target_state = KLP_UNDEFINED;
  19. static unsigned int klp_signals_cnt;
  20. /*
  21. * This work can be performed periodically to finish patching or unpatching any
  22. * "straggler" tasks which failed to transition in the first attempt.
  23. */
  24. static void klp_transition_work_fn(struct work_struct *work)
  25. {
  26. mutex_lock(&klp_mutex);
  27. if (klp_transition_patch)
  28. klp_try_complete_transition();
  29. mutex_unlock(&klp_mutex);
  30. }
  31. static DECLARE_DELAYED_WORK(klp_transition_work, klp_transition_work_fn);
  32. /*
  33. * This function is just a stub to implement a hard force
  34. * of synchronize_rcu(). This requires synchronizing
  35. * tasks even in userspace and idle.
  36. */
  37. static void klp_sync(struct work_struct *work)
  38. {
  39. }
  40. /*
  41. * We allow to patch also functions where RCU is not watching,
  42. * e.g. before user_exit(). We can not rely on the RCU infrastructure
  43. * to do the synchronization. Instead hard force the sched synchronization.
  44. *
  45. * This approach allows to use RCU functions for manipulating func_stack
  46. * safely.
  47. */
  48. static void klp_synchronize_transition(void)
  49. {
  50. schedule_on_each_cpu(klp_sync);
  51. }
  52. /*
  53. * The transition to the target patch state is complete. Clean up the data
  54. * structures.
  55. */
  56. static void klp_complete_transition(void)
  57. {
  58. struct klp_object *obj;
  59. struct klp_func *func;
  60. struct task_struct *g, *task;
  61. unsigned int cpu;
  62. pr_debug("'%s': completing %s transition\n",
  63. klp_transition_patch->mod->name,
  64. klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
  65. if (klp_transition_patch->replace && klp_target_state == KLP_PATCHED) {
  66. klp_unpatch_replaced_patches(klp_transition_patch);
  67. klp_discard_nops(klp_transition_patch);
  68. }
  69. if (klp_target_state == KLP_UNPATCHED) {
  70. /*
  71. * All tasks have transitioned to KLP_UNPATCHED so we can now
  72. * remove the new functions from the func_stack.
  73. */
  74. klp_unpatch_objects(klp_transition_patch);
  75. /*
  76. * Make sure klp_ftrace_handler() can no longer see functions
  77. * from this patch on the ops->func_stack. Otherwise, after
  78. * func->transition gets cleared, the handler may choose a
  79. * removed function.
  80. */
  81. klp_synchronize_transition();
  82. }
  83. klp_for_each_object(klp_transition_patch, obj)
  84. klp_for_each_func(obj, func)
  85. func->transition = false;
  86. /* Prevent klp_ftrace_handler() from seeing KLP_UNDEFINED state */
  87. if (klp_target_state == KLP_PATCHED)
  88. klp_synchronize_transition();
  89. read_lock(&tasklist_lock);
  90. for_each_process_thread(g, task) {
  91. WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
  92. task->patch_state = KLP_UNDEFINED;
  93. }
  94. read_unlock(&tasklist_lock);
  95. for_each_possible_cpu(cpu) {
  96. task = idle_task(cpu);
  97. WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
  98. task->patch_state = KLP_UNDEFINED;
  99. }
  100. klp_for_each_object(klp_transition_patch, obj) {
  101. if (!klp_is_object_loaded(obj))
  102. continue;
  103. if (klp_target_state == KLP_PATCHED)
  104. klp_post_patch_callback(obj);
  105. else if (klp_target_state == KLP_UNPATCHED)
  106. klp_post_unpatch_callback(obj);
  107. }
  108. pr_notice("'%s': %s complete\n", klp_transition_patch->mod->name,
  109. klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
  110. klp_target_state = KLP_UNDEFINED;
  111. klp_transition_patch = NULL;
  112. }
  113. /*
  114. * This is called in the error path, to cancel a transition before it has
  115. * started, i.e. klp_init_transition() has been called but
  116. * klp_start_transition() hasn't. If the transition *has* been started,
  117. * klp_reverse_transition() should be used instead.
  118. */
  119. void klp_cancel_transition(void)
  120. {
  121. if (WARN_ON_ONCE(klp_target_state != KLP_PATCHED))
  122. return;
  123. pr_debug("'%s': canceling patching transition, going to unpatch\n",
  124. klp_transition_patch->mod->name);
  125. klp_target_state = KLP_UNPATCHED;
  126. klp_complete_transition();
  127. }
  128. /*
  129. * Switch the patched state of the task to the set of functions in the target
  130. * patch state.
  131. *
  132. * NOTE: If task is not 'current', the caller must ensure the task is inactive.
  133. * Otherwise klp_ftrace_handler() might read the wrong 'patch_state' value.
  134. */
  135. void klp_update_patch_state(struct task_struct *task)
  136. {
  137. /*
  138. * A variant of synchronize_rcu() is used to allow patching functions
  139. * where RCU is not watching, see klp_synchronize_transition().
  140. */
  141. preempt_disable_notrace();
  142. /*
  143. * This test_and_clear_tsk_thread_flag() call also serves as a read
  144. * barrier (smp_rmb) for two cases:
  145. *
  146. * 1) Enforce the order of the TIF_PATCH_PENDING read and the
  147. * klp_target_state read. The corresponding write barrier is in
  148. * klp_init_transition().
  149. *
  150. * 2) Enforce the order of the TIF_PATCH_PENDING read and a future read
  151. * of func->transition, if klp_ftrace_handler() is called later on
  152. * the same CPU. See __klp_disable_patch().
  153. */
  154. if (test_and_clear_tsk_thread_flag(task, TIF_PATCH_PENDING))
  155. task->patch_state = READ_ONCE(klp_target_state);
  156. preempt_enable_notrace();
  157. }
  158. /*
  159. * Determine whether the given stack trace includes any references to a
  160. * to-be-patched or to-be-unpatched function.
  161. */
  162. static int klp_check_stack_func(struct klp_func *func, unsigned long *entries,
  163. unsigned int nr_entries)
  164. {
  165. unsigned long func_addr, func_size, address;
  166. struct klp_ops *ops;
  167. int i;
  168. for (i = 0; i < nr_entries; i++) {
  169. address = entries[i];
  170. if (klp_target_state == KLP_UNPATCHED) {
  171. /*
  172. * Check for the to-be-unpatched function
  173. * (the func itself).
  174. */
  175. func_addr = (unsigned long)func->new_func;
  176. func_size = func->new_size;
  177. } else {
  178. /*
  179. * Check for the to-be-patched function
  180. * (the previous func).
  181. */
  182. ops = klp_find_ops(func->old_func);
  183. if (list_is_singular(&ops->func_stack)) {
  184. /* original function */
  185. func_addr = (unsigned long)func->old_func;
  186. func_size = func->old_size;
  187. } else {
  188. /* previously patched function */
  189. struct klp_func *prev;
  190. prev = list_next_entry(func, stack_node);
  191. func_addr = (unsigned long)prev->new_func;
  192. func_size = prev->new_size;
  193. }
  194. }
  195. if (address >= func_addr && address < func_addr + func_size)
  196. return -EAGAIN;
  197. }
  198. return 0;
  199. }
  200. /*
  201. * Determine whether it's safe to transition the task to the target patch state
  202. * by looking for any to-be-patched or to-be-unpatched functions on its stack.
  203. */
  204. static int klp_check_stack(struct task_struct *task, char *err_buf)
  205. {
  206. static unsigned long entries[MAX_STACK_ENTRIES];
  207. struct klp_object *obj;
  208. struct klp_func *func;
  209. int ret, nr_entries;
  210. ret = stack_trace_save_tsk_reliable(task, entries, ARRAY_SIZE(entries));
  211. if (ret < 0) {
  212. snprintf(err_buf, STACK_ERR_BUF_SIZE,
  213. "%s: %s:%d has an unreliable stack\n",
  214. __func__, task->comm, task->pid);
  215. return ret;
  216. }
  217. nr_entries = ret;
  218. klp_for_each_object(klp_transition_patch, obj) {
  219. if (!obj->patched)
  220. continue;
  221. klp_for_each_func(obj, func) {
  222. ret = klp_check_stack_func(func, entries, nr_entries);
  223. if (ret) {
  224. snprintf(err_buf, STACK_ERR_BUF_SIZE,
  225. "%s: %s:%d is sleeping on function %s\n",
  226. __func__, task->comm, task->pid,
  227. func->old_name);
  228. return ret;
  229. }
  230. }
  231. }
  232. return 0;
  233. }
  234. /*
  235. * Try to safely switch a task to the target patch state. If it's currently
  236. * running, or it's sleeping on a to-be-patched or to-be-unpatched function, or
  237. * if the stack is unreliable, return false.
  238. */
  239. static bool klp_try_switch_task(struct task_struct *task)
  240. {
  241. static char err_buf[STACK_ERR_BUF_SIZE];
  242. struct rq *rq;
  243. struct rq_flags flags;
  244. int ret;
  245. bool success = false;
  246. err_buf[0] = '\0';
  247. /* check if this task has already switched over */
  248. if (task->patch_state == klp_target_state)
  249. return true;
  250. /*
  251. * For arches which don't have reliable stack traces, we have to rely
  252. * on other methods (e.g., switching tasks at kernel exit).
  253. */
  254. if (!klp_have_reliable_stack())
  255. return false;
  256. /*
  257. * Now try to check the stack for any to-be-patched or to-be-unpatched
  258. * functions. If all goes well, switch the task to the target patch
  259. * state.
  260. */
  261. rq = task_rq_lock(task, &flags);
  262. if (task_running(rq, task) && task != current) {
  263. snprintf(err_buf, STACK_ERR_BUF_SIZE,
  264. "%s: %s:%d is running\n", __func__, task->comm,
  265. task->pid);
  266. goto done;
  267. }
  268. ret = klp_check_stack(task, err_buf);
  269. if (ret)
  270. goto done;
  271. success = true;
  272. clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
  273. task->patch_state = klp_target_state;
  274. done:
  275. task_rq_unlock(rq, task, &flags);
  276. /*
  277. * Due to console deadlock issues, pr_debug() can't be used while
  278. * holding the task rq lock. Instead we have to use a temporary buffer
  279. * and print the debug message after releasing the lock.
  280. */
  281. if (err_buf[0] != '\0')
  282. pr_debug("%s", err_buf);
  283. return success;
  284. }
  285. /*
  286. * Sends a fake signal to all non-kthread tasks with TIF_PATCH_PENDING set.
  287. * Kthreads with TIF_PATCH_PENDING set are woken up.
  288. */
  289. static void klp_send_signals(void)
  290. {
  291. struct task_struct *g, *task;
  292. if (klp_signals_cnt == SIGNALS_TIMEOUT)
  293. pr_notice("signaling remaining tasks\n");
  294. read_lock(&tasklist_lock);
  295. for_each_process_thread(g, task) {
  296. if (!klp_patch_pending(task))
  297. continue;
  298. /*
  299. * There is a small race here. We could see TIF_PATCH_PENDING
  300. * set and decide to wake up a kthread or send a fake signal.
  301. * Meanwhile the task could migrate itself and the action
  302. * would be meaningless. It is not serious though.
  303. */
  304. if (task->flags & PF_KTHREAD) {
  305. /*
  306. * Wake up a kthread which sleeps interruptedly and
  307. * still has not been migrated.
  308. */
  309. wake_up_state(task, TASK_INTERRUPTIBLE);
  310. } else {
  311. /*
  312. * Send fake signal to all non-kthread tasks which are
  313. * still not migrated.
  314. */
  315. spin_lock_irq(&task->sighand->siglock);
  316. signal_wake_up(task, 0);
  317. spin_unlock_irq(&task->sighand->siglock);
  318. }
  319. }
  320. read_unlock(&tasklist_lock);
  321. }
  322. /*
  323. * Try to switch all remaining tasks to the target patch state by walking the
  324. * stacks of sleeping tasks and looking for any to-be-patched or
  325. * to-be-unpatched functions. If such functions are found, the task can't be
  326. * switched yet.
  327. *
  328. * If any tasks are still stuck in the initial patch state, schedule a retry.
  329. */
  330. void klp_try_complete_transition(void)
  331. {
  332. unsigned int cpu;
  333. struct task_struct *g, *task;
  334. struct klp_patch *patch;
  335. bool complete = true;
  336. WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
  337. /*
  338. * Try to switch the tasks to the target patch state by walking their
  339. * stacks and looking for any to-be-patched or to-be-unpatched
  340. * functions. If such functions are found on a stack, or if the stack
  341. * is deemed unreliable, the task can't be switched yet.
  342. *
  343. * Usually this will transition most (or all) of the tasks on a system
  344. * unless the patch includes changes to a very common function.
  345. */
  346. read_lock(&tasklist_lock);
  347. for_each_process_thread(g, task)
  348. if (!klp_try_switch_task(task))
  349. complete = false;
  350. read_unlock(&tasklist_lock);
  351. /*
  352. * Ditto for the idle "swapper" tasks.
  353. */
  354. get_online_cpus();
  355. for_each_possible_cpu(cpu) {
  356. task = idle_task(cpu);
  357. if (cpu_online(cpu)) {
  358. if (!klp_try_switch_task(task))
  359. complete = false;
  360. } else if (task->patch_state != klp_target_state) {
  361. /* offline idle tasks can be switched immediately */
  362. clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
  363. task->patch_state = klp_target_state;
  364. }
  365. }
  366. put_online_cpus();
  367. if (!complete) {
  368. if (klp_signals_cnt && !(klp_signals_cnt % SIGNALS_TIMEOUT))
  369. klp_send_signals();
  370. klp_signals_cnt++;
  371. /*
  372. * Some tasks weren't able to be switched over. Try again
  373. * later and/or wait for other methods like kernel exit
  374. * switching.
  375. */
  376. schedule_delayed_work(&klp_transition_work,
  377. round_jiffies_relative(HZ));
  378. return;
  379. }
  380. /* we're done, now cleanup the data structures */
  381. patch = klp_transition_patch;
  382. klp_complete_transition();
  383. /*
  384. * It would make more sense to free the unused patches in
  385. * klp_complete_transition() but it is called also
  386. * from klp_cancel_transition().
  387. */
  388. if (!patch->enabled)
  389. klp_free_patch_async(patch);
  390. else if (patch->replace)
  391. klp_free_replaced_patches_async(patch);
  392. }
  393. /*
  394. * Start the transition to the specified target patch state so tasks can begin
  395. * switching to it.
  396. */
  397. void klp_start_transition(void)
  398. {
  399. struct task_struct *g, *task;
  400. unsigned int cpu;
  401. WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
  402. pr_notice("'%s': starting %s transition\n",
  403. klp_transition_patch->mod->name,
  404. klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
  405. /*
  406. * Mark all normal tasks as needing a patch state update. They'll
  407. * switch either in klp_try_complete_transition() or as they exit the
  408. * kernel.
  409. */
  410. read_lock(&tasklist_lock);
  411. for_each_process_thread(g, task)
  412. if (task->patch_state != klp_target_state)
  413. set_tsk_thread_flag(task, TIF_PATCH_PENDING);
  414. read_unlock(&tasklist_lock);
  415. /*
  416. * Mark all idle tasks as needing a patch state update. They'll switch
  417. * either in klp_try_complete_transition() or at the idle loop switch
  418. * point.
  419. */
  420. for_each_possible_cpu(cpu) {
  421. task = idle_task(cpu);
  422. if (task->patch_state != klp_target_state)
  423. set_tsk_thread_flag(task, TIF_PATCH_PENDING);
  424. }
  425. klp_signals_cnt = 0;
  426. }
  427. /*
  428. * Initialize the global target patch state and all tasks to the initial patch
  429. * state, and initialize all function transition states to true in preparation
  430. * for patching or unpatching.
  431. */
  432. void klp_init_transition(struct klp_patch *patch, int state)
  433. {
  434. struct task_struct *g, *task;
  435. unsigned int cpu;
  436. struct klp_object *obj;
  437. struct klp_func *func;
  438. int initial_state = !state;
  439. WARN_ON_ONCE(klp_target_state != KLP_UNDEFINED);
  440. klp_transition_patch = patch;
  441. /*
  442. * Set the global target patch state which tasks will switch to. This
  443. * has no effect until the TIF_PATCH_PENDING flags get set later.
  444. */
  445. klp_target_state = state;
  446. pr_debug("'%s': initializing %s transition\n", patch->mod->name,
  447. klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
  448. /*
  449. * Initialize all tasks to the initial patch state to prepare them for
  450. * switching to the target state.
  451. */
  452. read_lock(&tasklist_lock);
  453. for_each_process_thread(g, task) {
  454. WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
  455. task->patch_state = initial_state;
  456. }
  457. read_unlock(&tasklist_lock);
  458. /*
  459. * Ditto for the idle "swapper" tasks.
  460. */
  461. for_each_possible_cpu(cpu) {
  462. task = idle_task(cpu);
  463. WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
  464. task->patch_state = initial_state;
  465. }
  466. /*
  467. * Enforce the order of the task->patch_state initializations and the
  468. * func->transition updates to ensure that klp_ftrace_handler() doesn't
  469. * see a func in transition with a task->patch_state of KLP_UNDEFINED.
  470. *
  471. * Also enforce the order of the klp_target_state write and future
  472. * TIF_PATCH_PENDING writes to ensure klp_update_patch_state() doesn't
  473. * set a task->patch_state to KLP_UNDEFINED.
  474. */
  475. smp_wmb();
  476. /*
  477. * Set the func transition states so klp_ftrace_handler() will know to
  478. * switch to the transition logic.
  479. *
  480. * When patching, the funcs aren't yet in the func_stack and will be
  481. * made visible to the ftrace handler shortly by the calls to
  482. * klp_patch_object().
  483. *
  484. * When unpatching, the funcs are already in the func_stack and so are
  485. * already visible to the ftrace handler.
  486. */
  487. klp_for_each_object(patch, obj)
  488. klp_for_each_func(obj, func)
  489. func->transition = true;
  490. }
  491. /*
  492. * This function can be called in the middle of an existing transition to
  493. * reverse the direction of the target patch state. This can be done to
  494. * effectively cancel an existing enable or disable operation if there are any
  495. * tasks which are stuck in the initial patch state.
  496. */
  497. void klp_reverse_transition(void)
  498. {
  499. unsigned int cpu;
  500. struct task_struct *g, *task;
  501. pr_debug("'%s': reversing transition from %s\n",
  502. klp_transition_patch->mod->name,
  503. klp_target_state == KLP_PATCHED ? "patching to unpatching" :
  504. "unpatching to patching");
  505. klp_transition_patch->enabled = !klp_transition_patch->enabled;
  506. klp_target_state = !klp_target_state;
  507. /*
  508. * Clear all TIF_PATCH_PENDING flags to prevent races caused by
  509. * klp_update_patch_state() running in parallel with
  510. * klp_start_transition().
  511. */
  512. read_lock(&tasklist_lock);
  513. for_each_process_thread(g, task)
  514. clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
  515. read_unlock(&tasklist_lock);
  516. for_each_possible_cpu(cpu)
  517. clear_tsk_thread_flag(idle_task(cpu), TIF_PATCH_PENDING);
  518. /* Let any remaining calls to klp_update_patch_state() complete */
  519. klp_synchronize_transition();
  520. klp_start_transition();
  521. }
  522. /* Called from copy_process() during fork */
  523. void klp_copy_process(struct task_struct *child)
  524. {
  525. child->patch_state = current->patch_state;
  526. /* TIF_PATCH_PENDING gets copied in setup_thread_stack() */
  527. }
  528. /*
  529. * Drop TIF_PATCH_PENDING of all tasks on admin's request. This forces an
  530. * existing transition to finish.
  531. *
  532. * NOTE: klp_update_patch_state(task) requires the task to be inactive or
  533. * 'current'. This is not the case here and the consistency model could be
  534. * broken. Administrator, who is the only one to execute the
  535. * klp_force_transitions(), has to be aware of this.
  536. */
  537. void klp_force_transition(void)
  538. {
  539. struct klp_patch *patch;
  540. struct task_struct *g, *task;
  541. unsigned int cpu;
  542. pr_warn("forcing remaining tasks to the patched state\n");
  543. read_lock(&tasklist_lock);
  544. for_each_process_thread(g, task)
  545. klp_update_patch_state(task);
  546. read_unlock(&tasklist_lock);
  547. for_each_possible_cpu(cpu)
  548. klp_update_patch_state(idle_task(cpu));
  549. klp_for_each_patch(patch)
  550. patch->forced = true;
  551. }