kthread.c 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Kernel thread helper functions.
  3. * Copyright (C) 2004 IBM Corporation, Rusty Russell.
  4. * Copyright (C) 2009 Red Hat, Inc.
  5. *
  6. * Creation is done via kthreadd, so that we get a clean environment
  7. * even if we're invoked from userspace (think modprobe, hotplug cpu,
  8. * etc.).
  9. */
  10. #include <uapi/linux/sched/types.h>
  11. #include <linux/mm.h>
  12. #include <linux/mmu_context.h>
  13. #include <linux/sched.h>
  14. #include <linux/sched/mm.h>
  15. #include <linux/sched/task.h>
  16. #include <linux/kthread.h>
  17. #include <linux/completion.h>
  18. #include <linux/err.h>
  19. #include <linux/cgroup.h>
  20. #include <linux/cpuset.h>
  21. #include <linux/unistd.h>
  22. #include <linux/file.h>
  23. #include <linux/export.h>
  24. #include <linux/mutex.h>
  25. #include <linux/slab.h>
  26. #include <linux/freezer.h>
  27. #include <linux/ptrace.h>
  28. #include <linux/uaccess.h>
  29. #include <linux/numa.h>
  30. #include <linux/sched/isolation.h>
  31. #include <trace/events/sched.h>
  32. static DEFINE_SPINLOCK(kthread_create_lock);
  33. static LIST_HEAD(kthread_create_list);
  34. struct task_struct *kthreadd_task;
  35. struct kthread_create_info
  36. {
  37. /* Information passed to kthread() from kthreadd. */
  38. int (*threadfn)(void *data);
  39. void *data;
  40. int node;
  41. /* Result passed back to kthread_create() from kthreadd. */
  42. struct task_struct *result;
  43. struct completion *done;
  44. struct list_head list;
  45. };
  46. struct kthread {
  47. unsigned long flags;
  48. unsigned int cpu;
  49. int (*threadfn)(void *);
  50. void *data;
  51. mm_segment_t oldfs;
  52. struct completion parked;
  53. struct completion exited;
  54. #ifdef CONFIG_BLK_CGROUP
  55. struct cgroup_subsys_state *blkcg_css;
  56. #endif
  57. };
  58. enum KTHREAD_BITS {
  59. KTHREAD_IS_PER_CPU = 0,
  60. KTHREAD_SHOULD_STOP,
  61. KTHREAD_SHOULD_PARK,
  62. };
  63. static inline void set_kthread_struct(void *kthread)
  64. {
  65. /*
  66. * We abuse ->set_child_tid to avoid the new member and because it
  67. * can't be wrongly copied by copy_process(). We also rely on fact
  68. * that the caller can't exec, so PF_KTHREAD can't be cleared.
  69. */
  70. current->set_child_tid = (__force void __user *)kthread;
  71. }
  72. static inline struct kthread *to_kthread(struct task_struct *k)
  73. {
  74. WARN_ON(!(k->flags & PF_KTHREAD));
  75. return (__force void *)k->set_child_tid;
  76. }
  77. /*
  78. * Variant of to_kthread() that doesn't assume @p is a kthread.
  79. *
  80. * Per construction; when:
  81. *
  82. * (p->flags & PF_KTHREAD) && p->set_child_tid
  83. *
  84. * the task is both a kthread and struct kthread is persistent. However
  85. * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and
  86. * begin_new_exec()).
  87. */
  88. static inline struct kthread *__to_kthread(struct task_struct *p)
  89. {
  90. void *kthread = (__force void *)p->set_child_tid;
  91. if (kthread && !(p->flags & PF_KTHREAD))
  92. kthread = NULL;
  93. return kthread;
  94. }
  95. void free_kthread_struct(struct task_struct *k)
  96. {
  97. struct kthread *kthread;
  98. /*
  99. * Can be NULL if this kthread was created by kernel_thread()
  100. * or if kmalloc() in kthread() failed.
  101. */
  102. kthread = to_kthread(k);
  103. #ifdef CONFIG_BLK_CGROUP
  104. WARN_ON_ONCE(kthread && kthread->blkcg_css);
  105. #endif
  106. kfree(kthread);
  107. }
  108. /**
  109. * kthread_should_stop - should this kthread return now?
  110. *
  111. * When someone calls kthread_stop() on your kthread, it will be woken
  112. * and this will return true. You should then return, and your return
  113. * value will be passed through to kthread_stop().
  114. */
  115. bool kthread_should_stop(void)
  116. {
  117. return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
  118. }
  119. EXPORT_SYMBOL(kthread_should_stop);
  120. bool __kthread_should_park(struct task_struct *k)
  121. {
  122. return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
  123. }
  124. EXPORT_SYMBOL_GPL(__kthread_should_park);
  125. /**
  126. * kthread_should_park - should this kthread park now?
  127. *
  128. * When someone calls kthread_park() on your kthread, it will be woken
  129. * and this will return true. You should then do the necessary
  130. * cleanup and call kthread_parkme()
  131. *
  132. * Similar to kthread_should_stop(), but this keeps the thread alive
  133. * and in a park position. kthread_unpark() "restarts" the thread and
  134. * calls the thread function again.
  135. */
  136. bool kthread_should_park(void)
  137. {
  138. return __kthread_should_park(current);
  139. }
  140. EXPORT_SYMBOL_GPL(kthread_should_park);
  141. /**
  142. * kthread_freezable_should_stop - should this freezable kthread return now?
  143. * @was_frozen: optional out parameter, indicates whether %current was frozen
  144. *
  145. * kthread_should_stop() for freezable kthreads, which will enter
  146. * refrigerator if necessary. This function is safe from kthread_stop() /
  147. * freezer deadlock and freezable kthreads should use this function instead
  148. * of calling try_to_freeze() directly.
  149. */
  150. bool kthread_freezable_should_stop(bool *was_frozen)
  151. {
  152. bool frozen = false;
  153. might_sleep();
  154. if (unlikely(freezing(current)))
  155. frozen = __refrigerator(true);
  156. if (was_frozen)
  157. *was_frozen = frozen;
  158. return kthread_should_stop();
  159. }
  160. EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
  161. /**
  162. * kthread_func - return the function specified on kthread creation
  163. * @task: kthread task in question
  164. *
  165. * Returns NULL if the task is not a kthread.
  166. */
  167. void *kthread_func(struct task_struct *task)
  168. {
  169. struct kthread *kthread = __to_kthread(task);
  170. if (kthread)
  171. return kthread->threadfn;
  172. return NULL;
  173. }
  174. EXPORT_SYMBOL_GPL(kthread_func);
  175. /**
  176. * kthread_data - return data value specified on kthread creation
  177. * @task: kthread task in question
  178. *
  179. * Return the data value specified when kthread @task was created.
  180. * The caller is responsible for ensuring the validity of @task when
  181. * calling this function.
  182. */
  183. void *kthread_data(struct task_struct *task)
  184. {
  185. return to_kthread(task)->data;
  186. }
  187. EXPORT_SYMBOL_GPL(kthread_data);
  188. /**
  189. * kthread_probe_data - speculative version of kthread_data()
  190. * @task: possible kthread task in question
  191. *
  192. * @task could be a kthread task. Return the data value specified when it
  193. * was created if accessible. If @task isn't a kthread task or its data is
  194. * inaccessible for any reason, %NULL is returned. This function requires
  195. * that @task itself is safe to dereference.
  196. */
  197. void *kthread_probe_data(struct task_struct *task)
  198. {
  199. struct kthread *kthread = __to_kthread(task);
  200. void *data = NULL;
  201. if (kthread)
  202. copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));
  203. return data;
  204. }
  205. static void __kthread_parkme(struct kthread *self)
  206. {
  207. for (;;) {
  208. /*
  209. * TASK_PARKED is a special state; we must serialize against
  210. * possible pending wakeups to avoid store-store collisions on
  211. * task->state.
  212. *
  213. * Such a collision might possibly result in the task state
  214. * changin from TASK_PARKED and us failing the
  215. * wait_task_inactive() in kthread_park().
  216. */
  217. set_special_state(TASK_PARKED);
  218. if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
  219. break;
  220. /*
  221. * Thread is going to call schedule(), do not preempt it,
  222. * or the caller of kthread_park() may spend more time in
  223. * wait_task_inactive().
  224. */
  225. preempt_disable();
  226. complete(&self->parked);
  227. schedule_preempt_disabled();
  228. preempt_enable();
  229. }
  230. __set_current_state(TASK_RUNNING);
  231. }
  232. void kthread_parkme(void)
  233. {
  234. __kthread_parkme(to_kthread(current));
  235. }
  236. EXPORT_SYMBOL_GPL(kthread_parkme);
  237. static int kthread(void *_create)
  238. {
  239. /* Copy data: it's on kthread's stack */
  240. struct kthread_create_info *create = _create;
  241. int (*threadfn)(void *data) = create->threadfn;
  242. void *data = create->data;
  243. struct completion *done;
  244. struct kthread *self;
  245. int ret;
  246. self = kzalloc(sizeof(*self), GFP_KERNEL);
  247. set_kthread_struct(self);
  248. /* If user was SIGKILLed, I release the structure. */
  249. done = xchg(&create->done, NULL);
  250. if (!done) {
  251. kfree(create);
  252. do_exit(-EINTR);
  253. }
  254. if (!self) {
  255. create->result = ERR_PTR(-ENOMEM);
  256. complete(done);
  257. do_exit(-ENOMEM);
  258. }
  259. self->threadfn = threadfn;
  260. self->data = data;
  261. init_completion(&self->exited);
  262. init_completion(&self->parked);
  263. current->vfork_done = &self->exited;
  264. /* OK, tell user we're spawned, wait for stop or wakeup */
  265. __set_current_state(TASK_UNINTERRUPTIBLE);
  266. create->result = current;
  267. /*
  268. * Thread is going to call schedule(), do not preempt it,
  269. * or the creator may spend more time in wait_task_inactive().
  270. */
  271. preempt_disable();
  272. complete(done);
  273. schedule_preempt_disabled();
  274. preempt_enable();
  275. ret = -EINTR;
  276. if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
  277. cgroup_kthread_ready();
  278. __kthread_parkme(self);
  279. ret = threadfn(data);
  280. }
  281. do_exit(ret);
  282. }
  283. /* called from do_fork() to get node information for about to be created task */
  284. int tsk_fork_get_node(struct task_struct *tsk)
  285. {
  286. #ifdef CONFIG_NUMA
  287. if (tsk == kthreadd_task)
  288. return tsk->pref_node_fork;
  289. #endif
  290. return NUMA_NO_NODE;
  291. }
  292. static void create_kthread(struct kthread_create_info *create)
  293. {
  294. int pid;
  295. #ifdef CONFIG_NUMA
  296. current->pref_node_fork = create->node;
  297. #endif
  298. /* We want our own signal handler (we take no signals by default). */
  299. pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
  300. if (pid < 0) {
  301. /* If user was SIGKILLed, I release the structure. */
  302. struct completion *done = xchg(&create->done, NULL);
  303. if (!done) {
  304. kfree(create);
  305. return;
  306. }
  307. create->result = ERR_PTR(pid);
  308. complete(done);
  309. }
  310. }
  311. static __printf(4, 0)
  312. struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
  313. void *data, int node,
  314. const char namefmt[],
  315. va_list args)
  316. {
  317. DECLARE_COMPLETION_ONSTACK(done);
  318. struct task_struct *task;
  319. struct kthread_create_info *create = kmalloc(sizeof(*create),
  320. GFP_KERNEL);
  321. if (!create)
  322. return ERR_PTR(-ENOMEM);
  323. create->threadfn = threadfn;
  324. create->data = data;
  325. create->node = node;
  326. create->done = &done;
  327. spin_lock(&kthread_create_lock);
  328. list_add_tail(&create->list, &kthread_create_list);
  329. spin_unlock(&kthread_create_lock);
  330. wake_up_process(kthreadd_task);
  331. /*
  332. * Wait for completion in killable state, for I might be chosen by
  333. * the OOM killer while kthreadd is trying to allocate memory for
  334. * new kernel thread.
  335. */
  336. if (unlikely(wait_for_completion_killable(&done))) {
  337. /*
  338. * If I was SIGKILLed before kthreadd (or new kernel thread)
  339. * calls complete(), leave the cleanup of this structure to
  340. * that thread.
  341. */
  342. if (xchg(&create->done, NULL))
  343. return ERR_PTR(-EINTR);
  344. /*
  345. * kthreadd (or new kernel thread) will call complete()
  346. * shortly.
  347. */
  348. wait_for_completion(&done);
  349. }
  350. task = create->result;
  351. if (!IS_ERR(task)) {
  352. static const struct sched_param param = { .sched_priority = 0 };
  353. char name[TASK_COMM_LEN];
  354. /*
  355. * task is already visible to other tasks, so updating
  356. * COMM must be protected.
  357. */
  358. vsnprintf(name, sizeof(name), namefmt, args);
  359. set_task_comm(task, name);
  360. /*
  361. * root may have changed our (kthreadd's) priority or CPU mask.
  362. * The kernel thread should not inherit these properties.
  363. */
  364. sched_setscheduler_nocheck(task, SCHED_NORMAL, &param);
  365. set_cpus_allowed_ptr(task,
  366. housekeeping_cpumask(HK_FLAG_KTHREAD));
  367. }
  368. kfree(create);
  369. return task;
  370. }
  371. /**
  372. * kthread_create_on_node - create a kthread.
  373. * @threadfn: the function to run until signal_pending(current).
  374. * @data: data ptr for @threadfn.
  375. * @node: task and thread structures for the thread are allocated on this node
  376. * @namefmt: printf-style name for the thread.
  377. *
  378. * Description: This helper function creates and names a kernel
  379. * thread. The thread will be stopped: use wake_up_process() to start
  380. * it. See also kthread_run(). The new thread has SCHED_NORMAL policy and
  381. * is affine to all CPUs.
  382. *
  383. * If thread is going to be bound on a particular cpu, give its node
  384. * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
  385. * When woken, the thread will run @threadfn() with @data as its
  386. * argument. @threadfn() can either call do_exit() directly if it is a
  387. * standalone thread for which no one will call kthread_stop(), or
  388. * return when 'kthread_should_stop()' is true (which means
  389. * kthread_stop() has been called). The return value should be zero
  390. * or a negative error number; it will be passed to kthread_stop().
  391. *
  392. * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
  393. */
  394. struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
  395. void *data, int node,
  396. const char namefmt[],
  397. ...)
  398. {
  399. struct task_struct *task;
  400. va_list args;
  401. va_start(args, namefmt);
  402. task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
  403. va_end(args);
  404. return task;
  405. }
  406. EXPORT_SYMBOL(kthread_create_on_node);
  407. static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, long state)
  408. {
  409. unsigned long flags;
  410. if (!wait_task_inactive(p, state)) {
  411. WARN_ON(1);
  412. return;
  413. }
  414. /* It's safe because the task is inactive. */
  415. raw_spin_lock_irqsave(&p->pi_lock, flags);
  416. do_set_cpus_allowed(p, mask);
  417. p->flags |= PF_NO_SETAFFINITY;
  418. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  419. }
  420. static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
  421. {
  422. __kthread_bind_mask(p, cpumask_of(cpu), state);
  423. }
  424. void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
  425. {
  426. __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
  427. }
  428. EXPORT_SYMBOL_GPL(kthread_bind_mask);
  429. /**
  430. * kthread_bind - bind a just-created kthread to a cpu.
  431. * @p: thread created by kthread_create().
  432. * @cpu: cpu (might not be online, must be possible) for @k to run on.
  433. *
  434. * Description: This function is equivalent to set_cpus_allowed(),
  435. * except that @cpu doesn't need to be online, and the thread must be
  436. * stopped (i.e., just returned from kthread_create()).
  437. */
  438. void kthread_bind(struct task_struct *p, unsigned int cpu)
  439. {
  440. __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
  441. }
  442. EXPORT_SYMBOL(kthread_bind);
  443. /**
  444. * kthread_create_on_cpu - Create a cpu bound kthread
  445. * @threadfn: the function to run until signal_pending(current).
  446. * @data: data ptr for @threadfn.
  447. * @cpu: The cpu on which the thread should be bound,
  448. * @namefmt: printf-style name for the thread. Format is restricted
  449. * to "name.*%u". Code fills in cpu number.
  450. *
  451. * Description: This helper function creates and names a kernel thread
  452. */
  453. struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
  454. void *data, unsigned int cpu,
  455. const char *namefmt)
  456. {
  457. struct task_struct *p;
  458. p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
  459. cpu);
  460. if (IS_ERR(p))
  461. return p;
  462. kthread_bind(p, cpu);
  463. /* CPU hotplug need to bind once again when unparking the thread. */
  464. to_kthread(p)->cpu = cpu;
  465. return p;
  466. }
  467. void kthread_set_per_cpu(struct task_struct *k, int cpu)
  468. {
  469. struct kthread *kthread = to_kthread(k);
  470. if (!kthread)
  471. return;
  472. WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY));
  473. if (cpu < 0) {
  474. clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
  475. return;
  476. }
  477. kthread->cpu = cpu;
  478. set_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
  479. }
  480. bool kthread_is_per_cpu(struct task_struct *p)
  481. {
  482. struct kthread *kthread = __to_kthread(p);
  483. if (!kthread)
  484. return false;
  485. return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
  486. }
  487. /**
  488. * kthread_unpark - unpark a thread created by kthread_create().
  489. * @k: thread created by kthread_create().
  490. *
  491. * Sets kthread_should_park() for @k to return false, wakes it, and
  492. * waits for it to return. If the thread is marked percpu then its
  493. * bound to the cpu again.
  494. */
  495. void kthread_unpark(struct task_struct *k)
  496. {
  497. struct kthread *kthread = to_kthread(k);
  498. /*
  499. * Newly created kthread was parked when the CPU was offline.
  500. * The binding was lost and we need to set it again.
  501. */
  502. if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
  503. __kthread_bind(k, kthread->cpu, TASK_PARKED);
  504. clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
  505. /*
  506. * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup.
  507. */
  508. wake_up_state(k, TASK_PARKED);
  509. }
  510. EXPORT_SYMBOL_GPL(kthread_unpark);
  511. /**
  512. * kthread_park - park a thread created by kthread_create().
  513. * @k: thread created by kthread_create().
  514. *
  515. * Sets kthread_should_park() for @k to return true, wakes it, and
  516. * waits for it to return. This can also be called after kthread_create()
  517. * instead of calling wake_up_process(): the thread will park without
  518. * calling threadfn().
  519. *
  520. * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
  521. * If called by the kthread itself just the park bit is set.
  522. */
  523. int kthread_park(struct task_struct *k)
  524. {
  525. struct kthread *kthread = to_kthread(k);
  526. if (WARN_ON(k->flags & PF_EXITING))
  527. return -ENOSYS;
  528. if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)))
  529. return -EBUSY;
  530. set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
  531. if (k != current) {
  532. wake_up_process(k);
  533. /*
  534. * Wait for __kthread_parkme() to complete(), this means we
  535. * _will_ have TASK_PARKED and are about to call schedule().
  536. */
  537. wait_for_completion(&kthread->parked);
  538. /*
  539. * Now wait for that schedule() to complete and the task to
  540. * get scheduled out.
  541. */
  542. WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED));
  543. }
  544. return 0;
  545. }
  546. EXPORT_SYMBOL_GPL(kthread_park);
  547. /**
  548. * kthread_stop - stop a thread created by kthread_create().
  549. * @k: thread created by kthread_create().
  550. *
  551. * Sets kthread_should_stop() for @k to return true, wakes it, and
  552. * waits for it to exit. This can also be called after kthread_create()
  553. * instead of calling wake_up_process(): the thread will exit without
  554. * calling threadfn().
  555. *
  556. * If threadfn() may call do_exit() itself, the caller must ensure
  557. * task_struct can't go away.
  558. *
  559. * Returns the result of threadfn(), or %-EINTR if wake_up_process()
  560. * was never called.
  561. */
  562. int kthread_stop(struct task_struct *k)
  563. {
  564. struct kthread *kthread;
  565. int ret;
  566. trace_sched_kthread_stop(k);
  567. get_task_struct(k);
  568. kthread = to_kthread(k);
  569. set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
  570. kthread_unpark(k);
  571. wake_up_process(k);
  572. wait_for_completion(&kthread->exited);
  573. ret = k->exit_code;
  574. put_task_struct(k);
  575. trace_sched_kthread_stop_ret(ret);
  576. return ret;
  577. }
  578. EXPORT_SYMBOL(kthread_stop);
  579. int kthreadd(void *unused)
  580. {
  581. struct task_struct *tsk = current;
  582. /* Setup a clean context for our children to inherit. */
  583. set_task_comm(tsk, "kthreadd");
  584. ignore_signals(tsk);
  585. set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_FLAG_KTHREAD));
  586. set_mems_allowed(node_states[N_MEMORY]);
  587. current->flags |= PF_NOFREEZE;
  588. cgroup_init_kthreadd();
  589. for (;;) {
  590. set_current_state(TASK_INTERRUPTIBLE);
  591. if (list_empty(&kthread_create_list))
  592. schedule();
  593. __set_current_state(TASK_RUNNING);
  594. spin_lock(&kthread_create_lock);
  595. while (!list_empty(&kthread_create_list)) {
  596. struct kthread_create_info *create;
  597. create = list_entry(kthread_create_list.next,
  598. struct kthread_create_info, list);
  599. list_del_init(&create->list);
  600. spin_unlock(&kthread_create_lock);
  601. create_kthread(create);
  602. spin_lock(&kthread_create_lock);
  603. }
  604. spin_unlock(&kthread_create_lock);
  605. }
  606. return 0;
  607. }
  608. void __kthread_init_worker(struct kthread_worker *worker,
  609. const char *name,
  610. struct lock_class_key *key)
  611. {
  612. memset(worker, 0, sizeof(struct kthread_worker));
  613. raw_spin_lock_init(&worker->lock);
  614. lockdep_set_class_and_name(&worker->lock, key, name);
  615. INIT_LIST_HEAD(&worker->work_list);
  616. INIT_LIST_HEAD(&worker->delayed_work_list);
  617. }
  618. EXPORT_SYMBOL_GPL(__kthread_init_worker);
  619. /**
  620. * kthread_worker_fn - kthread function to process kthread_worker
  621. * @worker_ptr: pointer to initialized kthread_worker
  622. *
  623. * This function implements the main cycle of kthread worker. It processes
  624. * work_list until it is stopped with kthread_stop(). It sleeps when the queue
  625. * is empty.
  626. *
  627. * The works are not allowed to keep any locks, disable preemption or interrupts
  628. * when they finish. There is defined a safe point for freezing when one work
  629. * finishes and before a new one is started.
  630. *
  631. * Also the works must not be handled by more than one worker at the same time,
  632. * see also kthread_queue_work().
  633. */
  634. int kthread_worker_fn(void *worker_ptr)
  635. {
  636. struct kthread_worker *worker = worker_ptr;
  637. struct kthread_work *work;
  638. /*
  639. * FIXME: Update the check and remove the assignment when all kthread
  640. * worker users are created using kthread_create_worker*() functions.
  641. */
  642. WARN_ON(worker->task && worker->task != current);
  643. worker->task = current;
  644. if (worker->flags & KTW_FREEZABLE)
  645. set_freezable();
  646. repeat:
  647. set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */
  648. if (kthread_should_stop()) {
  649. __set_current_state(TASK_RUNNING);
  650. raw_spin_lock_irq(&worker->lock);
  651. worker->task = NULL;
  652. raw_spin_unlock_irq(&worker->lock);
  653. return 0;
  654. }
  655. work = NULL;
  656. raw_spin_lock_irq(&worker->lock);
  657. if (!list_empty(&worker->work_list)) {
  658. work = list_first_entry(&worker->work_list,
  659. struct kthread_work, node);
  660. list_del_init(&work->node);
  661. }
  662. worker->current_work = work;
  663. raw_spin_unlock_irq(&worker->lock);
  664. if (work) {
  665. __set_current_state(TASK_RUNNING);
  666. work->func(work);
  667. } else if (!freezing(current))
  668. schedule();
  669. try_to_freeze();
  670. cond_resched();
  671. goto repeat;
  672. }
  673. EXPORT_SYMBOL_GPL(kthread_worker_fn);
  674. static __printf(3, 0) struct kthread_worker *
  675. __kthread_create_worker(int cpu, unsigned int flags,
  676. const char namefmt[], va_list args)
  677. {
  678. struct kthread_worker *worker;
  679. struct task_struct *task;
  680. int node = NUMA_NO_NODE;
  681. worker = kzalloc(sizeof(*worker), GFP_KERNEL);
  682. if (!worker)
  683. return ERR_PTR(-ENOMEM);
  684. kthread_init_worker(worker);
  685. if (cpu >= 0)
  686. node = cpu_to_node(cpu);
  687. task = __kthread_create_on_node(kthread_worker_fn, worker,
  688. node, namefmt, args);
  689. if (IS_ERR(task))
  690. goto fail_task;
  691. if (cpu >= 0)
  692. kthread_bind(task, cpu);
  693. worker->flags = flags;
  694. worker->task = task;
  695. wake_up_process(task);
  696. return worker;
  697. fail_task:
  698. kfree(worker);
  699. return ERR_CAST(task);
  700. }
  701. /**
  702. * kthread_create_worker - create a kthread worker
  703. * @flags: flags modifying the default behavior of the worker
  704. * @namefmt: printf-style name for the kthread worker (task).
  705. *
  706. * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
  707. * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
  708. * when the worker was SIGKILLed.
  709. */
  710. struct kthread_worker *
  711. kthread_create_worker(unsigned int flags, const char namefmt[], ...)
  712. {
  713. struct kthread_worker *worker;
  714. va_list args;
  715. va_start(args, namefmt);
  716. worker = __kthread_create_worker(-1, flags, namefmt, args);
  717. va_end(args);
  718. return worker;
  719. }
  720. EXPORT_SYMBOL(kthread_create_worker);
  721. /**
  722. * kthread_create_worker_on_cpu - create a kthread worker and bind it
  723. * to a given CPU and the associated NUMA node.
  724. * @cpu: CPU number
  725. * @flags: flags modifying the default behavior of the worker
  726. * @namefmt: printf-style name for the kthread worker (task).
  727. *
  728. * Use a valid CPU number if you want to bind the kthread worker
  729. * to the given CPU and the associated NUMA node.
  730. *
  731. * A good practice is to add the cpu number also into the worker name.
  732. * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
  733. *
  734. * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
  735. * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
  736. * when the worker was SIGKILLed.
  737. */
  738. struct kthread_worker *
  739. kthread_create_worker_on_cpu(int cpu, unsigned int flags,
  740. const char namefmt[], ...)
  741. {
  742. struct kthread_worker *worker;
  743. va_list args;
  744. va_start(args, namefmt);
  745. worker = __kthread_create_worker(cpu, flags, namefmt, args);
  746. va_end(args);
  747. return worker;
  748. }
  749. EXPORT_SYMBOL(kthread_create_worker_on_cpu);
  750. /*
  751. * Returns true when the work could not be queued at the moment.
  752. * It happens when it is already pending in a worker list
  753. * or when it is being cancelled.
  754. */
  755. static inline bool queuing_blocked(struct kthread_worker *worker,
  756. struct kthread_work *work)
  757. {
  758. lockdep_assert_held(&worker->lock);
  759. return !list_empty(&work->node) || work->canceling;
  760. }
  761. static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
  762. struct kthread_work *work)
  763. {
  764. lockdep_assert_held(&worker->lock);
  765. WARN_ON_ONCE(!list_empty(&work->node));
  766. /* Do not use a work with >1 worker, see kthread_queue_work() */
  767. WARN_ON_ONCE(work->worker && work->worker != worker);
  768. }
  769. /* insert @work before @pos in @worker */
  770. static void kthread_insert_work(struct kthread_worker *worker,
  771. struct kthread_work *work,
  772. struct list_head *pos)
  773. {
  774. kthread_insert_work_sanity_check(worker, work);
  775. list_add_tail(&work->node, pos);
  776. work->worker = worker;
  777. if (!worker->current_work && likely(worker->task))
  778. wake_up_process(worker->task);
  779. }
  780. /**
  781. * kthread_queue_work - queue a kthread_work
  782. * @worker: target kthread_worker
  783. * @work: kthread_work to queue
  784. *
  785. * Queue @work to work processor @task for async execution. @task
  786. * must have been created with kthread_worker_create(). Returns %true
  787. * if @work was successfully queued, %false if it was already pending.
  788. *
  789. * Reinitialize the work if it needs to be used by another worker.
  790. * For example, when the worker was stopped and started again.
  791. */
  792. bool kthread_queue_work(struct kthread_worker *worker,
  793. struct kthread_work *work)
  794. {
  795. bool ret = false;
  796. unsigned long flags;
  797. raw_spin_lock_irqsave(&worker->lock, flags);
  798. if (!queuing_blocked(worker, work)) {
  799. kthread_insert_work(worker, work, &worker->work_list);
  800. ret = true;
  801. }
  802. raw_spin_unlock_irqrestore(&worker->lock, flags);
  803. return ret;
  804. }
  805. EXPORT_SYMBOL_GPL(kthread_queue_work);
  806. /**
  807. * kthread_delayed_work_timer_fn - callback that queues the associated kthread
  808. * delayed work when the timer expires.
  809. * @t: pointer to the expired timer
  810. *
  811. * The format of the function is defined by struct timer_list.
  812. * It should have been called from irqsafe timer with irq already off.
  813. */
  814. void kthread_delayed_work_timer_fn(struct timer_list *t)
  815. {
  816. struct kthread_delayed_work *dwork = from_timer(dwork, t, timer);
  817. struct kthread_work *work = &dwork->work;
  818. struct kthread_worker *worker = work->worker;
  819. unsigned long flags;
  820. /*
  821. * This might happen when a pending work is reinitialized.
  822. * It means that it is used a wrong way.
  823. */
  824. if (WARN_ON_ONCE(!worker))
  825. return;
  826. raw_spin_lock_irqsave(&worker->lock, flags);
  827. /* Work must not be used with >1 worker, see kthread_queue_work(). */
  828. WARN_ON_ONCE(work->worker != worker);
  829. /* Move the work from worker->delayed_work_list. */
  830. WARN_ON_ONCE(list_empty(&work->node));
  831. list_del_init(&work->node);
  832. if (!work->canceling)
  833. kthread_insert_work(worker, work, &worker->work_list);
  834. raw_spin_unlock_irqrestore(&worker->lock, flags);
  835. }
  836. EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
  837. static void __kthread_queue_delayed_work(struct kthread_worker *worker,
  838. struct kthread_delayed_work *dwork,
  839. unsigned long delay)
  840. {
  841. struct timer_list *timer = &dwork->timer;
  842. struct kthread_work *work = &dwork->work;
  843. /*
  844. * With CFI, timer->function can point to a jump table entry in a module,
  845. * which fails the comparison. Disable the warning if CFI and modules are
  846. * both enabled.
  847. */
  848. if (!IS_ENABLED(CONFIG_CFI_CLANG) || !IS_ENABLED(CONFIG_MODULES))
  849. WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
  850. /*
  851. * If @delay is 0, queue @dwork->work immediately. This is for
  852. * both optimization and correctness. The earliest @timer can
  853. * expire is on the closest next tick and delayed_work users depend
  854. * on that there's no such delay when @delay is 0.
  855. */
  856. if (!delay) {
  857. kthread_insert_work(worker, work, &worker->work_list);
  858. return;
  859. }
  860. /* Be paranoid and try to detect possible races already now. */
  861. kthread_insert_work_sanity_check(worker, work);
  862. list_add(&work->node, &worker->delayed_work_list);
  863. work->worker = worker;
  864. timer->expires = jiffies + delay;
  865. add_timer(timer);
  866. }
  867. /**
  868. * kthread_queue_delayed_work - queue the associated kthread work
  869. * after a delay.
  870. * @worker: target kthread_worker
  871. * @dwork: kthread_delayed_work to queue
  872. * @delay: number of jiffies to wait before queuing
  873. *
  874. * If the work has not been pending it starts a timer that will queue
  875. * the work after the given @delay. If @delay is zero, it queues the
  876. * work immediately.
  877. *
  878. * Return: %false if the @work has already been pending. It means that
  879. * either the timer was running or the work was queued. It returns %true
  880. * otherwise.
  881. */
  882. bool kthread_queue_delayed_work(struct kthread_worker *worker,
  883. struct kthread_delayed_work *dwork,
  884. unsigned long delay)
  885. {
  886. struct kthread_work *work = &dwork->work;
  887. unsigned long flags;
  888. bool ret = false;
  889. raw_spin_lock_irqsave(&worker->lock, flags);
  890. if (!queuing_blocked(worker, work)) {
  891. __kthread_queue_delayed_work(worker, dwork, delay);
  892. ret = true;
  893. }
  894. raw_spin_unlock_irqrestore(&worker->lock, flags);
  895. return ret;
  896. }
  897. EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
  898. struct kthread_flush_work {
  899. struct kthread_work work;
  900. struct completion done;
  901. };
  902. static void kthread_flush_work_fn(struct kthread_work *work)
  903. {
  904. struct kthread_flush_work *fwork =
  905. container_of(work, struct kthread_flush_work, work);
  906. complete(&fwork->done);
  907. }
  908. /**
  909. * kthread_flush_work - flush a kthread_work
  910. * @work: work to flush
  911. *
  912. * If @work is queued or executing, wait for it to finish execution.
  913. */
  914. void kthread_flush_work(struct kthread_work *work)
  915. {
  916. struct kthread_flush_work fwork = {
  917. KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
  918. COMPLETION_INITIALIZER_ONSTACK(fwork.done),
  919. };
  920. struct kthread_worker *worker;
  921. bool noop = false;
  922. worker = work->worker;
  923. if (!worker)
  924. return;
  925. raw_spin_lock_irq(&worker->lock);
  926. /* Work must not be used with >1 worker, see kthread_queue_work(). */
  927. WARN_ON_ONCE(work->worker != worker);
  928. if (!list_empty(&work->node))
  929. kthread_insert_work(worker, &fwork.work, work->node.next);
  930. else if (worker->current_work == work)
  931. kthread_insert_work(worker, &fwork.work,
  932. worker->work_list.next);
  933. else
  934. noop = true;
  935. raw_spin_unlock_irq(&worker->lock);
  936. if (!noop)
  937. wait_for_completion(&fwork.done);
  938. }
  939. EXPORT_SYMBOL_GPL(kthread_flush_work);
  940. /*
  941. * Make sure that the timer is neither set nor running and could
  942. * not manipulate the work list_head any longer.
  943. *
  944. * The function is called under worker->lock. The lock is temporary
  945. * released but the timer can't be set again in the meantime.
  946. */
  947. static void kthread_cancel_delayed_work_timer(struct kthread_work *work,
  948. unsigned long *flags)
  949. {
  950. struct kthread_delayed_work *dwork =
  951. container_of(work, struct kthread_delayed_work, work);
  952. struct kthread_worker *worker = work->worker;
  953. /*
  954. * del_timer_sync() must be called to make sure that the timer
  955. * callback is not running. The lock must be temporary released
  956. * to avoid a deadlock with the callback. In the meantime,
  957. * any queuing is blocked by setting the canceling counter.
  958. */
  959. work->canceling++;
  960. raw_spin_unlock_irqrestore(&worker->lock, *flags);
  961. del_timer_sync(&dwork->timer);
  962. raw_spin_lock_irqsave(&worker->lock, *flags);
  963. work->canceling--;
  964. }
  965. /*
  966. * This function removes the work from the worker queue.
  967. *
  968. * It is called under worker->lock. The caller must make sure that
  969. * the timer used by delayed work is not running, e.g. by calling
  970. * kthread_cancel_delayed_work_timer().
  971. *
  972. * The work might still be in use when this function finishes. See the
  973. * current_work proceed by the worker.
  974. *
  975. * Return: %true if @work was pending and successfully canceled,
  976. * %false if @work was not pending
  977. */
  978. static bool __kthread_cancel_work(struct kthread_work *work)
  979. {
  980. /*
  981. * Try to remove the work from a worker list. It might either
  982. * be from worker->work_list or from worker->delayed_work_list.
  983. */
  984. if (!list_empty(&work->node)) {
  985. list_del_init(&work->node);
  986. return true;
  987. }
  988. return false;
  989. }
  990. /**
  991. * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
  992. * @worker: kthread worker to use
  993. * @dwork: kthread delayed work to queue
  994. * @delay: number of jiffies to wait before queuing
  995. *
  996. * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
  997. * modify @dwork's timer so that it expires after @delay. If @delay is zero,
  998. * @work is guaranteed to be queued immediately.
  999. *
  1000. * Return: %false if @dwork was idle and queued, %true otherwise.
  1001. *
  1002. * A special case is when the work is being canceled in parallel.
  1003. * It might be caused either by the real kthread_cancel_delayed_work_sync()
  1004. * or yet another kthread_mod_delayed_work() call. We let the other command
  1005. * win and return %true here. The return value can be used for reference
  1006. * counting and the number of queued works stays the same. Anyway, the caller
  1007. * is supposed to synchronize these operations a reasonable way.
  1008. *
  1009. * This function is safe to call from any context including IRQ handler.
  1010. * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
  1011. * for details.
  1012. */
  1013. bool kthread_mod_delayed_work(struct kthread_worker *worker,
  1014. struct kthread_delayed_work *dwork,
  1015. unsigned long delay)
  1016. {
  1017. struct kthread_work *work = &dwork->work;
  1018. unsigned long flags;
  1019. int ret;
  1020. raw_spin_lock_irqsave(&worker->lock, flags);
  1021. /* Do not bother with canceling when never queued. */
  1022. if (!work->worker) {
  1023. ret = false;
  1024. goto fast_queue;
  1025. }
  1026. /* Work must not be used with >1 worker, see kthread_queue_work() */
  1027. WARN_ON_ONCE(work->worker != worker);
  1028. /*
  1029. * Temporary cancel the work but do not fight with another command
  1030. * that is canceling the work as well.
  1031. *
  1032. * It is a bit tricky because of possible races with another
  1033. * mod_delayed_work() and cancel_delayed_work() callers.
  1034. *
  1035. * The timer must be canceled first because worker->lock is released
  1036. * when doing so. But the work can be removed from the queue (list)
  1037. * only when it can be queued again so that the return value can
  1038. * be used for reference counting.
  1039. */
  1040. kthread_cancel_delayed_work_timer(work, &flags);
  1041. if (work->canceling) {
  1042. /* The number of works in the queue does not change. */
  1043. ret = true;
  1044. goto out;
  1045. }
  1046. ret = __kthread_cancel_work(work);
  1047. /*
  1048. * Canceling could run in parallel from kthread_cancel_delayed_work_sync
  1049. * and change work's canceling count as the spinlock is released and regain
  1050. * in __kthread_cancel_work so we need to check the count again. Otherwise,
  1051. * we might incorrectly queue the dwork and further cause
  1052. * cancel_delayed_work_sync thread waiting for flush dwork endlessly.
  1053. */
  1054. if (work->canceling) {
  1055. ret = false;
  1056. goto out;
  1057. }
  1058. fast_queue:
  1059. __kthread_queue_delayed_work(worker, dwork, delay);
  1060. out:
  1061. raw_spin_unlock_irqrestore(&worker->lock, flags);
  1062. return ret;
  1063. }
  1064. EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
  1065. static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
  1066. {
  1067. struct kthread_worker *worker = work->worker;
  1068. unsigned long flags;
  1069. int ret = false;
  1070. if (!worker)
  1071. goto out;
  1072. raw_spin_lock_irqsave(&worker->lock, flags);
  1073. /* Work must not be used with >1 worker, see kthread_queue_work(). */
  1074. WARN_ON_ONCE(work->worker != worker);
  1075. if (is_dwork)
  1076. kthread_cancel_delayed_work_timer(work, &flags);
  1077. ret = __kthread_cancel_work(work);
  1078. if (worker->current_work != work)
  1079. goto out_fast;
  1080. /*
  1081. * The work is in progress and we need to wait with the lock released.
  1082. * In the meantime, block any queuing by setting the canceling counter.
  1083. */
  1084. work->canceling++;
  1085. raw_spin_unlock_irqrestore(&worker->lock, flags);
  1086. kthread_flush_work(work);
  1087. raw_spin_lock_irqsave(&worker->lock, flags);
  1088. work->canceling--;
  1089. out_fast:
  1090. raw_spin_unlock_irqrestore(&worker->lock, flags);
  1091. out:
  1092. return ret;
  1093. }
  1094. /**
  1095. * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
  1096. * @work: the kthread work to cancel
  1097. *
  1098. * Cancel @work and wait for its execution to finish. This function
  1099. * can be used even if the work re-queues itself. On return from this
  1100. * function, @work is guaranteed to be not pending or executing on any CPU.
  1101. *
  1102. * kthread_cancel_work_sync(&delayed_work->work) must not be used for
  1103. * delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
  1104. *
  1105. * The caller must ensure that the worker on which @work was last
  1106. * queued can't be destroyed before this function returns.
  1107. *
  1108. * Return: %true if @work was pending, %false otherwise.
  1109. */
  1110. bool kthread_cancel_work_sync(struct kthread_work *work)
  1111. {
  1112. return __kthread_cancel_work_sync(work, false);
  1113. }
  1114. EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);
  1115. /**
  1116. * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
  1117. * wait for it to finish.
  1118. * @dwork: the kthread delayed work to cancel
  1119. *
  1120. * This is kthread_cancel_work_sync() for delayed works.
  1121. *
  1122. * Return: %true if @dwork was pending, %false otherwise.
  1123. */
  1124. bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
  1125. {
  1126. return __kthread_cancel_work_sync(&dwork->work, true);
  1127. }
  1128. EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);
  1129. /**
  1130. * kthread_flush_worker - flush all current works on a kthread_worker
  1131. * @worker: worker to flush
  1132. *
  1133. * Wait until all currently executing or pending works on @worker are
  1134. * finished.
  1135. */
  1136. void kthread_flush_worker(struct kthread_worker *worker)
  1137. {
  1138. struct kthread_flush_work fwork = {
  1139. KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
  1140. COMPLETION_INITIALIZER_ONSTACK(fwork.done),
  1141. };
  1142. kthread_queue_work(worker, &fwork.work);
  1143. wait_for_completion(&fwork.done);
  1144. }
  1145. EXPORT_SYMBOL_GPL(kthread_flush_worker);
  1146. /**
  1147. * kthread_destroy_worker - destroy a kthread worker
  1148. * @worker: worker to be destroyed
  1149. *
  1150. * Flush and destroy @worker. The simple flush is enough because the kthread
  1151. * worker API is used only in trivial scenarios. There are no multi-step state
  1152. * machines needed.
  1153. */
  1154. void kthread_destroy_worker(struct kthread_worker *worker)
  1155. {
  1156. struct task_struct *task;
  1157. task = worker->task;
  1158. if (WARN_ON(!task))
  1159. return;
  1160. kthread_flush_worker(worker);
  1161. kthread_stop(task);
  1162. WARN_ON(!list_empty(&worker->work_list));
  1163. kfree(worker);
  1164. }
  1165. EXPORT_SYMBOL(kthread_destroy_worker);
  1166. /**
  1167. * kthread_use_mm - make the calling kthread operate on an address space
  1168. * @mm: address space to operate on
  1169. */
  1170. void kthread_use_mm(struct mm_struct *mm)
  1171. {
  1172. struct mm_struct *active_mm;
  1173. struct task_struct *tsk = current;
  1174. WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
  1175. WARN_ON_ONCE(tsk->mm);
  1176. task_lock(tsk);
  1177. /* Hold off tlb flush IPIs while switching mm's */
  1178. local_irq_disable();
  1179. active_mm = tsk->active_mm;
  1180. if (active_mm != mm) {
  1181. mmgrab(mm);
  1182. tsk->active_mm = mm;
  1183. }
  1184. tsk->mm = mm;
  1185. switch_mm_irqs_off(active_mm, mm, tsk);
  1186. local_irq_enable();
  1187. task_unlock(tsk);
  1188. #ifdef finish_arch_post_lock_switch
  1189. finish_arch_post_lock_switch();
  1190. #endif
  1191. if (active_mm != mm)
  1192. mmdrop(active_mm);
  1193. to_kthread(tsk)->oldfs = force_uaccess_begin();
  1194. }
  1195. EXPORT_SYMBOL_GPL(kthread_use_mm);
  1196. /**
  1197. * kthread_unuse_mm - reverse the effect of kthread_use_mm()
  1198. * @mm: address space to operate on
  1199. */
  1200. void kthread_unuse_mm(struct mm_struct *mm)
  1201. {
  1202. struct task_struct *tsk = current;
  1203. WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
  1204. WARN_ON_ONCE(!tsk->mm);
  1205. force_uaccess_end(to_kthread(tsk)->oldfs);
  1206. task_lock(tsk);
  1207. sync_mm_rss(mm);
  1208. local_irq_disable();
  1209. tsk->mm = NULL;
  1210. /* active_mm is still 'mm' */
  1211. enter_lazy_tlb(mm, tsk);
  1212. local_irq_enable();
  1213. task_unlock(tsk);
  1214. }
  1215. EXPORT_SYMBOL_GPL(kthread_unuse_mm);
  1216. #ifdef CONFIG_BLK_CGROUP
  1217. /**
  1218. * kthread_associate_blkcg - associate blkcg to current kthread
  1219. * @css: the cgroup info
  1220. *
  1221. * Current thread must be a kthread. The thread is running jobs on behalf of
  1222. * other threads. In some cases, we expect the jobs attach cgroup info of
  1223. * original threads instead of that of current thread. This function stores
  1224. * original thread's cgroup info in current kthread context for later
  1225. * retrieval.
  1226. */
  1227. void kthread_associate_blkcg(struct cgroup_subsys_state *css)
  1228. {
  1229. struct kthread *kthread;
  1230. if (!(current->flags & PF_KTHREAD))
  1231. return;
  1232. kthread = to_kthread(current);
  1233. if (!kthread)
  1234. return;
  1235. if (kthread->blkcg_css) {
  1236. css_put(kthread->blkcg_css);
  1237. kthread->blkcg_css = NULL;
  1238. }
  1239. if (css) {
  1240. css_get(css);
  1241. kthread->blkcg_css = css;
  1242. }
  1243. }
  1244. EXPORT_SYMBOL(kthread_associate_blkcg);
  1245. /**
  1246. * kthread_blkcg - get associated blkcg css of current kthread
  1247. *
  1248. * Current thread must be a kthread.
  1249. */
  1250. struct cgroup_subsys_state *kthread_blkcg(void)
  1251. {
  1252. struct kthread *kthread;
  1253. if (current->flags & PF_KTHREAD) {
  1254. kthread = to_kthread(current);
  1255. if (kthread)
  1256. return kthread->blkcg_css;
  1257. }
  1258. return NULL;
  1259. }
  1260. EXPORT_SYMBOL(kthread_blkcg);
  1261. #endif