runtime.c 52 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * drivers/base/power/runtime.c - Helper functions for device runtime PM
  4. *
  5. * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
  6. * Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu>
  7. */
  8. #include <linux/sched/mm.h>
  9. #include <linux/ktime.h>
  10. #include <linux/hrtimer.h>
  11. #include <linux/export.h>
  12. #include <linux/pm_runtime.h>
  13. #include <linux/pm_wakeirq.h>
  14. #include <trace/events/rpm.h>
  15. #include "../base.h"
  16. #include "power.h"
  17. typedef int (*pm_callback_t)(struct device *);
  18. static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
  19. {
  20. pm_callback_t cb;
  21. const struct dev_pm_ops *ops;
  22. if (dev->pm_domain)
  23. ops = &dev->pm_domain->ops;
  24. else if (dev->type && dev->type->pm)
  25. ops = dev->type->pm;
  26. else if (dev->class && dev->class->pm)
  27. ops = dev->class->pm;
  28. else if (dev->bus && dev->bus->pm)
  29. ops = dev->bus->pm;
  30. else
  31. ops = NULL;
  32. if (ops)
  33. cb = *(pm_callback_t *)((void *)ops + cb_offset);
  34. else
  35. cb = NULL;
  36. if (!cb && dev->driver && dev->driver->pm)
  37. cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
  38. return cb;
  39. }
  40. #define RPM_GET_CALLBACK(dev, callback) \
  41. __rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
  42. static int rpm_resume(struct device *dev, int rpmflags);
  43. static int rpm_suspend(struct device *dev, int rpmflags);
  44. /**
  45. * update_pm_runtime_accounting - Update the time accounting of power states
  46. * @dev: Device to update the accounting for
  47. *
  48. * In order to be able to have time accounting of the various power states
  49. * (as used by programs such as PowerTOP to show the effectiveness of runtime
  50. * PM), we need to track the time spent in each state.
  51. * update_pm_runtime_accounting must be called each time before the
  52. * runtime_status field is updated, to account the time in the old state
  53. * correctly.
  54. */
  55. static void update_pm_runtime_accounting(struct device *dev)
  56. {
  57. u64 now, last, delta;
  58. if (dev->power.disable_depth > 0)
  59. return;
  60. last = dev->power.accounting_timestamp;
  61. now = ktime_get_mono_fast_ns();
  62. dev->power.accounting_timestamp = now;
  63. /*
  64. * Because ktime_get_mono_fast_ns() is not monotonic during
  65. * timekeeping updates, ensure that 'now' is after the last saved
  66. * timesptamp.
  67. */
  68. if (now < last)
  69. return;
  70. delta = now - last;
  71. if (dev->power.runtime_status == RPM_SUSPENDED)
  72. dev->power.suspended_time += delta;
  73. else
  74. dev->power.active_time += delta;
  75. }
  76. static void __update_runtime_status(struct device *dev, enum rpm_status status)
  77. {
  78. update_pm_runtime_accounting(dev);
  79. dev->power.runtime_status = status;
  80. }
  81. static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
  82. {
  83. u64 time;
  84. unsigned long flags;
  85. spin_lock_irqsave(&dev->power.lock, flags);
  86. update_pm_runtime_accounting(dev);
  87. time = suspended ? dev->power.suspended_time : dev->power.active_time;
  88. spin_unlock_irqrestore(&dev->power.lock, flags);
  89. return time;
  90. }
  91. u64 pm_runtime_active_time(struct device *dev)
  92. {
  93. return rpm_get_accounted_time(dev, false);
  94. }
  95. u64 pm_runtime_suspended_time(struct device *dev)
  96. {
  97. return rpm_get_accounted_time(dev, true);
  98. }
  99. EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
  100. /**
  101. * pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
  102. * @dev: Device to handle.
  103. */
  104. static void pm_runtime_deactivate_timer(struct device *dev)
  105. {
  106. if (dev->power.timer_expires > 0) {
  107. hrtimer_try_to_cancel(&dev->power.suspend_timer);
  108. dev->power.timer_expires = 0;
  109. }
  110. }
  111. /**
  112. * pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
  113. * @dev: Device to handle.
  114. */
  115. static void pm_runtime_cancel_pending(struct device *dev)
  116. {
  117. pm_runtime_deactivate_timer(dev);
  118. /*
  119. * In case there's a request pending, make sure its work function will
  120. * return without doing anything.
  121. */
  122. dev->power.request = RPM_REQ_NONE;
  123. }
  124. /*
  125. * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
  126. * @dev: Device to handle.
  127. *
  128. * Compute the autosuspend-delay expiration time based on the device's
  129. * power.last_busy time. If the delay has already expired or is disabled
  130. * (negative) or the power.use_autosuspend flag isn't set, return 0.
  131. * Otherwise return the expiration time in nanoseconds (adjusted to be nonzero).
  132. *
  133. * This function may be called either with or without dev->power.lock held.
  134. * Either way it can be racy, since power.last_busy may be updated at any time.
  135. */
  136. u64 pm_runtime_autosuspend_expiration(struct device *dev)
  137. {
  138. int autosuspend_delay;
  139. u64 expires;
  140. if (!dev->power.use_autosuspend)
  141. return 0;
  142. autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
  143. if (autosuspend_delay < 0)
  144. return 0;
  145. expires = READ_ONCE(dev->power.last_busy);
  146. expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
  147. if (expires > ktime_get_mono_fast_ns())
  148. return expires; /* Expires in the future */
  149. return 0;
  150. }
  151. EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
  152. static int dev_memalloc_noio(struct device *dev, void *data)
  153. {
  154. return dev->power.memalloc_noio;
  155. }
  156. /*
  157. * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
  158. * @dev: Device to handle.
  159. * @enable: True for setting the flag and False for clearing the flag.
  160. *
  161. * Set the flag for all devices in the path from the device to the
  162. * root device in the device tree if @enable is true, otherwise clear
  163. * the flag for devices in the path whose siblings don't set the flag.
  164. *
  165. * The function should only be called by block device, or network
  166. * device driver for solving the deadlock problem during runtime
  167. * resume/suspend:
  168. *
  169. * If memory allocation with GFP_KERNEL is called inside runtime
  170. * resume/suspend callback of any one of its ancestors(or the
  171. * block device itself), the deadlock may be triggered inside the
  172. * memory allocation since it might not complete until the block
  173. * device becomes active and the involed page I/O finishes. The
  174. * situation is pointed out first by Alan Stern. Network device
  175. * are involved in iSCSI kind of situation.
  176. *
  177. * The lock of dev_hotplug_mutex is held in the function for handling
  178. * hotplug race because pm_runtime_set_memalloc_noio() may be called
  179. * in async probe().
  180. *
  181. * The function should be called between device_add() and device_del()
  182. * on the affected device(block/network device).
  183. */
  184. void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
  185. {
  186. static DEFINE_MUTEX(dev_hotplug_mutex);
  187. mutex_lock(&dev_hotplug_mutex);
  188. for (;;) {
  189. bool enabled;
  190. /* hold power lock since bitfield is not SMP-safe. */
  191. spin_lock_irq(&dev->power.lock);
  192. enabled = dev->power.memalloc_noio;
  193. dev->power.memalloc_noio = enable;
  194. spin_unlock_irq(&dev->power.lock);
  195. /*
  196. * not need to enable ancestors any more if the device
  197. * has been enabled.
  198. */
  199. if (enabled && enable)
  200. break;
  201. dev = dev->parent;
  202. /*
  203. * clear flag of the parent device only if all the
  204. * children don't set the flag because ancestor's
  205. * flag was set by any one of the descendants.
  206. */
  207. if (!dev || (!enable &&
  208. device_for_each_child(dev, NULL,
  209. dev_memalloc_noio)))
  210. break;
  211. }
  212. mutex_unlock(&dev_hotplug_mutex);
  213. }
  214. EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
  215. /**
  216. * rpm_check_suspend_allowed - Test whether a device may be suspended.
  217. * @dev: Device to test.
  218. */
  219. static int rpm_check_suspend_allowed(struct device *dev)
  220. {
  221. int retval = 0;
  222. if (dev->power.runtime_error)
  223. retval = -EINVAL;
  224. else if (dev->power.disable_depth > 0)
  225. retval = -EACCES;
  226. else if (atomic_read(&dev->power.usage_count) > 0)
  227. retval = -EAGAIN;
  228. else if (!dev->power.ignore_children &&
  229. atomic_read(&dev->power.child_count))
  230. retval = -EBUSY;
  231. /* Pending resume requests take precedence over suspends. */
  232. else if ((dev->power.deferred_resume
  233. && dev->power.runtime_status == RPM_SUSPENDING)
  234. || (dev->power.request_pending
  235. && dev->power.request == RPM_REQ_RESUME))
  236. retval = -EAGAIN;
  237. else if (__dev_pm_qos_resume_latency(dev) == 0)
  238. retval = -EPERM;
  239. else if (dev->power.runtime_status == RPM_SUSPENDED)
  240. retval = 1;
  241. return retval;
  242. }
  243. static int rpm_get_suppliers(struct device *dev)
  244. {
  245. struct device_link *link;
  246. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
  247. device_links_read_lock_held()) {
  248. int retval;
  249. if (!(link->flags & DL_FLAG_PM_RUNTIME))
  250. continue;
  251. retval = pm_runtime_get_sync(link->supplier);
  252. /* Ignore suppliers with disabled runtime PM. */
  253. if (retval < 0 && retval != -EACCES) {
  254. pm_runtime_put_noidle(link->supplier);
  255. return retval;
  256. }
  257. refcount_inc(&link->rpm_active);
  258. }
  259. return 0;
  260. }
  261. /**
  262. * pm_runtime_release_supplier - Drop references to device link's supplier.
  263. * @link: Target device link.
  264. * @check_idle: Whether or not to check if the supplier device is idle.
  265. *
  266. * Drop all runtime PM references associated with @link to its supplier device
  267. * and if @check_idle is set, check if that device is idle (and so it can be
  268. * suspended).
  269. */
  270. void pm_runtime_release_supplier(struct device_link *link, bool check_idle)
  271. {
  272. struct device *supplier = link->supplier;
  273. /*
  274. * The additional power.usage_count check is a safety net in case
  275. * the rpm_active refcount becomes saturated, in which case
  276. * refcount_dec_not_one() would return true forever, but it is not
  277. * strictly necessary.
  278. */
  279. while (refcount_dec_not_one(&link->rpm_active) &&
  280. atomic_read(&supplier->power.usage_count) > 0)
  281. pm_runtime_put_noidle(supplier);
  282. if (check_idle)
  283. pm_request_idle(supplier);
  284. }
  285. static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
  286. {
  287. struct device_link *link;
  288. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
  289. device_links_read_lock_held())
  290. pm_runtime_release_supplier(link, try_to_suspend);
  291. }
  292. static void rpm_put_suppliers(struct device *dev)
  293. {
  294. __rpm_put_suppliers(dev, true);
  295. }
  296. static void rpm_suspend_suppliers(struct device *dev)
  297. {
  298. struct device_link *link;
  299. int idx = device_links_read_lock();
  300. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
  301. device_links_read_lock_held())
  302. pm_request_idle(link->supplier);
  303. device_links_read_unlock(idx);
  304. }
  305. /**
  306. * __rpm_callback - Run a given runtime PM callback for a given device.
  307. * @cb: Runtime PM callback to run.
  308. * @dev: Device to run the callback for.
  309. */
  310. static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
  311. __releases(&dev->power.lock) __acquires(&dev->power.lock)
  312. {
  313. int retval, idx;
  314. bool use_links = dev->power.links_count > 0;
  315. if (dev->power.irq_safe) {
  316. spin_unlock(&dev->power.lock);
  317. } else {
  318. spin_unlock_irq(&dev->power.lock);
  319. /*
  320. * Resume suppliers if necessary.
  321. *
  322. * The device's runtime PM status cannot change until this
  323. * routine returns, so it is safe to read the status outside of
  324. * the lock.
  325. */
  326. if (use_links && dev->power.runtime_status == RPM_RESUMING) {
  327. idx = device_links_read_lock();
  328. retval = rpm_get_suppliers(dev);
  329. if (retval) {
  330. rpm_put_suppliers(dev);
  331. goto fail;
  332. }
  333. device_links_read_unlock(idx);
  334. }
  335. }
  336. retval = cb(dev);
  337. if (dev->power.irq_safe) {
  338. spin_lock(&dev->power.lock);
  339. } else {
  340. /*
  341. * If the device is suspending and the callback has returned
  342. * success, drop the usage counters of the suppliers that have
  343. * been reference counted on its resume.
  344. *
  345. * Do that if resume fails too.
  346. */
  347. if (use_links
  348. && ((dev->power.runtime_status == RPM_SUSPENDING && !retval)
  349. || (dev->power.runtime_status == RPM_RESUMING && retval))) {
  350. idx = device_links_read_lock();
  351. __rpm_put_suppliers(dev, false);
  352. fail:
  353. device_links_read_unlock(idx);
  354. }
  355. spin_lock_irq(&dev->power.lock);
  356. }
  357. return retval;
  358. }
  359. /**
  360. * rpm_idle - Notify device bus type if the device can be suspended.
  361. * @dev: Device to notify the bus type about.
  362. * @rpmflags: Flag bits.
  363. *
  364. * Check if the device's runtime PM status allows it to be suspended. If
  365. * another idle notification has been started earlier, return immediately. If
  366. * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
  367. * run the ->runtime_idle() callback directly. If the ->runtime_idle callback
  368. * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
  369. *
  370. * This function must be called under dev->power.lock with interrupts disabled.
  371. */
  372. static int rpm_idle(struct device *dev, int rpmflags)
  373. {
  374. int (*callback)(struct device *);
  375. int retval;
  376. trace_rpm_idle_rcuidle(dev, rpmflags);
  377. retval = rpm_check_suspend_allowed(dev);
  378. if (retval < 0)
  379. ; /* Conditions are wrong. */
  380. /* Idle notifications are allowed only in the RPM_ACTIVE state. */
  381. else if (dev->power.runtime_status != RPM_ACTIVE)
  382. retval = -EAGAIN;
  383. /*
  384. * Any pending request other than an idle notification takes
  385. * precedence over us, except that the timer may be running.
  386. */
  387. else if (dev->power.request_pending &&
  388. dev->power.request > RPM_REQ_IDLE)
  389. retval = -EAGAIN;
  390. /* Act as though RPM_NOWAIT is always set. */
  391. else if (dev->power.idle_notification)
  392. retval = -EINPROGRESS;
  393. if (retval)
  394. goto out;
  395. /* Pending requests need to be canceled. */
  396. dev->power.request = RPM_REQ_NONE;
  397. if (dev->power.no_callbacks)
  398. goto out;
  399. /* Carry out an asynchronous or a synchronous idle notification. */
  400. if (rpmflags & RPM_ASYNC) {
  401. dev->power.request = RPM_REQ_IDLE;
  402. if (!dev->power.request_pending) {
  403. dev->power.request_pending = true;
  404. queue_work(pm_wq, &dev->power.work);
  405. }
  406. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0);
  407. return 0;
  408. }
  409. dev->power.idle_notification = true;
  410. callback = RPM_GET_CALLBACK(dev, runtime_idle);
  411. if (callback)
  412. retval = __rpm_callback(callback, dev);
  413. dev->power.idle_notification = false;
  414. wake_up_all(&dev->power.wait_queue);
  415. out:
  416. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
  417. return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
  418. }
  419. /**
  420. * rpm_callback - Run a given runtime PM callback for a given device.
  421. * @cb: Runtime PM callback to run.
  422. * @dev: Device to run the callback for.
  423. */
  424. static int rpm_callback(int (*cb)(struct device *), struct device *dev)
  425. {
  426. int retval;
  427. if (!cb)
  428. return -ENOSYS;
  429. if (dev->power.memalloc_noio) {
  430. unsigned int noio_flag;
  431. /*
  432. * Deadlock might be caused if memory allocation with
  433. * GFP_KERNEL happens inside runtime_suspend and
  434. * runtime_resume callbacks of one block device's
  435. * ancestor or the block device itself. Network
  436. * device might be thought as part of iSCSI block
  437. * device, so network device and its ancestor should
  438. * be marked as memalloc_noio too.
  439. */
  440. noio_flag = memalloc_noio_save();
  441. retval = __rpm_callback(cb, dev);
  442. memalloc_noio_restore(noio_flag);
  443. } else {
  444. retval = __rpm_callback(cb, dev);
  445. }
  446. dev->power.runtime_error = retval;
  447. return retval != -EACCES ? retval : -EIO;
  448. }
  449. /**
  450. * rpm_suspend - Carry out runtime suspend of given device.
  451. * @dev: Device to suspend.
  452. * @rpmflags: Flag bits.
  453. *
  454. * Check if the device's runtime PM status allows it to be suspended.
  455. * Cancel a pending idle notification, autosuspend or suspend. If
  456. * another suspend has been started earlier, either return immediately
  457. * or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
  458. * flags. If the RPM_ASYNC flag is set then queue a suspend request;
  459. * otherwise run the ->runtime_suspend() callback directly. When
  460. * ->runtime_suspend succeeded, if a deferred resume was requested while
  461. * the callback was running then carry it out, otherwise send an idle
  462. * notification for its parent (if the suspend succeeded and both
  463. * ignore_children of parent->power and irq_safe of dev->power are not set).
  464. * If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
  465. * flag is set and the next autosuspend-delay expiration time is in the
  466. * future, schedule another autosuspend attempt.
  467. *
  468. * This function must be called under dev->power.lock with interrupts disabled.
  469. */
  470. static int rpm_suspend(struct device *dev, int rpmflags)
  471. __releases(&dev->power.lock) __acquires(&dev->power.lock)
  472. {
  473. int (*callback)(struct device *);
  474. struct device *parent = NULL;
  475. int retval;
  476. trace_rpm_suspend_rcuidle(dev, rpmflags);
  477. repeat:
  478. retval = rpm_check_suspend_allowed(dev);
  479. if (retval < 0)
  480. goto out; /* Conditions are wrong. */
  481. /* Synchronous suspends are not allowed in the RPM_RESUMING state. */
  482. if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
  483. retval = -EAGAIN;
  484. if (retval)
  485. goto out;
  486. /* If the autosuspend_delay time hasn't expired yet, reschedule. */
  487. if ((rpmflags & RPM_AUTO)
  488. && dev->power.runtime_status != RPM_SUSPENDING) {
  489. u64 expires = pm_runtime_autosuspend_expiration(dev);
  490. if (expires != 0) {
  491. /* Pending requests need to be canceled. */
  492. dev->power.request = RPM_REQ_NONE;
  493. /*
  494. * Optimization: If the timer is already running and is
  495. * set to expire at or before the autosuspend delay,
  496. * avoid the overhead of resetting it. Just let it
  497. * expire; pm_suspend_timer_fn() will take care of the
  498. * rest.
  499. */
  500. if (!(dev->power.timer_expires &&
  501. dev->power.timer_expires <= expires)) {
  502. /*
  503. * We add a slack of 25% to gather wakeups
  504. * without sacrificing the granularity.
  505. */
  506. u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
  507. (NSEC_PER_MSEC >> 2);
  508. dev->power.timer_expires = expires;
  509. hrtimer_start_range_ns(&dev->power.suspend_timer,
  510. ns_to_ktime(expires),
  511. slack,
  512. HRTIMER_MODE_ABS);
  513. }
  514. dev->power.timer_autosuspends = 1;
  515. goto out;
  516. }
  517. }
  518. /* Other scheduled or pending requests need to be canceled. */
  519. pm_runtime_cancel_pending(dev);
  520. if (dev->power.runtime_status == RPM_SUSPENDING) {
  521. DEFINE_WAIT(wait);
  522. if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
  523. retval = -EINPROGRESS;
  524. goto out;
  525. }
  526. if (dev->power.irq_safe) {
  527. spin_unlock(&dev->power.lock);
  528. cpu_relax();
  529. spin_lock(&dev->power.lock);
  530. goto repeat;
  531. }
  532. /* Wait for the other suspend running in parallel with us. */
  533. for (;;) {
  534. prepare_to_wait(&dev->power.wait_queue, &wait,
  535. TASK_UNINTERRUPTIBLE);
  536. if (dev->power.runtime_status != RPM_SUSPENDING)
  537. break;
  538. spin_unlock_irq(&dev->power.lock);
  539. schedule();
  540. spin_lock_irq(&dev->power.lock);
  541. }
  542. finish_wait(&dev->power.wait_queue, &wait);
  543. goto repeat;
  544. }
  545. if (dev->power.no_callbacks)
  546. goto no_callback; /* Assume success. */
  547. /* Carry out an asynchronous or a synchronous suspend. */
  548. if (rpmflags & RPM_ASYNC) {
  549. dev->power.request = (rpmflags & RPM_AUTO) ?
  550. RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
  551. if (!dev->power.request_pending) {
  552. dev->power.request_pending = true;
  553. queue_work(pm_wq, &dev->power.work);
  554. }
  555. goto out;
  556. }
  557. __update_runtime_status(dev, RPM_SUSPENDING);
  558. callback = RPM_GET_CALLBACK(dev, runtime_suspend);
  559. dev_pm_enable_wake_irq_check(dev, true);
  560. retval = rpm_callback(callback, dev);
  561. if (retval)
  562. goto fail;
  563. no_callback:
  564. __update_runtime_status(dev, RPM_SUSPENDED);
  565. pm_runtime_deactivate_timer(dev);
  566. if (dev->parent) {
  567. parent = dev->parent;
  568. atomic_add_unless(&parent->power.child_count, -1, 0);
  569. }
  570. wake_up_all(&dev->power.wait_queue);
  571. if (dev->power.deferred_resume) {
  572. dev->power.deferred_resume = false;
  573. rpm_resume(dev, 0);
  574. retval = -EAGAIN;
  575. goto out;
  576. }
  577. if (dev->power.irq_safe)
  578. goto out;
  579. /* Maybe the parent is now able to suspend. */
  580. if (parent && !parent->power.ignore_children) {
  581. spin_unlock(&dev->power.lock);
  582. spin_lock(&parent->power.lock);
  583. rpm_idle(parent, RPM_ASYNC);
  584. spin_unlock(&parent->power.lock);
  585. spin_lock(&dev->power.lock);
  586. }
  587. /* Maybe the suppliers are now able to suspend. */
  588. if (dev->power.links_count > 0) {
  589. spin_unlock_irq(&dev->power.lock);
  590. rpm_suspend_suppliers(dev);
  591. spin_lock_irq(&dev->power.lock);
  592. }
  593. out:
  594. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
  595. return retval;
  596. fail:
  597. dev_pm_disable_wake_irq_check(dev);
  598. __update_runtime_status(dev, RPM_ACTIVE);
  599. dev->power.deferred_resume = false;
  600. wake_up_all(&dev->power.wait_queue);
  601. if (retval == -EAGAIN || retval == -EBUSY) {
  602. dev->power.runtime_error = 0;
  603. /*
  604. * If the callback routine failed an autosuspend, and
  605. * if the last_busy time has been updated so that there
  606. * is a new autosuspend expiration time, automatically
  607. * reschedule another autosuspend.
  608. */
  609. if ((rpmflags & RPM_AUTO) &&
  610. pm_runtime_autosuspend_expiration(dev) != 0)
  611. goto repeat;
  612. } else {
  613. pm_runtime_cancel_pending(dev);
  614. }
  615. goto out;
  616. }
  617. /**
  618. * rpm_resume - Carry out runtime resume of given device.
  619. * @dev: Device to resume.
  620. * @rpmflags: Flag bits.
  621. *
  622. * Check if the device's runtime PM status allows it to be resumed. Cancel
  623. * any scheduled or pending requests. If another resume has been started
  624. * earlier, either return immediately or wait for it to finish, depending on the
  625. * RPM_NOWAIT and RPM_ASYNC flags. Similarly, if there's a suspend running in
  626. * parallel with this function, either tell the other process to resume after
  627. * suspending (deferred_resume) or wait for it to finish. If the RPM_ASYNC
  628. * flag is set then queue a resume request; otherwise run the
  629. * ->runtime_resume() callback directly. Queue an idle notification for the
  630. * device if the resume succeeded.
  631. *
  632. * This function must be called under dev->power.lock with interrupts disabled.
  633. */
  634. static int rpm_resume(struct device *dev, int rpmflags)
  635. __releases(&dev->power.lock) __acquires(&dev->power.lock)
  636. {
  637. int (*callback)(struct device *);
  638. struct device *parent = NULL;
  639. int retval = 0;
  640. trace_rpm_resume_rcuidle(dev, rpmflags);
  641. repeat:
  642. if (dev->power.runtime_error)
  643. retval = -EINVAL;
  644. else if (dev->power.disable_depth == 1 && dev->power.is_suspended
  645. && dev->power.runtime_status == RPM_ACTIVE)
  646. retval = 1;
  647. else if (dev->power.disable_depth > 0)
  648. retval = -EACCES;
  649. if (retval)
  650. goto out;
  651. /*
  652. * Other scheduled or pending requests need to be canceled. Small
  653. * optimization: If an autosuspend timer is running, leave it running
  654. * rather than cancelling it now only to restart it again in the near
  655. * future.
  656. */
  657. dev->power.request = RPM_REQ_NONE;
  658. if (!dev->power.timer_autosuspends)
  659. pm_runtime_deactivate_timer(dev);
  660. if (dev->power.runtime_status == RPM_ACTIVE) {
  661. retval = 1;
  662. goto out;
  663. }
  664. if (dev->power.runtime_status == RPM_RESUMING
  665. || dev->power.runtime_status == RPM_SUSPENDING) {
  666. DEFINE_WAIT(wait);
  667. if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
  668. if (dev->power.runtime_status == RPM_SUSPENDING)
  669. dev->power.deferred_resume = true;
  670. else
  671. retval = -EINPROGRESS;
  672. goto out;
  673. }
  674. if (dev->power.irq_safe) {
  675. spin_unlock(&dev->power.lock);
  676. cpu_relax();
  677. spin_lock(&dev->power.lock);
  678. goto repeat;
  679. }
  680. /* Wait for the operation carried out in parallel with us. */
  681. for (;;) {
  682. prepare_to_wait(&dev->power.wait_queue, &wait,
  683. TASK_UNINTERRUPTIBLE);
  684. if (dev->power.runtime_status != RPM_RESUMING
  685. && dev->power.runtime_status != RPM_SUSPENDING)
  686. break;
  687. spin_unlock_irq(&dev->power.lock);
  688. schedule();
  689. spin_lock_irq(&dev->power.lock);
  690. }
  691. finish_wait(&dev->power.wait_queue, &wait);
  692. goto repeat;
  693. }
  694. /*
  695. * See if we can skip waking up the parent. This is safe only if
  696. * power.no_callbacks is set, because otherwise we don't know whether
  697. * the resume will actually succeed.
  698. */
  699. if (dev->power.no_callbacks && !parent && dev->parent) {
  700. spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
  701. if (dev->parent->power.disable_depth > 0
  702. || dev->parent->power.ignore_children
  703. || dev->parent->power.runtime_status == RPM_ACTIVE) {
  704. atomic_inc(&dev->parent->power.child_count);
  705. spin_unlock(&dev->parent->power.lock);
  706. retval = 1;
  707. goto no_callback; /* Assume success. */
  708. }
  709. spin_unlock(&dev->parent->power.lock);
  710. }
  711. /* Carry out an asynchronous or a synchronous resume. */
  712. if (rpmflags & RPM_ASYNC) {
  713. dev->power.request = RPM_REQ_RESUME;
  714. if (!dev->power.request_pending) {
  715. dev->power.request_pending = true;
  716. queue_work(pm_wq, &dev->power.work);
  717. }
  718. retval = 0;
  719. goto out;
  720. }
  721. if (!parent && dev->parent) {
  722. /*
  723. * Increment the parent's usage counter and resume it if
  724. * necessary. Not needed if dev is irq-safe; then the
  725. * parent is permanently resumed.
  726. */
  727. parent = dev->parent;
  728. if (dev->power.irq_safe)
  729. goto skip_parent;
  730. spin_unlock(&dev->power.lock);
  731. pm_runtime_get_noresume(parent);
  732. spin_lock(&parent->power.lock);
  733. /*
  734. * Resume the parent if it has runtime PM enabled and not been
  735. * set to ignore its children.
  736. */
  737. if (!parent->power.disable_depth
  738. && !parent->power.ignore_children) {
  739. rpm_resume(parent, 0);
  740. if (parent->power.runtime_status != RPM_ACTIVE)
  741. retval = -EBUSY;
  742. }
  743. spin_unlock(&parent->power.lock);
  744. spin_lock(&dev->power.lock);
  745. if (retval)
  746. goto out;
  747. goto repeat;
  748. }
  749. skip_parent:
  750. if (dev->power.no_callbacks)
  751. goto no_callback; /* Assume success. */
  752. __update_runtime_status(dev, RPM_RESUMING);
  753. callback = RPM_GET_CALLBACK(dev, runtime_resume);
  754. dev_pm_disable_wake_irq_check(dev);
  755. retval = rpm_callback(callback, dev);
  756. if (retval) {
  757. __update_runtime_status(dev, RPM_SUSPENDED);
  758. pm_runtime_cancel_pending(dev);
  759. dev_pm_enable_wake_irq_check(dev, false);
  760. } else {
  761. no_callback:
  762. __update_runtime_status(dev, RPM_ACTIVE);
  763. pm_runtime_mark_last_busy(dev);
  764. if (parent)
  765. atomic_inc(&parent->power.child_count);
  766. }
  767. wake_up_all(&dev->power.wait_queue);
  768. if (retval >= 0)
  769. rpm_idle(dev, RPM_ASYNC);
  770. out:
  771. if (parent && !dev->power.irq_safe) {
  772. spin_unlock_irq(&dev->power.lock);
  773. pm_runtime_put(parent);
  774. spin_lock_irq(&dev->power.lock);
  775. }
  776. trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
  777. return retval;
  778. }
  779. /**
  780. * pm_runtime_work - Universal runtime PM work function.
  781. * @work: Work structure used for scheduling the execution of this function.
  782. *
  783. * Use @work to get the device object the work is to be done for, determine what
  784. * is to be done and execute the appropriate runtime PM function.
  785. */
  786. static void pm_runtime_work(struct work_struct *work)
  787. {
  788. struct device *dev = container_of(work, struct device, power.work);
  789. enum rpm_request req;
  790. spin_lock_irq(&dev->power.lock);
  791. if (!dev->power.request_pending)
  792. goto out;
  793. req = dev->power.request;
  794. dev->power.request = RPM_REQ_NONE;
  795. dev->power.request_pending = false;
  796. switch (req) {
  797. case RPM_REQ_NONE:
  798. break;
  799. case RPM_REQ_IDLE:
  800. rpm_idle(dev, RPM_NOWAIT);
  801. break;
  802. case RPM_REQ_SUSPEND:
  803. rpm_suspend(dev, RPM_NOWAIT);
  804. break;
  805. case RPM_REQ_AUTOSUSPEND:
  806. rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
  807. break;
  808. case RPM_REQ_RESUME:
  809. rpm_resume(dev, RPM_NOWAIT);
  810. break;
  811. }
  812. out:
  813. spin_unlock_irq(&dev->power.lock);
  814. }
  815. /**
  816. * pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
  817. * @data: Device pointer passed by pm_schedule_suspend().
  818. *
  819. * Check if the time is right and queue a suspend request.
  820. */
  821. static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer)
  822. {
  823. struct device *dev = container_of(timer, struct device, power.suspend_timer);
  824. unsigned long flags;
  825. u64 expires;
  826. spin_lock_irqsave(&dev->power.lock, flags);
  827. expires = dev->power.timer_expires;
  828. /*
  829. * If 'expires' is after the current time, we've been called
  830. * too early.
  831. */
  832. if (expires > 0 && expires < ktime_get_mono_fast_ns()) {
  833. dev->power.timer_expires = 0;
  834. rpm_suspend(dev, dev->power.timer_autosuspends ?
  835. (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
  836. }
  837. spin_unlock_irqrestore(&dev->power.lock, flags);
  838. return HRTIMER_NORESTART;
  839. }
  840. /**
  841. * pm_schedule_suspend - Set up a timer to submit a suspend request in future.
  842. * @dev: Device to suspend.
  843. * @delay: Time to wait before submitting a suspend request, in milliseconds.
  844. */
  845. int pm_schedule_suspend(struct device *dev, unsigned int delay)
  846. {
  847. unsigned long flags;
  848. u64 expires;
  849. int retval;
  850. spin_lock_irqsave(&dev->power.lock, flags);
  851. if (!delay) {
  852. retval = rpm_suspend(dev, RPM_ASYNC);
  853. goto out;
  854. }
  855. retval = rpm_check_suspend_allowed(dev);
  856. if (retval)
  857. goto out;
  858. /* Other scheduled or pending requests need to be canceled. */
  859. pm_runtime_cancel_pending(dev);
  860. expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
  861. dev->power.timer_expires = expires;
  862. dev->power.timer_autosuspends = 0;
  863. hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);
  864. out:
  865. spin_unlock_irqrestore(&dev->power.lock, flags);
  866. return retval;
  867. }
  868. EXPORT_SYMBOL_GPL(pm_schedule_suspend);
  869. /**
  870. * __pm_runtime_idle - Entry point for runtime idle operations.
  871. * @dev: Device to send idle notification for.
  872. * @rpmflags: Flag bits.
  873. *
  874. * If the RPM_GET_PUT flag is set, decrement the device's usage count and
  875. * return immediately if it is larger than zero. Then carry out an idle
  876. * notification, either synchronous or asynchronous.
  877. *
  878. * This routine may be called in atomic context if the RPM_ASYNC flag is set,
  879. * or if pm_runtime_irq_safe() has been called.
  880. */
  881. int __pm_runtime_idle(struct device *dev, int rpmflags)
  882. {
  883. unsigned long flags;
  884. int retval;
  885. if (rpmflags & RPM_GET_PUT) {
  886. if (!atomic_dec_and_test(&dev->power.usage_count)) {
  887. trace_rpm_usage_rcuidle(dev, rpmflags);
  888. return 0;
  889. }
  890. }
  891. might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
  892. spin_lock_irqsave(&dev->power.lock, flags);
  893. retval = rpm_idle(dev, rpmflags);
  894. spin_unlock_irqrestore(&dev->power.lock, flags);
  895. return retval;
  896. }
  897. EXPORT_SYMBOL_GPL(__pm_runtime_idle);
  898. /**
  899. * __pm_runtime_suspend - Entry point for runtime put/suspend operations.
  900. * @dev: Device to suspend.
  901. * @rpmflags: Flag bits.
  902. *
  903. * If the RPM_GET_PUT flag is set, decrement the device's usage count and
  904. * return immediately if it is larger than zero. Then carry out a suspend,
  905. * either synchronous or asynchronous.
  906. *
  907. * This routine may be called in atomic context if the RPM_ASYNC flag is set,
  908. * or if pm_runtime_irq_safe() has been called.
  909. */
  910. int __pm_runtime_suspend(struct device *dev, int rpmflags)
  911. {
  912. unsigned long flags;
  913. int retval;
  914. if (rpmflags & RPM_GET_PUT) {
  915. if (!atomic_dec_and_test(&dev->power.usage_count)) {
  916. trace_rpm_usage_rcuidle(dev, rpmflags);
  917. return 0;
  918. }
  919. }
  920. might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
  921. spin_lock_irqsave(&dev->power.lock, flags);
  922. retval = rpm_suspend(dev, rpmflags);
  923. spin_unlock_irqrestore(&dev->power.lock, flags);
  924. return retval;
  925. }
  926. EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
  927. /**
  928. * __pm_runtime_resume - Entry point for runtime resume operations.
  929. * @dev: Device to resume.
  930. * @rpmflags: Flag bits.
  931. *
  932. * If the RPM_GET_PUT flag is set, increment the device's usage count. Then
  933. * carry out a resume, either synchronous or asynchronous.
  934. *
  935. * This routine may be called in atomic context if the RPM_ASYNC flag is set,
  936. * or if pm_runtime_irq_safe() has been called.
  937. */
  938. int __pm_runtime_resume(struct device *dev, int rpmflags)
  939. {
  940. unsigned long flags;
  941. int retval;
  942. might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
  943. dev->power.runtime_status != RPM_ACTIVE);
  944. if (rpmflags & RPM_GET_PUT)
  945. atomic_inc(&dev->power.usage_count);
  946. spin_lock_irqsave(&dev->power.lock, flags);
  947. retval = rpm_resume(dev, rpmflags);
  948. spin_unlock_irqrestore(&dev->power.lock, flags);
  949. return retval;
  950. }
  951. EXPORT_SYMBOL_GPL(__pm_runtime_resume);
  952. /**
  953. * pm_runtime_get_if_active - Conditionally bump up device usage counter.
  954. * @dev: Device to handle.
  955. * @ign_usage_count: Whether or not to look at the current usage counter value.
  956. *
  957. * Return -EINVAL if runtime PM is disabled for @dev.
  958. *
  959. * Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either
  960. * @ign_usage_count is %true or the runtime PM usage counter of @dev is not
  961. * zero, increment the usage counter of @dev and return 1. Otherwise, return 0
  962. * without changing the usage counter.
  963. *
  964. * If @ign_usage_count is %true, this function can be used to prevent suspending
  965. * the device when its runtime PM status is %RPM_ACTIVE.
  966. *
  967. * If @ign_usage_count is %false, this function can be used to prevent
  968. * suspending the device when both its runtime PM status is %RPM_ACTIVE and its
  969. * runtime PM usage counter is not zero.
  970. *
  971. * The caller is resposible for decrementing the runtime PM usage counter of
  972. * @dev after this function has returned a positive value for it.
  973. */
  974. int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count)
  975. {
  976. unsigned long flags;
  977. int retval;
  978. spin_lock_irqsave(&dev->power.lock, flags);
  979. if (dev->power.disable_depth > 0) {
  980. retval = -EINVAL;
  981. } else if (dev->power.runtime_status != RPM_ACTIVE) {
  982. retval = 0;
  983. } else if (ign_usage_count) {
  984. retval = 1;
  985. atomic_inc(&dev->power.usage_count);
  986. } else {
  987. retval = atomic_inc_not_zero(&dev->power.usage_count);
  988. }
  989. trace_rpm_usage_rcuidle(dev, 0);
  990. spin_unlock_irqrestore(&dev->power.lock, flags);
  991. return retval;
  992. }
  993. EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
  994. /**
  995. * __pm_runtime_set_status - Set runtime PM status of a device.
  996. * @dev: Device to handle.
  997. * @status: New runtime PM status of the device.
  998. *
  999. * If runtime PM of the device is disabled or its power.runtime_error field is
  1000. * different from zero, the status may be changed either to RPM_ACTIVE, or to
  1001. * RPM_SUSPENDED, as long as that reflects the actual state of the device.
  1002. * However, if the device has a parent and the parent is not active, and the
  1003. * parent's power.ignore_children flag is unset, the device's status cannot be
  1004. * set to RPM_ACTIVE, so -EBUSY is returned in that case.
  1005. *
  1006. * If successful, __pm_runtime_set_status() clears the power.runtime_error field
  1007. * and the device parent's counter of unsuspended children is modified to
  1008. * reflect the new status. If the new status is RPM_SUSPENDED, an idle
  1009. * notification request for the parent is submitted.
  1010. *
  1011. * If @dev has any suppliers (as reflected by device links to them), and @status
  1012. * is RPM_ACTIVE, they will be activated upfront and if the activation of one
  1013. * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
  1014. * of the @status value) and the suppliers will be deacticated on exit. The
  1015. * error returned by the failing supplier activation will be returned in that
  1016. * case.
  1017. */
  1018. int __pm_runtime_set_status(struct device *dev, unsigned int status)
  1019. {
  1020. struct device *parent = dev->parent;
  1021. bool notify_parent = false;
  1022. int error = 0;
  1023. if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
  1024. return -EINVAL;
  1025. spin_lock_irq(&dev->power.lock);
  1026. /*
  1027. * Prevent PM-runtime from being enabled for the device or return an
  1028. * error if it is enabled already and working.
  1029. */
  1030. if (dev->power.runtime_error || dev->power.disable_depth)
  1031. dev->power.disable_depth++;
  1032. else
  1033. error = -EAGAIN;
  1034. spin_unlock_irq(&dev->power.lock);
  1035. if (error)
  1036. return error;
  1037. /*
  1038. * If the new status is RPM_ACTIVE, the suppliers can be activated
  1039. * upfront regardless of the current status, because next time
  1040. * rpm_put_suppliers() runs, the rpm_active refcounts of the links
  1041. * involved will be dropped down to one anyway.
  1042. */
  1043. if (status == RPM_ACTIVE) {
  1044. int idx = device_links_read_lock();
  1045. error = rpm_get_suppliers(dev);
  1046. if (error)
  1047. status = RPM_SUSPENDED;
  1048. device_links_read_unlock(idx);
  1049. }
  1050. spin_lock_irq(&dev->power.lock);
  1051. if (dev->power.runtime_status == status || !parent)
  1052. goto out_set;
  1053. if (status == RPM_SUSPENDED) {
  1054. atomic_add_unless(&parent->power.child_count, -1, 0);
  1055. notify_parent = !parent->power.ignore_children;
  1056. } else {
  1057. spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
  1058. /*
  1059. * It is invalid to put an active child under a parent that is
  1060. * not active, has runtime PM enabled and the
  1061. * 'power.ignore_children' flag unset.
  1062. */
  1063. if (!parent->power.disable_depth
  1064. && !parent->power.ignore_children
  1065. && parent->power.runtime_status != RPM_ACTIVE) {
  1066. dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
  1067. dev_name(dev),
  1068. dev_name(parent));
  1069. error = -EBUSY;
  1070. } else if (dev->power.runtime_status == RPM_SUSPENDED) {
  1071. atomic_inc(&parent->power.child_count);
  1072. }
  1073. spin_unlock(&parent->power.lock);
  1074. if (error) {
  1075. status = RPM_SUSPENDED;
  1076. goto out;
  1077. }
  1078. }
  1079. out_set:
  1080. __update_runtime_status(dev, status);
  1081. if (!error)
  1082. dev->power.runtime_error = 0;
  1083. out:
  1084. spin_unlock_irq(&dev->power.lock);
  1085. if (notify_parent)
  1086. pm_request_idle(parent);
  1087. if (status == RPM_SUSPENDED) {
  1088. int idx = device_links_read_lock();
  1089. rpm_put_suppliers(dev);
  1090. device_links_read_unlock(idx);
  1091. }
  1092. pm_runtime_enable(dev);
  1093. return error;
  1094. }
  1095. EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
  1096. /**
  1097. * __pm_runtime_barrier - Cancel pending requests and wait for completions.
  1098. * @dev: Device to handle.
  1099. *
  1100. * Flush all pending requests for the device from pm_wq and wait for all
  1101. * runtime PM operations involving the device in progress to complete.
  1102. *
  1103. * Should be called under dev->power.lock with interrupts disabled.
  1104. */
  1105. static void __pm_runtime_barrier(struct device *dev)
  1106. {
  1107. pm_runtime_deactivate_timer(dev);
  1108. if (dev->power.request_pending) {
  1109. dev->power.request = RPM_REQ_NONE;
  1110. spin_unlock_irq(&dev->power.lock);
  1111. cancel_work_sync(&dev->power.work);
  1112. spin_lock_irq(&dev->power.lock);
  1113. dev->power.request_pending = false;
  1114. }
  1115. if (dev->power.runtime_status == RPM_SUSPENDING
  1116. || dev->power.runtime_status == RPM_RESUMING
  1117. || dev->power.idle_notification) {
  1118. DEFINE_WAIT(wait);
  1119. /* Suspend, wake-up or idle notification in progress. */
  1120. for (;;) {
  1121. prepare_to_wait(&dev->power.wait_queue, &wait,
  1122. TASK_UNINTERRUPTIBLE);
  1123. if (dev->power.runtime_status != RPM_SUSPENDING
  1124. && dev->power.runtime_status != RPM_RESUMING
  1125. && !dev->power.idle_notification)
  1126. break;
  1127. spin_unlock_irq(&dev->power.lock);
  1128. schedule();
  1129. spin_lock_irq(&dev->power.lock);
  1130. }
  1131. finish_wait(&dev->power.wait_queue, &wait);
  1132. }
  1133. }
  1134. /**
  1135. * pm_runtime_barrier - Flush pending requests and wait for completions.
  1136. * @dev: Device to handle.
  1137. *
  1138. * Prevent the device from being suspended by incrementing its usage counter and
  1139. * if there's a pending resume request for the device, wake the device up.
  1140. * Next, make sure that all pending requests for the device have been flushed
  1141. * from pm_wq and wait for all runtime PM operations involving the device in
  1142. * progress to complete.
  1143. *
  1144. * Return value:
  1145. * 1, if there was a resume request pending and the device had to be woken up,
  1146. * 0, otherwise
  1147. */
  1148. int pm_runtime_barrier(struct device *dev)
  1149. {
  1150. int retval = 0;
  1151. pm_runtime_get_noresume(dev);
  1152. spin_lock_irq(&dev->power.lock);
  1153. if (dev->power.request_pending
  1154. && dev->power.request == RPM_REQ_RESUME) {
  1155. rpm_resume(dev, 0);
  1156. retval = 1;
  1157. }
  1158. __pm_runtime_barrier(dev);
  1159. spin_unlock_irq(&dev->power.lock);
  1160. pm_runtime_put_noidle(dev);
  1161. return retval;
  1162. }
  1163. EXPORT_SYMBOL_GPL(pm_runtime_barrier);
  1164. /**
  1165. * __pm_runtime_disable - Disable runtime PM of a device.
  1166. * @dev: Device to handle.
  1167. * @check_resume: If set, check if there's a resume request for the device.
  1168. *
  1169. * Increment power.disable_depth for the device and if it was zero previously,
  1170. * cancel all pending runtime PM requests for the device and wait for all
  1171. * operations in progress to complete. The device can be either active or
  1172. * suspended after its runtime PM has been disabled.
  1173. *
  1174. * If @check_resume is set and there's a resume request pending when
  1175. * __pm_runtime_disable() is called and power.disable_depth is zero, the
  1176. * function will wake up the device before disabling its runtime PM.
  1177. */
  1178. void __pm_runtime_disable(struct device *dev, bool check_resume)
  1179. {
  1180. spin_lock_irq(&dev->power.lock);
  1181. if (dev->power.disable_depth > 0) {
  1182. dev->power.disable_depth++;
  1183. goto out;
  1184. }
  1185. /*
  1186. * Wake up the device if there's a resume request pending, because that
  1187. * means there probably is some I/O to process and disabling runtime PM
  1188. * shouldn't prevent the device from processing the I/O.
  1189. */
  1190. if (check_resume && dev->power.request_pending
  1191. && dev->power.request == RPM_REQ_RESUME) {
  1192. /*
  1193. * Prevent suspends and idle notifications from being carried
  1194. * out after we have woken up the device.
  1195. */
  1196. pm_runtime_get_noresume(dev);
  1197. rpm_resume(dev, 0);
  1198. pm_runtime_put_noidle(dev);
  1199. }
  1200. /* Update time accounting before disabling PM-runtime. */
  1201. update_pm_runtime_accounting(dev);
  1202. if (!dev->power.disable_depth++)
  1203. __pm_runtime_barrier(dev);
  1204. out:
  1205. spin_unlock_irq(&dev->power.lock);
  1206. }
  1207. EXPORT_SYMBOL_GPL(__pm_runtime_disable);
  1208. /**
  1209. * pm_runtime_enable - Enable runtime PM of a device.
  1210. * @dev: Device to handle.
  1211. */
  1212. void pm_runtime_enable(struct device *dev)
  1213. {
  1214. unsigned long flags;
  1215. spin_lock_irqsave(&dev->power.lock, flags);
  1216. if (dev->power.disable_depth > 0) {
  1217. dev->power.disable_depth--;
  1218. /* About to enable runtime pm, set accounting_timestamp to now */
  1219. if (!dev->power.disable_depth)
  1220. dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
  1221. } else {
  1222. dev_warn(dev, "Unbalanced %s!\n", __func__);
  1223. }
  1224. WARN(!dev->power.disable_depth &&
  1225. dev->power.runtime_status == RPM_SUSPENDED &&
  1226. !dev->power.ignore_children &&
  1227. atomic_read(&dev->power.child_count) > 0,
  1228. "Enabling runtime PM for inactive device (%s) with active children\n",
  1229. dev_name(dev));
  1230. spin_unlock_irqrestore(&dev->power.lock, flags);
  1231. }
  1232. EXPORT_SYMBOL_GPL(pm_runtime_enable);
  1233. /**
  1234. * pm_runtime_forbid - Block runtime PM of a device.
  1235. * @dev: Device to handle.
  1236. *
  1237. * Increase the device's usage count and clear its power.runtime_auto flag,
  1238. * so that it cannot be suspended at run time until pm_runtime_allow() is called
  1239. * for it.
  1240. */
  1241. void pm_runtime_forbid(struct device *dev)
  1242. {
  1243. spin_lock_irq(&dev->power.lock);
  1244. if (!dev->power.runtime_auto)
  1245. goto out;
  1246. dev->power.runtime_auto = false;
  1247. atomic_inc(&dev->power.usage_count);
  1248. rpm_resume(dev, 0);
  1249. out:
  1250. spin_unlock_irq(&dev->power.lock);
  1251. }
  1252. EXPORT_SYMBOL_GPL(pm_runtime_forbid);
  1253. /**
  1254. * pm_runtime_allow - Unblock runtime PM of a device.
  1255. * @dev: Device to handle.
  1256. *
  1257. * Decrease the device's usage count and set its power.runtime_auto flag.
  1258. */
  1259. void pm_runtime_allow(struct device *dev)
  1260. {
  1261. spin_lock_irq(&dev->power.lock);
  1262. if (dev->power.runtime_auto)
  1263. goto out;
  1264. dev->power.runtime_auto = true;
  1265. if (atomic_dec_and_test(&dev->power.usage_count))
  1266. rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
  1267. else
  1268. trace_rpm_usage_rcuidle(dev, RPM_AUTO | RPM_ASYNC);
  1269. out:
  1270. spin_unlock_irq(&dev->power.lock);
  1271. }
  1272. EXPORT_SYMBOL_GPL(pm_runtime_allow);
  1273. /**
  1274. * pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
  1275. * @dev: Device to handle.
  1276. *
  1277. * Set the power.no_callbacks flag, which tells the PM core that this
  1278. * device is power-managed through its parent and has no runtime PM
  1279. * callbacks of its own. The runtime sysfs attributes will be removed.
  1280. */
  1281. void pm_runtime_no_callbacks(struct device *dev)
  1282. {
  1283. spin_lock_irq(&dev->power.lock);
  1284. dev->power.no_callbacks = 1;
  1285. spin_unlock_irq(&dev->power.lock);
  1286. if (device_is_registered(dev))
  1287. rpm_sysfs_remove(dev);
  1288. }
  1289. EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
  1290. /**
  1291. * pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
  1292. * @dev: Device to handle
  1293. *
  1294. * Set the power.irq_safe flag, which tells the PM core that the
  1295. * ->runtime_suspend() and ->runtime_resume() callbacks for this device should
  1296. * always be invoked with the spinlock held and interrupts disabled. It also
  1297. * causes the parent's usage counter to be permanently incremented, preventing
  1298. * the parent from runtime suspending -- otherwise an irq-safe child might have
  1299. * to wait for a non-irq-safe parent.
  1300. */
  1301. void pm_runtime_irq_safe(struct device *dev)
  1302. {
  1303. if (dev->parent)
  1304. pm_runtime_get_sync(dev->parent);
  1305. spin_lock_irq(&dev->power.lock);
  1306. dev->power.irq_safe = 1;
  1307. spin_unlock_irq(&dev->power.lock);
  1308. }
  1309. EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
  1310. /**
  1311. * update_autosuspend - Handle a change to a device's autosuspend settings.
  1312. * @dev: Device to handle.
  1313. * @old_delay: The former autosuspend_delay value.
  1314. * @old_use: The former use_autosuspend value.
  1315. *
  1316. * Prevent runtime suspend if the new delay is negative and use_autosuspend is
  1317. * set; otherwise allow it. Send an idle notification if suspends are allowed.
  1318. *
  1319. * This function must be called under dev->power.lock with interrupts disabled.
  1320. */
  1321. static void update_autosuspend(struct device *dev, int old_delay, int old_use)
  1322. {
  1323. int delay = dev->power.autosuspend_delay;
  1324. /* Should runtime suspend be prevented now? */
  1325. if (dev->power.use_autosuspend && delay < 0) {
  1326. /* If it used to be allowed then prevent it. */
  1327. if (!old_use || old_delay >= 0) {
  1328. atomic_inc(&dev->power.usage_count);
  1329. rpm_resume(dev, 0);
  1330. } else {
  1331. trace_rpm_usage_rcuidle(dev, 0);
  1332. }
  1333. }
  1334. /* Runtime suspend should be allowed now. */
  1335. else {
  1336. /* If it used to be prevented then allow it. */
  1337. if (old_use && old_delay < 0)
  1338. atomic_dec(&dev->power.usage_count);
  1339. /* Maybe we can autosuspend now. */
  1340. rpm_idle(dev, RPM_AUTO);
  1341. }
  1342. }
  1343. /**
  1344. * pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
  1345. * @dev: Device to handle.
  1346. * @delay: Value of the new delay in milliseconds.
  1347. *
  1348. * Set the device's power.autosuspend_delay value. If it changes to negative
  1349. * and the power.use_autosuspend flag is set, prevent runtime suspends. If it
  1350. * changes the other way, allow runtime suspends.
  1351. */
  1352. void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
  1353. {
  1354. int old_delay, old_use;
  1355. spin_lock_irq(&dev->power.lock);
  1356. old_delay = dev->power.autosuspend_delay;
  1357. old_use = dev->power.use_autosuspend;
  1358. dev->power.autosuspend_delay = delay;
  1359. update_autosuspend(dev, old_delay, old_use);
  1360. spin_unlock_irq(&dev->power.lock);
  1361. }
  1362. EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
  1363. /**
  1364. * __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
  1365. * @dev: Device to handle.
  1366. * @use: New value for use_autosuspend.
  1367. *
  1368. * Set the device's power.use_autosuspend flag, and allow or prevent runtime
  1369. * suspends as needed.
  1370. */
  1371. void __pm_runtime_use_autosuspend(struct device *dev, bool use)
  1372. {
  1373. int old_delay, old_use;
  1374. spin_lock_irq(&dev->power.lock);
  1375. old_delay = dev->power.autosuspend_delay;
  1376. old_use = dev->power.use_autosuspend;
  1377. dev->power.use_autosuspend = use;
  1378. update_autosuspend(dev, old_delay, old_use);
  1379. spin_unlock_irq(&dev->power.lock);
  1380. }
  1381. EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
  1382. /**
  1383. * pm_runtime_init - Initialize runtime PM fields in given device object.
  1384. * @dev: Device object to initialize.
  1385. */
  1386. void pm_runtime_init(struct device *dev)
  1387. {
  1388. dev->power.runtime_status = RPM_SUSPENDED;
  1389. dev->power.idle_notification = false;
  1390. dev->power.disable_depth = 1;
  1391. atomic_set(&dev->power.usage_count, 0);
  1392. dev->power.runtime_error = 0;
  1393. atomic_set(&dev->power.child_count, 0);
  1394. pm_suspend_ignore_children(dev, false);
  1395. dev->power.runtime_auto = true;
  1396. dev->power.request_pending = false;
  1397. dev->power.request = RPM_REQ_NONE;
  1398. dev->power.deferred_resume = false;
  1399. dev->power.needs_force_resume = 0;
  1400. INIT_WORK(&dev->power.work, pm_runtime_work);
  1401. dev->power.timer_expires = 0;
  1402. hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  1403. dev->power.suspend_timer.function = pm_suspend_timer_fn;
  1404. init_waitqueue_head(&dev->power.wait_queue);
  1405. }
  1406. /**
  1407. * pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
  1408. * @dev: Device object to re-initialize.
  1409. */
  1410. void pm_runtime_reinit(struct device *dev)
  1411. {
  1412. if (!pm_runtime_enabled(dev)) {
  1413. if (dev->power.runtime_status == RPM_ACTIVE)
  1414. pm_runtime_set_suspended(dev);
  1415. if (dev->power.irq_safe) {
  1416. spin_lock_irq(&dev->power.lock);
  1417. dev->power.irq_safe = 0;
  1418. spin_unlock_irq(&dev->power.lock);
  1419. if (dev->parent)
  1420. pm_runtime_put(dev->parent);
  1421. }
  1422. }
  1423. }
  1424. /**
  1425. * pm_runtime_remove - Prepare for removing a device from device hierarchy.
  1426. * @dev: Device object being removed from device hierarchy.
  1427. */
  1428. void pm_runtime_remove(struct device *dev)
  1429. {
  1430. __pm_runtime_disable(dev, false);
  1431. pm_runtime_reinit(dev);
  1432. }
  1433. /**
  1434. * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
  1435. * @dev: Consumer device.
  1436. */
  1437. void pm_runtime_get_suppliers(struct device *dev)
  1438. {
  1439. struct device_link *link;
  1440. int idx;
  1441. idx = device_links_read_lock();
  1442. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
  1443. device_links_read_lock_held())
  1444. if (link->flags & DL_FLAG_PM_RUNTIME) {
  1445. link->supplier_preactivated = true;
  1446. pm_runtime_get_sync(link->supplier);
  1447. refcount_inc(&link->rpm_active);
  1448. }
  1449. device_links_read_unlock(idx);
  1450. }
  1451. /**
  1452. * pm_runtime_put_suppliers - Drop references to supplier devices.
  1453. * @dev: Consumer device.
  1454. */
  1455. void pm_runtime_put_suppliers(struct device *dev)
  1456. {
  1457. struct device_link *link;
  1458. unsigned long flags;
  1459. bool put;
  1460. int idx;
  1461. idx = device_links_read_lock();
  1462. list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
  1463. device_links_read_lock_held())
  1464. if (link->supplier_preactivated) {
  1465. link->supplier_preactivated = false;
  1466. spin_lock_irqsave(&dev->power.lock, flags);
  1467. put = pm_runtime_status_suspended(dev) &&
  1468. refcount_dec_not_one(&link->rpm_active);
  1469. spin_unlock_irqrestore(&dev->power.lock, flags);
  1470. if (put)
  1471. pm_runtime_put(link->supplier);
  1472. }
  1473. device_links_read_unlock(idx);
  1474. }
  1475. void pm_runtime_new_link(struct device *dev)
  1476. {
  1477. spin_lock_irq(&dev->power.lock);
  1478. dev->power.links_count++;
  1479. spin_unlock_irq(&dev->power.lock);
  1480. }
  1481. static void pm_runtime_drop_link_count(struct device *dev)
  1482. {
  1483. spin_lock_irq(&dev->power.lock);
  1484. WARN_ON(dev->power.links_count == 0);
  1485. dev->power.links_count--;
  1486. spin_unlock_irq(&dev->power.lock);
  1487. }
  1488. /**
  1489. * pm_runtime_drop_link - Prepare for device link removal.
  1490. * @link: Device link going away.
  1491. *
  1492. * Drop the link count of the consumer end of @link and decrement the supplier
  1493. * device's runtime PM usage counter as many times as needed to drop all of the
  1494. * PM runtime reference to it from the consumer.
  1495. */
  1496. void pm_runtime_drop_link(struct device_link *link)
  1497. {
  1498. if (!(link->flags & DL_FLAG_PM_RUNTIME))
  1499. return;
  1500. pm_runtime_drop_link_count(link->consumer);
  1501. pm_runtime_release_supplier(link, true);
  1502. }
  1503. static bool pm_runtime_need_not_resume(struct device *dev)
  1504. {
  1505. return atomic_read(&dev->power.usage_count) <= 1 &&
  1506. (atomic_read(&dev->power.child_count) == 0 ||
  1507. dev->power.ignore_children);
  1508. }
  1509. /**
  1510. * pm_runtime_force_suspend - Force a device into suspend state if needed.
  1511. * @dev: Device to suspend.
  1512. *
  1513. * Disable runtime PM so we safely can check the device's runtime PM status and
  1514. * if it is active, invoke its ->runtime_suspend callback to suspend it and
  1515. * change its runtime PM status field to RPM_SUSPENDED. Also, if the device's
  1516. * usage and children counters don't indicate that the device was in use before
  1517. * the system-wide transition under way, decrement its parent's children counter
  1518. * (if there is a parent). Keep runtime PM disabled to preserve the state
  1519. * unless we encounter errors.
  1520. *
  1521. * Typically this function may be invoked from a system suspend callback to make
  1522. * sure the device is put into low power state and it should only be used during
  1523. * system-wide PM transitions to sleep states. It assumes that the analogous
  1524. * pm_runtime_force_resume() will be used to resume the device.
  1525. */
  1526. int pm_runtime_force_suspend(struct device *dev)
  1527. {
  1528. int (*callback)(struct device *);
  1529. int ret;
  1530. pm_runtime_disable(dev);
  1531. if (pm_runtime_status_suspended(dev))
  1532. return 0;
  1533. callback = RPM_GET_CALLBACK(dev, runtime_suspend);
  1534. ret = callback ? callback(dev) : 0;
  1535. if (ret)
  1536. goto err;
  1537. /*
  1538. * If the device can stay in suspend after the system-wide transition
  1539. * to the working state that will follow, drop the children counter of
  1540. * its parent, but set its status to RPM_SUSPENDED anyway in case this
  1541. * function will be called again for it in the meantime.
  1542. */
  1543. if (pm_runtime_need_not_resume(dev)) {
  1544. pm_runtime_set_suspended(dev);
  1545. } else {
  1546. __update_runtime_status(dev, RPM_SUSPENDED);
  1547. dev->power.needs_force_resume = 1;
  1548. }
  1549. return 0;
  1550. err:
  1551. pm_runtime_enable(dev);
  1552. return ret;
  1553. }
  1554. EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
  1555. /**
  1556. * pm_runtime_force_resume - Force a device into resume state if needed.
  1557. * @dev: Device to resume.
  1558. *
  1559. * Prior invoking this function we expect the user to have brought the device
  1560. * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
  1561. * those actions and bring the device into full power, if it is expected to be
  1562. * used on system resume. In the other case, we defer the resume to be managed
  1563. * via runtime PM.
  1564. *
  1565. * Typically this function may be invoked from a system resume callback.
  1566. */
  1567. int pm_runtime_force_resume(struct device *dev)
  1568. {
  1569. int (*callback)(struct device *);
  1570. int ret = 0;
  1571. if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume)
  1572. goto out;
  1573. /*
  1574. * The value of the parent's children counter is correct already, so
  1575. * just update the status of the device.
  1576. */
  1577. __update_runtime_status(dev, RPM_ACTIVE);
  1578. callback = RPM_GET_CALLBACK(dev, runtime_resume);
  1579. ret = callback ? callback(dev) : 0;
  1580. if (ret) {
  1581. pm_runtime_set_suspended(dev);
  1582. goto out;
  1583. }
  1584. pm_runtime_mark_last_busy(dev);
  1585. out:
  1586. dev->power.needs_force_resume = 0;
  1587. pm_runtime_enable(dev);
  1588. return ret;
  1589. }
  1590. EXPORT_SYMBOL_GPL(pm_runtime_force_resume);