rpmh-rsc.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
  4. */
  5. #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
  6. #include <linux/atomic.h>
  7. #include <linux/cpu_pm.h>
  8. #include <linux/delay.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/io.h>
  11. #include <linux/iopoll.h>
  12. #include <linux/kernel.h>
  13. #include <linux/list.h>
  14. #include <linux/module.h>
  15. #include <linux/of.h>
  16. #include <linux/of_irq.h>
  17. #include <linux/of_platform.h>
  18. #include <linux/platform_device.h>
  19. #include <linux/slab.h>
  20. #include <linux/spinlock.h>
  21. #include <linux/wait.h>
  22. #include <soc/qcom/cmd-db.h>
  23. #include <soc/qcom/tcs.h>
  24. #include <dt-bindings/soc/qcom,rpmh-rsc.h>
  25. #include "rpmh-internal.h"
  26. #define CREATE_TRACE_POINTS
  27. #include "trace-rpmh.h"
  28. #define RSC_DRV_TCS_OFFSET 672
  29. #define RSC_DRV_CMD_OFFSET 20
  30. /* DRV HW Solver Configuration Information Register */
  31. #define DRV_SOLVER_CONFIG 0x04
  32. #define DRV_HW_SOLVER_MASK 1
  33. #define DRV_HW_SOLVER_SHIFT 24
  34. /* DRV TCS Configuration Information Register */
  35. #define DRV_PRNT_CHLD_CONFIG 0x0C
  36. #define DRV_NUM_TCS_MASK 0x3F
  37. #define DRV_NUM_TCS_SHIFT 6
  38. #define DRV_NCPT_MASK 0x1F
  39. #define DRV_NCPT_SHIFT 27
  40. /* Offsets for common TCS Registers, one bit per TCS */
  41. #define RSC_DRV_IRQ_ENABLE 0x00
  42. #define RSC_DRV_IRQ_STATUS 0x04
  43. #define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */
  44. /*
  45. * Offsets for per TCS Registers.
  46. *
  47. * TCSes start at 0x10 from tcs_base and are stored one after another.
  48. * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one
  49. * of the below to find a register.
  50. */
  51. #define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */
  52. #define RSC_DRV_CONTROL 0x14
  53. #define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */
  54. #define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */
  55. /*
  56. * Offsets for per command in a TCS.
  57. *
  58. * Commands (up to 16) start at 0x30 in a TCS; multiply command index
  59. * by RSC_DRV_CMD_OFFSET and add one of the below to find a register.
  60. */
  61. #define RSC_DRV_CMD_MSGID 0x30
  62. #define RSC_DRV_CMD_ADDR 0x34
  63. #define RSC_DRV_CMD_DATA 0x38
  64. #define RSC_DRV_CMD_STATUS 0x3C
  65. #define RSC_DRV_CMD_RESP_DATA 0x40
  66. #define TCS_AMC_MODE_ENABLE BIT(16)
  67. #define TCS_AMC_MODE_TRIGGER BIT(24)
  68. /* TCS CMD register bit mask */
  69. #define CMD_MSGID_LEN 8
  70. #define CMD_MSGID_RESP_REQ BIT(8)
  71. #define CMD_MSGID_WRITE BIT(16)
  72. #define CMD_STATUS_ISSUED BIT(8)
  73. #define CMD_STATUS_COMPL BIT(16)
  74. /*
  75. * Here's a high level overview of how all the registers in RPMH work
  76. * together:
  77. *
  78. * - The main rpmh-rsc address is the base of a register space that can
  79. * be used to find overall configuration of the hardware
  80. * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
  81. * space are all the TCS blocks. The offset of the TCS blocks is
  82. * specified in the device tree by "qcom,tcs-offset" and used to
  83. * compute tcs_base.
  84. * - TCS blocks come one after another. Type, count, and order are
  85. * specified by the device tree as "qcom,tcs-config".
  86. * - Each TCS block has some registers, then space for up to 16 commands.
  87. * Note that though address space is reserved for 16 commands, fewer
  88. * might be present. See ncpt (num cmds per TCS).
  89. *
  90. * Here's a picture:
  91. *
  92. * +---------------------------------------------------+
  93. * |RSC |
  94. * | ctrl |
  95. * | |
  96. * | Drvs: |
  97. * | +-----------------------------------------------+ |
  98. * | |DRV0 | |
  99. * | | ctrl/config | |
  100. * | | IRQ | |
  101. * | | | |
  102. * | | TCSes: | |
  103. * | | +------------------------------------------+ | |
  104. * | | |TCS0 | | | | | | | | | | | | | | |
  105. * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
  106. * | | | | | | | | | | | | | | | | | |
  107. * | | +------------------------------------------+ | |
  108. * | | +------------------------------------------+ | |
  109. * | | |TCS1 | | | | | | | | | | | | | | |
  110. * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
  111. * | | | | | | | | | | | | | | | | | |
  112. * | | +------------------------------------------+ | |
  113. * | | +------------------------------------------+ | |
  114. * | | |TCS2 | | | | | | | | | | | | | | |
  115. * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
  116. * | | | | | | | | | | | | | | | | | |
  117. * | | +------------------------------------------+ | |
  118. * | | ...... | |
  119. * | +-----------------------------------------------+ |
  120. * | +-----------------------------------------------+ |
  121. * | |DRV1 | |
  122. * | | (same as DRV0) | |
  123. * | +-----------------------------------------------+ |
  124. * | ...... |
  125. * +---------------------------------------------------+
  126. */
  127. static inline void __iomem *
  128. tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
  129. {
  130. return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg;
  131. }
  132. static inline void __iomem *
  133. tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
  134. {
  135. return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id;
  136. }
  137. static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
  138. int cmd_id)
  139. {
  140. return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
  141. }
  142. static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
  143. {
  144. return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
  145. }
  146. static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
  147. int cmd_id, u32 data)
  148. {
  149. writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
  150. }
  151. static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
  152. u32 data)
  153. {
  154. writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
  155. }
  156. static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
  157. u32 data)
  158. {
  159. int i;
  160. writel(data, tcs_reg_addr(drv, reg, tcs_id));
  161. /*
  162. * Wait until we read back the same value. Use a counter rather than
  163. * ktime for timeout since this may be called after timekeeping stops.
  164. */
  165. for (i = 0; i < USEC_PER_SEC; i++) {
  166. if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)
  167. return;
  168. udelay(1);
  169. }
  170. pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
  171. data, tcs_id, reg);
  172. }
  173. /**
  174. * tcs_is_free() - Return if a TCS is totally free.
  175. * @drv: The RSC controller.
  176. * @tcs_id: The global ID of this TCS.
  177. *
  178. * Returns true if nobody has claimed this TCS (by setting tcs_in_use).
  179. *
  180. * Context: Must be called with the drv->lock held.
  181. *
  182. * Return: true if the given TCS is free.
  183. */
  184. static bool tcs_is_free(struct rsc_drv *drv, int tcs_id)
  185. {
  186. return !test_bit(tcs_id, drv->tcs_in_use);
  187. }
  188. /**
  189. * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
  190. * @drv: The RSC controller.
  191. * @type: SLEEP_TCS or WAKE_TCS
  192. *
  193. * This will clear the "slots" variable of the given tcs_group and also
  194. * tell the hardware to forget about all entries.
  195. *
  196. * The caller must ensure that no other RPMH actions are happening when this
  197. * function is called, since otherwise the device may immediately become
  198. * used again even before this function exits.
  199. */
  200. static void tcs_invalidate(struct rsc_drv *drv, int type)
  201. {
  202. int m;
  203. struct tcs_group *tcs = &drv->tcs[type];
  204. /* Caller ensures nobody else is running so no lock */
  205. if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))
  206. return;
  207. for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
  208. write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0);
  209. write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0);
  210. }
  211. bitmap_zero(tcs->slots, MAX_TCS_SLOTS);
  212. }
  213. /**
  214. * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
  215. * @drv: The RSC controller.
  216. *
  217. * The caller must ensure that no other RPMH actions are happening when this
  218. * function is called, since otherwise the device may immediately become
  219. * used again even before this function exits.
  220. */
  221. void rpmh_rsc_invalidate(struct rsc_drv *drv)
  222. {
  223. tcs_invalidate(drv, SLEEP_TCS);
  224. tcs_invalidate(drv, WAKE_TCS);
  225. }
  226. /**
  227. * get_tcs_for_msg() - Get the tcs_group used to send the given message.
  228. * @drv: The RSC controller.
  229. * @msg: The message we want to send.
  230. *
  231. * This is normally pretty straightforward except if we are trying to send
  232. * an ACTIVE_ONLY message but don't have any active_only TCSes.
  233. *
  234. * Return: A pointer to a tcs_group or an ERR_PTR.
  235. */
  236. static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
  237. const struct tcs_request *msg)
  238. {
  239. int type;
  240. struct tcs_group *tcs;
  241. switch (msg->state) {
  242. case RPMH_ACTIVE_ONLY_STATE:
  243. type = ACTIVE_TCS;
  244. break;
  245. case RPMH_WAKE_ONLY_STATE:
  246. type = WAKE_TCS;
  247. break;
  248. case RPMH_SLEEP_STATE:
  249. type = SLEEP_TCS;
  250. break;
  251. default:
  252. return ERR_PTR(-EINVAL);
  253. }
  254. /*
  255. * If we are making an active request on a RSC that does not have a
  256. * dedicated TCS for active state use, then re-purpose a wake TCS to
  257. * send active votes. This is safe because we ensure any active-only
  258. * transfers have finished before we use it (maybe by running from
  259. * the last CPU in PM code).
  260. */
  261. tcs = &drv->tcs[type];
  262. if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
  263. tcs = &drv->tcs[WAKE_TCS];
  264. return tcs;
  265. }
  266. /**
  267. * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
  268. * @drv: The RSC controller.
  269. * @tcs_id: The global ID of this TCS.
  270. *
  271. * For ACTIVE_ONLY transfers we want to call back into the client when the
  272. * transfer finishes. To do this we need the "request" that the client
  273. * originally provided us. This function grabs the request that we stashed
  274. * when we started the transfer.
  275. *
  276. * This only makes sense for ACTIVE_ONLY transfers since those are the only
  277. * ones we track sending (the only ones we enable interrupts for and the only
  278. * ones we call back to the client for).
  279. *
  280. * Return: The stashed request.
  281. */
  282. static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
  283. int tcs_id)
  284. {
  285. struct tcs_group *tcs;
  286. int i;
  287. for (i = 0; i < TCS_TYPE_NR; i++) {
  288. tcs = &drv->tcs[i];
  289. if (tcs->mask & BIT(tcs_id))
  290. return tcs->req[tcs_id - tcs->offset];
  291. }
  292. return NULL;
  293. }
  294. /**
  295. * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
  296. * @drv: The controller.
  297. * @tcs_id: The global ID of this TCS.
  298. * @trigger: If true then untrigger/retrigger. If false then just untrigger.
  299. *
  300. * In the normal case we only ever call with "trigger=true" to start a
  301. * transfer. That will un-trigger/disable the TCS from the last transfer
  302. * then trigger/enable for this transfer.
  303. *
  304. * If we borrowed a wake TCS for an active-only transfer we'll also call
  305. * this function with "trigger=false" to just do the un-trigger/disable
  306. * before using the TCS for wake purposes again.
  307. *
  308. * Note that the AP is only in charge of triggering active-only transfers.
  309. * The AP never triggers sleep/wake values using this function.
  310. */
  311. static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
  312. {
  313. u32 enable;
  314. /*
  315. * HW req: Clear the DRV_CONTROL and enable TCS again
  316. * While clearing ensure that the AMC mode trigger is cleared
  317. * and then the mode enable is cleared.
  318. */
  319. enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id);
  320. enable &= ~TCS_AMC_MODE_TRIGGER;
  321. write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
  322. enable &= ~TCS_AMC_MODE_ENABLE;
  323. write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
  324. if (trigger) {
  325. /* Enable the AMC mode on the TCS and then trigger the TCS */
  326. enable = TCS_AMC_MODE_ENABLE;
  327. write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
  328. enable |= TCS_AMC_MODE_TRIGGER;
  329. write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
  330. }
  331. }
  332. /**
  333. * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
  334. * @drv: The controller.
  335. * @tcs_id: The global ID of this TCS.
  336. * @enable: If true then enable; if false then disable
  337. *
  338. * We only ever call this when we borrow a wake TCS for an active-only
  339. * transfer. For active-only TCSes interrupts are always left enabled.
  340. */
  341. static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
  342. {
  343. u32 data;
  344. data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE);
  345. if (enable)
  346. data |= BIT(tcs_id);
  347. else
  348. data &= ~BIT(tcs_id);
  349. writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE);
  350. }
  351. /**
  352. * tcs_tx_done() - TX Done interrupt handler.
  353. * @irq: The IRQ number (ignored).
  354. * @p: Pointer to "struct rsc_drv".
  355. *
  356. * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
  357. * IRQ for) when a transfer is done.
  358. *
  359. * Return: IRQ_HANDLED
  360. */
  361. static irqreturn_t tcs_tx_done(int irq, void *p)
  362. {
  363. struct rsc_drv *drv = p;
  364. int i, j, err = 0;
  365. unsigned long irq_status;
  366. const struct tcs_request *req;
  367. struct tcs_cmd *cmd;
  368. irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS);
  369. for_each_set_bit(i, &irq_status, BITS_PER_LONG) {
  370. req = get_req_from_tcs(drv, i);
  371. if (!req) {
  372. WARN_ON(1);
  373. goto skip;
  374. }
  375. err = 0;
  376. for (j = 0; j < req->num_cmds; j++) {
  377. u32 sts;
  378. cmd = &req->cmds[j];
  379. sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j);
  380. if (!(sts & CMD_STATUS_ISSUED) ||
  381. ((req->wait_for_compl || cmd->wait) &&
  382. !(sts & CMD_STATUS_COMPL))) {
  383. pr_err("Incomplete request: %s: addr=%#x data=%#x",
  384. drv->name, cmd->addr, cmd->data);
  385. err = -EIO;
  386. }
  387. }
  388. trace_rpmh_tx_done(drv, i, req, err);
  389. /*
  390. * If wake tcs was re-purposed for sending active
  391. * votes, clear AMC trigger & enable modes and
  392. * disable interrupt for this TCS
  393. */
  394. if (!drv->tcs[ACTIVE_TCS].num_tcs)
  395. __tcs_set_trigger(drv, i, false);
  396. skip:
  397. /* Reclaim the TCS */
  398. write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0);
  399. write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0);
  400. writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR);
  401. spin_lock(&drv->lock);
  402. clear_bit(i, drv->tcs_in_use);
  403. /*
  404. * Disable interrupt for WAKE TCS to avoid being
  405. * spammed with interrupts coming when the solver
  406. * sends its wake votes.
  407. */
  408. if (!drv->tcs[ACTIVE_TCS].num_tcs)
  409. enable_tcs_irq(drv, i, false);
  410. spin_unlock(&drv->lock);
  411. wake_up(&drv->tcs_wait);
  412. if (req)
  413. rpmh_tx_done(req, err);
  414. }
  415. return IRQ_HANDLED;
  416. }
  417. /**
  418. * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
  419. * @drv: The controller.
  420. * @tcs_id: The global ID of this TCS.
  421. * @cmd_id: The index within the TCS to start writing.
  422. * @msg: The message we want to send, which will contain several addr/data
  423. * pairs to program (but few enough that they all fit in one TCS).
  424. *
  425. * This is used for all types of transfers (active, sleep, and wake).
  426. */
  427. static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
  428. const struct tcs_request *msg)
  429. {
  430. u32 msgid, cmd_msgid;
  431. u32 cmd_enable = 0;
  432. u32 cmd_complete;
  433. struct tcs_cmd *cmd;
  434. int i, j;
  435. cmd_msgid = CMD_MSGID_LEN;
  436. cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
  437. cmd_msgid |= CMD_MSGID_WRITE;
  438. cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id);
  439. for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
  440. cmd = &msg->cmds[i];
  441. cmd_enable |= BIT(j);
  442. cmd_complete |= cmd->wait << j;
  443. msgid = cmd_msgid;
  444. msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;
  445. write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid);
  446. write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr);
  447. write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data);
  448. // trace_rpmh_send_msg_rcuidle(drv, tcs_id, j, msgid, cmd);
  449. }
  450. write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete);
  451. cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
  452. write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable);
  453. }
  454. /**
  455. * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
  456. * @drv: The controller.
  457. * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
  458. * @msg: The message we want to send, which will contain several addr/data
  459. * pairs to program (but few enough that they all fit in one TCS).
  460. *
  461. * This will walk through the TCSes in the group and check if any of them
  462. * appear to be sending to addresses referenced in the message. If it finds
  463. * one it'll return -EBUSY.
  464. *
  465. * Only for use for active-only transfers.
  466. *
  467. * Must be called with the drv->lock held since that protects tcs_in_use.
  468. *
  469. * Return: 0 if nothing in flight or -EBUSY if we should try again later.
  470. * The caller must re-enable interrupts between tries since that's
  471. * the only way tcs_is_free() will ever return true and the only way
  472. * RSC_DRV_CMD_ENABLE will ever be cleared.
  473. */
  474. static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
  475. const struct tcs_request *msg)
  476. {
  477. unsigned long curr_enabled;
  478. u32 addr;
  479. int i, j, k;
  480. int tcs_id = tcs->offset;
  481. for (i = 0; i < tcs->num_tcs; i++, tcs_id++) {
  482. if (tcs_is_free(drv, tcs_id))
  483. continue;
  484. curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
  485. for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
  486. addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j);
  487. for (k = 0; k < msg->num_cmds; k++) {
  488. if (addr == msg->cmds[k].addr)
  489. return -EBUSY;
  490. }
  491. }
  492. }
  493. return 0;
  494. }
  495. /**
  496. * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
  497. * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
  498. * we borrowed it because there are zero active-only ones).
  499. *
  500. * Must be called with the drv->lock held since that protects tcs_in_use.
  501. *
  502. * Return: The first tcs that's free.
  503. */
  504. static int find_free_tcs(struct tcs_group *tcs)
  505. {
  506. int i;
  507. for (i = 0; i < tcs->num_tcs; i++) {
  508. if (tcs_is_free(tcs->drv, tcs->offset + i))
  509. return tcs->offset + i;
  510. }
  511. return -EBUSY;
  512. }
  513. /**
  514. * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
  515. * @drv: The controller.
  516. * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
  517. * @msg: The data to be sent.
  518. *
  519. * Claims a tcs in the given tcs_group while making sure that no existing cmd
  520. * is in flight that would conflict with the one in @msg.
  521. *
  522. * Context: Must be called with the drv->lock held since that protects
  523. * tcs_in_use.
  524. *
  525. * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
  526. * or the tcs_group is full.
  527. */
  528. static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
  529. const struct tcs_request *msg)
  530. {
  531. int ret;
  532. /*
  533. * The h/w does not like if we send a request to the same address,
  534. * when one is already in-flight or being processed.
  535. */
  536. ret = check_for_req_inflight(drv, tcs, msg);
  537. if (ret)
  538. return ret;
  539. return find_free_tcs(tcs);
  540. }
  541. /**
  542. * rpmh_rsc_send_data() - Write / trigger active-only message.
  543. * @drv: The controller.
  544. * @msg: The data to be sent.
  545. *
  546. * NOTES:
  547. * - This is only used for "ACTIVE_ONLY" since the limitations of this
  548. * function don't make sense for sleep/wake cases.
  549. * - To do the transfer, we will grab a whole TCS for ourselves--we don't
  550. * try to share. If there are none available we'll wait indefinitely
  551. * for a free one.
  552. * - This function will not wait for the commands to be finished, only for
  553. * data to be programmed into the RPMh. See rpmh_tx_done() which will
  554. * be called when the transfer is fully complete.
  555. * - This function must be called with interrupts enabled. If the hardware
  556. * is busy doing someone else's transfer we need that transfer to fully
  557. * finish so that we can have the hardware, and to fully finish it needs
  558. * the interrupt handler to run. If the interrupts is set to run on the
  559. * active CPU this can never happen if interrupts are disabled.
  560. *
  561. * Return: 0 on success, -EINVAL on error.
  562. */
  563. int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
  564. {
  565. struct tcs_group *tcs;
  566. int tcs_id;
  567. unsigned long flags;
  568. tcs = get_tcs_for_msg(drv, msg);
  569. if (IS_ERR(tcs))
  570. return PTR_ERR(tcs);
  571. spin_lock_irqsave(&drv->lock, flags);
  572. /* Wait forever for a free tcs. It better be there eventually! */
  573. wait_event_lock_irq(drv->tcs_wait,
  574. (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
  575. drv->lock);
  576. tcs->req[tcs_id - tcs->offset] = msg;
  577. set_bit(tcs_id, drv->tcs_in_use);
  578. if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {
  579. /*
  580. * Clear previously programmed WAKE commands in selected
  581. * repurposed TCS to avoid triggering them. tcs->slots will be
  582. * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
  583. */
  584. write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
  585. write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0);
  586. enable_tcs_irq(drv, tcs_id, true);
  587. }
  588. spin_unlock_irqrestore(&drv->lock, flags);
  589. /*
  590. * These two can be done after the lock is released because:
  591. * - We marked "tcs_in_use" under lock.
  592. * - Once "tcs_in_use" has been marked nobody else could be writing
  593. * to these registers until the interrupt goes off.
  594. * - The interrupt can't go off until we trigger w/ the last line
  595. * of __tcs_set_trigger() below.
  596. */
  597. __tcs_buffer_write(drv, tcs_id, 0, msg);
  598. __tcs_set_trigger(drv, tcs_id, true);
  599. return 0;
  600. }
  601. /**
  602. * find_slots() - Find a place to write the given message.
  603. * @tcs: The tcs group to search.
  604. * @msg: The message we want to find room for.
  605. * @tcs_id: If we return 0 from the function, we return the global ID of the
  606. * TCS to write to here.
  607. * @cmd_id: If we return 0 from the function, we return the index of
  608. * the command array of the returned TCS where the client should
  609. * start writing the message.
  610. *
  611. * Only for use on sleep/wake TCSes since those are the only ones we maintain
  612. * tcs->slots for.
  613. *
  614. * Return: -ENOMEM if there was no room, else 0.
  615. */
  616. static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
  617. int *tcs_id, int *cmd_id)
  618. {
  619. int slot, offset;
  620. int i = 0;
  621. /* Do over, until we can fit the full payload in a single TCS */
  622. do {
  623. slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,
  624. i, msg->num_cmds, 0);
  625. if (slot >= tcs->num_tcs * tcs->ncpt)
  626. return -ENOMEM;
  627. i += tcs->ncpt;
  628. } while (slot + msg->num_cmds - 1 >= i);
  629. bitmap_set(tcs->slots, slot, msg->num_cmds);
  630. offset = slot / tcs->ncpt;
  631. *tcs_id = offset + tcs->offset;
  632. *cmd_id = slot % tcs->ncpt;
  633. return 0;
  634. }
  635. /**
  636. * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
  637. * @drv: The controller.
  638. * @msg: The data to be written to the controller.
  639. *
  640. * This should only be called for for sleep/wake state, never active-only
  641. * state.
  642. *
  643. * The caller must ensure that no other RPMH actions are happening and the
  644. * controller is idle when this function is called since it runs lockless.
  645. *
  646. * Return: 0 if no error; else -error.
  647. */
  648. int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
  649. {
  650. struct tcs_group *tcs;
  651. int tcs_id = 0, cmd_id = 0;
  652. int ret;
  653. tcs = get_tcs_for_msg(drv, msg);
  654. if (IS_ERR(tcs))
  655. return PTR_ERR(tcs);
  656. /* find the TCS id and the command in the TCS to write to */
  657. ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
  658. if (!ret)
  659. __tcs_buffer_write(drv, tcs_id, cmd_id, msg);
  660. return ret;
  661. }
  662. /**
  663. * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
  664. * @drv: The controller
  665. *
  666. * Checks if any of the AMCs are busy in handling ACTIVE sets.
  667. * This is called from the last cpu powering down before flushing
  668. * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
  669. * power collapse, so deny from the last cpu's pm notification.
  670. *
  671. * Context: Must be called with the drv->lock held.
  672. *
  673. * Return:
  674. * * False - AMCs are idle
  675. * * True - AMCs are busy
  676. */
  677. static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
  678. {
  679. int m;
  680. struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
  681. /*
  682. * If we made an active request on a RSC that does not have a
  683. * dedicated TCS for active state use, then re-purposed wake TCSes
  684. * should be checked for not busy, because we used wake TCSes for
  685. * active requests in this case.
  686. */
  687. if (!tcs->num_tcs)
  688. tcs = &drv->tcs[WAKE_TCS];
  689. for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
  690. if (!tcs_is_free(drv, m))
  691. return true;
  692. }
  693. return false;
  694. }
  695. /**
  696. * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
  697. * @nfb: Pointer to the notifier block in struct rsc_drv.
  698. * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
  699. * @v: Unused
  700. *
  701. * This function is given to cpu_pm_register_notifier so we can be informed
  702. * about when CPUs go down. When all CPUs go down we know no more active
  703. * transfers will be started so we write sleep/wake sets. This function gets
  704. * called from cpuidle code paths and also at system suspend time.
  705. *
  706. * If its last CPU going down and AMCs are not busy then writes cached sleep
  707. * and wake messages to TCSes. The firmware then takes care of triggering
  708. * them when entering deepest low power modes.
  709. *
  710. * Return: See cpu_pm_register_notifier()
  711. */
  712. static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
  713. unsigned long action, void *v)
  714. {
  715. struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
  716. int ret = NOTIFY_OK;
  717. int cpus_in_pm;
  718. switch (action) {
  719. case CPU_PM_ENTER:
  720. cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
  721. /*
  722. * NOTE: comments for num_online_cpus() point out that it's
  723. * only a snapshot so we need to be careful. It should be OK
  724. * for us to use, though. It's important for us not to miss
  725. * if we're the last CPU going down so it would only be a
  726. * problem if a CPU went offline right after we did the check
  727. * AND that CPU was not idle AND that CPU was the last non-idle
  728. * CPU. That can't happen. CPUs would have to come out of idle
  729. * before the CPU could go offline.
  730. */
  731. if (cpus_in_pm < num_online_cpus())
  732. return NOTIFY_OK;
  733. break;
  734. case CPU_PM_ENTER_FAILED:
  735. case CPU_PM_EXIT:
  736. atomic_dec(&drv->cpus_in_pm);
  737. return NOTIFY_OK;
  738. default:
  739. return NOTIFY_DONE;
  740. }
  741. /*
  742. * It's likely we're on the last CPU. Grab the drv->lock and write
  743. * out the sleep/wake commands to RPMH hardware. Grabbing the lock
  744. * means that if we race with another CPU coming up we are still
  745. * guaranteed to be safe. If another CPU came up just after we checked
  746. * and has grabbed the lock or started an active transfer then we'll
  747. * notice we're busy and abort. If another CPU comes up after we start
  748. * flushing it will be blocked from starting an active transfer until
  749. * we're done flushing. If another CPU starts an active transfer after
  750. * we release the lock we're still OK because we're no longer the last
  751. * CPU.
  752. */
  753. if (spin_trylock(&drv->lock)) {
  754. if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
  755. ret = NOTIFY_BAD;
  756. spin_unlock(&drv->lock);
  757. } else {
  758. /* Another CPU must be up */
  759. return NOTIFY_OK;
  760. }
  761. if (ret == NOTIFY_BAD) {
  762. /* Double-check if we're here because someone else is up */
  763. if (cpus_in_pm < num_online_cpus())
  764. ret = NOTIFY_OK;
  765. else
  766. /* We won't be called w/ CPU_PM_ENTER_FAILED */
  767. atomic_dec(&drv->cpus_in_pm);
  768. }
  769. return ret;
  770. }
  771. static int rpmh_probe_tcs_config(struct platform_device *pdev,
  772. struct rsc_drv *drv, void __iomem *base)
  773. {
  774. struct tcs_type_config {
  775. u32 type;
  776. u32 n;
  777. } tcs_cfg[TCS_TYPE_NR] = { { 0 } };
  778. struct device_node *dn = pdev->dev.of_node;
  779. u32 config, max_tcs, ncpt, offset;
  780. int i, ret, n, st = 0;
  781. struct tcs_group *tcs;
  782. ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);
  783. if (ret)
  784. return ret;
  785. drv->tcs_base = base + offset;
  786. config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG);
  787. max_tcs = config;
  788. max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);
  789. max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);
  790. ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);
  791. ncpt = ncpt >> DRV_NCPT_SHIFT;
  792. n = of_property_count_u32_elems(dn, "qcom,tcs-config");
  793. if (n != 2 * TCS_TYPE_NR)
  794. return -EINVAL;
  795. for (i = 0; i < TCS_TYPE_NR; i++) {
  796. ret = of_property_read_u32_index(dn, "qcom,tcs-config",
  797. i * 2, &tcs_cfg[i].type);
  798. if (ret)
  799. return ret;
  800. if (tcs_cfg[i].type >= TCS_TYPE_NR)
  801. return -EINVAL;
  802. ret = of_property_read_u32_index(dn, "qcom,tcs-config",
  803. i * 2 + 1, &tcs_cfg[i].n);
  804. if (ret)
  805. return ret;
  806. if (tcs_cfg[i].n > MAX_TCS_PER_TYPE)
  807. return -EINVAL;
  808. }
  809. for (i = 0; i < TCS_TYPE_NR; i++) {
  810. tcs = &drv->tcs[tcs_cfg[i].type];
  811. if (tcs->drv)
  812. return -EINVAL;
  813. tcs->drv = drv;
  814. tcs->type = tcs_cfg[i].type;
  815. tcs->num_tcs = tcs_cfg[i].n;
  816. tcs->ncpt = ncpt;
  817. if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
  818. continue;
  819. if (st + tcs->num_tcs > max_tcs ||
  820. st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))
  821. return -EINVAL;
  822. tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
  823. tcs->offset = st;
  824. st += tcs->num_tcs;
  825. }
  826. drv->num_tcs = st;
  827. return 0;
  828. }
  829. static int rpmh_rsc_probe(struct platform_device *pdev)
  830. {
  831. struct device_node *dn = pdev->dev.of_node;
  832. struct rsc_drv *drv;
  833. struct resource *res;
  834. char drv_id[10] = {0};
  835. int ret, irq;
  836. u32 solver_config;
  837. void __iomem *base;
  838. /*
  839. * Even though RPMh doesn't directly use cmd-db, all of its children
  840. * do. To avoid adding this check to our children we'll do it now.
  841. */
  842. ret = cmd_db_ready();
  843. if (ret) {
  844. if (ret != -EPROBE_DEFER)
  845. dev_err(&pdev->dev, "Command DB not available (%d)\n",
  846. ret);
  847. return ret;
  848. }
  849. drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);
  850. if (!drv)
  851. return -ENOMEM;
  852. ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);
  853. if (ret)
  854. return ret;
  855. drv->name = of_get_property(dn, "label", NULL);
  856. if (!drv->name)
  857. drv->name = dev_name(&pdev->dev);
  858. snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
  859. res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id);
  860. base = devm_ioremap_resource(&pdev->dev, res);
  861. if (IS_ERR(base))
  862. return PTR_ERR(base);
  863. ret = rpmh_probe_tcs_config(pdev, drv, base);
  864. if (ret)
  865. return ret;
  866. spin_lock_init(&drv->lock);
  867. init_waitqueue_head(&drv->tcs_wait);
  868. bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);
  869. irq = platform_get_irq(pdev, drv->id);
  870. if (irq < 0)
  871. return irq;
  872. ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,
  873. IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND,
  874. drv->name, drv);
  875. if (ret)
  876. return ret;
  877. /*
  878. * CPU PM notification are not required for controllers that support
  879. * 'HW solver' mode where they can be in autonomous mode executing low
  880. * power mode to power down.
  881. */
  882. solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG);
  883. solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
  884. solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
  885. if (!solver_config) {
  886. drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
  887. cpu_pm_register_notifier(&drv->rsc_pm);
  888. }
  889. /* Enable the active TCS to send requests immediately */
  890. writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
  891. drv->tcs_base + RSC_DRV_IRQ_ENABLE);
  892. spin_lock_init(&drv->client.cache_lock);
  893. INIT_LIST_HEAD(&drv->client.cache);
  894. INIT_LIST_HEAD(&drv->client.batch_cache);
  895. dev_set_drvdata(&pdev->dev, drv);
  896. return devm_of_platform_populate(&pdev->dev);
  897. }
  898. static const struct of_device_id rpmh_drv_match[] = {
  899. { .compatible = "qcom,rpmh-rsc", },
  900. { }
  901. };
  902. MODULE_DEVICE_TABLE(of, rpmh_drv_match);
  903. static struct platform_driver rpmh_driver = {
  904. .probe = rpmh_rsc_probe,
  905. .driver = {
  906. .name = "rpmh",
  907. .of_match_table = rpmh_drv_match,
  908. .suppress_bind_attrs = true,
  909. },
  910. };
  911. static int __init rpmh_driver_init(void)
  912. {
  913. return platform_driver_register(&rpmh_driver);
  914. }
  915. arch_initcall(rpmh_driver_init);
  916. MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
  917. MODULE_LICENSE("GPL v2");