cvmx-cmd-queue.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Copyright (C) 2020 Marvell International Ltd.
  4. *
  5. * Support functions for managing command queues used for
  6. * various hardware blocks.
  7. *
  8. * The common command queue infrastructure abstracts out the
  9. * software necessary for adding to Octeon's chained queue
  10. * structures. These structures are used for commands to the
  11. * PKO, ZIP, DFA, RAID, HNA, and DMA engine blocks. Although each
  12. * hardware unit takes commands and CSRs of different types,
  13. * they all use basic linked command buffers to store the
  14. * pending request. In general, users of the CVMX API don't
  15. * call cvmx-cmd-queue functions directly. Instead the hardware
  16. * unit specific wrapper should be used. The wrappers perform
  17. * unit specific validation and CSR writes to submit the
  18. * commands.
  19. *
  20. * Even though most software will never directly interact with
  21. * cvmx-cmd-queue, knowledge of its internal workings can help
  22. * in diagnosing performance problems and help with debugging.
  23. *
  24. * Command queue pointers are stored in a global named block
  25. * called "cvmx_cmd_queues". Except for the PKO queues, each
  26. * hardware queue is stored in its own cache line to reduce SMP
  27. * contention on spin locks. The PKO queues are stored such that
  28. * every 16th queue is next to each other in memory. This scheme
  29. * allows for queues being in separate cache lines when there
  30. * are low number of queues per port. With 16 queues per port,
  31. * the first queue for each port is in the same cache area. The
  32. * second queues for each port are in another area, etc. This
  33. * allows software to implement very efficient lockless PKO with
  34. * 16 queues per port using a minimum of cache lines per core.
  35. * All queues for a given core will be isolated in the same
  36. * cache area.
  37. *
  38. * In addition to the memory pointer layout, cvmx-cmd-queue
  39. * provides an optimized fair ll/sc locking mechanism for the
  40. * queues. The lock uses a "ticket / now serving" model to
  41. * maintain fair order on contended locks. In addition, it uses
  42. * predicted locking time to limit cache contention. When a core
  43. * know it must wait in line for a lock, it spins on the
  44. * internal cycle counter to completely eliminate any causes of
  45. * bus traffic.
  46. */
  47. #ifndef __CVMX_CMD_QUEUE_H__
  48. #define __CVMX_CMD_QUEUE_H__
  49. /**
  50. * By default we disable the max depth support. Most programs
  51. * don't use it and it slows down the command queue processing
  52. * significantly.
  53. */
  54. #ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH
  55. #define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0
  56. #endif
  57. /**
  58. * Enumeration representing all hardware blocks that use command
  59. * queues. Each hardware block has up to 65536 sub identifiers for
  60. * multiple command queues. Not all chips support all hardware
  61. * units.
  62. */
  63. typedef enum {
  64. CVMX_CMD_QUEUE_PKO_BASE = 0x00000,
  65. #define CVMX_CMD_QUEUE_PKO(queue) \
  66. ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff & (queue))))
  67. CVMX_CMD_QUEUE_ZIP = 0x10000,
  68. #define CVMX_CMD_QUEUE_ZIP_QUE(queue) \
  69. ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_ZIP + (0xffff & (queue))))
  70. CVMX_CMD_QUEUE_DFA = 0x20000,
  71. CVMX_CMD_QUEUE_RAID = 0x30000,
  72. CVMX_CMD_QUEUE_DMA_BASE = 0x40000,
  73. #define CVMX_CMD_QUEUE_DMA(queue) \
  74. ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff & (queue))))
  75. CVMX_CMD_QUEUE_BCH = 0x50000,
  76. #define CVMX_CMD_QUEUE_BCH(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_BCH + (0xffff & (queue))))
  77. CVMX_CMD_QUEUE_HNA = 0x60000,
  78. CVMX_CMD_QUEUE_END = 0x70000,
  79. } cvmx_cmd_queue_id_t;
  80. #define CVMX_CMD_QUEUE_ZIP3_QUE(node, queue) \
  81. ((cvmx_cmd_queue_id_t)((node) << 24 | CVMX_CMD_QUEUE_ZIP | (0xffff & (queue))))
  82. /**
  83. * Command write operations can fail if the command queue needs
  84. * a new buffer and the associated FPA pool is empty. It can also
  85. * fail if the number of queued command words reaches the maximum
  86. * set at initialization.
  87. */
  88. typedef enum {
  89. CVMX_CMD_QUEUE_SUCCESS = 0,
  90. CVMX_CMD_QUEUE_NO_MEMORY = -1,
  91. CVMX_CMD_QUEUE_FULL = -2,
  92. CVMX_CMD_QUEUE_INVALID_PARAM = -3,
  93. CVMX_CMD_QUEUE_ALREADY_SETUP = -4,
  94. } cvmx_cmd_queue_result_t;
  95. typedef struct {
  96. /* First 64-bit word: */
  97. u64 fpa_pool : 16;
  98. u64 base_paddr : 48;
  99. s32 index;
  100. u16 max_depth;
  101. u16 pool_size_m1;
  102. } __cvmx_cmd_queue_state_t;
  103. /**
  104. * command-queue locking uses a fair ticket spinlock algo,
  105. * with 64-bit tickets for endianness-neutrality and
  106. * counter overflow protection.
  107. * Lock is free when both counters are of equal value.
  108. */
  109. typedef struct {
  110. u64 ticket;
  111. u64 now_serving;
  112. } __cvmx_cmd_queue_lock_t;
  113. /**
  114. * @INTERNAL
  115. * This structure contains the global state of all command queues.
  116. * It is stored in a bootmem named block and shared by all
  117. * applications running on Octeon. Tickets are stored in a different
  118. * cache line that queue information to reduce the contention on the
  119. * ll/sc used to get a ticket. If this is not the case, the update
  120. * of queue state causes the ll/sc to fail quite often.
  121. */
  122. typedef struct {
  123. __cvmx_cmd_queue_lock_t lock[(CVMX_CMD_QUEUE_END >> 16) * 256];
  124. __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) * 256];
  125. } __cvmx_cmd_queue_all_state_t;
  126. extern __cvmx_cmd_queue_all_state_t *__cvmx_cmd_queue_state_ptrs[CVMX_MAX_NODES];
  127. /**
  128. * @INTERNAL
  129. * Internal function to handle the corner cases
  130. * of adding command words to a queue when the current
  131. * block is getting full.
  132. */
  133. cvmx_cmd_queue_result_t __cvmx_cmd_queue_write_raw(cvmx_cmd_queue_id_t queue_id,
  134. __cvmx_cmd_queue_state_t *qptr, int cmd_count,
  135. const u64 *cmds);
  136. /**
  137. * Initialize a command queue for use. The initial FPA buffer is
  138. * allocated and the hardware unit is configured to point to the
  139. * new command queue.
  140. *
  141. * @param queue_id Hardware command queue to initialize.
  142. * @param max_depth Maximum outstanding commands that can be queued.
  143. * @param fpa_pool FPA pool the command queues should come from.
  144. * @param pool_size Size of each buffer in the FPA pool (bytes)
  145. *
  146. * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
  147. */
  148. cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, int max_depth,
  149. int fpa_pool, int pool_size);
  150. /**
  151. * Shutdown a queue a free it's command buffers to the FPA. The
  152. * hardware connected to the queue must be stopped before this
  153. * function is called.
  154. *
  155. * @param queue_id Queue to shutdown
  156. *
  157. * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
  158. */
  159. cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id);
  160. /**
  161. * Return the number of command words pending in the queue. This
  162. * function may be relatively slow for some hardware units.
  163. *
  164. * @param queue_id Hardware command queue to query
  165. *
  166. * Return: Number of outstanding commands
  167. */
  168. int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id);
  169. /**
  170. * Return the command buffer to be written to. The purpose of this
  171. * function is to allow CVMX routine access to the low level buffer
  172. * for initial hardware setup. User applications should not call this
  173. * function directly.
  174. *
  175. * @param queue_id Command queue to query
  176. *
  177. * Return: Command buffer or NULL on failure
  178. */
  179. void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id);
  180. /**
  181. * @INTERNAL
  182. * Retrieve or allocate command queue state named block
  183. */
  184. cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(unsigned int node);
  185. /**
  186. * @INTERNAL
  187. * Get the index into the state arrays for the supplied queue id.
  188. *
  189. * @param queue_id Queue ID to get an index for
  190. *
  191. * Return: Index into the state arrays
  192. */
  193. static inline unsigned int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)
  194. {
  195. /* Warning: This code currently only works with devices that have 256
  196. * queues or less. Devices with more than 16 queues are laid out in
  197. * memory to allow cores quick access to every 16th queue. This reduces
  198. * cache thrashing when you are running 16 queues per port to support
  199. * lockless operation
  200. */
  201. unsigned int unit = (queue_id >> 16) & 0xff;
  202. unsigned int q = (queue_id >> 4) & 0xf;
  203. unsigned int core = queue_id & 0xf;
  204. return (unit << 8) | (core << 4) | q;
  205. }
  206. static inline int __cvmx_cmd_queue_get_node(cvmx_cmd_queue_id_t queue_id)
  207. {
  208. unsigned int node = queue_id >> 24;
  209. return node;
  210. }
  211. /**
  212. * @INTERNAL
  213. * Lock the supplied queue so nobody else is updating it at the same
  214. * time as us.
  215. *
  216. * @param queue_id Queue ID to lock
  217. *
  218. */
  219. static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id)
  220. {
  221. }
  222. /**
  223. * @INTERNAL
  224. * Unlock the queue, flushing all writes.
  225. *
  226. * @param queue_id Queue ID to lock
  227. *
  228. */
  229. static inline void __cvmx_cmd_queue_unlock(cvmx_cmd_queue_id_t queue_id)
  230. {
  231. CVMX_SYNCWS; /* nudge out the unlock. */
  232. }
  233. /**
  234. * @INTERNAL
  235. * Initialize a command-queue lock to "unlocked" state.
  236. */
  237. static inline void __cvmx_cmd_queue_lock_init(cvmx_cmd_queue_id_t queue_id)
  238. {
  239. unsigned int index = __cvmx_cmd_queue_get_index(queue_id);
  240. unsigned int node = __cvmx_cmd_queue_get_node(queue_id);
  241. __cvmx_cmd_queue_state_ptrs[node]->lock[index] = (__cvmx_cmd_queue_lock_t){ 0, 0 };
  242. CVMX_SYNCWS;
  243. }
  244. /**
  245. * @INTERNAL
  246. * Get the queue state structure for the given queue id
  247. *
  248. * @param queue_id Queue id to get
  249. *
  250. * Return: Queue structure or NULL on failure
  251. */
  252. static inline __cvmx_cmd_queue_state_t *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)
  253. {
  254. unsigned int index;
  255. unsigned int node;
  256. __cvmx_cmd_queue_state_t *qptr;
  257. node = __cvmx_cmd_queue_get_node(queue_id);
  258. index = __cvmx_cmd_queue_get_index(queue_id);
  259. if (cvmx_unlikely(!__cvmx_cmd_queue_state_ptrs[node]))
  260. __cvmx_cmd_queue_init_state_ptr(node);
  261. qptr = &__cvmx_cmd_queue_state_ptrs[node]->state[index];
  262. return qptr;
  263. }
  264. /**
  265. * Write an arbitrary number of command words to a command queue.
  266. * This is a generic function; the fixed number of command word
  267. * functions yield higher performance.
  268. *
  269. * @param queue_id Hardware command queue to write to
  270. * @param use_locking
  271. * Use internal locking to ensure exclusive access for queue
  272. * updates. If you don't use this locking you must ensure
  273. * exclusivity some other way. Locking is strongly recommended.
  274. * @param cmd_count Number of command words to write
  275. * @param cmds Array of commands to write
  276. *
  277. * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
  278. */
  279. static inline cvmx_cmd_queue_result_t
  280. cvmx_cmd_queue_write(cvmx_cmd_queue_id_t queue_id, bool use_locking, int cmd_count, const u64 *cmds)
  281. {
  282. cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
  283. u64 *cmd_ptr;
  284. __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
  285. /* Make sure nobody else is updating the same queue */
  286. if (cvmx_likely(use_locking))
  287. __cvmx_cmd_queue_lock(queue_id);
  288. /* Most of the time there is lots of free words in current block */
  289. if (cvmx_unlikely((qptr->index + cmd_count) >= qptr->pool_size_m1)) {
  290. /* The rare case when nearing end of block */
  291. ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, cmd_count, cmds);
  292. } else {
  293. cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
  294. /* Loop easy for compiler to unroll for the likely case */
  295. while (cmd_count > 0) {
  296. cmd_ptr[qptr->index++] = *cmds++;
  297. cmd_count--;
  298. }
  299. }
  300. /* All updates are complete. Release the lock and return */
  301. if (cvmx_likely(use_locking))
  302. __cvmx_cmd_queue_unlock(queue_id);
  303. else
  304. CVMX_SYNCWS;
  305. return ret;
  306. }
  307. /**
  308. * Simple function to write two command words to a command queue.
  309. *
  310. * @param queue_id Hardware command queue to write to
  311. * @param use_locking
  312. * Use internal locking to ensure exclusive access for queue
  313. * updates. If you don't use this locking you must ensure
  314. * exclusivity some other way. Locking is strongly recommended.
  315. * @param cmd1 Command
  316. * @param cmd2 Command
  317. *
  318. * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
  319. */
  320. static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t queue_id,
  321. bool use_locking, u64 cmd1, u64 cmd2)
  322. {
  323. cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
  324. u64 *cmd_ptr;
  325. __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
  326. /* Make sure nobody else is updating the same queue */
  327. if (cvmx_likely(use_locking))
  328. __cvmx_cmd_queue_lock(queue_id);
  329. if (cvmx_unlikely((qptr->index + 2) >= qptr->pool_size_m1)) {
  330. /* The rare case when nearing end of block */
  331. u64 cmds[2];
  332. cmds[0] = cmd1;
  333. cmds[1] = cmd2;
  334. ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, 2, cmds);
  335. } else {
  336. /* Likely case to work fast */
  337. cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
  338. cmd_ptr += qptr->index;
  339. qptr->index += 2;
  340. cmd_ptr[0] = cmd1;
  341. cmd_ptr[1] = cmd2;
  342. }
  343. /* All updates are complete. Release the lock and return */
  344. if (cvmx_likely(use_locking))
  345. __cvmx_cmd_queue_unlock(queue_id);
  346. else
  347. CVMX_SYNCWS;
  348. return ret;
  349. }
  350. /**
  351. * Simple function to write three command words to a command queue.
  352. *
  353. * @param queue_id Hardware command queue to write to
  354. * @param use_locking
  355. * Use internal locking to ensure exclusive access for queue
  356. * updates. If you don't use this locking you must ensure
  357. * exclusivity some other way. Locking is strongly recommended.
  358. * @param cmd1 Command
  359. * @param cmd2 Command
  360. * @param cmd3 Command
  361. *
  362. * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
  363. */
  364. static inline cvmx_cmd_queue_result_t
  365. cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t queue_id, bool use_locking, u64 cmd1, u64 cmd2, u64 cmd3)
  366. {
  367. cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
  368. __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
  369. u64 *cmd_ptr;
  370. /* Make sure nobody else is updating the same queue */
  371. if (cvmx_likely(use_locking))
  372. __cvmx_cmd_queue_lock(queue_id);
  373. if (cvmx_unlikely((qptr->index + 3) >= qptr->pool_size_m1)) {
  374. /* Most of the time there is lots of free words in current block */
  375. u64 cmds[3];
  376. cmds[0] = cmd1;
  377. cmds[1] = cmd2;
  378. cmds[2] = cmd3;
  379. ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, 3, cmds);
  380. } else {
  381. cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
  382. cmd_ptr += qptr->index;
  383. qptr->index += 3;
  384. cmd_ptr[0] = cmd1;
  385. cmd_ptr[1] = cmd2;
  386. cmd_ptr[2] = cmd3;
  387. }
  388. /* All updates are complete. Release the lock and return */
  389. if (cvmx_likely(use_locking))
  390. __cvmx_cmd_queue_unlock(queue_id);
  391. else
  392. CVMX_SYNCWS;
  393. return ret;
  394. }
  395. #endif /* __CVMX_CMD_QUEUE_H__ */