123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441 |
- /* SPDX-License-Identifier: GPL-2.0 */
- /*
- * Copyright (C) 2020 Marvell International Ltd.
- *
- * Support functions for managing command queues used for
- * various hardware blocks.
- *
- * The common command queue infrastructure abstracts out the
- * software necessary for adding to Octeon's chained queue
- * structures. These structures are used for commands to the
- * PKO, ZIP, DFA, RAID, HNA, and DMA engine blocks. Although each
- * hardware unit takes commands and CSRs of different types,
- * they all use basic linked command buffers to store the
- * pending request. In general, users of the CVMX API don't
- * call cvmx-cmd-queue functions directly. Instead the hardware
- * unit specific wrapper should be used. The wrappers perform
- * unit specific validation and CSR writes to submit the
- * commands.
- *
- * Even though most software will never directly interact with
- * cvmx-cmd-queue, knowledge of its internal workings can help
- * in diagnosing performance problems and help with debugging.
- *
- * Command queue pointers are stored in a global named block
- * called "cvmx_cmd_queues". Except for the PKO queues, each
- * hardware queue is stored in its own cache line to reduce SMP
- * contention on spin locks. The PKO queues are stored such that
- * every 16th queue is next to each other in memory. This scheme
- * allows for queues being in separate cache lines when there
- * are low number of queues per port. With 16 queues per port,
- * the first queue for each port is in the same cache area. The
- * second queues for each port are in another area, etc. This
- * allows software to implement very efficient lockless PKO with
- * 16 queues per port using a minimum of cache lines per core.
- * All queues for a given core will be isolated in the same
- * cache area.
- *
- * In addition to the memory pointer layout, cvmx-cmd-queue
- * provides an optimized fair ll/sc locking mechanism for the
- * queues. The lock uses a "ticket / now serving" model to
- * maintain fair order on contended locks. In addition, it uses
- * predicted locking time to limit cache contention. When a core
- * know it must wait in line for a lock, it spins on the
- * internal cycle counter to completely eliminate any causes of
- * bus traffic.
- */
- #ifndef __CVMX_CMD_QUEUE_H__
- #define __CVMX_CMD_QUEUE_H__
- /**
- * By default we disable the max depth support. Most programs
- * don't use it and it slows down the command queue processing
- * significantly.
- */
- #ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH
- #define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0
- #endif
- /**
- * Enumeration representing all hardware blocks that use command
- * queues. Each hardware block has up to 65536 sub identifiers for
- * multiple command queues. Not all chips support all hardware
- * units.
- */
- typedef enum {
- CVMX_CMD_QUEUE_PKO_BASE = 0x00000,
- #define CVMX_CMD_QUEUE_PKO(queue) \
- ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff & (queue))))
- CVMX_CMD_QUEUE_ZIP = 0x10000,
- #define CVMX_CMD_QUEUE_ZIP_QUE(queue) \
- ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_ZIP + (0xffff & (queue))))
- CVMX_CMD_QUEUE_DFA = 0x20000,
- CVMX_CMD_QUEUE_RAID = 0x30000,
- CVMX_CMD_QUEUE_DMA_BASE = 0x40000,
- #define CVMX_CMD_QUEUE_DMA(queue) \
- ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff & (queue))))
- CVMX_CMD_QUEUE_BCH = 0x50000,
- #define CVMX_CMD_QUEUE_BCH(queue) ((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_BCH + (0xffff & (queue))))
- CVMX_CMD_QUEUE_HNA = 0x60000,
- CVMX_CMD_QUEUE_END = 0x70000,
- } cvmx_cmd_queue_id_t;
- #define CVMX_CMD_QUEUE_ZIP3_QUE(node, queue) \
- ((cvmx_cmd_queue_id_t)((node) << 24 | CVMX_CMD_QUEUE_ZIP | (0xffff & (queue))))
- /**
- * Command write operations can fail if the command queue needs
- * a new buffer and the associated FPA pool is empty. It can also
- * fail if the number of queued command words reaches the maximum
- * set at initialization.
- */
- typedef enum {
- CVMX_CMD_QUEUE_SUCCESS = 0,
- CVMX_CMD_QUEUE_NO_MEMORY = -1,
- CVMX_CMD_QUEUE_FULL = -2,
- CVMX_CMD_QUEUE_INVALID_PARAM = -3,
- CVMX_CMD_QUEUE_ALREADY_SETUP = -4,
- } cvmx_cmd_queue_result_t;
- typedef struct {
- /* First 64-bit word: */
- u64 fpa_pool : 16;
- u64 base_paddr : 48;
- s32 index;
- u16 max_depth;
- u16 pool_size_m1;
- } __cvmx_cmd_queue_state_t;
- /**
- * command-queue locking uses a fair ticket spinlock algo,
- * with 64-bit tickets for endianness-neutrality and
- * counter overflow protection.
- * Lock is free when both counters are of equal value.
- */
- typedef struct {
- u64 ticket;
- u64 now_serving;
- } __cvmx_cmd_queue_lock_t;
- /**
- * @INTERNAL
- * This structure contains the global state of all command queues.
- * It is stored in a bootmem named block and shared by all
- * applications running on Octeon. Tickets are stored in a different
- * cache line that queue information to reduce the contention on the
- * ll/sc used to get a ticket. If this is not the case, the update
- * of queue state causes the ll/sc to fail quite often.
- */
- typedef struct {
- __cvmx_cmd_queue_lock_t lock[(CVMX_CMD_QUEUE_END >> 16) * 256];
- __cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) * 256];
- } __cvmx_cmd_queue_all_state_t;
- extern __cvmx_cmd_queue_all_state_t *__cvmx_cmd_queue_state_ptrs[CVMX_MAX_NODES];
- /**
- * @INTERNAL
- * Internal function to handle the corner cases
- * of adding command words to a queue when the current
- * block is getting full.
- */
- cvmx_cmd_queue_result_t __cvmx_cmd_queue_write_raw(cvmx_cmd_queue_id_t queue_id,
- __cvmx_cmd_queue_state_t *qptr, int cmd_count,
- const u64 *cmds);
- /**
- * Initialize a command queue for use. The initial FPA buffer is
- * allocated and the hardware unit is configured to point to the
- * new command queue.
- *
- * @param queue_id Hardware command queue to initialize.
- * @param max_depth Maximum outstanding commands that can be queued.
- * @param fpa_pool FPA pool the command queues should come from.
- * @param pool_size Size of each buffer in the FPA pool (bytes)
- *
- * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
- */
- cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id, int max_depth,
- int fpa_pool, int pool_size);
- /**
- * Shutdown a queue a free it's command buffers to the FPA. The
- * hardware connected to the queue must be stopped before this
- * function is called.
- *
- * @param queue_id Queue to shutdown
- *
- * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
- */
- cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id);
- /**
- * Return the number of command words pending in the queue. This
- * function may be relatively slow for some hardware units.
- *
- * @param queue_id Hardware command queue to query
- *
- * Return: Number of outstanding commands
- */
- int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id);
- /**
- * Return the command buffer to be written to. The purpose of this
- * function is to allow CVMX routine access to the low level buffer
- * for initial hardware setup. User applications should not call this
- * function directly.
- *
- * @param queue_id Command queue to query
- *
- * Return: Command buffer or NULL on failure
- */
- void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id);
- /**
- * @INTERNAL
- * Retrieve or allocate command queue state named block
- */
- cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(unsigned int node);
- /**
- * @INTERNAL
- * Get the index into the state arrays for the supplied queue id.
- *
- * @param queue_id Queue ID to get an index for
- *
- * Return: Index into the state arrays
- */
- static inline unsigned int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)
- {
- /* Warning: This code currently only works with devices that have 256
- * queues or less. Devices with more than 16 queues are laid out in
- * memory to allow cores quick access to every 16th queue. This reduces
- * cache thrashing when you are running 16 queues per port to support
- * lockless operation
- */
- unsigned int unit = (queue_id >> 16) & 0xff;
- unsigned int q = (queue_id >> 4) & 0xf;
- unsigned int core = queue_id & 0xf;
- return (unit << 8) | (core << 4) | q;
- }
- static inline int __cvmx_cmd_queue_get_node(cvmx_cmd_queue_id_t queue_id)
- {
- unsigned int node = queue_id >> 24;
- return node;
- }
- /**
- * @INTERNAL
- * Lock the supplied queue so nobody else is updating it at the same
- * time as us.
- *
- * @param queue_id Queue ID to lock
- *
- */
- static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id)
- {
- }
- /**
- * @INTERNAL
- * Unlock the queue, flushing all writes.
- *
- * @param queue_id Queue ID to lock
- *
- */
- static inline void __cvmx_cmd_queue_unlock(cvmx_cmd_queue_id_t queue_id)
- {
- CVMX_SYNCWS; /* nudge out the unlock. */
- }
- /**
- * @INTERNAL
- * Initialize a command-queue lock to "unlocked" state.
- */
- static inline void __cvmx_cmd_queue_lock_init(cvmx_cmd_queue_id_t queue_id)
- {
- unsigned int index = __cvmx_cmd_queue_get_index(queue_id);
- unsigned int node = __cvmx_cmd_queue_get_node(queue_id);
- __cvmx_cmd_queue_state_ptrs[node]->lock[index] = (__cvmx_cmd_queue_lock_t){ 0, 0 };
- CVMX_SYNCWS;
- }
- /**
- * @INTERNAL
- * Get the queue state structure for the given queue id
- *
- * @param queue_id Queue id to get
- *
- * Return: Queue structure or NULL on failure
- */
- static inline __cvmx_cmd_queue_state_t *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)
- {
- unsigned int index;
- unsigned int node;
- __cvmx_cmd_queue_state_t *qptr;
- node = __cvmx_cmd_queue_get_node(queue_id);
- index = __cvmx_cmd_queue_get_index(queue_id);
- if (cvmx_unlikely(!__cvmx_cmd_queue_state_ptrs[node]))
- __cvmx_cmd_queue_init_state_ptr(node);
- qptr = &__cvmx_cmd_queue_state_ptrs[node]->state[index];
- return qptr;
- }
- /**
- * Write an arbitrary number of command words to a command queue.
- * This is a generic function; the fixed number of command word
- * functions yield higher performance.
- *
- * @param queue_id Hardware command queue to write to
- * @param use_locking
- * Use internal locking to ensure exclusive access for queue
- * updates. If you don't use this locking you must ensure
- * exclusivity some other way. Locking is strongly recommended.
- * @param cmd_count Number of command words to write
- * @param cmds Array of commands to write
- *
- * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
- */
- static inline cvmx_cmd_queue_result_t
- cvmx_cmd_queue_write(cvmx_cmd_queue_id_t queue_id, bool use_locking, int cmd_count, const u64 *cmds)
- {
- cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
- u64 *cmd_ptr;
- __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
- /* Make sure nobody else is updating the same queue */
- if (cvmx_likely(use_locking))
- __cvmx_cmd_queue_lock(queue_id);
- /* Most of the time there is lots of free words in current block */
- if (cvmx_unlikely((qptr->index + cmd_count) >= qptr->pool_size_m1)) {
- /* The rare case when nearing end of block */
- ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, cmd_count, cmds);
- } else {
- cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
- /* Loop easy for compiler to unroll for the likely case */
- while (cmd_count > 0) {
- cmd_ptr[qptr->index++] = *cmds++;
- cmd_count--;
- }
- }
- /* All updates are complete. Release the lock and return */
- if (cvmx_likely(use_locking))
- __cvmx_cmd_queue_unlock(queue_id);
- else
- CVMX_SYNCWS;
- return ret;
- }
- /**
- * Simple function to write two command words to a command queue.
- *
- * @param queue_id Hardware command queue to write to
- * @param use_locking
- * Use internal locking to ensure exclusive access for queue
- * updates. If you don't use this locking you must ensure
- * exclusivity some other way. Locking is strongly recommended.
- * @param cmd1 Command
- * @param cmd2 Command
- *
- * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
- */
- static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t queue_id,
- bool use_locking, u64 cmd1, u64 cmd2)
- {
- cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
- u64 *cmd_ptr;
- __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
- /* Make sure nobody else is updating the same queue */
- if (cvmx_likely(use_locking))
- __cvmx_cmd_queue_lock(queue_id);
- if (cvmx_unlikely((qptr->index + 2) >= qptr->pool_size_m1)) {
- /* The rare case when nearing end of block */
- u64 cmds[2];
- cmds[0] = cmd1;
- cmds[1] = cmd2;
- ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, 2, cmds);
- } else {
- /* Likely case to work fast */
- cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
- cmd_ptr += qptr->index;
- qptr->index += 2;
- cmd_ptr[0] = cmd1;
- cmd_ptr[1] = cmd2;
- }
- /* All updates are complete. Release the lock and return */
- if (cvmx_likely(use_locking))
- __cvmx_cmd_queue_unlock(queue_id);
- else
- CVMX_SYNCWS;
- return ret;
- }
- /**
- * Simple function to write three command words to a command queue.
- *
- * @param queue_id Hardware command queue to write to
- * @param use_locking
- * Use internal locking to ensure exclusive access for queue
- * updates. If you don't use this locking you must ensure
- * exclusivity some other way. Locking is strongly recommended.
- * @param cmd1 Command
- * @param cmd2 Command
- * @param cmd3 Command
- *
- * Return: CVMX_CMD_QUEUE_SUCCESS or a failure code
- */
- static inline cvmx_cmd_queue_result_t
- cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t queue_id, bool use_locking, u64 cmd1, u64 cmd2, u64 cmd3)
- {
- cvmx_cmd_queue_result_t ret = CVMX_CMD_QUEUE_SUCCESS;
- __cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
- u64 *cmd_ptr;
- /* Make sure nobody else is updating the same queue */
- if (cvmx_likely(use_locking))
- __cvmx_cmd_queue_lock(queue_id);
- if (cvmx_unlikely((qptr->index + 3) >= qptr->pool_size_m1)) {
- /* Most of the time there is lots of free words in current block */
- u64 cmds[3];
- cmds[0] = cmd1;
- cmds[1] = cmd2;
- cmds[2] = cmd3;
- ret = __cvmx_cmd_queue_write_raw(queue_id, qptr, 3, cmds);
- } else {
- cmd_ptr = (u64 *)cvmx_phys_to_ptr((u64)qptr->base_paddr);
- cmd_ptr += qptr->index;
- qptr->index += 3;
- cmd_ptr[0] = cmd1;
- cmd_ptr[1] = cmd2;
- cmd_ptr[2] = cmd3;
- }
- /* All updates are complete. Release the lock and return */
- if (cvmx_likely(use_locking))
- __cvmx_cmd_queue_unlock(queue_id);
- else
- CVMX_SYNCWS;
- return ret;
- }
- #endif /* __CVMX_CMD_QUEUE_H__ */
|