12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130 |
- // SPDX-License-Identifier: GPL-2.0+
- /*
- * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
- *
- * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
- */
- #include <common.h>
- #include <cpu_func.h>
- #include <log.h>
- #include <asm/cache.h>
- #include <asm/io.h>
- #include <malloc.h>
- #include <asm/bitops.h>
- #include <dm.h>
- #include <dm/device_compat.h>
- #include <dm/devres.h>
- #include <dm/read.h>
- #include <dm/uclass.h>
- #include <linux/bitops.h>
- #include <linux/compat.h>
- #include <linux/dma-mapping.h>
- #include <linux/err.h>
- #include <linux/soc/ti/k3-navss-ringacc.h>
- #include <linux/soc/ti/ti_sci_protocol.h>
- #include <linux/soc/ti/cppi5.h>
- #define set_bit(bit, bitmap) __set_bit(bit, bitmap)
- #define clear_bit(bit, bitmap) __clear_bit(bit, bitmap)
- #define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
- dma_free_coherent(cpu_addr)
- #define dma_zalloc_coherent(dev, size, dma_handle, flag) \
- ({ \
- void *ring_mem_virt; \
- ring_mem_virt = dma_alloc_coherent((size), \
- (unsigned long *)(dma_handle)); \
- if (ring_mem_virt) \
- memset(ring_mem_virt, 0, (size)); \
- ring_mem_virt; \
- })
- static LIST_HEAD(k3_nav_ringacc_list);
- static void ringacc_writel(u32 v, void __iomem *reg)
- {
- pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
- writel(v, reg);
- }
- static u32 ringacc_readl(void __iomem *reg)
- {
- u32 v;
- v = readl(reg);
- pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
- return v;
- }
- #define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
- #define K3_DMARING_RING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0)
- /**
- * struct k3_nav_ring_rt_regs - The RA Control/Status Registers region
- */
- struct k3_nav_ring_rt_regs {
- u32 resv_16[4];
- u32 db; /* RT Ring N Doorbell Register */
- u32 resv_4[1];
- u32 occ; /* RT Ring N Occupancy Register */
- u32 indx; /* RT Ring N Current Index Register */
- u32 hwocc; /* RT Ring N Hardware Occupancy Register */
- u32 hwindx; /* RT Ring N Current Index Register */
- };
- #define KNAV_RINGACC_RT_REGS_STEP 0x1000
- #define K3_DMARING_RING_RT_REGS_STEP 0x2000
- #define K3_DMARING_RING_RT_REGS_REVERSE_OFS 0x1000
- #define KNAV_RINGACC_RT_OCC_MASK GENMASK(20, 0)
- #define K3_DMARING_RING_RT_OCC_TDOWN_COMPLETE BIT(31)
- #define K3_DMARING_RING_RT_DB_ENTRY_MASK GENMASK(7, 0)
- #define K3_DMARING_RING_RT_DB_TDOWN_ACK BIT(31)
- /**
- * struct k3_nav_ring_fifo_regs - The Ring Accelerator Queues Registers region
- */
- struct k3_nav_ring_fifo_regs {
- u32 head_data[128]; /* Ring Head Entry Data Registers */
- u32 tail_data[128]; /* Ring Tail Entry Data Registers */
- u32 peek_head_data[128]; /* Ring Peek Head Entry Data Regs */
- u32 peek_tail_data[128]; /* Ring Peek Tail Entry Data Regs */
- };
- #define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
- #define KNAV_RINGACC_FIFO_REGS_STEP 0x1000
- #define KNAV_RINGACC_MAX_DB_RING_CNT (127U)
- /**
- * struct k3_nav_ring_ops - Ring operations
- */
- struct k3_nav_ring_ops {
- int (*push_tail)(struct k3_nav_ring *ring, void *elm);
- int (*push_head)(struct k3_nav_ring *ring, void *elm);
- int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
- int (*pop_head)(struct k3_nav_ring *ring, void *elm);
- };
- /**
- * struct k3_nav_ring_state - Internal state tracking structure
- *
- * @free: Number of free entries
- * @occ: Occupancy
- * @windex: Write index
- * @rindex: Read index
- */
- struct k3_nav_ring_state {
- u32 free;
- u32 occ;
- u32 windex;
- u32 rindex;
- u32 tdown_complete:1;
- };
- /**
- * struct k3_nav_ring - RA Ring descriptor
- *
- * @cfg - Ring configuration registers
- * @rt - Ring control/status registers
- * @fifos - Ring queues registers
- * @ring_mem_dma - Ring buffer dma address
- * @ring_mem_virt - Ring buffer virt address
- * @ops - Ring operations
- * @size - Ring size in elements
- * @elm_size - Size of the ring element
- * @mode - Ring mode
- * @flags - flags
- * @ring_id - Ring Id
- * @parent - Pointer on struct @k3_nav_ringacc
- * @use_count - Use count for shared rings
- */
- struct k3_nav_ring {
- struct k3_nav_ring_cfg_regs __iomem *cfg;
- struct k3_nav_ring_rt_regs __iomem *rt;
- struct k3_nav_ring_fifo_regs __iomem *fifos;
- dma_addr_t ring_mem_dma;
- void *ring_mem_virt;
- struct k3_nav_ring_ops *ops;
- u32 size;
- enum k3_nav_ring_size elm_size;
- enum k3_nav_ring_mode mode;
- u32 flags;
- #define KNAV_RING_FLAG_BUSY BIT(1)
- #define K3_NAV_RING_FLAG_SHARED BIT(2)
- #define K3_NAV_RING_FLAG_REVERSE BIT(3)
- struct k3_nav_ring_state state;
- u32 ring_id;
- struct k3_nav_ringacc *parent;
- u32 use_count;
- };
- struct k3_nav_ringacc_ops {
- int (*init)(struct udevice *dev, struct k3_nav_ringacc *ringacc);
- };
- /**
- * struct k3_nav_ringacc - Rings accelerator descriptor
- *
- * @dev - pointer on RA device
- * @num_rings - number of ring in RA
- * @rm_gp_range - general purpose rings range from tisci
- * @dma_ring_reset_quirk - DMA reset w/a enable
- * @num_proxies - number of RA proxies
- * @rings - array of rings descriptors (struct @k3_nav_ring)
- * @list - list of RAs in the system
- * @tisci - pointer ti-sci handle
- * @tisci_ring_ops - ti-sci rings ops
- * @tisci_dev_id - ti-sci device id
- * @ops: SoC specific ringacc operation
- * @dual_ring: indicate k3_dmaring dual ring support
- */
- struct k3_nav_ringacc {
- struct udevice *dev;
- u32 num_rings; /* number of rings in Ringacc module */
- unsigned long *rings_inuse;
- struct ti_sci_resource *rm_gp_range;
- bool dma_ring_reset_quirk;
- u32 num_proxies;
- struct k3_nav_ring *rings;
- struct list_head list;
- const struct ti_sci_handle *tisci;
- const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
- u32 tisci_dev_id;
- const struct k3_nav_ringacc_ops *ops;
- bool dual_ring;
- };
- #include "k3-navss-ringacc-u-boot.c"
- static int k3_nav_ringacc_ring_read_occ(struct k3_nav_ring *ring)
- {
- return readl(&ring->rt->occ) & KNAV_RINGACC_RT_OCC_MASK;
- }
- static void k3_nav_ringacc_ring_update_occ(struct k3_nav_ring *ring)
- {
- u32 val;
- val = readl(&ring->rt->occ);
- ring->state.occ = val & KNAV_RINGACC_RT_OCC_MASK;
- ring->state.tdown_complete = !!(val & K3_DMARING_RING_RT_OCC_TDOWN_COMPLETE);
- }
- static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
- {
- return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
- }
- static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
- static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
- static int k3_dmaring_ring_fwd_pop_mem(struct k3_nav_ring *ring, void *elem);
- static int k3_dmaring_ring_reverse_pop_mem(struct k3_nav_ring *ring, void *elem);
- static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
- .push_tail = k3_nav_ringacc_ring_push_mem,
- .pop_head = k3_nav_ringacc_ring_pop_mem,
- };
- static struct k3_nav_ring_ops k3_dmaring_fwd_ring_ops = {
- .push_tail = k3_nav_ringacc_ring_push_mem,
- .pop_head = k3_dmaring_ring_fwd_pop_mem,
- };
- static struct k3_nav_ring_ops k3_dmaring_reverse_ring_ops = {
- .pop_head = k3_dmaring_ring_reverse_pop_mem,
- };
- struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
- {
- return ringacc->dev;
- }
- struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
- int id)
- {
- if (id == K3_NAV_RINGACC_RING_ID_ANY) {
- /* Request for any general purpose ring */
- struct ti_sci_resource_desc *gp_rings =
- &ringacc->rm_gp_range->desc[0];
- unsigned long size;
- size = gp_rings->start + gp_rings->num;
- id = find_next_zero_bit(ringacc->rings_inuse,
- size, gp_rings->start);
- if (id == size)
- goto error;
- } else if (id < 0) {
- goto error;
- }
- if (test_bit(id, ringacc->rings_inuse) &&
- !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
- goto error;
- else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
- goto out;
- if (!try_module_get(ringacc->dev->driver->owner))
- goto error;
- pr_debug("Giving ring#%d\n", id);
- set_bit(id, ringacc->rings_inuse);
- out:
- ringacc->rings[id].use_count++;
- return &ringacc->rings[id];
- error:
- return NULL;
- }
- static int k3_dmaring_ring_request_rings_pair(struct k3_nav_ringacc *ringacc,
- int fwd_id, int compl_id,
- struct k3_nav_ring **fwd_ring,
- struct k3_nav_ring **compl_ring)
- {
- /* k3_dmaring: fwd_id == compl_id, so we ignore compl_id */
- if (fwd_id < 0)
- return -EINVAL;
- if (test_bit(fwd_id, ringacc->rings_inuse))
- return -EBUSY;
- *fwd_ring = &ringacc->rings[fwd_id];
- *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
- set_bit(fwd_id, ringacc->rings_inuse);
- ringacc->rings[fwd_id].use_count++;
- dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
- return 0;
- }
- int k3_nav_ringacc_request_rings_pair(struct k3_nav_ringacc *ringacc,
- int fwd_id, int compl_id,
- struct k3_nav_ring **fwd_ring,
- struct k3_nav_ring **compl_ring)
- {
- int ret = 0;
- if (!fwd_ring || !compl_ring)
- return -EINVAL;
- if (ringacc->dual_ring)
- return k3_dmaring_ring_request_rings_pair(ringacc, fwd_id, compl_id,
- fwd_ring, compl_ring);
- *fwd_ring = k3_nav_ringacc_request_ring(ringacc, fwd_id);
- if (!(*fwd_ring))
- return -ENODEV;
- *compl_ring = k3_nav_ringacc_request_ring(ringacc, compl_id);
- if (!(*compl_ring)) {
- k3_nav_ringacc_ring_free(*fwd_ring);
- ret = -ENODEV;
- }
- return ret;
- }
- static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
- {
- struct k3_nav_ringacc *ringacc = ring->parent;
- int ret;
- if (IS_ENABLED(CONFIG_K3_DM_FW))
- return k3_ringacc_ring_reset_raw(ring);
- ret = ringacc->tisci_ring_ops->config(
- ringacc->tisci,
- TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
- ringacc->tisci_dev_id,
- ring->ring_id,
- 0,
- 0,
- ring->size,
- 0,
- 0,
- 0);
- if (ret)
- dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
- ret, ring->ring_id);
- }
- void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
- {
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return;
- memset(&ring->state, 0, sizeof(ring->state));
- k3_ringacc_ring_reset_sci(ring);
- }
- static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
- enum k3_nav_ring_mode mode)
- {
- struct k3_nav_ringacc *ringacc = ring->parent;
- int ret;
- if (IS_ENABLED(CONFIG_K3_DM_FW))
- return k3_ringacc_ring_reconfig_qmode_raw(ring, mode);
- ret = ringacc->tisci_ring_ops->config(
- ringacc->tisci,
- TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
- ringacc->tisci_dev_id,
- ring->ring_id,
- 0,
- 0,
- 0,
- mode,
- 0,
- 0);
- if (ret)
- dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
- ret, ring->ring_id);
- }
- void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
- {
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return;
- if (!ring->parent->dma_ring_reset_quirk) {
- k3_nav_ringacc_ring_reset(ring);
- return;
- }
- if (!occ)
- occ = ringacc_readl(&ring->rt->occ);
- if (occ) {
- u32 db_ring_cnt, db_ring_cnt_cur;
- pr_debug("%s %u occ: %u\n", __func__,
- ring->ring_id, occ);
- /* 2. Reset the ring */
- k3_ringacc_ring_reset_sci(ring);
- /*
- * 3. Setup the ring in ring/doorbell mode
- * (if not already in this mode)
- */
- if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
- k3_ringacc_ring_reconfig_qmode_sci(
- ring, K3_NAV_RINGACC_RING_MODE_RING);
- /*
- * 4. Ring the doorbell 2**22 – ringOcc times.
- * This will wrap the internal UDMAP ring state occupancy
- * counter (which is 21-bits wide) to 0.
- */
- db_ring_cnt = (1U << 22) - occ;
- while (db_ring_cnt != 0) {
- /*
- * Ring the doorbell with the maximum count each
- * iteration if possible to minimize the total
- * of writes
- */
- if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
- db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
- else
- db_ring_cnt_cur = db_ring_cnt;
- writel(db_ring_cnt_cur, &ring->rt->db);
- db_ring_cnt -= db_ring_cnt_cur;
- }
- /* 5. Restore the original ring mode (if not ring mode) */
- if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
- k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
- }
- /* 2. Reset the ring */
- k3_nav_ringacc_ring_reset(ring);
- }
- static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
- {
- struct k3_nav_ringacc *ringacc = ring->parent;
- int ret;
- if (IS_ENABLED(CONFIG_K3_DM_FW))
- return k3_ringacc_ring_free_raw(ring);
- ret = ringacc->tisci_ring_ops->config(
- ringacc->tisci,
- TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
- ringacc->tisci_dev_id,
- ring->ring_id,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0);
- if (ret)
- dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
- ret, ring->ring_id);
- }
- int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
- {
- struct k3_nav_ringacc *ringacc;
- if (!ring)
- return -EINVAL;
- ringacc = ring->parent;
- /*
- * k3_dmaring: rings shared memory and configuration, only forward ring is
- * configured and reverse ring considered as slave.
- */
- if (ringacc->dual_ring && (ring->flags & K3_NAV_RING_FLAG_REVERSE))
- return 0;
- pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);
- if (!test_bit(ring->ring_id, ringacc->rings_inuse))
- return -EINVAL;
- if (--ring->use_count)
- goto out;
- if (!(ring->flags & KNAV_RING_FLAG_BUSY))
- goto no_init;
- k3_ringacc_ring_free_sci(ring);
- dma_free_coherent(ringacc->dev,
- ring->size * (4 << ring->elm_size),
- ring->ring_mem_virt, ring->ring_mem_dma);
- ring->flags &= ~KNAV_RING_FLAG_BUSY;
- ring->ops = NULL;
- no_init:
- clear_bit(ring->ring_id, ringacc->rings_inuse);
- module_put(ringacc->dev->driver->owner);
- out:
- return 0;
- }
- u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
- {
- if (!ring)
- return -EINVAL;
- return ring->ring_id;
- }
- static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
- {
- struct k3_nav_ringacc *ringacc = ring->parent;
- u32 ring_idx;
- int ret;
- if (!ringacc->tisci)
- return -EINVAL;
- ring_idx = ring->ring_id;
- ret = ringacc->tisci_ring_ops->config(
- ringacc->tisci,
- TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
- ringacc->tisci_dev_id,
- ring_idx,
- lower_32_bits(ring->ring_mem_dma),
- upper_32_bits(ring->ring_mem_dma),
- ring->size,
- ring->mode,
- ring->elm_size,
- 0);
- if (ret) {
- dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
- ret, ring_idx);
- return ret;
- }
- /*
- * Above TI SCI call handles firewall configuration, cfg
- * register configuration still has to be done locally in
- * absence of RM services.
- */
- if (IS_ENABLED(CONFIG_K3_DM_FW))
- k3_nav_ringacc_ring_cfg_raw(ring);
- return 0;
- }
- static int k3_dmaring_ring_cfg(struct k3_nav_ring *ring, struct k3_nav_ring_cfg *cfg)
- {
- struct k3_nav_ringacc *ringacc;
- struct k3_nav_ring *reverse_ring;
- int ret = 0;
- if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
- cfg->mode != K3_NAV_RINGACC_RING_MODE_RING ||
- cfg->size & ~K3_DMARING_RING_CFG_RING_SIZE_ELCNT_MASK)
- return -EINVAL;
- ringacc = ring->parent;
- /*
- * k3_dmaring: rings shared memory and configuration, only forward ring is
- * configured and reverse ring considered as slave.
- */
- if (ringacc->dual_ring && (ring->flags & K3_NAV_RING_FLAG_REVERSE))
- return 0;
- if (!test_bit(ring->ring_id, ringacc->rings_inuse))
- return -EINVAL;
- ring->size = cfg->size;
- ring->elm_size = cfg->elm_size;
- ring->mode = cfg->mode;
- memset(&ring->state, 0, sizeof(ring->state));
- ring->ops = &k3_dmaring_fwd_ring_ops;
- ring->ring_mem_virt =
- dma_alloc_coherent(ring->size * (4 << ring->elm_size),
- (unsigned long *)&ring->ring_mem_dma);
- if (!ring->ring_mem_virt) {
- dev_err(ringacc->dev, "Failed to alloc ring mem\n");
- ret = -ENOMEM;
- goto err_free_ops;
- }
- ret = k3_nav_ringacc_ring_cfg_sci(ring);
- if (ret)
- goto err_free_mem;
- ring->flags |= KNAV_RING_FLAG_BUSY;
- /* k3_dmaring: configure reverse ring */
- reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
- reverse_ring->size = cfg->size;
- reverse_ring->elm_size = cfg->elm_size;
- reverse_ring->mode = cfg->mode;
- memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
- reverse_ring->ops = &k3_dmaring_reverse_ring_ops;
- reverse_ring->ring_mem_virt = ring->ring_mem_virt;
- reverse_ring->ring_mem_dma = ring->ring_mem_dma;
- reverse_ring->flags |= KNAV_RING_FLAG_BUSY;
- return 0;
- err_free_mem:
- dma_free_coherent(ringacc->dev,
- ring->size * (4 << ring->elm_size),
- ring->ring_mem_virt,
- ring->ring_mem_dma);
- err_free_ops:
- ring->ops = NULL;
- return ret;
- }
- int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
- struct k3_nav_ring_cfg *cfg)
- {
- struct k3_nav_ringacc *ringacc = ring->parent;
- int ret = 0;
- if (!ring || !cfg)
- return -EINVAL;
- if (ringacc->dual_ring)
- return k3_dmaring_ring_cfg(ring, cfg);
- if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
- cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
- cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
- !test_bit(ring->ring_id, ringacc->rings_inuse))
- return -EINVAL;
- if (ring->use_count != 1)
- return 0;
- ring->size = cfg->size;
- ring->elm_size = cfg->elm_size;
- ring->mode = cfg->mode;
- memset(&ring->state, 0, sizeof(ring->state));
- switch (ring->mode) {
- case K3_NAV_RINGACC_RING_MODE_RING:
- ring->ops = &k3_nav_mode_ring_ops;
- break;
- default:
- ring->ops = NULL;
- ret = -EINVAL;
- goto err_free_ops;
- };
- ring->ring_mem_virt =
- dma_zalloc_coherent(ringacc->dev,
- ring->size * (4 << ring->elm_size),
- &ring->ring_mem_dma, GFP_KERNEL);
- if (!ring->ring_mem_virt) {
- dev_err(ringacc->dev, "Failed to alloc ring mem\n");
- ret = -ENOMEM;
- goto err_free_ops;
- }
- ret = k3_nav_ringacc_ring_cfg_sci(ring);
- if (ret)
- goto err_free_mem;
- ring->flags |= KNAV_RING_FLAG_BUSY;
- ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
- K3_NAV_RING_FLAG_SHARED : 0;
- return 0;
- err_free_mem:
- dma_free_coherent(ringacc->dev,
- ring->size * (4 << ring->elm_size),
- ring->ring_mem_virt,
- ring->ring_mem_dma);
- err_free_ops:
- ring->ops = NULL;
- return ret;
- }
- u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
- {
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- return ring->size;
- }
- u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
- {
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- if (!ring->state.free)
- ring->state.free = ring->size - ringacc_readl(&ring->rt->occ);
- return ring->state.free;
- }
- u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
- {
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- return ringacc_readl(&ring->rt->occ);
- }
- u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
- {
- return !k3_nav_ringacc_ring_get_free(ring);
- }
- enum k3_ringacc_access_mode {
- K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
- K3_RINGACC_ACCESS_MODE_POP_HEAD,
- K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
- K3_RINGACC_ACCESS_MODE_POP_TAIL,
- K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
- K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
- };
- static int k3_dmaring_ring_fwd_pop_mem(struct k3_nav_ring *ring, void *elem)
- {
- void *elem_ptr;
- u32 elem_idx;
- /*
- * k3_dmaring: forward ring is always tied DMA channel and HW does not
- * maintain any state data required for POP operation and its unknown
- * how much elements were consumed by HW. So, to actually
- * do POP, the read pointer has to be recalculated every time.
- */
- ring->state.occ = k3_nav_ringacc_ring_read_occ(ring);
- if (ring->state.windex >= ring->state.occ)
- elem_idx = ring->state.windex - ring->state.occ;
- else
- elem_idx = ring->size - (ring->state.occ - ring->state.windex);
- elem_ptr = k3_nav_ringacc_get_elm_addr(ring, elem_idx);
- invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
- ALIGN((unsigned long)ring->ring_mem_virt +
- ring->size * (4 << ring->elm_size),
- ARCH_DMA_MINALIGN));
- memcpy(elem, elem_ptr, (4 << ring->elm_size));
- ring->state.occ--;
- writel(-1, &ring->rt->db);
- dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
- __func__, ring->state.occ, ring->state.windex, elem_idx,
- elem_ptr);
- return 0;
- }
- static int k3_dmaring_ring_reverse_pop_mem(struct k3_nav_ring *ring, void *elem)
- {
- void *elem_ptr;
- elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);
- if (ring->state.occ) {
- invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
- ALIGN((unsigned long)ring->ring_mem_virt +
- ring->size * (4 << ring->elm_size),
- ARCH_DMA_MINALIGN));
- memcpy(elem, elem_ptr, (4 << ring->elm_size));
- ring->state.rindex = (ring->state.rindex + 1) % ring->size;
- ring->state.occ--;
- writel(-1 & K3_DMARING_RING_RT_DB_ENTRY_MASK, &ring->rt->db);
- }
- dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
- __func__, ring->state.occ, ring->state.rindex, elem_ptr);
- return 0;
- }
- static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
- {
- void *elem_ptr;
- elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.windex);
- memcpy(elem_ptr, elem, (4 << ring->elm_size));
- flush_dcache_range((unsigned long)ring->ring_mem_virt,
- ALIGN((unsigned long)ring->ring_mem_virt +
- ring->size * (4 << ring->elm_size),
- ARCH_DMA_MINALIGN));
- ring->state.windex = (ring->state.windex + 1) % ring->size;
- ring->state.free--;
- ringacc_writel(1, &ring->rt->db);
- pr_debug("ring_push_mem: free%d index%d\n",
- ring->state.free, ring->state.windex);
- return 0;
- }
- static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
- {
- void *elem_ptr;
- elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);
- invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
- ALIGN((unsigned long)ring->ring_mem_virt +
- ring->size * (4 << ring->elm_size),
- ARCH_DMA_MINALIGN));
- memcpy(elem, elem_ptr, (4 << ring->elm_size));
- ring->state.rindex = (ring->state.rindex + 1) % ring->size;
- ring->state.occ--;
- ringacc_writel(-1, &ring->rt->db);
- pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
- ring->state.occ, ring->state.rindex, elem_ptr);
- return 0;
- }
- int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
- {
- int ret = -EOPNOTSUPP;
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- pr_debug("ring_push%d: free%d index%d\n",
- ring->ring_id, ring->state.free, ring->state.windex);
- if (k3_nav_ringacc_ring_is_full(ring))
- return -ENOMEM;
- if (ring->ops && ring->ops->push_tail)
- ret = ring->ops->push_tail(ring, elem);
- return ret;
- }
- int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
- {
- int ret = -EOPNOTSUPP;
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- pr_debug("ring_push_head: free%d index%d\n",
- ring->state.free, ring->state.windex);
- if (k3_nav_ringacc_ring_is_full(ring))
- return -ENOMEM;
- if (ring->ops && ring->ops->push_head)
- ret = ring->ops->push_head(ring, elem);
- return ret;
- }
- int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
- {
- int ret = -EOPNOTSUPP;
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- if (!ring->state.occ)
- k3_nav_ringacc_ring_update_occ(ring);
- pr_debug("ring_pop%d: occ%d index%d\n",
- ring->ring_id, ring->state.occ, ring->state.rindex);
- if (!ring->state.occ && !ring->state.tdown_complete)
- return -ENODATA;
- if (ring->ops && ring->ops->pop_head)
- ret = ring->ops->pop_head(ring, elem);
- return ret;
- }
- int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
- {
- int ret = -EOPNOTSUPP;
- if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
- return -EINVAL;
- if (!ring->state.occ)
- k3_nav_ringacc_ring_update_occ(ring);
- pr_debug("ring_pop_tail: occ%d index%d\n",
- ring->state.occ, ring->state.rindex);
- if (!ring->state.occ)
- return -ENODATA;
- if (ring->ops && ring->ops->pop_tail)
- ret = ring->ops->pop_tail(ring, elem);
- return ret;
- }
- static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
- {
- struct udevice *dev = ringacc->dev;
- struct udevice *devp = dev;
- struct udevice *tisci_dev = NULL;
- int ret;
- ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
- if (!ringacc->num_rings) {
- dev_err(dev, "ti,num-rings read failure %d\n", ret);
- return -EINVAL;
- }
- ringacc->dma_ring_reset_quirk =
- dev_read_bool(dev, "ti,dma-ring-reset-quirk");
- ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, devp,
- "ti,sci", &tisci_dev);
- if (ret) {
- pr_debug("TISCI RA RM get failed (%d)\n", ret);
- ringacc->tisci = NULL;
- return -ENODEV;
- }
- ringacc->tisci = (struct ti_sci_handle *)
- (ti_sci_get_handle_from_sysfw(tisci_dev));
- ret = dev_read_u32_default(devp, "ti,sci", 0);
- if (!ret) {
- dev_err(dev, "TISCI RA RM disabled\n");
- ringacc->tisci = NULL;
- return ret;
- }
- ret = dev_read_u32(devp, "ti,sci-dev-id", &ringacc->tisci_dev_id);
- if (ret) {
- dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
- ringacc->tisci = NULL;
- return ret;
- }
- ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
- ringacc->tisci, dev,
- ringacc->tisci_dev_id,
- "ti,sci-rm-range-gp-rings");
- if (IS_ERR(ringacc->rm_gp_range))
- ret = PTR_ERR(ringacc->rm_gp_range);
- return 0;
- }
- static int k3_nav_ringacc_init(struct udevice *dev, struct k3_nav_ringacc *ringacc)
- {
- void __iomem *base_cfg, *base_rt;
- int ret, i;
- ret = k3_nav_ringacc_probe_dt(ringacc);
- if (ret)
- return ret;
- base_cfg = dev_remap_addr_name(dev, "cfg");
- pr_debug("cfg %p\n", base_cfg);
- if (!base_cfg)
- return -EINVAL;
- base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
- pr_debug("rt %p\n", base_rt);
- if (IS_ERR(base_rt))
- return PTR_ERR(base_rt);
- ringacc->rings = devm_kzalloc(dev,
- sizeof(*ringacc->rings) *
- ringacc->num_rings,
- GFP_KERNEL);
- ringacc->rings_inuse = devm_kcalloc(dev,
- BITS_TO_LONGS(ringacc->num_rings),
- sizeof(unsigned long), GFP_KERNEL);
- if (!ringacc->rings || !ringacc->rings_inuse)
- return -ENOMEM;
- for (i = 0; i < ringacc->num_rings; i++) {
- ringacc->rings[i].cfg = base_cfg +
- KNAV_RINGACC_CFG_REGS_STEP * i;
- ringacc->rings[i].rt = base_rt +
- KNAV_RINGACC_RT_REGS_STEP * i;
- ringacc->rings[i].parent = ringacc;
- ringacc->rings[i].ring_id = i;
- }
- dev_set_drvdata(dev, ringacc);
- ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
- list_add_tail(&ringacc->list, &k3_nav_ringacc_list);
- dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
- ringacc->num_rings,
- ringacc->rm_gp_range->desc[0].start,
- ringacc->rm_gp_range->desc[0].num,
- ringacc->tisci_dev_id);
- dev_info(dev, "dma-ring-reset-quirk: %s\n",
- ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
- return 0;
- }
- struct k3_nav_ringacc *k3_ringacc_dmarings_init(struct udevice *dev,
- struct k3_ringacc_init_data *data)
- {
- struct k3_nav_ringacc *ringacc;
- void __iomem *base_rt;
- int i;
- ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
- if (!ringacc)
- return ERR_PTR(-ENOMEM);
- ringacc->dual_ring = true;
- ringacc->dev = dev;
- ringacc->num_rings = data->num_rings;
- ringacc->tisci = data->tisci;
- ringacc->tisci_dev_id = data->tisci_dev_id;
- base_rt = (uint32_t *)devfdt_get_addr_name(dev, "ringrt");
- if (IS_ERR(base_rt))
- return base_rt;
- ringacc->rings = devm_kzalloc(dev,
- sizeof(*ringacc->rings) *
- ringacc->num_rings * 2,
- GFP_KERNEL);
- ringacc->rings_inuse = devm_kcalloc(dev,
- BITS_TO_LONGS(ringacc->num_rings),
- sizeof(unsigned long), GFP_KERNEL);
- if (!ringacc->rings || !ringacc->rings_inuse)
- return ERR_PTR(-ENOMEM);
- for (i = 0; i < ringacc->num_rings; i++) {
- struct k3_nav_ring *ring = &ringacc->rings[i];
- ring->rt = base_rt + K3_DMARING_RING_RT_REGS_STEP * i;
- ring->parent = ringacc;
- ring->ring_id = i;
- ring = &ringacc->rings[ringacc->num_rings + i];
- ring->rt = base_rt + K3_DMARING_RING_RT_REGS_STEP * i +
- K3_DMARING_RING_RT_REGS_REVERSE_OFS;
- ring->parent = ringacc;
- ring->ring_id = i;
- ring->flags = K3_NAV_RING_FLAG_REVERSE;
- }
- ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
- dev_info(dev, "k3_dmaring Ring probed rings:%u, sci-dev-id:%u\n",
- ringacc->num_rings,
- ringacc->tisci_dev_id);
- dev_info(dev, "dma-ring-reset-quirk: %s\n",
- ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
- return ringacc;
- }
- struct ringacc_match_data {
- struct k3_nav_ringacc_ops ops;
- };
- static struct ringacc_match_data k3_nav_ringacc_data = {
- .ops = {
- .init = k3_nav_ringacc_init,
- },
- };
- static const struct udevice_id knav_ringacc_ids[] = {
- { .compatible = "ti,am654-navss-ringacc", .data = (ulong)&k3_nav_ringacc_data, },
- {},
- };
- static int k3_nav_ringacc_probe(struct udevice *dev)
- {
- struct k3_nav_ringacc *ringacc;
- int ret;
- const struct ringacc_match_data *match_data;
- match_data = (struct ringacc_match_data *)dev_get_driver_data(dev);
- ringacc = dev_get_priv(dev);
- if (!ringacc)
- return -ENOMEM;
- ringacc->dev = dev;
- ringacc->ops = &match_data->ops;
- ret = ringacc->ops->init(dev, ringacc);
- if (ret)
- return ret;
- return 0;
- }
- U_BOOT_DRIVER(k3_navss_ringacc) = {
- .name = "k3-navss-ringacc",
- .id = UCLASS_MISC,
- .of_match = knav_ringacc_ids,
- .probe = k3_nav_ringacc_probe,
- .priv_auto = sizeof(struct k3_nav_ringacc),
- };
|