knav_qmss_acc.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Keystone accumulator queue manager
  4. *
  5. * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
  6. * Author: Sandeep Nair <sandeep_n@ti.com>
  7. * Cyril Chemparathy <cyril@ti.com>
  8. * Santosh Shilimkar <santosh.shilimkar@ti.com>
  9. */
  10. #include <linux/dma-mapping.h>
  11. #include <linux/io.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/module.h>
  14. #include <linux/of_address.h>
  15. #include <linux/soc/ti/knav_qmss.h>
  16. #include "knav_qmss.h"
  17. #define knav_range_offset_to_inst(kdev, range, q) \
  18. (range->queue_base_inst + (q << kdev->inst_shift))
  19. static void __knav_acc_notify(struct knav_range_info *range,
  20. struct knav_acc_channel *acc)
  21. {
  22. struct knav_device *kdev = range->kdev;
  23. struct knav_queue_inst *inst;
  24. int range_base, queue;
  25. range_base = kdev->base_id + range->queue_base;
  26. if (range->flags & RANGE_MULTI_QUEUE) {
  27. for (queue = 0; queue < range->num_queues; queue++) {
  28. inst = knav_range_offset_to_inst(kdev, range,
  29. queue);
  30. if (inst->notify_needed) {
  31. inst->notify_needed = 0;
  32. dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  33. range_base + queue);
  34. knav_queue_notify(inst);
  35. }
  36. }
  37. } else {
  38. queue = acc->channel - range->acc_info.start_channel;
  39. inst = knav_range_offset_to_inst(kdev, range, queue);
  40. dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  41. range_base + queue);
  42. knav_queue_notify(inst);
  43. }
  44. }
  45. static int knav_acc_set_notify(struct knav_range_info *range,
  46. struct knav_queue_inst *kq,
  47. bool enabled)
  48. {
  49. struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
  50. struct knav_device *kdev = range->kdev;
  51. u32 mask, offset;
  52. /*
  53. * when enabling, we need to re-trigger an interrupt if we
  54. * have descriptors pending
  55. */
  56. if (!enabled || atomic_read(&kq->desc_count) <= 0)
  57. return 0;
  58. kq->notify_needed = 1;
  59. atomic_inc(&kq->acc->retrigger_count);
  60. mask = BIT(kq->acc->channel % 32);
  61. offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
  62. dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
  63. kq->acc->name);
  64. writel_relaxed(mask, pdsp->intd + offset);
  65. return 0;
  66. }
  67. static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
  68. {
  69. struct knav_acc_channel *acc;
  70. struct knav_queue_inst *kq = NULL;
  71. struct knav_range_info *range;
  72. struct knav_pdsp_info *pdsp;
  73. struct knav_acc_info *info;
  74. struct knav_device *kdev;
  75. u32 *list, *list_cpu, val, idx, notifies;
  76. int range_base, channel, queue = 0;
  77. dma_addr_t list_dma;
  78. range = _instdata;
  79. info = &range->acc_info;
  80. kdev = range->kdev;
  81. pdsp = range->acc_info.pdsp;
  82. acc = range->acc;
  83. range_base = kdev->base_id + range->queue_base;
  84. if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
  85. for (queue = 0; queue < range->num_irqs; queue++)
  86. if (range->irqs[queue].irq == irq)
  87. break;
  88. kq = knav_range_offset_to_inst(kdev, range, queue);
  89. acc += queue;
  90. }
  91. channel = acc->channel;
  92. list_dma = acc->list_dma[acc->list_index];
  93. list_cpu = acc->list_cpu[acc->list_index];
  94. dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
  95. channel, acc->list_index, list_cpu, &list_dma);
  96. if (atomic_read(&acc->retrigger_count)) {
  97. atomic_dec(&acc->retrigger_count);
  98. __knav_acc_notify(range, acc);
  99. writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
  100. /* ack the interrupt */
  101. writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
  102. pdsp->intd + ACC_INTD_OFFSET_EOI);
  103. return IRQ_HANDLED;
  104. }
  105. notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
  106. WARN_ON(!notifies);
  107. dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
  108. DMA_FROM_DEVICE);
  109. for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
  110. list += ACC_LIST_ENTRY_WORDS) {
  111. if (ACC_LIST_ENTRY_WORDS == 1) {
  112. dev_dbg(kdev->dev,
  113. "acc-irq: list %d, entry @%p, %08x\n",
  114. acc->list_index, list, list[0]);
  115. } else if (ACC_LIST_ENTRY_WORDS == 2) {
  116. dev_dbg(kdev->dev,
  117. "acc-irq: list %d, entry @%p, %08x %08x\n",
  118. acc->list_index, list, list[0], list[1]);
  119. } else if (ACC_LIST_ENTRY_WORDS == 4) {
  120. dev_dbg(kdev->dev,
  121. "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
  122. acc->list_index, list, list[0], list[1],
  123. list[2], list[3]);
  124. }
  125. val = list[ACC_LIST_ENTRY_DESC_IDX];
  126. if (!val)
  127. break;
  128. if (range->flags & RANGE_MULTI_QUEUE) {
  129. queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
  130. if (queue < range_base ||
  131. queue >= range_base + range->num_queues) {
  132. dev_err(kdev->dev,
  133. "bad queue %d, expecting %d-%d\n",
  134. queue, range_base,
  135. range_base + range->num_queues);
  136. break;
  137. }
  138. queue -= range_base;
  139. kq = knav_range_offset_to_inst(kdev, range,
  140. queue);
  141. }
  142. if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
  143. atomic_dec(&kq->desc_count);
  144. dev_err(kdev->dev,
  145. "acc-irq: queue %d full, entry dropped\n",
  146. queue + range_base);
  147. continue;
  148. }
  149. idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
  150. kq->descs[idx] = val;
  151. kq->notify_needed = 1;
  152. dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
  153. val, idx, queue + range_base);
  154. }
  155. __knav_acc_notify(range, acc);
  156. memset(list_cpu, 0, info->list_size);
  157. dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
  158. DMA_TO_DEVICE);
  159. /* flip to the other list */
  160. acc->list_index ^= 1;
  161. /* reset the interrupt counter */
  162. writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
  163. /* ack the interrupt */
  164. writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
  165. pdsp->intd + ACC_INTD_OFFSET_EOI);
  166. return IRQ_HANDLED;
  167. }
  168. static int knav_range_setup_acc_irq(struct knav_range_info *range,
  169. int queue, bool enabled)
  170. {
  171. struct knav_device *kdev = range->kdev;
  172. struct knav_acc_channel *acc;
  173. struct cpumask *cpu_mask;
  174. int ret = 0, irq;
  175. u32 old, new;
  176. if (range->flags & RANGE_MULTI_QUEUE) {
  177. acc = range->acc;
  178. irq = range->irqs[0].irq;
  179. cpu_mask = range->irqs[0].cpu_mask;
  180. } else {
  181. acc = range->acc + queue;
  182. irq = range->irqs[queue].irq;
  183. cpu_mask = range->irqs[queue].cpu_mask;
  184. }
  185. old = acc->open_mask;
  186. if (enabled)
  187. new = old | BIT(queue);
  188. else
  189. new = old & ~BIT(queue);
  190. acc->open_mask = new;
  191. dev_dbg(kdev->dev,
  192. "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
  193. old, new, acc->name);
  194. if (likely(new == old))
  195. return 0;
  196. if (new && !old) {
  197. dev_dbg(kdev->dev,
  198. "setup-acc-irq: requesting %s for channel %s\n",
  199. acc->name, acc->name);
  200. ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
  201. range);
  202. if (!ret && cpu_mask) {
  203. ret = irq_set_affinity_hint(irq, cpu_mask);
  204. if (ret) {
  205. dev_warn(range->kdev->dev,
  206. "Failed to set IRQ affinity\n");
  207. return ret;
  208. }
  209. }
  210. }
  211. if (old && !new) {
  212. dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
  213. acc->name, acc->name);
  214. ret = irq_set_affinity_hint(irq, NULL);
  215. if (ret)
  216. dev_warn(range->kdev->dev,
  217. "Failed to set IRQ affinity\n");
  218. free_irq(irq, range);
  219. }
  220. return ret;
  221. }
  222. static const char *knav_acc_result_str(enum knav_acc_result result)
  223. {
  224. static const char * const result_str[] = {
  225. [ACC_RET_IDLE] = "idle",
  226. [ACC_RET_SUCCESS] = "success",
  227. [ACC_RET_INVALID_COMMAND] = "invalid command",
  228. [ACC_RET_INVALID_CHANNEL] = "invalid channel",
  229. [ACC_RET_INACTIVE_CHANNEL] = "inactive channel",
  230. [ACC_RET_ACTIVE_CHANNEL] = "active channel",
  231. [ACC_RET_INVALID_QUEUE] = "invalid queue",
  232. [ACC_RET_INVALID_RET] = "invalid return code",
  233. };
  234. if (result >= ARRAY_SIZE(result_str))
  235. return result_str[ACC_RET_INVALID_RET];
  236. else
  237. return result_str[result];
  238. }
  239. static enum knav_acc_result
  240. knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
  241. struct knav_reg_acc_command *cmd)
  242. {
  243. u32 result;
  244. dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
  245. cmd->command, cmd->queue_mask, cmd->list_dma,
  246. cmd->queue_num, cmd->timer_config);
  247. writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
  248. writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
  249. writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
  250. writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
  251. writel_relaxed(cmd->command, &pdsp->acc_command->command);
  252. /* wait for the command to clear */
  253. do {
  254. result = readl_relaxed(&pdsp->acc_command->command);
  255. } while ((result >> 8) & 0xff);
  256. return (result >> 24) & 0xff;
  257. }
  258. static void knav_acc_setup_cmd(struct knav_device *kdev,
  259. struct knav_range_info *range,
  260. struct knav_reg_acc_command *cmd,
  261. int queue)
  262. {
  263. struct knav_acc_info *info = &range->acc_info;
  264. struct knav_acc_channel *acc;
  265. int queue_base;
  266. u32 queue_mask;
  267. if (range->flags & RANGE_MULTI_QUEUE) {
  268. acc = range->acc;
  269. queue_base = range->queue_base;
  270. queue_mask = BIT(range->num_queues) - 1;
  271. } else {
  272. acc = range->acc + queue;
  273. queue_base = range->queue_base + queue;
  274. queue_mask = 0;
  275. }
  276. memset(cmd, 0, sizeof(*cmd));
  277. cmd->command = acc->channel;
  278. cmd->queue_mask = queue_mask;
  279. cmd->list_dma = (u32)acc->list_dma[0];
  280. cmd->queue_num = info->list_entries << 16;
  281. cmd->queue_num |= queue_base;
  282. cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
  283. if (range->flags & RANGE_MULTI_QUEUE)
  284. cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
  285. cmd->timer_config |= info->pacing_mode << 16;
  286. cmd->timer_config |= info->timer_count;
  287. }
  288. static void knav_acc_stop(struct knav_device *kdev,
  289. struct knav_range_info *range,
  290. int queue)
  291. {
  292. struct knav_reg_acc_command cmd;
  293. struct knav_acc_channel *acc;
  294. enum knav_acc_result result;
  295. acc = range->acc + queue;
  296. knav_acc_setup_cmd(kdev, range, &cmd, queue);
  297. cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
  298. result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
  299. dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
  300. acc->name, knav_acc_result_str(result));
  301. }
  302. static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
  303. struct knav_range_info *range,
  304. int queue)
  305. {
  306. struct knav_reg_acc_command cmd;
  307. struct knav_acc_channel *acc;
  308. enum knav_acc_result result;
  309. acc = range->acc + queue;
  310. knav_acc_setup_cmd(kdev, range, &cmd, queue);
  311. cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
  312. result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
  313. dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
  314. acc->name, knav_acc_result_str(result));
  315. return result;
  316. }
  317. static int knav_acc_init_range(struct knav_range_info *range)
  318. {
  319. struct knav_device *kdev = range->kdev;
  320. struct knav_acc_channel *acc;
  321. enum knav_acc_result result;
  322. int queue;
  323. for (queue = 0; queue < range->num_queues; queue++) {
  324. acc = range->acc + queue;
  325. knav_acc_stop(kdev, range, queue);
  326. acc->list_index = 0;
  327. result = knav_acc_start(kdev, range, queue);
  328. if (result != ACC_RET_SUCCESS)
  329. return -EIO;
  330. if (range->flags & RANGE_MULTI_QUEUE)
  331. return 0;
  332. }
  333. return 0;
  334. }
  335. static int knav_acc_init_queue(struct knav_range_info *range,
  336. struct knav_queue_inst *kq)
  337. {
  338. unsigned id = kq->id - range->queue_base;
  339. kq->descs = devm_kcalloc(range->kdev->dev,
  340. ACC_DESCS_MAX, sizeof(u32), GFP_KERNEL);
  341. if (!kq->descs)
  342. return -ENOMEM;
  343. kq->acc = range->acc;
  344. if ((range->flags & RANGE_MULTI_QUEUE) == 0)
  345. kq->acc += id;
  346. return 0;
  347. }
  348. static int knav_acc_open_queue(struct knav_range_info *range,
  349. struct knav_queue_inst *inst, unsigned flags)
  350. {
  351. unsigned id = inst->id - range->queue_base;
  352. return knav_range_setup_acc_irq(range, id, true);
  353. }
  354. static int knav_acc_close_queue(struct knav_range_info *range,
  355. struct knav_queue_inst *inst)
  356. {
  357. unsigned id = inst->id - range->queue_base;
  358. return knav_range_setup_acc_irq(range, id, false);
  359. }
  360. static int knav_acc_free_range(struct knav_range_info *range)
  361. {
  362. struct knav_device *kdev = range->kdev;
  363. struct knav_acc_channel *acc;
  364. struct knav_acc_info *info;
  365. int channel, channels;
  366. info = &range->acc_info;
  367. if (range->flags & RANGE_MULTI_QUEUE)
  368. channels = 1;
  369. else
  370. channels = range->num_queues;
  371. for (channel = 0; channel < channels; channel++) {
  372. acc = range->acc + channel;
  373. if (!acc->list_cpu[0])
  374. continue;
  375. dma_unmap_single(kdev->dev, acc->list_dma[0],
  376. info->mem_size, DMA_BIDIRECTIONAL);
  377. free_pages_exact(acc->list_cpu[0], info->mem_size);
  378. }
  379. devm_kfree(range->kdev->dev, range->acc);
  380. return 0;
  381. }
  382. static struct knav_range_ops knav_acc_range_ops = {
  383. .set_notify = knav_acc_set_notify,
  384. .init_queue = knav_acc_init_queue,
  385. .open_queue = knav_acc_open_queue,
  386. .close_queue = knav_acc_close_queue,
  387. .init_range = knav_acc_init_range,
  388. .free_range = knav_acc_free_range,
  389. };
  390. /**
  391. * knav_init_acc_range: Initialise accumulator ranges
  392. *
  393. * @kdev: qmss device
  394. * @node: device node
  395. * @range: qmms range information
  396. *
  397. * Return 0 on success or error
  398. */
  399. int knav_init_acc_range(struct knav_device *kdev,
  400. struct device_node *node,
  401. struct knav_range_info *range)
  402. {
  403. struct knav_acc_channel *acc;
  404. struct knav_pdsp_info *pdsp;
  405. struct knav_acc_info *info;
  406. int ret, channel, channels;
  407. int list_size, mem_size;
  408. dma_addr_t list_dma;
  409. void *list_mem;
  410. u32 config[5];
  411. range->flags |= RANGE_HAS_ACCUMULATOR;
  412. info = &range->acc_info;
  413. ret = of_property_read_u32_array(node, "accumulator", config, 5);
  414. if (ret)
  415. return ret;
  416. info->pdsp_id = config[0];
  417. info->start_channel = config[1];
  418. info->list_entries = config[2];
  419. info->pacing_mode = config[3];
  420. info->timer_count = config[4] / ACC_DEFAULT_PERIOD;
  421. if (info->start_channel > ACC_MAX_CHANNEL) {
  422. dev_err(kdev->dev, "channel %d invalid for range %s\n",
  423. info->start_channel, range->name);
  424. return -EINVAL;
  425. }
  426. if (info->pacing_mode > 3) {
  427. dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
  428. info->pacing_mode, range->name);
  429. return -EINVAL;
  430. }
  431. pdsp = knav_find_pdsp(kdev, info->pdsp_id);
  432. if (!pdsp) {
  433. dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
  434. info->pdsp_id, range->name);
  435. return -EINVAL;
  436. }
  437. if (!pdsp->started) {
  438. dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
  439. info->pdsp_id, range->name);
  440. return -ENODEV;
  441. }
  442. info->pdsp = pdsp;
  443. channels = range->num_queues;
  444. if (of_get_property(node, "multi-queue", NULL)) {
  445. range->flags |= RANGE_MULTI_QUEUE;
  446. channels = 1;
  447. if (range->queue_base & (32 - 1)) {
  448. dev_err(kdev->dev,
  449. "misaligned multi-queue accumulator range %s\n",
  450. range->name);
  451. return -EINVAL;
  452. }
  453. if (range->num_queues > 32) {
  454. dev_err(kdev->dev,
  455. "too many queues in accumulator range %s\n",
  456. range->name);
  457. return -EINVAL;
  458. }
  459. }
  460. /* figure out list size */
  461. list_size = info->list_entries;
  462. list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
  463. info->list_size = list_size;
  464. mem_size = PAGE_ALIGN(list_size * 2);
  465. info->mem_size = mem_size;
  466. range->acc = devm_kcalloc(kdev->dev, channels, sizeof(*range->acc),
  467. GFP_KERNEL);
  468. if (!range->acc)
  469. return -ENOMEM;
  470. for (channel = 0; channel < channels; channel++) {
  471. acc = range->acc + channel;
  472. acc->channel = info->start_channel + channel;
  473. /* allocate memory for the two lists */
  474. list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
  475. if (!list_mem)
  476. return -ENOMEM;
  477. list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
  478. DMA_BIDIRECTIONAL);
  479. if (dma_mapping_error(kdev->dev, list_dma)) {
  480. free_pages_exact(list_mem, mem_size);
  481. return -ENOMEM;
  482. }
  483. memset(list_mem, 0, mem_size);
  484. dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
  485. DMA_TO_DEVICE);
  486. scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
  487. acc->channel);
  488. acc->list_cpu[0] = list_mem;
  489. acc->list_cpu[1] = list_mem + list_size;
  490. acc->list_dma[0] = list_dma;
  491. acc->list_dma[1] = list_dma + list_size;
  492. dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
  493. acc->name, acc->channel, &list_dma, list_mem);
  494. }
  495. range->ops = &knav_acc_range_ops;
  496. return 0;
  497. }
  498. EXPORT_SYMBOL_GPL(knav_init_acc_range);