ivc.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2016, NVIDIA CORPORATION.
  4. */
  5. #include <common.h>
  6. #include <cpu_func.h>
  7. #include <asm/io.h>
  8. #include <asm/arch-tegra/ivc.h>
  9. #define TEGRA_IVC_ALIGN 64
  10. /*
  11. * IVC channel reset protocol.
  12. *
  13. * Each end uses its tx_channel.state to indicate its synchronization state.
  14. */
  15. enum ivc_state {
  16. /*
  17. * This value is zero for backwards compatibility with services that
  18. * assume channels to be initially zeroed. Such channels are in an
  19. * initially valid state, but cannot be asynchronously reset, and must
  20. * maintain a valid state at all times.
  21. *
  22. * The transmitting end can enter the established state from the sync or
  23. * ack state when it observes the receiving endpoint in the ack or
  24. * established state, indicating that has cleared the counters in our
  25. * rx_channel.
  26. */
  27. ivc_state_established = 0,
  28. /*
  29. * If an endpoint is observed in the sync state, the remote endpoint is
  30. * allowed to clear the counters it owns asynchronously with respect to
  31. * the current endpoint. Therefore, the current endpoint is no longer
  32. * allowed to communicate.
  33. */
  34. ivc_state_sync,
  35. /*
  36. * When the transmitting end observes the receiving end in the sync
  37. * state, it can clear the w_count and r_count and transition to the ack
  38. * state. If the remote endpoint observes us in the ack state, it can
  39. * return to the established state once it has cleared its counters.
  40. */
  41. ivc_state_ack
  42. };
  43. /*
  44. * This structure is divided into two-cache aligned parts, the first is only
  45. * written through the tx_channel pointer, while the second is only written
  46. * through the rx_channel pointer. This delineates ownership of the cache lines,
  47. * which is critical to performance and necessary in non-cache coherent
  48. * implementations.
  49. */
  50. struct tegra_ivc_channel_header {
  51. union {
  52. /* fields owned by the transmitting end */
  53. struct {
  54. uint32_t w_count;
  55. uint32_t state;
  56. };
  57. uint8_t w_align[TEGRA_IVC_ALIGN];
  58. };
  59. union {
  60. /* fields owned by the receiving end */
  61. uint32_t r_count;
  62. uint8_t r_align[TEGRA_IVC_ALIGN];
  63. };
  64. };
  65. static inline void tegra_ivc_invalidate_counter(struct tegra_ivc *ivc,
  66. struct tegra_ivc_channel_header *h,
  67. ulong offset)
  68. {
  69. ulong base = ((ulong)h) + offset;
  70. invalidate_dcache_range(base, base + TEGRA_IVC_ALIGN);
  71. }
  72. static inline void tegra_ivc_flush_counter(struct tegra_ivc *ivc,
  73. struct tegra_ivc_channel_header *h,
  74. ulong offset)
  75. {
  76. ulong base = ((ulong)h) + offset;
  77. flush_dcache_range(base, base + TEGRA_IVC_ALIGN);
  78. }
  79. static inline ulong tegra_ivc_frame_addr(struct tegra_ivc *ivc,
  80. struct tegra_ivc_channel_header *h,
  81. uint32_t frame)
  82. {
  83. BUG_ON(frame >= ivc->nframes);
  84. return ((ulong)h) + sizeof(struct tegra_ivc_channel_header) +
  85. (ivc->frame_size * frame);
  86. }
  87. static inline void *tegra_ivc_frame_pointer(struct tegra_ivc *ivc,
  88. struct tegra_ivc_channel_header *ch,
  89. uint32_t frame)
  90. {
  91. return (void *)tegra_ivc_frame_addr(ivc, ch, frame);
  92. }
  93. static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,
  94. struct tegra_ivc_channel_header *h,
  95. unsigned frame)
  96. {
  97. ulong base = tegra_ivc_frame_addr(ivc, h, frame);
  98. invalidate_dcache_range(base, base + ivc->frame_size);
  99. }
  100. static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,
  101. struct tegra_ivc_channel_header *h,
  102. unsigned frame)
  103. {
  104. ulong base = tegra_ivc_frame_addr(ivc, h, frame);
  105. flush_dcache_range(base, base + ivc->frame_size);
  106. }
  107. static inline int tegra_ivc_channel_empty(struct tegra_ivc *ivc,
  108. struct tegra_ivc_channel_header *ch)
  109. {
  110. /*
  111. * This function performs multiple checks on the same values with
  112. * security implications, so create snapshots with ACCESS_ONCE() to
  113. * ensure that these checks use the same values.
  114. */
  115. uint32_t w_count = ACCESS_ONCE(ch->w_count);
  116. uint32_t r_count = ACCESS_ONCE(ch->r_count);
  117. /*
  118. * Perform an over-full check to prevent denial of service attacks where
  119. * a server could be easily fooled into believing that there's an
  120. * extremely large number of frames ready, since receivers are not
  121. * expected to check for full or over-full conditions.
  122. *
  123. * Although the channel isn't empty, this is an invalid case caused by
  124. * a potentially malicious peer, so returning empty is safer, because it
  125. * gives the impression that the channel has gone silent.
  126. */
  127. if (w_count - r_count > ivc->nframes)
  128. return 1;
  129. return w_count == r_count;
  130. }
  131. static inline int tegra_ivc_channel_full(struct tegra_ivc *ivc,
  132. struct tegra_ivc_channel_header *ch)
  133. {
  134. /*
  135. * Invalid cases where the counters indicate that the queue is over
  136. * capacity also appear full.
  137. */
  138. return (ACCESS_ONCE(ch->w_count) - ACCESS_ONCE(ch->r_count)) >=
  139. ivc->nframes;
  140. }
  141. static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)
  142. {
  143. ACCESS_ONCE(ivc->rx_channel->r_count) =
  144. ACCESS_ONCE(ivc->rx_channel->r_count) + 1;
  145. if (ivc->r_pos == ivc->nframes - 1)
  146. ivc->r_pos = 0;
  147. else
  148. ivc->r_pos++;
  149. }
  150. static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)
  151. {
  152. ACCESS_ONCE(ivc->tx_channel->w_count) =
  153. ACCESS_ONCE(ivc->tx_channel->w_count) + 1;
  154. if (ivc->w_pos == ivc->nframes - 1)
  155. ivc->w_pos = 0;
  156. else
  157. ivc->w_pos++;
  158. }
  159. static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)
  160. {
  161. ulong offset;
  162. /*
  163. * tx_channel->state is set locally, so it is not synchronized with
  164. * state from the remote peer. The remote peer cannot reset its
  165. * transmit counters until we've acknowledged its synchronization
  166. * request, so no additional synchronization is required because an
  167. * asynchronous transition of rx_channel->state to ivc_state_ack is not
  168. * allowed.
  169. */
  170. if (ivc->tx_channel->state != ivc_state_established)
  171. return -ECONNRESET;
  172. /*
  173. * Avoid unnecessary invalidations when performing repeated accesses to
  174. * an IVC channel by checking the old queue pointers first.
  175. * Synchronization is only necessary when these pointers indicate empty
  176. * or full.
  177. */
  178. if (!tegra_ivc_channel_empty(ivc, ivc->rx_channel))
  179. return 0;
  180. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  181. tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
  182. return tegra_ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;
  183. }
  184. static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)
  185. {
  186. ulong offset;
  187. if (ivc->tx_channel->state != ivc_state_established)
  188. return -ECONNRESET;
  189. if (!tegra_ivc_channel_full(ivc, ivc->tx_channel))
  190. return 0;
  191. offset = offsetof(struct tegra_ivc_channel_header, r_count);
  192. tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
  193. return tegra_ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;
  194. }
  195. static inline uint32_t tegra_ivc_channel_avail_count(struct tegra_ivc *ivc,
  196. struct tegra_ivc_channel_header *ch)
  197. {
  198. /*
  199. * This function isn't expected to be used in scenarios where an
  200. * over-full situation can lead to denial of service attacks. See the
  201. * comment in tegra_ivc_channel_empty() for an explanation about
  202. * special over-full considerations.
  203. */
  204. return ACCESS_ONCE(ch->w_count) - ACCESS_ONCE(ch->r_count);
  205. }
  206. int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, void **frame)
  207. {
  208. int result = tegra_ivc_check_read(ivc);
  209. if (result < 0)
  210. return result;
  211. /*
  212. * Order observation of w_pos potentially indicating new data before
  213. * data read.
  214. */
  215. mb();
  216. tegra_ivc_invalidate_frame(ivc, ivc->rx_channel, ivc->r_pos);
  217. *frame = tegra_ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
  218. return 0;
  219. }
  220. int tegra_ivc_read_advance(struct tegra_ivc *ivc)
  221. {
  222. ulong offset;
  223. int result;
  224. /*
  225. * No read barriers or synchronization here: the caller is expected to
  226. * have already observed the channel non-empty. This check is just to
  227. * catch programming errors.
  228. */
  229. result = tegra_ivc_check_read(ivc);
  230. if (result)
  231. return result;
  232. tegra_ivc_advance_rx(ivc);
  233. offset = offsetof(struct tegra_ivc_channel_header, r_count);
  234. tegra_ivc_flush_counter(ivc, ivc->rx_channel, offset);
  235. /*
  236. * Ensure our write to r_pos occurs before our read from w_pos.
  237. */
  238. mb();
  239. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  240. tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
  241. if (tegra_ivc_channel_avail_count(ivc, ivc->rx_channel) ==
  242. ivc->nframes - 1)
  243. ivc->notify(ivc);
  244. return 0;
  245. }
  246. int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, void **frame)
  247. {
  248. int result = tegra_ivc_check_write(ivc);
  249. if (result)
  250. return result;
  251. *frame = tegra_ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
  252. return 0;
  253. }
  254. int tegra_ivc_write_advance(struct tegra_ivc *ivc)
  255. {
  256. ulong offset;
  257. int result;
  258. result = tegra_ivc_check_write(ivc);
  259. if (result)
  260. return result;
  261. tegra_ivc_flush_frame(ivc, ivc->tx_channel, ivc->w_pos);
  262. /*
  263. * Order any possible stores to the frame before update of w_pos.
  264. */
  265. mb();
  266. tegra_ivc_advance_tx(ivc);
  267. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  268. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  269. /*
  270. * Ensure our write to w_pos occurs before our read from r_pos.
  271. */
  272. mb();
  273. offset = offsetof(struct tegra_ivc_channel_header, r_count);
  274. tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
  275. if (tegra_ivc_channel_avail_count(ivc, ivc->tx_channel) == 1)
  276. ivc->notify(ivc);
  277. return 0;
  278. }
  279. /*
  280. * ===============================================================
  281. * IVC State Transition Table - see tegra_ivc_channel_notified()
  282. * ===============================================================
  283. *
  284. * local remote action
  285. * ----- ------ -----------------------------------
  286. * SYNC EST <none>
  287. * SYNC ACK reset counters; move to EST; notify
  288. * SYNC SYNC reset counters; move to ACK; notify
  289. * ACK EST move to EST; notify
  290. * ACK ACK move to EST; notify
  291. * ACK SYNC reset counters; move to ACK; notify
  292. * EST EST <none>
  293. * EST ACK <none>
  294. * EST SYNC reset counters; move to ACK; notify
  295. *
  296. * ===============================================================
  297. */
  298. int tegra_ivc_channel_notified(struct tegra_ivc *ivc)
  299. {
  300. ulong offset;
  301. enum ivc_state peer_state;
  302. /* Copy the receiver's state out of shared memory. */
  303. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  304. tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
  305. peer_state = ACCESS_ONCE(ivc->rx_channel->state);
  306. if (peer_state == ivc_state_sync) {
  307. /*
  308. * Order observation of ivc_state_sync before stores clearing
  309. * tx_channel.
  310. */
  311. mb();
  312. /*
  313. * Reset tx_channel counters. The remote end is in the SYNC
  314. * state and won't make progress until we change our state,
  315. * so the counters are not in use at this time.
  316. */
  317. ivc->tx_channel->w_count = 0;
  318. ivc->rx_channel->r_count = 0;
  319. ivc->w_pos = 0;
  320. ivc->r_pos = 0;
  321. /*
  322. * Ensure that counters appear cleared before new state can be
  323. * observed.
  324. */
  325. mb();
  326. /*
  327. * Move to ACK state. We have just cleared our counters, so it
  328. * is now safe for the remote end to start using these values.
  329. */
  330. ivc->tx_channel->state = ivc_state_ack;
  331. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  332. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  333. /*
  334. * Notify remote end to observe state transition.
  335. */
  336. ivc->notify(ivc);
  337. } else if (ivc->tx_channel->state == ivc_state_sync &&
  338. peer_state == ivc_state_ack) {
  339. /*
  340. * Order observation of ivc_state_sync before stores clearing
  341. * tx_channel.
  342. */
  343. mb();
  344. /*
  345. * Reset tx_channel counters. The remote end is in the ACK
  346. * state and won't make progress until we change our state,
  347. * so the counters are not in use at this time.
  348. */
  349. ivc->tx_channel->w_count = 0;
  350. ivc->rx_channel->r_count = 0;
  351. ivc->w_pos = 0;
  352. ivc->r_pos = 0;
  353. /*
  354. * Ensure that counters appear cleared before new state can be
  355. * observed.
  356. */
  357. mb();
  358. /*
  359. * Move to ESTABLISHED state. We know that the remote end has
  360. * already cleared its counters, so it is safe to start
  361. * writing/reading on this channel.
  362. */
  363. ivc->tx_channel->state = ivc_state_established;
  364. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  365. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  366. /*
  367. * Notify remote end to observe state transition.
  368. */
  369. ivc->notify(ivc);
  370. } else if (ivc->tx_channel->state == ivc_state_ack) {
  371. /*
  372. * At this point, we have observed the peer to be in either
  373. * the ACK or ESTABLISHED state. Next, order observation of
  374. * peer state before storing to tx_channel.
  375. */
  376. mb();
  377. /*
  378. * Move to ESTABLISHED state. We know that we have previously
  379. * cleared our counters, and we know that the remote end has
  380. * cleared its counters, so it is safe to start writing/reading
  381. * on this channel.
  382. */
  383. ivc->tx_channel->state = ivc_state_established;
  384. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  385. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  386. /*
  387. * Notify remote end to observe state transition.
  388. */
  389. ivc->notify(ivc);
  390. } else {
  391. /*
  392. * There is no need to handle any further action. Either the
  393. * channel is already fully established, or we are waiting for
  394. * the remote end to catch up with our current state. Refer
  395. * to the diagram in "IVC State Transition Table" above.
  396. */
  397. }
  398. if (ivc->tx_channel->state != ivc_state_established)
  399. return -EAGAIN;
  400. return 0;
  401. }
  402. void tegra_ivc_channel_reset(struct tegra_ivc *ivc)
  403. {
  404. ulong offset;
  405. ivc->tx_channel->state = ivc_state_sync;
  406. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  407. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  408. ivc->notify(ivc);
  409. }
  410. static int check_ivc_params(ulong qbase1, ulong qbase2, uint32_t nframes,
  411. uint32_t frame_size)
  412. {
  413. int ret = 0;
  414. BUG_ON(offsetof(struct tegra_ivc_channel_header, w_count) &
  415. (TEGRA_IVC_ALIGN - 1));
  416. BUG_ON(offsetof(struct tegra_ivc_channel_header, r_count) &
  417. (TEGRA_IVC_ALIGN - 1));
  418. BUG_ON(sizeof(struct tegra_ivc_channel_header) &
  419. (TEGRA_IVC_ALIGN - 1));
  420. if ((uint64_t)nframes * (uint64_t)frame_size >= 0x100000000) {
  421. pr_err("tegra_ivc: nframes * frame_size overflows\n");
  422. return -EINVAL;
  423. }
  424. /*
  425. * The headers must at least be aligned enough for counters
  426. * to be accessed atomically.
  427. */
  428. if ((qbase1 & (TEGRA_IVC_ALIGN - 1)) ||
  429. (qbase2 & (TEGRA_IVC_ALIGN - 1))) {
  430. pr_err("tegra_ivc: channel start not aligned\n");
  431. return -EINVAL;
  432. }
  433. if (frame_size & (TEGRA_IVC_ALIGN - 1)) {
  434. pr_err("tegra_ivc: frame size not adequately aligned\n");
  435. return -EINVAL;
  436. }
  437. if (qbase1 < qbase2) {
  438. if (qbase1 + frame_size * nframes > qbase2)
  439. ret = -EINVAL;
  440. } else {
  441. if (qbase2 + frame_size * nframes > qbase1)
  442. ret = -EINVAL;
  443. }
  444. if (ret) {
  445. pr_err("tegra_ivc: queue regions overlap\n");
  446. return ret;
  447. }
  448. return 0;
  449. }
  450. int tegra_ivc_init(struct tegra_ivc *ivc, ulong rx_base, ulong tx_base,
  451. uint32_t nframes, uint32_t frame_size,
  452. void (*notify)(struct tegra_ivc *))
  453. {
  454. int ret;
  455. if (!ivc)
  456. return -EINVAL;
  457. ret = check_ivc_params(rx_base, tx_base, nframes, frame_size);
  458. if (ret)
  459. return ret;
  460. ivc->rx_channel = (struct tegra_ivc_channel_header *)rx_base;
  461. ivc->tx_channel = (struct tegra_ivc_channel_header *)tx_base;
  462. ivc->w_pos = 0;
  463. ivc->r_pos = 0;
  464. ivc->nframes = nframes;
  465. ivc->frame_size = frame_size;
  466. ivc->notify = notify;
  467. return 0;
  468. }