ivc.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2016, NVIDIA CORPORATION.
  4. */
  5. #include <common.h>
  6. #include <cpu_func.h>
  7. #include <asm/io.h>
  8. #include <asm/arch-tegra/ivc.h>
  9. #include <linux/bug.h>
  10. #define TEGRA_IVC_ALIGN 64
  11. /*
  12. * IVC channel reset protocol.
  13. *
  14. * Each end uses its tx_channel.state to indicate its synchronization state.
  15. */
  16. enum ivc_state {
  17. /*
  18. * This value is zero for backwards compatibility with services that
  19. * assume channels to be initially zeroed. Such channels are in an
  20. * initially valid state, but cannot be asynchronously reset, and must
  21. * maintain a valid state at all times.
  22. *
  23. * The transmitting end can enter the established state from the sync or
  24. * ack state when it observes the receiving endpoint in the ack or
  25. * established state, indicating that has cleared the counters in our
  26. * rx_channel.
  27. */
  28. ivc_state_established = 0,
  29. /*
  30. * If an endpoint is observed in the sync state, the remote endpoint is
  31. * allowed to clear the counters it owns asynchronously with respect to
  32. * the current endpoint. Therefore, the current endpoint is no longer
  33. * allowed to communicate.
  34. */
  35. ivc_state_sync,
  36. /*
  37. * When the transmitting end observes the receiving end in the sync
  38. * state, it can clear the w_count and r_count and transition to the ack
  39. * state. If the remote endpoint observes us in the ack state, it can
  40. * return to the established state once it has cleared its counters.
  41. */
  42. ivc_state_ack
  43. };
  44. /*
  45. * This structure is divided into two-cache aligned parts, the first is only
  46. * written through the tx_channel pointer, while the second is only written
  47. * through the rx_channel pointer. This delineates ownership of the cache lines,
  48. * which is critical to performance and necessary in non-cache coherent
  49. * implementations.
  50. */
  51. struct tegra_ivc_channel_header {
  52. union {
  53. /* fields owned by the transmitting end */
  54. struct {
  55. uint32_t w_count;
  56. uint32_t state;
  57. };
  58. uint8_t w_align[TEGRA_IVC_ALIGN];
  59. };
  60. union {
  61. /* fields owned by the receiving end */
  62. uint32_t r_count;
  63. uint8_t r_align[TEGRA_IVC_ALIGN];
  64. };
  65. };
  66. static inline void tegra_ivc_invalidate_counter(struct tegra_ivc *ivc,
  67. struct tegra_ivc_channel_header *h,
  68. ulong offset)
  69. {
  70. ulong base = ((ulong)h) + offset;
  71. invalidate_dcache_range(base, base + TEGRA_IVC_ALIGN);
  72. }
  73. static inline void tegra_ivc_flush_counter(struct tegra_ivc *ivc,
  74. struct tegra_ivc_channel_header *h,
  75. ulong offset)
  76. {
  77. ulong base = ((ulong)h) + offset;
  78. flush_dcache_range(base, base + TEGRA_IVC_ALIGN);
  79. }
  80. static inline ulong tegra_ivc_frame_addr(struct tegra_ivc *ivc,
  81. struct tegra_ivc_channel_header *h,
  82. uint32_t frame)
  83. {
  84. BUG_ON(frame >= ivc->nframes);
  85. return ((ulong)h) + sizeof(struct tegra_ivc_channel_header) +
  86. (ivc->frame_size * frame);
  87. }
  88. static inline void *tegra_ivc_frame_pointer(struct tegra_ivc *ivc,
  89. struct tegra_ivc_channel_header *ch,
  90. uint32_t frame)
  91. {
  92. return (void *)tegra_ivc_frame_addr(ivc, ch, frame);
  93. }
  94. static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,
  95. struct tegra_ivc_channel_header *h,
  96. unsigned frame)
  97. {
  98. ulong base = tegra_ivc_frame_addr(ivc, h, frame);
  99. invalidate_dcache_range(base, base + ivc->frame_size);
  100. }
  101. static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,
  102. struct tegra_ivc_channel_header *h,
  103. unsigned frame)
  104. {
  105. ulong base = tegra_ivc_frame_addr(ivc, h, frame);
  106. flush_dcache_range(base, base + ivc->frame_size);
  107. }
  108. static inline int tegra_ivc_channel_empty(struct tegra_ivc *ivc,
  109. struct tegra_ivc_channel_header *ch)
  110. {
  111. /*
  112. * This function performs multiple checks on the same values with
  113. * security implications, so create snapshots with ACCESS_ONCE() to
  114. * ensure that these checks use the same values.
  115. */
  116. uint32_t w_count = ACCESS_ONCE(ch->w_count);
  117. uint32_t r_count = ACCESS_ONCE(ch->r_count);
  118. /*
  119. * Perform an over-full check to prevent denial of service attacks where
  120. * a server could be easily fooled into believing that there's an
  121. * extremely large number of frames ready, since receivers are not
  122. * expected to check for full or over-full conditions.
  123. *
  124. * Although the channel isn't empty, this is an invalid case caused by
  125. * a potentially malicious peer, so returning empty is safer, because it
  126. * gives the impression that the channel has gone silent.
  127. */
  128. if (w_count - r_count > ivc->nframes)
  129. return 1;
  130. return w_count == r_count;
  131. }
  132. static inline int tegra_ivc_channel_full(struct tegra_ivc *ivc,
  133. struct tegra_ivc_channel_header *ch)
  134. {
  135. /*
  136. * Invalid cases where the counters indicate that the queue is over
  137. * capacity also appear full.
  138. */
  139. return (ACCESS_ONCE(ch->w_count) - ACCESS_ONCE(ch->r_count)) >=
  140. ivc->nframes;
  141. }
  142. static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)
  143. {
  144. ACCESS_ONCE(ivc->rx_channel->r_count) =
  145. ACCESS_ONCE(ivc->rx_channel->r_count) + 1;
  146. if (ivc->r_pos == ivc->nframes - 1)
  147. ivc->r_pos = 0;
  148. else
  149. ivc->r_pos++;
  150. }
  151. static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)
  152. {
  153. ACCESS_ONCE(ivc->tx_channel->w_count) =
  154. ACCESS_ONCE(ivc->tx_channel->w_count) + 1;
  155. if (ivc->w_pos == ivc->nframes - 1)
  156. ivc->w_pos = 0;
  157. else
  158. ivc->w_pos++;
  159. }
  160. static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)
  161. {
  162. ulong offset;
  163. /*
  164. * tx_channel->state is set locally, so it is not synchronized with
  165. * state from the remote peer. The remote peer cannot reset its
  166. * transmit counters until we've acknowledged its synchronization
  167. * request, so no additional synchronization is required because an
  168. * asynchronous transition of rx_channel->state to ivc_state_ack is not
  169. * allowed.
  170. */
  171. if (ivc->tx_channel->state != ivc_state_established)
  172. return -ECONNRESET;
  173. /*
  174. * Avoid unnecessary invalidations when performing repeated accesses to
  175. * an IVC channel by checking the old queue pointers first.
  176. * Synchronization is only necessary when these pointers indicate empty
  177. * or full.
  178. */
  179. if (!tegra_ivc_channel_empty(ivc, ivc->rx_channel))
  180. return 0;
  181. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  182. tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
  183. return tegra_ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;
  184. }
  185. static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)
  186. {
  187. ulong offset;
  188. if (ivc->tx_channel->state != ivc_state_established)
  189. return -ECONNRESET;
  190. if (!tegra_ivc_channel_full(ivc, ivc->tx_channel))
  191. return 0;
  192. offset = offsetof(struct tegra_ivc_channel_header, r_count);
  193. tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
  194. return tegra_ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;
  195. }
  196. static inline uint32_t tegra_ivc_channel_avail_count(struct tegra_ivc *ivc,
  197. struct tegra_ivc_channel_header *ch)
  198. {
  199. /*
  200. * This function isn't expected to be used in scenarios where an
  201. * over-full situation can lead to denial of service attacks. See the
  202. * comment in tegra_ivc_channel_empty() for an explanation about
  203. * special over-full considerations.
  204. */
  205. return ACCESS_ONCE(ch->w_count) - ACCESS_ONCE(ch->r_count);
  206. }
  207. int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, void **frame)
  208. {
  209. int result = tegra_ivc_check_read(ivc);
  210. if (result < 0)
  211. return result;
  212. /*
  213. * Order observation of w_pos potentially indicating new data before
  214. * data read.
  215. */
  216. mb();
  217. tegra_ivc_invalidate_frame(ivc, ivc->rx_channel, ivc->r_pos);
  218. *frame = tegra_ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
  219. return 0;
  220. }
  221. int tegra_ivc_read_advance(struct tegra_ivc *ivc)
  222. {
  223. ulong offset;
  224. int result;
  225. /*
  226. * No read barriers or synchronization here: the caller is expected to
  227. * have already observed the channel non-empty. This check is just to
  228. * catch programming errors.
  229. */
  230. result = tegra_ivc_check_read(ivc);
  231. if (result)
  232. return result;
  233. tegra_ivc_advance_rx(ivc);
  234. offset = offsetof(struct tegra_ivc_channel_header, r_count);
  235. tegra_ivc_flush_counter(ivc, ivc->rx_channel, offset);
  236. /*
  237. * Ensure our write to r_pos occurs before our read from w_pos.
  238. */
  239. mb();
  240. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  241. tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
  242. if (tegra_ivc_channel_avail_count(ivc, ivc->rx_channel) ==
  243. ivc->nframes - 1)
  244. ivc->notify(ivc);
  245. return 0;
  246. }
  247. int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, void **frame)
  248. {
  249. int result = tegra_ivc_check_write(ivc);
  250. if (result)
  251. return result;
  252. *frame = tegra_ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
  253. return 0;
  254. }
  255. int tegra_ivc_write_advance(struct tegra_ivc *ivc)
  256. {
  257. ulong offset;
  258. int result;
  259. result = tegra_ivc_check_write(ivc);
  260. if (result)
  261. return result;
  262. tegra_ivc_flush_frame(ivc, ivc->tx_channel, ivc->w_pos);
  263. /*
  264. * Order any possible stores to the frame before update of w_pos.
  265. */
  266. mb();
  267. tegra_ivc_advance_tx(ivc);
  268. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  269. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  270. /*
  271. * Ensure our write to w_pos occurs before our read from r_pos.
  272. */
  273. mb();
  274. offset = offsetof(struct tegra_ivc_channel_header, r_count);
  275. tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
  276. if (tegra_ivc_channel_avail_count(ivc, ivc->tx_channel) == 1)
  277. ivc->notify(ivc);
  278. return 0;
  279. }
  280. /*
  281. * ===============================================================
  282. * IVC State Transition Table - see tegra_ivc_channel_notified()
  283. * ===============================================================
  284. *
  285. * local remote action
  286. * ----- ------ -----------------------------------
  287. * SYNC EST <none>
  288. * SYNC ACK reset counters; move to EST; notify
  289. * SYNC SYNC reset counters; move to ACK; notify
  290. * ACK EST move to EST; notify
  291. * ACK ACK move to EST; notify
  292. * ACK SYNC reset counters; move to ACK; notify
  293. * EST EST <none>
  294. * EST ACK <none>
  295. * EST SYNC reset counters; move to ACK; notify
  296. *
  297. * ===============================================================
  298. */
  299. int tegra_ivc_channel_notified(struct tegra_ivc *ivc)
  300. {
  301. ulong offset;
  302. enum ivc_state peer_state;
  303. /* Copy the receiver's state out of shared memory. */
  304. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  305. tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
  306. peer_state = ACCESS_ONCE(ivc->rx_channel->state);
  307. if (peer_state == ivc_state_sync) {
  308. /*
  309. * Order observation of ivc_state_sync before stores clearing
  310. * tx_channel.
  311. */
  312. mb();
  313. /*
  314. * Reset tx_channel counters. The remote end is in the SYNC
  315. * state and won't make progress until we change our state,
  316. * so the counters are not in use at this time.
  317. */
  318. ivc->tx_channel->w_count = 0;
  319. ivc->rx_channel->r_count = 0;
  320. ivc->w_pos = 0;
  321. ivc->r_pos = 0;
  322. /*
  323. * Ensure that counters appear cleared before new state can be
  324. * observed.
  325. */
  326. mb();
  327. /*
  328. * Move to ACK state. We have just cleared our counters, so it
  329. * is now safe for the remote end to start using these values.
  330. */
  331. ivc->tx_channel->state = ivc_state_ack;
  332. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  333. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  334. /*
  335. * Notify remote end to observe state transition.
  336. */
  337. ivc->notify(ivc);
  338. } else if (ivc->tx_channel->state == ivc_state_sync &&
  339. peer_state == ivc_state_ack) {
  340. /*
  341. * Order observation of ivc_state_sync before stores clearing
  342. * tx_channel.
  343. */
  344. mb();
  345. /*
  346. * Reset tx_channel counters. The remote end is in the ACK
  347. * state and won't make progress until we change our state,
  348. * so the counters are not in use at this time.
  349. */
  350. ivc->tx_channel->w_count = 0;
  351. ivc->rx_channel->r_count = 0;
  352. ivc->w_pos = 0;
  353. ivc->r_pos = 0;
  354. /*
  355. * Ensure that counters appear cleared before new state can be
  356. * observed.
  357. */
  358. mb();
  359. /*
  360. * Move to ESTABLISHED state. We know that the remote end has
  361. * already cleared its counters, so it is safe to start
  362. * writing/reading on this channel.
  363. */
  364. ivc->tx_channel->state = ivc_state_established;
  365. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  366. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  367. /*
  368. * Notify remote end to observe state transition.
  369. */
  370. ivc->notify(ivc);
  371. } else if (ivc->tx_channel->state == ivc_state_ack) {
  372. /*
  373. * At this point, we have observed the peer to be in either
  374. * the ACK or ESTABLISHED state. Next, order observation of
  375. * peer state before storing to tx_channel.
  376. */
  377. mb();
  378. /*
  379. * Move to ESTABLISHED state. We know that we have previously
  380. * cleared our counters, and we know that the remote end has
  381. * cleared its counters, so it is safe to start writing/reading
  382. * on this channel.
  383. */
  384. ivc->tx_channel->state = ivc_state_established;
  385. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  386. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  387. /*
  388. * Notify remote end to observe state transition.
  389. */
  390. ivc->notify(ivc);
  391. } else {
  392. /*
  393. * There is no need to handle any further action. Either the
  394. * channel is already fully established, or we are waiting for
  395. * the remote end to catch up with our current state. Refer
  396. * to the diagram in "IVC State Transition Table" above.
  397. */
  398. }
  399. if (ivc->tx_channel->state != ivc_state_established)
  400. return -EAGAIN;
  401. return 0;
  402. }
  403. void tegra_ivc_channel_reset(struct tegra_ivc *ivc)
  404. {
  405. ulong offset;
  406. ivc->tx_channel->state = ivc_state_sync;
  407. offset = offsetof(struct tegra_ivc_channel_header, w_count);
  408. tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
  409. ivc->notify(ivc);
  410. }
  411. static int check_ivc_params(ulong qbase1, ulong qbase2, uint32_t nframes,
  412. uint32_t frame_size)
  413. {
  414. int ret = 0;
  415. BUG_ON(offsetof(struct tegra_ivc_channel_header, w_count) &
  416. (TEGRA_IVC_ALIGN - 1));
  417. BUG_ON(offsetof(struct tegra_ivc_channel_header, r_count) &
  418. (TEGRA_IVC_ALIGN - 1));
  419. BUG_ON(sizeof(struct tegra_ivc_channel_header) &
  420. (TEGRA_IVC_ALIGN - 1));
  421. if ((uint64_t)nframes * (uint64_t)frame_size >= 0x100000000) {
  422. pr_err("tegra_ivc: nframes * frame_size overflows\n");
  423. return -EINVAL;
  424. }
  425. /*
  426. * The headers must at least be aligned enough for counters
  427. * to be accessed atomically.
  428. */
  429. if ((qbase1 & (TEGRA_IVC_ALIGN - 1)) ||
  430. (qbase2 & (TEGRA_IVC_ALIGN - 1))) {
  431. pr_err("tegra_ivc: channel start not aligned\n");
  432. return -EINVAL;
  433. }
  434. if (frame_size & (TEGRA_IVC_ALIGN - 1)) {
  435. pr_err("tegra_ivc: frame size not adequately aligned\n");
  436. return -EINVAL;
  437. }
  438. if (qbase1 < qbase2) {
  439. if (qbase1 + frame_size * nframes > qbase2)
  440. ret = -EINVAL;
  441. } else {
  442. if (qbase2 + frame_size * nframes > qbase1)
  443. ret = -EINVAL;
  444. }
  445. if (ret) {
  446. pr_err("tegra_ivc: queue regions overlap\n");
  447. return ret;
  448. }
  449. return 0;
  450. }
  451. int tegra_ivc_init(struct tegra_ivc *ivc, ulong rx_base, ulong tx_base,
  452. uint32_t nframes, uint32_t frame_size,
  453. void (*notify)(struct tegra_ivc *))
  454. {
  455. int ret;
  456. if (!ivc)
  457. return -EINVAL;
  458. ret = check_ivc_params(rx_base, tx_base, nframes, frame_size);
  459. if (ret)
  460. return ret;
  461. ivc->rx_channel = (struct tegra_ivc_channel_header *)rx_base;
  462. ivc->tx_channel = (struct tegra_ivc_channel_header *)tx_base;
  463. ivc->w_pos = 0;
  464. ivc->r_pos = 0;
  465. ivc->nframes = nframes;
  466. ivc->frame_size = frame_size;
  467. ivc->notify = notify;
  468. return 0;
  469. }