ring_buffer.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright (c) 2009, Microsoft Corporation.
  5. *
  6. * Authors:
  7. * Haiyang Zhang <haiyangz@microsoft.com>
  8. * Hank Janssen <hjanssen@microsoft.com>
  9. * K. Y. Srinivasan <kys@microsoft.com>
  10. */
  11. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12. #include <linux/kernel.h>
  13. #include <linux/mm.h>
  14. #include <linux/hyperv.h>
  15. #include <linux/uio.h>
  16. #include <linux/vmalloc.h>
  17. #include <linux/slab.h>
  18. #include <linux/prefetch.h>
  19. #include "hyperv_vmbus.h"
  20. #define VMBUS_PKT_TRAILER 8
  21. /*
  22. * When we write to the ring buffer, check if the host needs to
  23. * be signaled. Here is the details of this protocol:
  24. *
  25. * 1. The host guarantees that while it is draining the
  26. * ring buffer, it will set the interrupt_mask to
  27. * indicate it does not need to be interrupted when
  28. * new data is placed.
  29. *
  30. * 2. The host guarantees that it will completely drain
  31. * the ring buffer before exiting the read loop. Further,
  32. * once the ring buffer is empty, it will clear the
  33. * interrupt_mask and re-check to see if new data has
  34. * arrived.
  35. *
  36. * KYS: Oct. 30, 2016:
  37. * It looks like Windows hosts have logic to deal with DOS attacks that
  38. * can be triggered if it receives interrupts when it is not expecting
  39. * the interrupt. The host expects interrupts only when the ring
  40. * transitions from empty to non-empty (or full to non full on the guest
  41. * to host ring).
  42. * So, base the signaling decision solely on the ring state until the
  43. * host logic is fixed.
  44. */
  45. static void hv_signal_on_write(u32 old_write, struct vmbus_channel *channel)
  46. {
  47. struct hv_ring_buffer_info *rbi = &channel->outbound;
  48. virt_mb();
  49. if (READ_ONCE(rbi->ring_buffer->interrupt_mask))
  50. return;
  51. /* check interrupt_mask before read_index */
  52. virt_rmb();
  53. /*
  54. * This is the only case we need to signal when the
  55. * ring transitions from being empty to non-empty.
  56. */
  57. if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) {
  58. ++channel->intr_out_empty;
  59. vmbus_setevent(channel);
  60. }
  61. }
  62. /* Get the next write location for the specified ring buffer. */
  63. static inline u32
  64. hv_get_next_write_location(struct hv_ring_buffer_info *ring_info)
  65. {
  66. u32 next = ring_info->ring_buffer->write_index;
  67. return next;
  68. }
  69. /* Set the next write location for the specified ring buffer. */
  70. static inline void
  71. hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
  72. u32 next_write_location)
  73. {
  74. ring_info->ring_buffer->write_index = next_write_location;
  75. }
  76. /* Set the next read location for the specified ring buffer. */
  77. static inline void
  78. hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
  79. u32 next_read_location)
  80. {
  81. ring_info->ring_buffer->read_index = next_read_location;
  82. ring_info->priv_read_index = next_read_location;
  83. }
  84. /* Get the size of the ring buffer. */
  85. static inline u32
  86. hv_get_ring_buffersize(const struct hv_ring_buffer_info *ring_info)
  87. {
  88. return ring_info->ring_datasize;
  89. }
  90. /* Get the read and write indices as u64 of the specified ring buffer. */
  91. static inline u64
  92. hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
  93. {
  94. return (u64)ring_info->ring_buffer->write_index << 32;
  95. }
  96. /*
  97. * Helper routine to copy from source to ring buffer.
  98. * Assume there is enough room. Handles wrap-around in dest case only!!
  99. */
  100. static u32 hv_copyto_ringbuffer(
  101. struct hv_ring_buffer_info *ring_info,
  102. u32 start_write_offset,
  103. const void *src,
  104. u32 srclen)
  105. {
  106. void *ring_buffer = hv_get_ring_buffer(ring_info);
  107. u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
  108. memcpy(ring_buffer + start_write_offset, src, srclen);
  109. start_write_offset += srclen;
  110. if (start_write_offset >= ring_buffer_size)
  111. start_write_offset -= ring_buffer_size;
  112. return start_write_offset;
  113. }
  114. /*
  115. *
  116. * hv_get_ringbuffer_availbytes()
  117. *
  118. * Get number of bytes available to read and to write to
  119. * for the specified ring buffer
  120. */
  121. static void
  122. hv_get_ringbuffer_availbytes(const struct hv_ring_buffer_info *rbi,
  123. u32 *read, u32 *write)
  124. {
  125. u32 read_loc, write_loc, dsize;
  126. /* Capture the read/write indices before they changed */
  127. read_loc = READ_ONCE(rbi->ring_buffer->read_index);
  128. write_loc = READ_ONCE(rbi->ring_buffer->write_index);
  129. dsize = rbi->ring_datasize;
  130. *write = write_loc >= read_loc ? dsize - (write_loc - read_loc) :
  131. read_loc - write_loc;
  132. *read = dsize - *write;
  133. }
  134. /* Get various debug metrics for the specified ring buffer. */
  135. int hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
  136. struct hv_ring_buffer_debug_info *debug_info)
  137. {
  138. u32 bytes_avail_towrite;
  139. u32 bytes_avail_toread;
  140. mutex_lock(&ring_info->ring_buffer_mutex);
  141. if (!ring_info->ring_buffer) {
  142. mutex_unlock(&ring_info->ring_buffer_mutex);
  143. return -EINVAL;
  144. }
  145. hv_get_ringbuffer_availbytes(ring_info,
  146. &bytes_avail_toread,
  147. &bytes_avail_towrite);
  148. debug_info->bytes_avail_toread = bytes_avail_toread;
  149. debug_info->bytes_avail_towrite = bytes_avail_towrite;
  150. debug_info->current_read_index = ring_info->ring_buffer->read_index;
  151. debug_info->current_write_index = ring_info->ring_buffer->write_index;
  152. debug_info->current_interrupt_mask
  153. = ring_info->ring_buffer->interrupt_mask;
  154. mutex_unlock(&ring_info->ring_buffer_mutex);
  155. return 0;
  156. }
  157. EXPORT_SYMBOL_GPL(hv_ringbuffer_get_debuginfo);
  158. /* Initialize a channel's ring buffer info mutex locks */
  159. void hv_ringbuffer_pre_init(struct vmbus_channel *channel)
  160. {
  161. mutex_init(&channel->inbound.ring_buffer_mutex);
  162. mutex_init(&channel->outbound.ring_buffer_mutex);
  163. }
  164. /* Initialize the ring buffer. */
  165. int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
  166. struct page *pages, u32 page_cnt)
  167. {
  168. int i;
  169. struct page **pages_wraparound;
  170. BUILD_BUG_ON((sizeof(struct hv_ring_buffer) != PAGE_SIZE));
  171. /*
  172. * First page holds struct hv_ring_buffer, do wraparound mapping for
  173. * the rest.
  174. */
  175. pages_wraparound = kcalloc(page_cnt * 2 - 1, sizeof(struct page *),
  176. GFP_KERNEL);
  177. if (!pages_wraparound)
  178. return -ENOMEM;
  179. pages_wraparound[0] = pages;
  180. for (i = 0; i < 2 * (page_cnt - 1); i++)
  181. pages_wraparound[i + 1] = &pages[i % (page_cnt - 1) + 1];
  182. ring_info->ring_buffer = (struct hv_ring_buffer *)
  183. vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, PAGE_KERNEL);
  184. kfree(pages_wraparound);
  185. if (!ring_info->ring_buffer)
  186. return -ENOMEM;
  187. ring_info->ring_buffer->read_index =
  188. ring_info->ring_buffer->write_index = 0;
  189. /* Set the feature bit for enabling flow control. */
  190. ring_info->ring_buffer->feature_bits.value = 1;
  191. ring_info->ring_size = page_cnt << PAGE_SHIFT;
  192. ring_info->ring_size_div10_reciprocal =
  193. reciprocal_value(ring_info->ring_size / 10);
  194. ring_info->ring_datasize = ring_info->ring_size -
  195. sizeof(struct hv_ring_buffer);
  196. ring_info->priv_read_index = 0;
  197. spin_lock_init(&ring_info->ring_lock);
  198. return 0;
  199. }
  200. /* Cleanup the ring buffer. */
  201. void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
  202. {
  203. mutex_lock(&ring_info->ring_buffer_mutex);
  204. vunmap(ring_info->ring_buffer);
  205. ring_info->ring_buffer = NULL;
  206. mutex_unlock(&ring_info->ring_buffer_mutex);
  207. }
  208. /* Write to the ring buffer. */
  209. int hv_ringbuffer_write(struct vmbus_channel *channel,
  210. const struct kvec *kv_list, u32 kv_count)
  211. {
  212. int i;
  213. u32 bytes_avail_towrite;
  214. u32 totalbytes_towrite = sizeof(u64);
  215. u32 next_write_location;
  216. u32 old_write;
  217. u64 prev_indices;
  218. unsigned long flags;
  219. struct hv_ring_buffer_info *outring_info = &channel->outbound;
  220. if (channel->rescind)
  221. return -ENODEV;
  222. for (i = 0; i < kv_count; i++)
  223. totalbytes_towrite += kv_list[i].iov_len;
  224. spin_lock_irqsave(&outring_info->ring_lock, flags);
  225. bytes_avail_towrite = hv_get_bytes_to_write(outring_info);
  226. /*
  227. * If there is only room for the packet, assume it is full.
  228. * Otherwise, the next time around, we think the ring buffer
  229. * is empty since the read index == write index.
  230. */
  231. if (bytes_avail_towrite <= totalbytes_towrite) {
  232. ++channel->out_full_total;
  233. if (!channel->out_full_flag) {
  234. ++channel->out_full_first;
  235. channel->out_full_flag = true;
  236. }
  237. spin_unlock_irqrestore(&outring_info->ring_lock, flags);
  238. return -EAGAIN;
  239. }
  240. channel->out_full_flag = false;
  241. /* Write to the ring buffer */
  242. next_write_location = hv_get_next_write_location(outring_info);
  243. old_write = next_write_location;
  244. for (i = 0; i < kv_count; i++) {
  245. next_write_location = hv_copyto_ringbuffer(outring_info,
  246. next_write_location,
  247. kv_list[i].iov_base,
  248. kv_list[i].iov_len);
  249. }
  250. /* Set previous packet start */
  251. prev_indices = hv_get_ring_bufferindices(outring_info);
  252. next_write_location = hv_copyto_ringbuffer(outring_info,
  253. next_write_location,
  254. &prev_indices,
  255. sizeof(u64));
  256. /* Issue a full memory barrier before updating the write index */
  257. virt_mb();
  258. /* Now, update the write location */
  259. hv_set_next_write_location(outring_info, next_write_location);
  260. spin_unlock_irqrestore(&outring_info->ring_lock, flags);
  261. hv_signal_on_write(old_write, channel);
  262. if (channel->rescind)
  263. return -ENODEV;
  264. return 0;
  265. }
  266. int hv_ringbuffer_read(struct vmbus_channel *channel,
  267. void *buffer, u32 buflen, u32 *buffer_actual_len,
  268. u64 *requestid, bool raw)
  269. {
  270. struct vmpacket_descriptor *desc;
  271. u32 packetlen, offset;
  272. if (unlikely(buflen == 0))
  273. return -EINVAL;
  274. *buffer_actual_len = 0;
  275. *requestid = 0;
  276. /* Make sure there is something to read */
  277. desc = hv_pkt_iter_first(channel);
  278. if (desc == NULL) {
  279. /*
  280. * No error is set when there is even no header, drivers are
  281. * supposed to analyze buffer_actual_len.
  282. */
  283. return 0;
  284. }
  285. offset = raw ? 0 : (desc->offset8 << 3);
  286. packetlen = (desc->len8 << 3) - offset;
  287. *buffer_actual_len = packetlen;
  288. *requestid = desc->trans_id;
  289. if (unlikely(packetlen > buflen))
  290. return -ENOBUFS;
  291. /* since ring is double mapped, only one copy is necessary */
  292. memcpy(buffer, (const char *)desc + offset, packetlen);
  293. /* Advance ring index to next packet descriptor */
  294. __hv_pkt_iter_next(channel, desc);
  295. /* Notify host of update */
  296. hv_pkt_iter_close(channel);
  297. return 0;
  298. }
  299. /*
  300. * Determine number of bytes available in ring buffer after
  301. * the current iterator (priv_read_index) location.
  302. *
  303. * This is similar to hv_get_bytes_to_read but with private
  304. * read index instead.
  305. */
  306. static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi)
  307. {
  308. u32 priv_read_loc = rbi->priv_read_index;
  309. u32 write_loc;
  310. /*
  311. * The Hyper-V host writes the packet data, then uses
  312. * store_release() to update the write_index. Use load_acquire()
  313. * here to prevent loads of the packet data from being re-ordered
  314. * before the read of the write_index and potentially getting
  315. * stale data.
  316. */
  317. write_loc = virt_load_acquire(&rbi->ring_buffer->write_index);
  318. if (write_loc >= priv_read_loc)
  319. return write_loc - priv_read_loc;
  320. else
  321. return (rbi->ring_datasize - priv_read_loc) + write_loc;
  322. }
  323. /*
  324. * Get first vmbus packet from ring buffer after read_index
  325. *
  326. * If ring buffer is empty, returns NULL and no other action needed.
  327. */
  328. struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel)
  329. {
  330. struct hv_ring_buffer_info *rbi = &channel->inbound;
  331. struct vmpacket_descriptor *desc;
  332. hv_debug_delay_test(channel, MESSAGE_DELAY);
  333. if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor))
  334. return NULL;
  335. desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index;
  336. if (desc)
  337. prefetch((char *)desc + (desc->len8 << 3));
  338. return desc;
  339. }
  340. EXPORT_SYMBOL_GPL(hv_pkt_iter_first);
  341. /*
  342. * Get next vmbus packet from ring buffer.
  343. *
  344. * Advances the current location (priv_read_index) and checks for more
  345. * data. If the end of the ring buffer is reached, then return NULL.
  346. */
  347. struct vmpacket_descriptor *
  348. __hv_pkt_iter_next(struct vmbus_channel *channel,
  349. const struct vmpacket_descriptor *desc)
  350. {
  351. struct hv_ring_buffer_info *rbi = &channel->inbound;
  352. u32 packetlen = desc->len8 << 3;
  353. u32 dsize = rbi->ring_datasize;
  354. hv_debug_delay_test(channel, MESSAGE_DELAY);
  355. /* bump offset to next potential packet */
  356. rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER;
  357. if (rbi->priv_read_index >= dsize)
  358. rbi->priv_read_index -= dsize;
  359. /* more data? */
  360. return hv_pkt_iter_first(channel);
  361. }
  362. EXPORT_SYMBOL_GPL(__hv_pkt_iter_next);
  363. /* How many bytes were read in this iterator cycle */
  364. static u32 hv_pkt_iter_bytes_read(const struct hv_ring_buffer_info *rbi,
  365. u32 start_read_index)
  366. {
  367. if (rbi->priv_read_index >= start_read_index)
  368. return rbi->priv_read_index - start_read_index;
  369. else
  370. return rbi->ring_datasize - start_read_index +
  371. rbi->priv_read_index;
  372. }
  373. /*
  374. * Update host ring buffer after iterating over packets. If the host has
  375. * stopped queuing new entries because it found the ring buffer full, and
  376. * sufficient space is being freed up, signal the host. But be careful to
  377. * only signal the host when necessary, both for performance reasons and
  378. * because Hyper-V protects itself by throttling guests that signal
  379. * inappropriately.
  380. *
  381. * Determining when to signal is tricky. There are three key data inputs
  382. * that must be handled in this order to avoid race conditions:
  383. *
  384. * 1. Update the read_index
  385. * 2. Read the pending_send_sz
  386. * 3. Read the current write_index
  387. *
  388. * The interrupt_mask is not used to determine when to signal. The
  389. * interrupt_mask is used only on the guest->host ring buffer when
  390. * sending requests to the host. The host does not use it on the host->
  391. * guest ring buffer to indicate whether it should be signaled.
  392. */
  393. void hv_pkt_iter_close(struct vmbus_channel *channel)
  394. {
  395. struct hv_ring_buffer_info *rbi = &channel->inbound;
  396. u32 curr_write_sz, pending_sz, bytes_read, start_read_index;
  397. /*
  398. * Make sure all reads are done before we update the read index since
  399. * the writer may start writing to the read area once the read index
  400. * is updated.
  401. */
  402. virt_rmb();
  403. start_read_index = rbi->ring_buffer->read_index;
  404. rbi->ring_buffer->read_index = rbi->priv_read_index;
  405. /*
  406. * Older versions of Hyper-V (before WS2102 and Win8) do not
  407. * implement pending_send_sz and simply poll if the host->guest
  408. * ring buffer is full. No signaling is needed or expected.
  409. */
  410. if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz)
  411. return;
  412. /*
  413. * Issue a full memory barrier before making the signaling decision.
  414. * If reading pending_send_sz were to be reordered and happen
  415. * before we commit the new read_index, a race could occur. If the
  416. * host were to set the pending_send_sz after we have sampled
  417. * pending_send_sz, and the ring buffer blocks before we commit the
  418. * read index, we could miss sending the interrupt. Issue a full
  419. * memory barrier to address this.
  420. */
  421. virt_mb();
  422. /*
  423. * If the pending_send_sz is zero, then the ring buffer is not
  424. * blocked and there is no need to signal. This is far by the
  425. * most common case, so exit quickly for best performance.
  426. */
  427. pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
  428. if (!pending_sz)
  429. return;
  430. /*
  431. * Ensure the read of write_index in hv_get_bytes_to_write()
  432. * happens after the read of pending_send_sz.
  433. */
  434. virt_rmb();
  435. curr_write_sz = hv_get_bytes_to_write(rbi);
  436. bytes_read = hv_pkt_iter_bytes_read(rbi, start_read_index);
  437. /*
  438. * We want to signal the host only if we're transitioning
  439. * from a "not enough free space" state to a "enough free
  440. * space" state. For example, it's possible that this function
  441. * could run and free up enough space to signal the host, and then
  442. * run again and free up additional space before the host has a
  443. * chance to clear the pending_send_sz. The 2nd invocation would
  444. * be a null transition from "enough free space" to "enough free
  445. * space", which doesn't warrant a signal.
  446. *
  447. * Exactly filling the ring buffer is treated as "not enough
  448. * space". The ring buffer always must have at least one byte
  449. * empty so the empty and full conditions are distinguishable.
  450. * hv_get_bytes_to_write() doesn't fully tell the truth in
  451. * this regard.
  452. *
  453. * So first check if we were in the "enough free space" state
  454. * before we began the iteration. If so, the host was not
  455. * blocked, and there's no need to signal.
  456. */
  457. if (curr_write_sz - bytes_read > pending_sz)
  458. return;
  459. /*
  460. * Similarly, if the new state is "not enough space", then
  461. * there's no need to signal.
  462. */
  463. if (curr_write_sz <= pending_sz)
  464. return;
  465. ++channel->intr_in_full;
  466. vmbus_setevent(channel);
  467. }
  468. EXPORT_SYMBOL_GPL(hv_pkt_iter_close);