rdmavt_mr.h 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155
  1. /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
  2. /*
  3. * Copyright(c) 2016 Intel Corporation.
  4. */
  5. #ifndef DEF_RDMAVT_INCMR_H
  6. #define DEF_RDMAVT_INCMR_H
  7. /*
  8. * For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
  9. * drivers no longer need access to the MR directly.
  10. */
  11. #include <linux/percpu-refcount.h>
  12. /*
  13. * A segment is a linear region of low physical memory.
  14. * Used by the verbs layer.
  15. */
  16. struct rvt_seg {
  17. void *vaddr;
  18. size_t length;
  19. };
  20. /* The number of rvt_segs that fit in a page. */
  21. #define RVT_SEGSZ (PAGE_SIZE / sizeof(struct rvt_seg))
  22. struct rvt_segarray {
  23. struct rvt_seg segs[RVT_SEGSZ];
  24. };
  25. struct rvt_mregion {
  26. struct ib_pd *pd; /* shares refcnt of ibmr.pd */
  27. u64 user_base; /* User's address for this region */
  28. u64 iova; /* IB start address of this region */
  29. size_t length;
  30. u32 lkey;
  31. u32 offset; /* offset (bytes) to start of region */
  32. int access_flags;
  33. u32 max_segs; /* number of rvt_segs in all the arrays */
  34. u32 mapsz; /* size of the map array */
  35. atomic_t lkey_invalid; /* true if current lkey is invalid */
  36. u8 page_shift; /* 0 - non unform/non powerof2 sizes */
  37. u8 lkey_published; /* in global table */
  38. struct percpu_ref refcount;
  39. struct completion comp; /* complete when refcount goes to zero */
  40. struct rvt_segarray *map[]; /* the segments */
  41. };
  42. #define RVT_MAX_LKEY_TABLE_BITS 23
  43. struct rvt_lkey_table {
  44. /* read mostly fields */
  45. u32 max; /* size of the table */
  46. u32 shift; /* lkey/rkey shift */
  47. struct rvt_mregion __rcu **table;
  48. /* writeable fields */
  49. /* protect changes in this struct */
  50. spinlock_t lock ____cacheline_aligned_in_smp;
  51. u32 next; /* next unused index (speeds search) */
  52. u32 gen; /* generation count */
  53. };
  54. /*
  55. * These keep track of the copy progress within a memory region.
  56. * Used by the verbs layer.
  57. */
  58. struct rvt_sge {
  59. struct rvt_mregion *mr;
  60. void *vaddr; /* kernel virtual address of segment */
  61. u32 sge_length; /* length of the SGE */
  62. u32 length; /* remaining length of the segment */
  63. u16 m; /* current index: mr->map[m] */
  64. u16 n; /* current index: mr->map[m]->segs[n] */
  65. };
  66. struct rvt_sge_state {
  67. struct rvt_sge *sg_list; /* next SGE to be used if any */
  68. struct rvt_sge sge; /* progress state for the current SGE */
  69. u32 total_len;
  70. u8 num_sge;
  71. };
  72. static inline void rvt_put_mr(struct rvt_mregion *mr)
  73. {
  74. percpu_ref_put(&mr->refcount);
  75. }
  76. static inline void rvt_get_mr(struct rvt_mregion *mr)
  77. {
  78. percpu_ref_get(&mr->refcount);
  79. }
  80. static inline void rvt_put_ss(struct rvt_sge_state *ss)
  81. {
  82. while (ss->num_sge) {
  83. rvt_put_mr(ss->sge.mr);
  84. if (--ss->num_sge)
  85. ss->sge = *ss->sg_list++;
  86. }
  87. }
  88. static inline u32 rvt_get_sge_length(struct rvt_sge *sge, u32 length)
  89. {
  90. u32 len = sge->length;
  91. if (len > length)
  92. len = length;
  93. if (len > sge->sge_length)
  94. len = sge->sge_length;
  95. return len;
  96. }
  97. static inline void rvt_update_sge(struct rvt_sge_state *ss, u32 length,
  98. bool release)
  99. {
  100. struct rvt_sge *sge = &ss->sge;
  101. sge->vaddr += length;
  102. sge->length -= length;
  103. sge->sge_length -= length;
  104. if (sge->sge_length == 0) {
  105. if (release)
  106. rvt_put_mr(sge->mr);
  107. if (--ss->num_sge)
  108. *sge = *ss->sg_list++;
  109. } else if (sge->length == 0 && sge->mr->lkey) {
  110. if (++sge->n >= RVT_SEGSZ) {
  111. if (++sge->m >= sge->mr->mapsz)
  112. return;
  113. sge->n = 0;
  114. }
  115. sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
  116. sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
  117. }
  118. }
  119. static inline void rvt_skip_sge(struct rvt_sge_state *ss, u32 length,
  120. bool release)
  121. {
  122. struct rvt_sge *sge = &ss->sge;
  123. while (length) {
  124. u32 len = rvt_get_sge_length(sge, length);
  125. WARN_ON_ONCE(len == 0);
  126. rvt_update_sge(ss, len, release);
  127. length -= len;
  128. }
  129. }
  130. bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey);
  131. bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey);
  132. #endif /* DEF_RDMAVT_INCMRH */