swap_cgroup.c 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/swap_cgroup.h>
  3. #include <linux/vmalloc.h>
  4. #include <linux/mm.h>
  5. #include <linux/swapops.h> /* depends on mm.h include */
  6. static DEFINE_MUTEX(swap_cgroup_mutex);
  7. struct swap_cgroup_ctrl {
  8. struct page **map;
  9. unsigned long length;
  10. spinlock_t lock;
  11. };
  12. static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
  13. struct swap_cgroup {
  14. unsigned short id;
  15. };
  16. #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
  17. /*
  18. * SwapCgroup implements "lookup" and "exchange" operations.
  19. * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
  20. * against SwapCache. At swap_free(), this is accessed directly from swap.
  21. *
  22. * This means,
  23. * - we have no race in "exchange" when we're accessed via SwapCache because
  24. * SwapCache(and its swp_entry) is under lock.
  25. * - When called via swap_free(), there is no user of this entry and no race.
  26. * Then, we don't need lock around "exchange".
  27. *
  28. * TODO: we can push these buffers out to HIGHMEM.
  29. */
  30. /*
  31. * allocate buffer for swap_cgroup.
  32. */
  33. static int swap_cgroup_prepare(int type)
  34. {
  35. struct page *page;
  36. struct swap_cgroup_ctrl *ctrl;
  37. unsigned long idx, max;
  38. ctrl = &swap_cgroup_ctrl[type];
  39. for (idx = 0; idx < ctrl->length; idx++) {
  40. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  41. if (!page)
  42. goto not_enough_page;
  43. ctrl->map[idx] = page;
  44. if (!(idx % SWAP_CLUSTER_MAX))
  45. cond_resched();
  46. }
  47. return 0;
  48. not_enough_page:
  49. max = idx;
  50. for (idx = 0; idx < max; idx++)
  51. __free_page(ctrl->map[idx]);
  52. return -ENOMEM;
  53. }
  54. static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
  55. pgoff_t offset)
  56. {
  57. struct page *mappage;
  58. struct swap_cgroup *sc;
  59. mappage = ctrl->map[offset / SC_PER_PAGE];
  60. sc = page_address(mappage);
  61. return sc + offset % SC_PER_PAGE;
  62. }
  63. static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
  64. struct swap_cgroup_ctrl **ctrlp)
  65. {
  66. pgoff_t offset = swp_offset(ent);
  67. struct swap_cgroup_ctrl *ctrl;
  68. ctrl = &swap_cgroup_ctrl[swp_type(ent)];
  69. if (ctrlp)
  70. *ctrlp = ctrl;
  71. return __lookup_swap_cgroup(ctrl, offset);
  72. }
  73. /**
  74. * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
  75. * @ent: swap entry to be cmpxchged
  76. * @old: old id
  77. * @new: new id
  78. *
  79. * Returns old id at success, 0 at failure.
  80. * (There is no mem_cgroup using 0 as its id)
  81. */
  82. unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
  83. unsigned short old, unsigned short new)
  84. {
  85. struct swap_cgroup_ctrl *ctrl;
  86. struct swap_cgroup *sc;
  87. unsigned long flags;
  88. unsigned short retval;
  89. sc = lookup_swap_cgroup(ent, &ctrl);
  90. spin_lock_irqsave(&ctrl->lock, flags);
  91. retval = sc->id;
  92. if (retval == old)
  93. sc->id = new;
  94. else
  95. retval = 0;
  96. spin_unlock_irqrestore(&ctrl->lock, flags);
  97. return retval;
  98. }
  99. /**
  100. * swap_cgroup_record - record mem_cgroup for a set of swap entries
  101. * @ent: the first swap entry to be recorded into
  102. * @id: mem_cgroup to be recorded
  103. * @nr_ents: number of swap entries to be recorded
  104. *
  105. * Returns old value at success, 0 at failure.
  106. * (Of course, old value can be 0.)
  107. */
  108. unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
  109. unsigned int nr_ents)
  110. {
  111. struct swap_cgroup_ctrl *ctrl;
  112. struct swap_cgroup *sc;
  113. unsigned short old;
  114. unsigned long flags;
  115. pgoff_t offset = swp_offset(ent);
  116. pgoff_t end = offset + nr_ents;
  117. sc = lookup_swap_cgroup(ent, &ctrl);
  118. spin_lock_irqsave(&ctrl->lock, flags);
  119. old = sc->id;
  120. for (;;) {
  121. VM_BUG_ON(sc->id != old);
  122. sc->id = id;
  123. offset++;
  124. if (offset == end)
  125. break;
  126. if (offset % SC_PER_PAGE)
  127. sc++;
  128. else
  129. sc = __lookup_swap_cgroup(ctrl, offset);
  130. }
  131. spin_unlock_irqrestore(&ctrl->lock, flags);
  132. return old;
  133. }
  134. /**
  135. * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
  136. * @ent: swap entry to be looked up.
  137. *
  138. * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
  139. */
  140. unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
  141. {
  142. return lookup_swap_cgroup(ent, NULL)->id;
  143. }
  144. int swap_cgroup_swapon(int type, unsigned long max_pages)
  145. {
  146. void *array;
  147. unsigned long array_size;
  148. unsigned long length;
  149. struct swap_cgroup_ctrl *ctrl;
  150. length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
  151. array_size = length * sizeof(void *);
  152. array = vzalloc(array_size);
  153. if (!array)
  154. goto nomem;
  155. ctrl = &swap_cgroup_ctrl[type];
  156. mutex_lock(&swap_cgroup_mutex);
  157. ctrl->length = length;
  158. ctrl->map = array;
  159. spin_lock_init(&ctrl->lock);
  160. if (swap_cgroup_prepare(type)) {
  161. /* memory shortage */
  162. ctrl->map = NULL;
  163. ctrl->length = 0;
  164. mutex_unlock(&swap_cgroup_mutex);
  165. vfree(array);
  166. goto nomem;
  167. }
  168. mutex_unlock(&swap_cgroup_mutex);
  169. return 0;
  170. nomem:
  171. pr_info("couldn't allocate enough memory for swap_cgroup\n");
  172. pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
  173. return -ENOMEM;
  174. }
  175. void swap_cgroup_swapoff(int type)
  176. {
  177. struct page **map;
  178. unsigned long i, length;
  179. struct swap_cgroup_ctrl *ctrl;
  180. mutex_lock(&swap_cgroup_mutex);
  181. ctrl = &swap_cgroup_ctrl[type];
  182. map = ctrl->map;
  183. length = ctrl->length;
  184. ctrl->map = NULL;
  185. ctrl->length = 0;
  186. mutex_unlock(&swap_cgroup_mutex);
  187. if (map) {
  188. for (i = 0; i < length; i++) {
  189. struct page *page = map[i];
  190. if (page)
  191. __free_page(page);
  192. if (!(i % SWAP_CLUSTER_MAX))
  193. cond_resched();
  194. }
  195. vfree(map);
  196. }
  197. }