slob.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * SLOB Allocator: Simple List Of Blocks
  4. *
  5. * Matt Mackall <mpm@selenic.com> 12/30/03
  6. *
  7. * NUMA support by Paul Mundt, 2007.
  8. *
  9. * How SLOB works:
  10. *
  11. * The core of SLOB is a traditional K&R style heap allocator, with
  12. * support for returning aligned objects. The granularity of this
  13. * allocator is as little as 2 bytes, however typically most architectures
  14. * will require 4 bytes on 32-bit and 8 bytes on 64-bit.
  15. *
  16. * The slob heap is a set of linked list of pages from alloc_pages(),
  17. * and within each page, there is a singly-linked list of free blocks
  18. * (slob_t). The heap is grown on demand. To reduce fragmentation,
  19. * heap pages are segregated into three lists, with objects less than
  20. * 256 bytes, objects less than 1024 bytes, and all other objects.
  21. *
  22. * Allocation from heap involves first searching for a page with
  23. * sufficient free blocks (using a next-fit-like approach) followed by
  24. * a first-fit scan of the page. Deallocation inserts objects back
  25. * into the free list in address order, so this is effectively an
  26. * address-ordered first fit.
  27. *
  28. * Above this is an implementation of kmalloc/kfree. Blocks returned
  29. * from kmalloc are prepended with a 4-byte header with the kmalloc size.
  30. * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls
  31. * alloc_pages() directly, allocating compound pages so the page order
  32. * does not have to be separately tracked.
  33. * These objects are detected in kfree() because PageSlab()
  34. * is false for them.
  35. *
  36. * SLAB is emulated on top of SLOB by simply calling constructors and
  37. * destructors for every SLAB allocation. Objects are returned with the
  38. * 4-byte alignment unless the SLAB_HWCACHE_ALIGN flag is set, in which
  39. * case the low-level allocator will fragment blocks to create the proper
  40. * alignment. Again, objects of page-size or greater are allocated by
  41. * calling alloc_pages(). As SLAB objects know their size, no separate
  42. * size bookkeeping is necessary and there is essentially no allocation
  43. * space overhead, and compound pages aren't needed for multi-page
  44. * allocations.
  45. *
  46. * NUMA support in SLOB is fairly simplistic, pushing most of the real
  47. * logic down to the page allocator, and simply doing the node accounting
  48. * on the upper levels. In the event that a node id is explicitly
  49. * provided, __alloc_pages_node() with the specified node id is used
  50. * instead. The common case (or when the node id isn't explicitly provided)
  51. * will default to the current node, as per numa_node_id().
  52. *
  53. * Node aware pages are still inserted in to the global freelist, and
  54. * these are scanned for by matching against the node id encoded in the
  55. * page flags. As a result, block allocations that can be satisfied from
  56. * the freelist will only be done so on pages residing on the same node,
  57. * in order to prevent random node placement.
  58. */
  59. #include <linux/kernel.h>
  60. #include <linux/slab.h>
  61. #include <linux/mm.h>
  62. #include <linux/swap.h> /* struct reclaim_state */
  63. #include <linux/cache.h>
  64. #include <linux/init.h>
  65. #include <linux/export.h>
  66. #include <linux/rcupdate.h>
  67. #include <linux/list.h>
  68. #include <linux/kmemleak.h>
  69. #include <trace/events/kmem.h>
  70. #include <linux/atomic.h>
  71. #include "slab.h"
  72. /*
  73. * slob_block has a field 'units', which indicates size of block if +ve,
  74. * or offset of next block if -ve (in SLOB_UNITs).
  75. *
  76. * Free blocks of size 1 unit simply contain the offset of the next block.
  77. * Those with larger size contain their size in the first SLOB_UNIT of
  78. * memory, and the offset of the next free block in the second SLOB_UNIT.
  79. */
  80. #if PAGE_SIZE <= (32767 * 2)
  81. typedef s16 slobidx_t;
  82. #else
  83. typedef s32 slobidx_t;
  84. #endif
  85. struct slob_block {
  86. slobidx_t units;
  87. };
  88. typedef struct slob_block slob_t;
  89. /*
  90. * All partially free slob pages go on these lists.
  91. */
  92. #define SLOB_BREAK1 256
  93. #define SLOB_BREAK2 1024
  94. static LIST_HEAD(free_slob_small);
  95. static LIST_HEAD(free_slob_medium);
  96. static LIST_HEAD(free_slob_large);
  97. /*
  98. * slob_page_free: true for pages on free_slob_pages list.
  99. */
  100. static inline int slob_page_free(struct page *sp)
  101. {
  102. return PageSlobFree(sp);
  103. }
  104. static void set_slob_page_free(struct page *sp, struct list_head *list)
  105. {
  106. list_add(&sp->slab_list, list);
  107. __SetPageSlobFree(sp);
  108. }
  109. static inline void clear_slob_page_free(struct page *sp)
  110. {
  111. list_del(&sp->slab_list);
  112. __ClearPageSlobFree(sp);
  113. }
  114. #define SLOB_UNIT sizeof(slob_t)
  115. #define SLOB_UNITS(size) DIV_ROUND_UP(size, SLOB_UNIT)
  116. /*
  117. * struct slob_rcu is inserted at the tail of allocated slob blocks, which
  118. * were created with a SLAB_TYPESAFE_BY_RCU slab. slob_rcu is used to free
  119. * the block using call_rcu.
  120. */
  121. struct slob_rcu {
  122. struct rcu_head head;
  123. int size;
  124. };
  125. /*
  126. * slob_lock protects all slob allocator structures.
  127. */
  128. static DEFINE_SPINLOCK(slob_lock);
  129. /*
  130. * Encode the given size and next info into a free slob block s.
  131. */
  132. static void set_slob(slob_t *s, slobidx_t size, slob_t *next)
  133. {
  134. slob_t *base = (slob_t *)((unsigned long)s & PAGE_MASK);
  135. slobidx_t offset = next - base;
  136. if (size > 1) {
  137. s[0].units = size;
  138. s[1].units = offset;
  139. } else
  140. s[0].units = -offset;
  141. }
  142. /*
  143. * Return the size of a slob block.
  144. */
  145. static slobidx_t slob_units(slob_t *s)
  146. {
  147. if (s->units > 0)
  148. return s->units;
  149. return 1;
  150. }
  151. /*
  152. * Return the next free slob block pointer after this one.
  153. */
  154. static slob_t *slob_next(slob_t *s)
  155. {
  156. slob_t *base = (slob_t *)((unsigned long)s & PAGE_MASK);
  157. slobidx_t next;
  158. if (s[0].units < 0)
  159. next = -s[0].units;
  160. else
  161. next = s[1].units;
  162. return base+next;
  163. }
  164. /*
  165. * Returns true if s is the last free block in its page.
  166. */
  167. static int slob_last(slob_t *s)
  168. {
  169. return !((unsigned long)slob_next(s) & ~PAGE_MASK);
  170. }
  171. static void *slob_new_pages(gfp_t gfp, int order, int node)
  172. {
  173. struct page *page;
  174. #ifdef CONFIG_NUMA
  175. if (node != NUMA_NO_NODE)
  176. page = __alloc_pages_node(node, gfp, order);
  177. else
  178. #endif
  179. page = alloc_pages(gfp, order);
  180. if (!page)
  181. return NULL;
  182. mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B,
  183. PAGE_SIZE << order);
  184. return page_address(page);
  185. }
  186. static void slob_free_pages(void *b, int order)
  187. {
  188. struct page *sp = virt_to_page(b);
  189. if (current->reclaim_state)
  190. current->reclaim_state->reclaimed_slab += 1 << order;
  191. mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE_B,
  192. -(PAGE_SIZE << order));
  193. __free_pages(sp, order);
  194. }
  195. /*
  196. * slob_page_alloc() - Allocate a slob block within a given slob_page sp.
  197. * @sp: Page to look in.
  198. * @size: Size of the allocation.
  199. * @align: Allocation alignment.
  200. * @align_offset: Offset in the allocated block that will be aligned.
  201. * @page_removed_from_list: Return parameter.
  202. *
  203. * Tries to find a chunk of memory at least @size bytes big within @page.
  204. *
  205. * Return: Pointer to memory if allocated, %NULL otherwise. If the
  206. * allocation fills up @page then the page is removed from the
  207. * freelist, in this case @page_removed_from_list will be set to
  208. * true (set to false otherwise).
  209. */
  210. static void *slob_page_alloc(struct page *sp, size_t size, int align,
  211. int align_offset, bool *page_removed_from_list)
  212. {
  213. slob_t *prev, *cur, *aligned = NULL;
  214. int delta = 0, units = SLOB_UNITS(size);
  215. *page_removed_from_list = false;
  216. for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) {
  217. slobidx_t avail = slob_units(cur);
  218. /*
  219. * 'aligned' will hold the address of the slob block so that the
  220. * address 'aligned'+'align_offset' is aligned according to the
  221. * 'align' parameter. This is for kmalloc() which prepends the
  222. * allocated block with its size, so that the block itself is
  223. * aligned when needed.
  224. */
  225. if (align) {
  226. aligned = (slob_t *)
  227. (ALIGN((unsigned long)cur + align_offset, align)
  228. - align_offset);
  229. delta = aligned - cur;
  230. }
  231. if (avail >= units + delta) { /* room enough? */
  232. slob_t *next;
  233. if (delta) { /* need to fragment head to align? */
  234. next = slob_next(cur);
  235. set_slob(aligned, avail - delta, next);
  236. set_slob(cur, delta, aligned);
  237. prev = cur;
  238. cur = aligned;
  239. avail = slob_units(cur);
  240. }
  241. next = slob_next(cur);
  242. if (avail == units) { /* exact fit? unlink. */
  243. if (prev)
  244. set_slob(prev, slob_units(prev), next);
  245. else
  246. sp->freelist = next;
  247. } else { /* fragment */
  248. if (prev)
  249. set_slob(prev, slob_units(prev), cur + units);
  250. else
  251. sp->freelist = cur + units;
  252. set_slob(cur + units, avail - units, next);
  253. }
  254. sp->units -= units;
  255. if (!sp->units) {
  256. clear_slob_page_free(sp);
  257. *page_removed_from_list = true;
  258. }
  259. return cur;
  260. }
  261. if (slob_last(cur))
  262. return NULL;
  263. }
  264. }
  265. /*
  266. * slob_alloc: entry point into the slob allocator.
  267. */
  268. static void *slob_alloc(size_t size, gfp_t gfp, int align, int node,
  269. int align_offset)
  270. {
  271. struct page *sp;
  272. struct list_head *slob_list;
  273. slob_t *b = NULL;
  274. unsigned long flags;
  275. bool _unused;
  276. if (size < SLOB_BREAK1)
  277. slob_list = &free_slob_small;
  278. else if (size < SLOB_BREAK2)
  279. slob_list = &free_slob_medium;
  280. else
  281. slob_list = &free_slob_large;
  282. spin_lock_irqsave(&slob_lock, flags);
  283. /* Iterate through each partially free page, try to find room */
  284. list_for_each_entry(sp, slob_list, slab_list) {
  285. bool page_removed_from_list = false;
  286. #ifdef CONFIG_NUMA
  287. /*
  288. * If there's a node specification, search for a partial
  289. * page with a matching node id in the freelist.
  290. */
  291. if (node != NUMA_NO_NODE && page_to_nid(sp) != node)
  292. continue;
  293. #endif
  294. /* Enough room on this page? */
  295. if (sp->units < SLOB_UNITS(size))
  296. continue;
  297. b = slob_page_alloc(sp, size, align, align_offset, &page_removed_from_list);
  298. if (!b)
  299. continue;
  300. /*
  301. * If slob_page_alloc() removed sp from the list then we
  302. * cannot call list functions on sp. If so allocation
  303. * did not fragment the page anyway so optimisation is
  304. * unnecessary.
  305. */
  306. if (!page_removed_from_list) {
  307. /*
  308. * Improve fragment distribution and reduce our average
  309. * search time by starting our next search here. (see
  310. * Knuth vol 1, sec 2.5, pg 449)
  311. */
  312. if (!list_is_first(&sp->slab_list, slob_list))
  313. list_rotate_to_front(&sp->slab_list, slob_list);
  314. }
  315. break;
  316. }
  317. spin_unlock_irqrestore(&slob_lock, flags);
  318. /* Not enough space: must allocate a new page */
  319. if (!b) {
  320. b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node);
  321. if (!b)
  322. return NULL;
  323. sp = virt_to_page(b);
  324. __SetPageSlab(sp);
  325. spin_lock_irqsave(&slob_lock, flags);
  326. sp->units = SLOB_UNITS(PAGE_SIZE);
  327. sp->freelist = b;
  328. INIT_LIST_HEAD(&sp->slab_list);
  329. set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE));
  330. set_slob_page_free(sp, slob_list);
  331. b = slob_page_alloc(sp, size, align, align_offset, &_unused);
  332. BUG_ON(!b);
  333. spin_unlock_irqrestore(&slob_lock, flags);
  334. }
  335. if (unlikely(gfp & __GFP_ZERO))
  336. memset(b, 0, size);
  337. return b;
  338. }
  339. /*
  340. * slob_free: entry point into the slob allocator.
  341. */
  342. static void slob_free(void *block, int size)
  343. {
  344. struct page *sp;
  345. slob_t *prev, *next, *b = (slob_t *)block;
  346. slobidx_t units;
  347. unsigned long flags;
  348. struct list_head *slob_list;
  349. if (unlikely(ZERO_OR_NULL_PTR(block)))
  350. return;
  351. BUG_ON(!size);
  352. sp = virt_to_page(block);
  353. units = SLOB_UNITS(size);
  354. spin_lock_irqsave(&slob_lock, flags);
  355. if (sp->units + units == SLOB_UNITS(PAGE_SIZE)) {
  356. /* Go directly to page allocator. Do not pass slob allocator */
  357. if (slob_page_free(sp))
  358. clear_slob_page_free(sp);
  359. spin_unlock_irqrestore(&slob_lock, flags);
  360. __ClearPageSlab(sp);
  361. page_mapcount_reset(sp);
  362. slob_free_pages(b, 0);
  363. return;
  364. }
  365. if (!slob_page_free(sp)) {
  366. /* This slob page is about to become partially free. Easy! */
  367. sp->units = units;
  368. sp->freelist = b;
  369. set_slob(b, units,
  370. (void *)((unsigned long)(b +
  371. SLOB_UNITS(PAGE_SIZE)) & PAGE_MASK));
  372. if (size < SLOB_BREAK1)
  373. slob_list = &free_slob_small;
  374. else if (size < SLOB_BREAK2)
  375. slob_list = &free_slob_medium;
  376. else
  377. slob_list = &free_slob_large;
  378. set_slob_page_free(sp, slob_list);
  379. goto out;
  380. }
  381. /*
  382. * Otherwise the page is already partially free, so find reinsertion
  383. * point.
  384. */
  385. sp->units += units;
  386. if (b < (slob_t *)sp->freelist) {
  387. if (b + units == sp->freelist) {
  388. units += slob_units(sp->freelist);
  389. sp->freelist = slob_next(sp->freelist);
  390. }
  391. set_slob(b, units, sp->freelist);
  392. sp->freelist = b;
  393. } else {
  394. prev = sp->freelist;
  395. next = slob_next(prev);
  396. while (b > next) {
  397. prev = next;
  398. next = slob_next(prev);
  399. }
  400. if (!slob_last(prev) && b + units == next) {
  401. units += slob_units(next);
  402. set_slob(b, units, slob_next(next));
  403. } else
  404. set_slob(b, units, next);
  405. if (prev + slob_units(prev) == b) {
  406. units = slob_units(b) + slob_units(prev);
  407. set_slob(prev, units, slob_next(b));
  408. } else
  409. set_slob(prev, slob_units(prev), b);
  410. }
  411. out:
  412. spin_unlock_irqrestore(&slob_lock, flags);
  413. }
  414. /*
  415. * End of slob allocator proper. Begin kmem_cache_alloc and kmalloc frontend.
  416. */
  417. static __always_inline void *
  418. __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
  419. {
  420. unsigned int *m;
  421. int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
  422. void *ret;
  423. gfp &= gfp_allowed_mask;
  424. fs_reclaim_acquire(gfp);
  425. fs_reclaim_release(gfp);
  426. if (size < PAGE_SIZE - minalign) {
  427. int align = minalign;
  428. /*
  429. * For power of two sizes, guarantee natural alignment for
  430. * kmalloc()'d objects.
  431. */
  432. if (is_power_of_2(size))
  433. align = max(minalign, (int) size);
  434. if (!size)
  435. return ZERO_SIZE_PTR;
  436. m = slob_alloc(size + minalign, gfp, align, node, minalign);
  437. if (!m)
  438. return NULL;
  439. *m = size;
  440. ret = (void *)m + minalign;
  441. trace_kmalloc_node(caller, ret,
  442. size, size + minalign, gfp, node);
  443. } else {
  444. unsigned int order = get_order(size);
  445. if (likely(order))
  446. gfp |= __GFP_COMP;
  447. ret = slob_new_pages(gfp, order, node);
  448. trace_kmalloc_node(caller, ret,
  449. size, PAGE_SIZE << order, gfp, node);
  450. }
  451. kmemleak_alloc(ret, size, 1, gfp);
  452. return ret;
  453. }
  454. void *__kmalloc(size_t size, gfp_t gfp)
  455. {
  456. return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, _RET_IP_);
  457. }
  458. EXPORT_SYMBOL(__kmalloc);
  459. void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller)
  460. {
  461. return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, caller);
  462. }
  463. EXPORT_SYMBOL(__kmalloc_track_caller);
  464. #ifdef CONFIG_NUMA
  465. void *__kmalloc_node_track_caller(size_t size, gfp_t gfp,
  466. int node, unsigned long caller)
  467. {
  468. return __do_kmalloc_node(size, gfp, node, caller);
  469. }
  470. EXPORT_SYMBOL(__kmalloc_node_track_caller);
  471. #endif
  472. void kfree(const void *block)
  473. {
  474. struct page *sp;
  475. trace_kfree(_RET_IP_, block);
  476. if (unlikely(ZERO_OR_NULL_PTR(block)))
  477. return;
  478. kmemleak_free(block);
  479. sp = virt_to_page(block);
  480. if (PageSlab(sp)) {
  481. int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
  482. unsigned int *m = (unsigned int *)(block - align);
  483. slob_free(m, *m + align);
  484. } else {
  485. unsigned int order = compound_order(sp);
  486. mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE_B,
  487. -(PAGE_SIZE << order));
  488. __free_pages(sp, order);
  489. }
  490. }
  491. EXPORT_SYMBOL(kfree);
  492. /* can't use ksize for kmem_cache_alloc memory, only kmalloc */
  493. size_t __ksize(const void *block)
  494. {
  495. struct page *sp;
  496. int align;
  497. unsigned int *m;
  498. BUG_ON(!block);
  499. if (unlikely(block == ZERO_SIZE_PTR))
  500. return 0;
  501. sp = virt_to_page(block);
  502. if (unlikely(!PageSlab(sp)))
  503. return page_size(sp);
  504. align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
  505. m = (unsigned int *)(block - align);
  506. return SLOB_UNITS(*m) * SLOB_UNIT;
  507. }
  508. EXPORT_SYMBOL(__ksize);
  509. int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags)
  510. {
  511. if (flags & SLAB_TYPESAFE_BY_RCU) {
  512. /* leave room for rcu footer at the end of object */
  513. c->size += sizeof(struct slob_rcu);
  514. }
  515. c->flags = flags;
  516. return 0;
  517. }
  518. static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
  519. {
  520. void *b;
  521. flags &= gfp_allowed_mask;
  522. fs_reclaim_acquire(flags);
  523. fs_reclaim_release(flags);
  524. if (c->size < PAGE_SIZE) {
  525. b = slob_alloc(c->size, flags, c->align, node, 0);
  526. trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
  527. SLOB_UNITS(c->size) * SLOB_UNIT,
  528. flags, node);
  529. } else {
  530. b = slob_new_pages(flags, get_order(c->size), node);
  531. trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
  532. PAGE_SIZE << get_order(c->size),
  533. flags, node);
  534. }
  535. if (b && c->ctor) {
  536. WARN_ON_ONCE(flags & __GFP_ZERO);
  537. c->ctor(b);
  538. }
  539. kmemleak_alloc_recursive(b, c->size, 1, c->flags, flags);
  540. return b;
  541. }
  542. void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
  543. {
  544. return slob_alloc_node(cachep, flags, NUMA_NO_NODE);
  545. }
  546. EXPORT_SYMBOL(kmem_cache_alloc);
  547. #ifdef CONFIG_NUMA
  548. void *__kmalloc_node(size_t size, gfp_t gfp, int node)
  549. {
  550. return __do_kmalloc_node(size, gfp, node, _RET_IP_);
  551. }
  552. EXPORT_SYMBOL(__kmalloc_node);
  553. void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node)
  554. {
  555. return slob_alloc_node(cachep, gfp, node);
  556. }
  557. EXPORT_SYMBOL(kmem_cache_alloc_node);
  558. #endif
  559. static void __kmem_cache_free(void *b, int size)
  560. {
  561. if (size < PAGE_SIZE)
  562. slob_free(b, size);
  563. else
  564. slob_free_pages(b, get_order(size));
  565. }
  566. static void kmem_rcu_free(struct rcu_head *head)
  567. {
  568. struct slob_rcu *slob_rcu = (struct slob_rcu *)head;
  569. void *b = (void *)slob_rcu - (slob_rcu->size - sizeof(struct slob_rcu));
  570. __kmem_cache_free(b, slob_rcu->size);
  571. }
  572. void kmem_cache_free(struct kmem_cache *c, void *b)
  573. {
  574. kmemleak_free_recursive(b, c->flags);
  575. if (unlikely(c->flags & SLAB_TYPESAFE_BY_RCU)) {
  576. struct slob_rcu *slob_rcu;
  577. slob_rcu = b + (c->size - sizeof(struct slob_rcu));
  578. slob_rcu->size = c->size;
  579. call_rcu(&slob_rcu->head, kmem_rcu_free);
  580. } else {
  581. __kmem_cache_free(b, c->size);
  582. }
  583. trace_kmem_cache_free(_RET_IP_, b);
  584. }
  585. EXPORT_SYMBOL(kmem_cache_free);
  586. void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
  587. {
  588. __kmem_cache_free_bulk(s, size, p);
  589. }
  590. EXPORT_SYMBOL(kmem_cache_free_bulk);
  591. int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
  592. void **p)
  593. {
  594. return __kmem_cache_alloc_bulk(s, flags, size, p);
  595. }
  596. EXPORT_SYMBOL(kmem_cache_alloc_bulk);
  597. int __kmem_cache_shutdown(struct kmem_cache *c)
  598. {
  599. /* No way to check for remaining objects */
  600. return 0;
  601. }
  602. void __kmem_cache_release(struct kmem_cache *c)
  603. {
  604. }
  605. int __kmem_cache_shrink(struct kmem_cache *d)
  606. {
  607. return 0;
  608. }
  609. struct kmem_cache kmem_cache_boot = {
  610. .name = "kmem_cache",
  611. .size = sizeof(struct kmem_cache),
  612. .flags = SLAB_PANIC,
  613. .align = ARCH_KMALLOC_MINALIGN,
  614. };
  615. void __init kmem_cache_init(void)
  616. {
  617. kmem_cache = &kmem_cache_boot;
  618. slab_state = UP;
  619. }
  620. void __init kmem_cache_init_late(void)
  621. {
  622. slab_state = FULL;
  623. }