slab.h 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef MM_SLAB_H
  3. #define MM_SLAB_H
  4. /*
  5. * Internal slab definitions
  6. */
  7. #ifdef CONFIG_SLOB
  8. /*
  9. * Common fields provided in kmem_cache by all slab allocators
  10. * This struct is either used directly by the allocator (SLOB)
  11. * or the allocator must include definitions for all fields
  12. * provided in kmem_cache_common in their definition of kmem_cache.
  13. *
  14. * Once we can do anonymous structs (C11 standard) we could put a
  15. * anonymous struct definition in these allocators so that the
  16. * separate allocations in the kmem_cache structure of SLAB and
  17. * SLUB is no longer needed.
  18. */
  19. struct kmem_cache {
  20. unsigned int object_size;/* The original size of the object */
  21. unsigned int size; /* The aligned/padded/added on size */
  22. unsigned int align; /* Alignment as calculated */
  23. slab_flags_t flags; /* Active flags on the slab */
  24. unsigned int useroffset;/* Usercopy region offset */
  25. unsigned int usersize; /* Usercopy region size */
  26. const char *name; /* Slab name for sysfs */
  27. int refcount; /* Use counter */
  28. void (*ctor)(void *); /* Called on object slot creation */
  29. struct list_head list; /* List of all slab caches on the system */
  30. };
  31. #endif /* CONFIG_SLOB */
  32. #ifdef CONFIG_SLAB
  33. #include <linux/slab_def.h>
  34. #endif
  35. #ifdef CONFIG_SLUB
  36. #include <linux/slub_def.h>
  37. #endif
  38. #include <linux/memcontrol.h>
  39. #include <linux/fault-inject.h>
  40. #include <linux/kasan.h>
  41. #include <linux/kmemleak.h>
  42. #include <linux/random.h>
  43. #include <linux/sched/mm.h>
  44. #include <linux/android_vendor.h>
  45. /*
  46. * State of the slab allocator.
  47. *
  48. * This is used to describe the states of the allocator during bootup.
  49. * Allocators use this to gradually bootstrap themselves. Most allocators
  50. * have the problem that the structures used for managing slab caches are
  51. * allocated from slab caches themselves.
  52. */
  53. enum slab_state {
  54. DOWN, /* No slab functionality yet */
  55. PARTIAL, /* SLUB: kmem_cache_node available */
  56. PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
  57. UP, /* Slab caches usable but not all extras yet */
  58. FULL /* Everything is working */
  59. };
  60. extern enum slab_state slab_state;
  61. /* The slab cache mutex protects the management structures during changes */
  62. extern struct mutex slab_mutex;
  63. /* The list of all slab caches on the system */
  64. extern struct list_head slab_caches;
  65. /* The slab cache that manages slab cache information */
  66. extern struct kmem_cache *kmem_cache;
  67. /* A table of kmalloc cache names and sizes */
  68. extern const struct kmalloc_info_struct {
  69. const char *name[NR_KMALLOC_TYPES];
  70. unsigned int size;
  71. } kmalloc_info[];
  72. #ifndef CONFIG_SLOB
  73. /* Kmalloc array related functions */
  74. void setup_kmalloc_cache_index_table(void);
  75. void create_kmalloc_caches(slab_flags_t);
  76. /* Find the kmalloc slab corresponding for a certain size */
  77. struct kmem_cache *kmalloc_slab(size_t, gfp_t);
  78. #endif
  79. gfp_t kmalloc_fix_flags(gfp_t flags);
  80. #ifdef CONFIG_SLUB
  81. /*
  82. * Tracking user of a slab.
  83. */
  84. #define TRACK_ADDRS_COUNT 16
  85. struct track {
  86. unsigned long addr; /* Called from address */
  87. #ifdef CONFIG_STACKTRACE
  88. unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
  89. #endif
  90. int cpu; /* Was running on cpu */
  91. int pid; /* Pid context */
  92. unsigned long when; /* When did the operation occur */
  93. #ifdef CONFIG_STACKTRACE
  94. ANDROID_OEM_DATA(1);
  95. #endif
  96. };
  97. enum track_item { TRACK_ALLOC, TRACK_FREE };
  98. #endif
  99. /* Functions provided by the slab allocators */
  100. int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
  101. struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size,
  102. slab_flags_t flags, unsigned int useroffset,
  103. unsigned int usersize);
  104. extern void create_boot_cache(struct kmem_cache *, const char *name,
  105. unsigned int size, slab_flags_t flags,
  106. unsigned int useroffset, unsigned int usersize);
  107. int slab_unmergeable(struct kmem_cache *s);
  108. struct kmem_cache *find_mergeable(unsigned size, unsigned align,
  109. slab_flags_t flags, const char *name, void (*ctor)(void *));
  110. #ifndef CONFIG_SLOB
  111. struct kmem_cache *
  112. __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
  113. slab_flags_t flags, void (*ctor)(void *));
  114. slab_flags_t kmem_cache_flags(unsigned int object_size,
  115. slab_flags_t flags, const char *name);
  116. #else
  117. static inline struct kmem_cache *
  118. __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
  119. slab_flags_t flags, void (*ctor)(void *))
  120. { return NULL; }
  121. static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
  122. slab_flags_t flags, const char *name)
  123. {
  124. return flags;
  125. }
  126. #endif
  127. /* Legal flag mask for kmem_cache_create(), for various configurations */
  128. #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
  129. SLAB_CACHE_DMA32 | SLAB_PANIC | \
  130. SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
  131. #if defined(CONFIG_DEBUG_SLAB)
  132. #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
  133. #elif defined(CONFIG_SLUB_DEBUG)
  134. #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
  135. SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
  136. #else
  137. #define SLAB_DEBUG_FLAGS (0)
  138. #endif
  139. #if defined(CONFIG_SLAB)
  140. #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
  141. SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \
  142. SLAB_ACCOUNT)
  143. #elif defined(CONFIG_SLUB)
  144. #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
  145. SLAB_TEMPORARY | SLAB_ACCOUNT)
  146. #else
  147. #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE)
  148. #endif
  149. /* Common flags available with current configuration */
  150. #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
  151. /* Common flags permitted for kmem_cache_create */
  152. #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
  153. SLAB_RED_ZONE | \
  154. SLAB_POISON | \
  155. SLAB_STORE_USER | \
  156. SLAB_TRACE | \
  157. SLAB_CONSISTENCY_CHECKS | \
  158. SLAB_MEM_SPREAD | \
  159. SLAB_NOLEAKTRACE | \
  160. SLAB_RECLAIM_ACCOUNT | \
  161. SLAB_TEMPORARY | \
  162. SLAB_ACCOUNT)
  163. bool __kmem_cache_empty(struct kmem_cache *);
  164. int __kmem_cache_shutdown(struct kmem_cache *);
  165. void __kmem_cache_release(struct kmem_cache *);
  166. int __kmem_cache_shrink(struct kmem_cache *);
  167. void slab_kmem_cache_release(struct kmem_cache *);
  168. struct seq_file;
  169. struct file;
  170. struct slabinfo {
  171. unsigned long active_objs;
  172. unsigned long num_objs;
  173. unsigned long active_slabs;
  174. unsigned long num_slabs;
  175. unsigned long shared_avail;
  176. unsigned int limit;
  177. unsigned int batchcount;
  178. unsigned int shared;
  179. unsigned int objects_per_slab;
  180. unsigned int cache_order;
  181. };
  182. void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
  183. void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
  184. ssize_t slabinfo_write(struct file *file, const char __user *buffer,
  185. size_t count, loff_t *ppos);
  186. /*
  187. * Generic implementation of bulk operations
  188. * These are useful for situations in which the allocator cannot
  189. * perform optimizations. In that case segments of the object listed
  190. * may be allocated or freed using these operations.
  191. */
  192. void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
  193. int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
  194. static inline int cache_vmstat_idx(struct kmem_cache *s)
  195. {
  196. return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
  197. NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
  198. }
  199. #ifdef CONFIG_SLUB_DEBUG
  200. #ifdef CONFIG_SLUB_DEBUG_ON
  201. DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
  202. #else
  203. DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
  204. #endif
  205. extern void print_tracking(struct kmem_cache *s, void *object);
  206. extern unsigned long get_each_object_track(struct kmem_cache *s,
  207. struct page *page, enum track_item alloc,
  208. int (*fn)(const struct kmem_cache *, const void *,
  209. const struct track *, void *), void *private);
  210. extern slab_flags_t slub_debug;
  211. static inline bool __slub_debug_enabled(void)
  212. {
  213. return static_branch_unlikely(&slub_debug_enabled);
  214. }
  215. #else
  216. static inline void print_tracking(struct kmem_cache *s, void *object)
  217. {
  218. }
  219. static inline bool __slub_debug_enabled(void)
  220. {
  221. return false;
  222. }
  223. #ifdef CONFIG_SLUB
  224. static inline unsigned long get_each_object_track(struct kmem_cache *s,
  225. struct page *page, enum track_item alloc,
  226. int (*fn)(const struct kmem_cache *, const void *,
  227. const struct track *, void *), void *private)
  228. {
  229. return 0;
  230. }
  231. #endif
  232. #endif
  233. /*
  234. * Returns true if any of the specified slub_debug flags is enabled for the
  235. * cache. Use only for flags parsed by setup_slub_debug() as it also enables
  236. * the static key.
  237. */
  238. static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
  239. {
  240. if (IS_ENABLED(CONFIG_SLUB_DEBUG))
  241. VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
  242. if (__slub_debug_enabled())
  243. return s->flags & flags;
  244. return false;
  245. }
  246. #ifdef CONFIG_MEMCG_KMEM
  247. static inline struct obj_cgroup **page_obj_cgroups(struct page *page)
  248. {
  249. /*
  250. * page->mem_cgroup and page->obj_cgroups are sharing the same
  251. * space. To distinguish between them in case we don't know for sure
  252. * that the page is a slab page (e.g. page_cgroup_ino()), let's
  253. * always set the lowest bit of obj_cgroups.
  254. */
  255. return (struct obj_cgroup **)
  256. ((unsigned long)page->obj_cgroups & ~0x1UL);
  257. }
  258. static inline bool page_has_obj_cgroups(struct page *page)
  259. {
  260. return ((unsigned long)page->obj_cgroups & 0x1UL);
  261. }
  262. int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
  263. gfp_t gfp);
  264. static inline void memcg_free_page_obj_cgroups(struct page *page)
  265. {
  266. kfree(page_obj_cgroups(page));
  267. page->obj_cgroups = NULL;
  268. }
  269. static inline size_t obj_full_size(struct kmem_cache *s)
  270. {
  271. /*
  272. * For each accounted object there is an extra space which is used
  273. * to store obj_cgroup membership. Charge it too.
  274. */
  275. return s->size + sizeof(struct obj_cgroup *);
  276. }
  277. /*
  278. * Returns false if the allocation should fail.
  279. */
  280. static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
  281. struct obj_cgroup **objcgp,
  282. size_t objects, gfp_t flags)
  283. {
  284. struct obj_cgroup *objcg;
  285. if (!memcg_kmem_enabled())
  286. return true;
  287. if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
  288. return true;
  289. objcg = get_obj_cgroup_from_current();
  290. if (!objcg)
  291. return true;
  292. if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {
  293. obj_cgroup_put(objcg);
  294. return false;
  295. }
  296. *objcgp = objcg;
  297. return true;
  298. }
  299. static inline void mod_objcg_state(struct obj_cgroup *objcg,
  300. struct pglist_data *pgdat,
  301. int idx, int nr)
  302. {
  303. struct mem_cgroup *memcg;
  304. struct lruvec *lruvec;
  305. rcu_read_lock();
  306. memcg = obj_cgroup_memcg(objcg);
  307. lruvec = mem_cgroup_lruvec(memcg, pgdat);
  308. mod_memcg_lruvec_state(lruvec, idx, nr);
  309. rcu_read_unlock();
  310. }
  311. static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
  312. struct obj_cgroup *objcg,
  313. gfp_t flags, size_t size,
  314. void **p)
  315. {
  316. struct page *page;
  317. unsigned long off;
  318. size_t i;
  319. if (!memcg_kmem_enabled() || !objcg)
  320. return;
  321. for (i = 0; i < size; i++) {
  322. if (likely(p[i])) {
  323. page = virt_to_head_page(p[i]);
  324. if (!page_has_obj_cgroups(page) &&
  325. memcg_alloc_page_obj_cgroups(page, s, flags)) {
  326. obj_cgroup_uncharge(objcg, obj_full_size(s));
  327. continue;
  328. }
  329. off = obj_to_index(s, page, p[i]);
  330. obj_cgroup_get(objcg);
  331. page_obj_cgroups(page)[off] = objcg;
  332. mod_objcg_state(objcg, page_pgdat(page),
  333. cache_vmstat_idx(s), obj_full_size(s));
  334. } else {
  335. obj_cgroup_uncharge(objcg, obj_full_size(s));
  336. }
  337. }
  338. obj_cgroup_put(objcg);
  339. }
  340. static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,
  341. void **p, int objects)
  342. {
  343. struct kmem_cache *s;
  344. struct obj_cgroup *objcg;
  345. struct page *page;
  346. unsigned int off;
  347. int i;
  348. if (!memcg_kmem_enabled())
  349. return;
  350. for (i = 0; i < objects; i++) {
  351. if (unlikely(!p[i]))
  352. continue;
  353. page = virt_to_head_page(p[i]);
  354. if (!page_has_obj_cgroups(page))
  355. continue;
  356. if (!s_orig)
  357. s = page->slab_cache;
  358. else
  359. s = s_orig;
  360. off = obj_to_index(s, page, p[i]);
  361. objcg = page_obj_cgroups(page)[off];
  362. if (!objcg)
  363. continue;
  364. page_obj_cgroups(page)[off] = NULL;
  365. obj_cgroup_uncharge(objcg, obj_full_size(s));
  366. mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s),
  367. -obj_full_size(s));
  368. obj_cgroup_put(objcg);
  369. }
  370. }
  371. #else /* CONFIG_MEMCG_KMEM */
  372. static inline bool page_has_obj_cgroups(struct page *page)
  373. {
  374. return false;
  375. }
  376. static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
  377. {
  378. return NULL;
  379. }
  380. static inline int memcg_alloc_page_obj_cgroups(struct page *page,
  381. struct kmem_cache *s, gfp_t gfp)
  382. {
  383. return 0;
  384. }
  385. static inline void memcg_free_page_obj_cgroups(struct page *page)
  386. {
  387. }
  388. static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
  389. struct obj_cgroup **objcgp,
  390. size_t objects, gfp_t flags)
  391. {
  392. return true;
  393. }
  394. static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
  395. struct obj_cgroup *objcg,
  396. gfp_t flags, size_t size,
  397. void **p)
  398. {
  399. }
  400. static inline void memcg_slab_free_hook(struct kmem_cache *s,
  401. void **p, int objects)
  402. {
  403. }
  404. #endif /* CONFIG_MEMCG_KMEM */
  405. static inline struct kmem_cache *virt_to_cache(const void *obj)
  406. {
  407. struct page *page;
  408. page = virt_to_head_page(obj);
  409. if (WARN_ONCE(!PageSlab(page), "%s: Object is not a Slab page!\n",
  410. __func__))
  411. return NULL;
  412. return page->slab_cache;
  413. }
  414. static __always_inline void account_slab_page(struct page *page, int order,
  415. struct kmem_cache *s)
  416. {
  417. mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
  418. PAGE_SIZE << order);
  419. }
  420. static __always_inline void unaccount_slab_page(struct page *page, int order,
  421. struct kmem_cache *s)
  422. {
  423. if (memcg_kmem_enabled())
  424. memcg_free_page_obj_cgroups(page);
  425. mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
  426. -(PAGE_SIZE << order));
  427. }
  428. static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
  429. {
  430. struct kmem_cache *cachep;
  431. if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
  432. !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
  433. return s;
  434. cachep = virt_to_cache(x);
  435. if (WARN(cachep && cachep != s,
  436. "%s: Wrong slab cache. %s but object is from %s\n",
  437. __func__, s->name, cachep->name))
  438. print_tracking(cachep, x);
  439. return cachep;
  440. }
  441. static inline size_t slab_ksize(const struct kmem_cache *s)
  442. {
  443. #ifndef CONFIG_SLUB
  444. return s->object_size;
  445. #else /* CONFIG_SLUB */
  446. # ifdef CONFIG_SLUB_DEBUG
  447. /*
  448. * Debugging requires use of the padding between object
  449. * and whatever may come after it.
  450. */
  451. if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
  452. return s->object_size;
  453. # endif
  454. if (s->flags & SLAB_KASAN)
  455. return s->object_size;
  456. /*
  457. * If we have the need to store the freelist pointer
  458. * back there or track user information then we can
  459. * only use the space before that information.
  460. */
  461. if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
  462. return s->inuse;
  463. /*
  464. * Else we can use all the padding etc for the allocation
  465. */
  466. return s->size;
  467. #endif
  468. }
  469. static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
  470. struct obj_cgroup **objcgp,
  471. size_t size, gfp_t flags)
  472. {
  473. flags &= gfp_allowed_mask;
  474. fs_reclaim_acquire(flags);
  475. fs_reclaim_release(flags);
  476. might_sleep_if(gfpflags_allow_blocking(flags));
  477. if (should_failslab(s, flags))
  478. return NULL;
  479. if (!memcg_slab_pre_alloc_hook(s, objcgp, size, flags))
  480. return NULL;
  481. return s;
  482. }
  483. static inline void slab_post_alloc_hook(struct kmem_cache *s,
  484. struct obj_cgroup *objcg, gfp_t flags,
  485. size_t size, void **p, bool init)
  486. {
  487. size_t i;
  488. flags &= gfp_allowed_mask;
  489. /*
  490. * As memory initialization might be integrated into KASAN,
  491. * kasan_slab_alloc and initialization memset must be
  492. * kept together to avoid discrepancies in behavior.
  493. *
  494. * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
  495. */
  496. for (i = 0; i < size; i++) {
  497. p[i] = kasan_slab_alloc(s, p[i], flags, init);
  498. if (p[i] && init && !kasan_has_integrated_init())
  499. memset(p[i], 0, s->object_size);
  500. kmemleak_alloc_recursive(p[i], s->object_size, 1,
  501. s->flags, flags);
  502. }
  503. memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
  504. }
  505. #ifndef CONFIG_SLOB
  506. /*
  507. * The slab lists for all objects.
  508. */
  509. struct kmem_cache_node {
  510. spinlock_t list_lock;
  511. #ifdef CONFIG_SLAB
  512. struct list_head slabs_partial; /* partial list first, better asm code */
  513. struct list_head slabs_full;
  514. struct list_head slabs_free;
  515. unsigned long total_slabs; /* length of all slab lists */
  516. unsigned long free_slabs; /* length of free slab list only */
  517. unsigned long free_objects;
  518. unsigned int free_limit;
  519. unsigned int colour_next; /* Per-node cache coloring */
  520. struct array_cache *shared; /* shared per node */
  521. struct alien_cache **alien; /* on other nodes */
  522. unsigned long next_reap; /* updated without locking */
  523. int free_touched; /* updated without locking */
  524. #endif
  525. #ifdef CONFIG_SLUB
  526. unsigned long nr_partial;
  527. struct list_head partial;
  528. #ifdef CONFIG_SLUB_DEBUG
  529. atomic_long_t nr_slabs;
  530. atomic_long_t total_objects;
  531. struct list_head full;
  532. #endif
  533. #endif
  534. };
  535. static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
  536. {
  537. return s->node[node];
  538. }
  539. /*
  540. * Iterator over all nodes. The body will be executed for each node that has
  541. * a kmem_cache_node structure allocated (which is true for all online nodes)
  542. */
  543. #define for_each_kmem_cache_node(__s, __node, __n) \
  544. for (__node = 0; __node < nr_node_ids; __node++) \
  545. if ((__n = get_node(__s, __node)))
  546. #endif
  547. void *slab_start(struct seq_file *m, loff_t *pos);
  548. void *slab_next(struct seq_file *m, void *p, loff_t *pos);
  549. void slab_stop(struct seq_file *m, void *p);
  550. int memcg_slab_show(struct seq_file *m, void *p);
  551. #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
  552. void dump_unreclaimable_slab(void);
  553. #else
  554. static inline void dump_unreclaimable_slab(void)
  555. {
  556. }
  557. #endif
  558. void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
  559. #ifdef CONFIG_SLAB_FREELIST_RANDOM
  560. int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
  561. gfp_t gfp);
  562. void cache_random_seq_destroy(struct kmem_cache *cachep);
  563. #else
  564. static inline int cache_random_seq_create(struct kmem_cache *cachep,
  565. unsigned int count, gfp_t gfp)
  566. {
  567. return 0;
  568. }
  569. static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
  570. #endif /* CONFIG_SLAB_FREELIST_RANDOM */
  571. static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
  572. {
  573. if (static_branch_unlikely(&init_on_alloc)) {
  574. if (c->ctor)
  575. return false;
  576. if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
  577. return flags & __GFP_ZERO;
  578. return true;
  579. }
  580. return flags & __GFP_ZERO;
  581. }
  582. static inline bool slab_want_init_on_free(struct kmem_cache *c)
  583. {
  584. if (static_branch_unlikely(&init_on_free))
  585. return !(c->ctor ||
  586. (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
  587. return false;
  588. }
  589. #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
  590. void debugfs_slab_release(struct kmem_cache *);
  591. #else
  592. static inline void debugfs_slab_release(struct kmem_cache *s) { }
  593. #endif
  594. #endif /* MM_SLAB_H */