kmemleak.c 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * mm/kmemleak.c
  4. *
  5. * Copyright (C) 2008 ARM Limited
  6. * Written by Catalin Marinas <catalin.marinas@arm.com>
  7. *
  8. * For more information on the algorithm and kmemleak usage, please see
  9. * Documentation/dev-tools/kmemleak.rst.
  10. *
  11. * Notes on locking
  12. * ----------------
  13. *
  14. * The following locks and mutexes are used by kmemleak:
  15. *
  16. * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
  17. * accesses to the object_tree_root. The object_list is the main list
  18. * holding the metadata (struct kmemleak_object) for the allocated memory
  19. * blocks. The object_tree_root is a red black tree used to look-up
  20. * metadata based on a pointer to the corresponding memory block. The
  21. * kmemleak_object structures are added to the object_list and
  22. * object_tree_root in the create_object() function called from the
  23. * kmemleak_alloc() callback and removed in delete_object() called from the
  24. * kmemleak_free() callback
  25. * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
  26. * Accesses to the metadata (e.g. count) are protected by this lock. Note
  27. * that some members of this structure may be protected by other means
  28. * (atomic or kmemleak_lock). This lock is also held when scanning the
  29. * corresponding memory block to avoid the kernel freeing it via the
  30. * kmemleak_free() callback. This is less heavyweight than holding a global
  31. * lock like kmemleak_lock during scanning.
  32. * - scan_mutex (mutex): ensures that only one thread may scan the memory for
  33. * unreferenced objects at a time. The gray_list contains the objects which
  34. * are already referenced or marked as false positives and need to be
  35. * scanned. This list is only modified during a scanning episode when the
  36. * scan_mutex is held. At the end of a scan, the gray_list is always empty.
  37. * Note that the kmemleak_object.use_count is incremented when an object is
  38. * added to the gray_list and therefore cannot be freed. This mutex also
  39. * prevents multiple users of the "kmemleak" debugfs file together with
  40. * modifications to the memory scanning parameters including the scan_thread
  41. * pointer
  42. *
  43. * Locks and mutexes are acquired/nested in the following order:
  44. *
  45. * scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
  46. *
  47. * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
  48. * regions.
  49. *
  50. * The kmemleak_object structures have a use_count incremented or decremented
  51. * using the get_object()/put_object() functions. When the use_count becomes
  52. * 0, this count can no longer be incremented and put_object() schedules the
  53. * kmemleak_object freeing via an RCU callback. All calls to the get_object()
  54. * function must be protected by rcu_read_lock() to avoid accessing a freed
  55. * structure.
  56. */
  57. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  58. #include <linux/init.h>
  59. #include <linux/kernel.h>
  60. #include <linux/list.h>
  61. #include <linux/sched/signal.h>
  62. #include <linux/sched/task.h>
  63. #include <linux/sched/task_stack.h>
  64. #include <linux/jiffies.h>
  65. #include <linux/delay.h>
  66. #include <linux/export.h>
  67. #include <linux/kthread.h>
  68. #include <linux/rbtree.h>
  69. #include <linux/fs.h>
  70. #include <linux/debugfs.h>
  71. #include <linux/seq_file.h>
  72. #include <linux/cpumask.h>
  73. #include <linux/spinlock.h>
  74. #include <linux/module.h>
  75. #include <linux/mutex.h>
  76. #include <linux/rcupdate.h>
  77. #include <linux/stacktrace.h>
  78. #include <linux/cache.h>
  79. #include <linux/percpu.h>
  80. #include <linux/memblock.h>
  81. #include <linux/pfn.h>
  82. #include <linux/mmzone.h>
  83. #include <linux/slab.h>
  84. #include <linux/thread_info.h>
  85. #include <linux/err.h>
  86. #include <linux/uaccess.h>
  87. #include <linux/string.h>
  88. #include <linux/nodemask.h>
  89. #include <linux/mm.h>
  90. #include <linux/workqueue.h>
  91. #include <linux/crc32.h>
  92. #include <asm/sections.h>
  93. #include <asm/processor.h>
  94. #include <linux/atomic.h>
  95. #include <linux/kasan.h>
  96. #include <linux/kfence.h>
  97. #include <linux/kmemleak.h>
  98. #include <linux/memory_hotplug.h>
  99. /*
  100. * Kmemleak configuration and common defines.
  101. */
  102. #define MAX_TRACE 16 /* stack trace length */
  103. #define MSECS_MIN_AGE 5000 /* minimum object age for reporting */
  104. #define SECS_FIRST_SCAN 60 /* delay before the first scan */
  105. #define SECS_SCAN_WAIT 600 /* subsequent auto scanning delay */
  106. #define MAX_SCAN_SIZE 4096 /* maximum size of a scanned block */
  107. #define BYTES_PER_POINTER sizeof(void *)
  108. /* GFP bitmask for kmemleak internal allocations */
  109. #define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
  110. __GFP_NORETRY | __GFP_NOMEMALLOC | \
  111. __GFP_NOWARN)
  112. /* scanning area inside a memory block */
  113. struct kmemleak_scan_area {
  114. struct hlist_node node;
  115. unsigned long start;
  116. size_t size;
  117. };
  118. #define KMEMLEAK_GREY 0
  119. #define KMEMLEAK_BLACK -1
  120. /*
  121. * Structure holding the metadata for each allocated memory block.
  122. * Modifications to such objects should be made while holding the
  123. * object->lock. Insertions or deletions from object_list, gray_list or
  124. * rb_node are already protected by the corresponding locks or mutex (see
  125. * the notes on locking above). These objects are reference-counted
  126. * (use_count) and freed using the RCU mechanism.
  127. */
  128. struct kmemleak_object {
  129. raw_spinlock_t lock;
  130. unsigned int flags; /* object status flags */
  131. struct list_head object_list;
  132. struct list_head gray_list;
  133. struct rb_node rb_node;
  134. struct rcu_head rcu; /* object_list lockless traversal */
  135. /* object usage count; object freed when use_count == 0 */
  136. atomic_t use_count;
  137. unsigned long pointer;
  138. size_t size;
  139. /* pass surplus references to this pointer */
  140. unsigned long excess_ref;
  141. /* minimum number of a pointers found before it is considered leak */
  142. int min_count;
  143. /* the total number of pointers found pointing to this object */
  144. int count;
  145. /* checksum for detecting modified objects */
  146. u32 checksum;
  147. /* memory ranges to be scanned inside an object (empty for all) */
  148. struct hlist_head area_list;
  149. unsigned long trace[MAX_TRACE];
  150. unsigned int trace_len;
  151. unsigned long jiffies; /* creation timestamp */
  152. pid_t pid; /* pid of the current task */
  153. char comm[TASK_COMM_LEN]; /* executable name */
  154. };
  155. /* flag representing the memory block allocation status */
  156. #define OBJECT_ALLOCATED (1 << 0)
  157. /* flag set after the first reporting of an unreference object */
  158. #define OBJECT_REPORTED (1 << 1)
  159. /* flag set to not scan the object */
  160. #define OBJECT_NO_SCAN (1 << 2)
  161. /* flag set to fully scan the object when scan_area allocation failed */
  162. #define OBJECT_FULL_SCAN (1 << 3)
  163. #define HEX_PREFIX " "
  164. /* number of bytes to print per line; must be 16 or 32 */
  165. #define HEX_ROW_SIZE 16
  166. /* number of bytes to print at a time (1, 2, 4, 8) */
  167. #define HEX_GROUP_SIZE 1
  168. /* include ASCII after the hex output */
  169. #define HEX_ASCII 1
  170. /* max number of lines to be printed */
  171. #define HEX_MAX_LINES 2
  172. /* the list of all allocated objects */
  173. static LIST_HEAD(object_list);
  174. /* the list of gray-colored objects (see color_gray comment below) */
  175. static LIST_HEAD(gray_list);
  176. /* memory pool allocation */
  177. static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
  178. static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
  179. static LIST_HEAD(mem_pool_free_list);
  180. /* search tree for object boundaries */
  181. static struct rb_root object_tree_root = RB_ROOT;
  182. /* protecting the access to object_list and object_tree_root */
  183. static DEFINE_RAW_SPINLOCK(kmemleak_lock);
  184. /* allocation caches for kmemleak internal data */
  185. static struct kmem_cache *object_cache;
  186. static struct kmem_cache *scan_area_cache;
  187. /* set if tracing memory operations is enabled */
  188. static int kmemleak_enabled = 1;
  189. /* same as above but only for the kmemleak_free() callback */
  190. static int kmemleak_free_enabled = 1;
  191. /* set in the late_initcall if there were no errors */
  192. static int kmemleak_initialized;
  193. /* set if a kmemleak warning was issued */
  194. static int kmemleak_warning;
  195. /* set if a fatal kmemleak error has occurred */
  196. static int kmemleak_error;
  197. /* minimum and maximum address that may be valid pointers */
  198. static unsigned long min_addr = ULONG_MAX;
  199. static unsigned long max_addr;
  200. static struct task_struct *scan_thread;
  201. /* used to avoid reporting of recently allocated objects */
  202. static unsigned long jiffies_min_age;
  203. static unsigned long jiffies_last_scan;
  204. /* delay between automatic memory scannings */
  205. static signed long jiffies_scan_wait;
  206. /* enables or disables the task stacks scanning */
  207. static int kmemleak_stack_scan = 1;
  208. /* protects the memory scanning, parameters and debug/kmemleak file access */
  209. static DEFINE_MUTEX(scan_mutex);
  210. /* setting kmemleak=on, will set this var, skipping the disable */
  211. static int kmemleak_skip_disable;
  212. /* If there are leaks that can be reported */
  213. static bool kmemleak_found_leaks;
  214. static bool kmemleak_verbose;
  215. module_param_named(verbose, kmemleak_verbose, bool, 0600);
  216. static void kmemleak_disable(void);
  217. /*
  218. * Print a warning and dump the stack trace.
  219. */
  220. #define kmemleak_warn(x...) do { \
  221. pr_warn(x); \
  222. dump_stack(); \
  223. kmemleak_warning = 1; \
  224. } while (0)
  225. /*
  226. * Macro invoked when a serious kmemleak condition occurred and cannot be
  227. * recovered from. Kmemleak will be disabled and further allocation/freeing
  228. * tracing no longer available.
  229. */
  230. #define kmemleak_stop(x...) do { \
  231. kmemleak_warn(x); \
  232. kmemleak_disable(); \
  233. } while (0)
  234. #define warn_or_seq_printf(seq, fmt, ...) do { \
  235. if (seq) \
  236. seq_printf(seq, fmt, ##__VA_ARGS__); \
  237. else \
  238. pr_warn(fmt, ##__VA_ARGS__); \
  239. } while (0)
  240. static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
  241. int rowsize, int groupsize, const void *buf,
  242. size_t len, bool ascii)
  243. {
  244. if (seq)
  245. seq_hex_dump(seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
  246. buf, len, ascii);
  247. else
  248. print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
  249. rowsize, groupsize, buf, len, ascii);
  250. }
  251. /*
  252. * Printing of the objects hex dump to the seq file. The number of lines to be
  253. * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
  254. * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
  255. * with the object->lock held.
  256. */
  257. static void hex_dump_object(struct seq_file *seq,
  258. struct kmemleak_object *object)
  259. {
  260. const u8 *ptr = (const u8 *)object->pointer;
  261. size_t len;
  262. /* limit the number of lines to HEX_MAX_LINES */
  263. len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
  264. warn_or_seq_printf(seq, " hex dump (first %zu bytes):\n", len);
  265. kasan_disable_current();
  266. warn_or_seq_hex_dump(seq, DUMP_PREFIX_NONE, HEX_ROW_SIZE,
  267. HEX_GROUP_SIZE, kasan_reset_tag((void *)ptr), len, HEX_ASCII);
  268. kasan_enable_current();
  269. }
  270. /*
  271. * Object colors, encoded with count and min_count:
  272. * - white - orphan object, not enough references to it (count < min_count)
  273. * - gray - not orphan, not marked as false positive (min_count == 0) or
  274. * sufficient references to it (count >= min_count)
  275. * - black - ignore, it doesn't contain references (e.g. text section)
  276. * (min_count == -1). No function defined for this color.
  277. * Newly created objects don't have any color assigned (object->count == -1)
  278. * before the next memory scan when they become white.
  279. */
  280. static bool color_white(const struct kmemleak_object *object)
  281. {
  282. return object->count != KMEMLEAK_BLACK &&
  283. object->count < object->min_count;
  284. }
  285. static bool color_gray(const struct kmemleak_object *object)
  286. {
  287. return object->min_count != KMEMLEAK_BLACK &&
  288. object->count >= object->min_count;
  289. }
  290. /*
  291. * Objects are considered unreferenced only if their color is white, they have
  292. * not be deleted and have a minimum age to avoid false positives caused by
  293. * pointers temporarily stored in CPU registers.
  294. */
  295. static bool unreferenced_object(struct kmemleak_object *object)
  296. {
  297. return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
  298. time_before_eq(object->jiffies + jiffies_min_age,
  299. jiffies_last_scan);
  300. }
  301. /*
  302. * Printing of the unreferenced objects information to the seq file. The
  303. * print_unreferenced function must be called with the object->lock held.
  304. */
  305. static void print_unreferenced(struct seq_file *seq,
  306. struct kmemleak_object *object)
  307. {
  308. int i;
  309. unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
  310. warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
  311. object->pointer, object->size);
  312. warn_or_seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
  313. object->comm, object->pid, object->jiffies,
  314. msecs_age / 1000, msecs_age % 1000);
  315. hex_dump_object(seq, object);
  316. warn_or_seq_printf(seq, " backtrace:\n");
  317. for (i = 0; i < object->trace_len; i++) {
  318. void *ptr = (void *)object->trace[i];
  319. warn_or_seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
  320. }
  321. }
  322. /*
  323. * Print the kmemleak_object information. This function is used mainly for
  324. * debugging special cases when kmemleak operations. It must be called with
  325. * the object->lock held.
  326. */
  327. static void dump_object_info(struct kmemleak_object *object)
  328. {
  329. pr_notice("Object 0x%08lx (size %zu):\n",
  330. object->pointer, object->size);
  331. pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
  332. object->comm, object->pid, object->jiffies);
  333. pr_notice(" min_count = %d\n", object->min_count);
  334. pr_notice(" count = %d\n", object->count);
  335. pr_notice(" flags = 0x%x\n", object->flags);
  336. pr_notice(" checksum = %u\n", object->checksum);
  337. pr_notice(" backtrace:\n");
  338. stack_trace_print(object->trace, object->trace_len, 4);
  339. }
  340. /*
  341. * Look-up a memory block metadata (kmemleak_object) in the object search
  342. * tree based on a pointer value. If alias is 0, only values pointing to the
  343. * beginning of the memory block are allowed. The kmemleak_lock must be held
  344. * when calling this function.
  345. */
  346. static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
  347. {
  348. struct rb_node *rb = object_tree_root.rb_node;
  349. while (rb) {
  350. struct kmemleak_object *object =
  351. rb_entry(rb, struct kmemleak_object, rb_node);
  352. if (ptr < object->pointer)
  353. rb = object->rb_node.rb_left;
  354. else if (object->pointer + object->size <= ptr)
  355. rb = object->rb_node.rb_right;
  356. else if (object->pointer == ptr || alias)
  357. return object;
  358. else {
  359. kmemleak_warn("Found object by alias at 0x%08lx\n",
  360. ptr);
  361. dump_object_info(object);
  362. break;
  363. }
  364. }
  365. return NULL;
  366. }
  367. /*
  368. * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
  369. * that once an object's use_count reached 0, the RCU freeing was already
  370. * registered and the object should no longer be used. This function must be
  371. * called under the protection of rcu_read_lock().
  372. */
  373. static int get_object(struct kmemleak_object *object)
  374. {
  375. return atomic_inc_not_zero(&object->use_count);
  376. }
  377. /*
  378. * Memory pool allocation and freeing. kmemleak_lock must not be held.
  379. */
  380. static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
  381. {
  382. unsigned long flags;
  383. struct kmemleak_object *object;
  384. /* try the slab allocator first */
  385. if (object_cache) {
  386. object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
  387. if (object)
  388. return object;
  389. }
  390. /* slab allocation failed, try the memory pool */
  391. raw_spin_lock_irqsave(&kmemleak_lock, flags);
  392. object = list_first_entry_or_null(&mem_pool_free_list,
  393. typeof(*object), object_list);
  394. if (object)
  395. list_del(&object->object_list);
  396. else if (mem_pool_free_count)
  397. object = &mem_pool[--mem_pool_free_count];
  398. else
  399. pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
  400. raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
  401. return object;
  402. }
  403. /*
  404. * Return the object to either the slab allocator or the memory pool.
  405. */
  406. static void mem_pool_free(struct kmemleak_object *object)
  407. {
  408. unsigned long flags;
  409. if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
  410. kmem_cache_free(object_cache, object);
  411. return;
  412. }
  413. /* add the object to the memory pool free list */
  414. raw_spin_lock_irqsave(&kmemleak_lock, flags);
  415. list_add(&object->object_list, &mem_pool_free_list);
  416. raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
  417. }
  418. /*
  419. * RCU callback to free a kmemleak_object.
  420. */
  421. static void free_object_rcu(struct rcu_head *rcu)
  422. {
  423. struct hlist_node *tmp;
  424. struct kmemleak_scan_area *area;
  425. struct kmemleak_object *object =
  426. container_of(rcu, struct kmemleak_object, rcu);
  427. /*
  428. * Once use_count is 0 (guaranteed by put_object), there is no other
  429. * code accessing this object, hence no need for locking.
  430. */
  431. hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
  432. hlist_del(&area->node);
  433. kmem_cache_free(scan_area_cache, area);
  434. }
  435. mem_pool_free(object);
  436. }
  437. /*
  438. * Decrement the object use_count. Once the count is 0, free the object using
  439. * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
  440. * delete_object() path, the delayed RCU freeing ensures that there is no
  441. * recursive call to the kernel allocator. Lock-less RCU object_list traversal
  442. * is also possible.
  443. */
  444. static void put_object(struct kmemleak_object *object)
  445. {
  446. if (!atomic_dec_and_test(&object->use_count))
  447. return;
  448. /* should only get here after delete_object was called */
  449. WARN_ON(object->flags & OBJECT_ALLOCATED);
  450. /*
  451. * It may be too early for the RCU callbacks, however, there is no
  452. * concurrent object_list traversal when !object_cache and all objects
  453. * came from the memory pool. Free the object directly.
  454. */
  455. if (object_cache)
  456. call_rcu(&object->rcu, free_object_rcu);
  457. else
  458. free_object_rcu(&object->rcu);
  459. }
  460. /*
  461. * Look up an object in the object search tree and increase its use_count.
  462. */
  463. static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
  464. {
  465. unsigned long flags;
  466. struct kmemleak_object *object;
  467. rcu_read_lock();
  468. raw_spin_lock_irqsave(&kmemleak_lock, flags);
  469. object = lookup_object(ptr, alias);
  470. raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
  471. /* check whether the object is still available */
  472. if (object && !get_object(object))
  473. object = NULL;
  474. rcu_read_unlock();
  475. return object;
  476. }
  477. /*
  478. * Remove an object from the object_tree_root and object_list. Must be called
  479. * with the kmemleak_lock held _if_ kmemleak is still enabled.
  480. */
  481. static void __remove_object(struct kmemleak_object *object)
  482. {
  483. rb_erase(&object->rb_node, &object_tree_root);
  484. list_del_rcu(&object->object_list);
  485. }
  486. /*
  487. * Look up an object in the object search tree and remove it from both
  488. * object_tree_root and object_list. The returned object's use_count should be
  489. * at least 1, as initially set by create_object().
  490. */
  491. static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
  492. {
  493. unsigned long flags;
  494. struct kmemleak_object *object;
  495. raw_spin_lock_irqsave(&kmemleak_lock, flags);
  496. object = lookup_object(ptr, alias);
  497. if (object)
  498. __remove_object(object);
  499. raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
  500. return object;
  501. }
  502. /*
  503. * Save stack trace to the given array of MAX_TRACE size.
  504. */
  505. static int __save_stack_trace(unsigned long *trace)
  506. {
  507. return stack_trace_save(trace, MAX_TRACE, 2);
  508. }
  509. /*
  510. * Create the metadata (struct kmemleak_object) corresponding to an allocated
  511. * memory block and add it to the object_list and object_tree_root.
  512. */
  513. static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
  514. int min_count, gfp_t gfp)
  515. {
  516. unsigned long flags;
  517. struct kmemleak_object *object, *parent;
  518. struct rb_node **link, *rb_parent;
  519. unsigned long untagged_ptr;
  520. object = mem_pool_alloc(gfp);
  521. if (!object) {
  522. pr_warn("Cannot allocate a kmemleak_object structure\n");
  523. kmemleak_disable();
  524. return NULL;
  525. }
  526. INIT_LIST_HEAD(&object->object_list);
  527. INIT_LIST_HEAD(&object->gray_list);
  528. INIT_HLIST_HEAD(&object->area_list);
  529. raw_spin_lock_init(&object->lock);
  530. atomic_set(&object->use_count, 1);
  531. object->flags = OBJECT_ALLOCATED;
  532. object->pointer = ptr;
  533. object->size = kfence_ksize((void *)ptr) ?: size;
  534. object->excess_ref = 0;
  535. object->min_count = min_count;
  536. object->count = 0; /* white color initially */
  537. object->jiffies = jiffies;
  538. object->checksum = 0;
  539. /* task information */
  540. if (in_irq()) {
  541. object->pid = 0;
  542. strncpy(object->comm, "hardirq", sizeof(object->comm));
  543. } else if (in_serving_softirq()) {
  544. object->pid = 0;
  545. strncpy(object->comm, "softirq", sizeof(object->comm));
  546. } else {
  547. object->pid = current->pid;
  548. /*
  549. * There is a small chance of a race with set_task_comm(),
  550. * however using get_task_comm() here may cause locking
  551. * dependency issues with current->alloc_lock. In the worst
  552. * case, the command line is not correct.
  553. */
  554. strncpy(object->comm, current->comm, sizeof(object->comm));
  555. }
  556. /* kernel backtrace */
  557. object->trace_len = __save_stack_trace(object->trace);
  558. raw_spin_lock_irqsave(&kmemleak_lock, flags);
  559. untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
  560. min_addr = min(min_addr, untagged_ptr);
  561. max_addr = max(max_addr, untagged_ptr + size);
  562. link = &object_tree_root.rb_node;
  563. rb_parent = NULL;
  564. while (*link) {
  565. rb_parent = *link;
  566. parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
  567. if (ptr + size <= parent->pointer)
  568. link = &parent->rb_node.rb_left;
  569. else if (parent->pointer + parent->size <= ptr)
  570. link = &parent->rb_node.rb_right;
  571. else {
  572. kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
  573. ptr);
  574. /*
  575. * No need for parent->lock here since "parent" cannot
  576. * be freed while the kmemleak_lock is held.
  577. */
  578. dump_object_info(parent);
  579. kmem_cache_free(object_cache, object);
  580. object = NULL;
  581. goto out;
  582. }
  583. }
  584. rb_link_node(&object->rb_node, rb_parent, link);
  585. rb_insert_color(&object->rb_node, &object_tree_root);
  586. list_add_tail_rcu(&object->object_list, &object_list);
  587. out:
  588. raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
  589. return object;
  590. }
  591. /*
  592. * Mark the object as not allocated and schedule RCU freeing via put_object().
  593. */
  594. static void __delete_object(struct kmemleak_object *object)
  595. {
  596. unsigned long flags;
  597. WARN_ON(!(object->flags & OBJECT_ALLOCATED));
  598. WARN_ON(atomic_read(&object->use_count) < 1);
  599. /*
  600. * Locking here also ensures that the corresponding memory block
  601. * cannot be freed when it is being scanned.
  602. */
  603. raw_spin_lock_irqsave(&object->lock, flags);
  604. object->flags &= ~OBJECT_ALLOCATED;
  605. raw_spin_unlock_irqrestore(&object->lock, flags);
  606. put_object(object);
  607. }
  608. /*
  609. * Look up the metadata (struct kmemleak_object) corresponding to ptr and
  610. * delete it.
  611. */
  612. static void delete_object_full(unsigned long ptr)
  613. {
  614. struct kmemleak_object *object;
  615. object = find_and_remove_object(ptr, 0);
  616. if (!object) {
  617. #ifdef DEBUG
  618. kmemleak_warn("Freeing unknown object at 0x%08lx\n",
  619. ptr);
  620. #endif
  621. return;
  622. }
  623. __delete_object(object);
  624. }
  625. /*
  626. * Look up the metadata (struct kmemleak_object) corresponding to ptr and
  627. * delete it. If the memory block is partially freed, the function may create
  628. * additional metadata for the remaining parts of the block.
  629. */
  630. static void delete_object_part(unsigned long ptr, size_t size)
  631. {
  632. struct kmemleak_object *object;
  633. unsigned long start, end;
  634. object = find_and_remove_object(ptr, 1);
  635. if (!object) {
  636. #ifdef DEBUG
  637. kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
  638. ptr, size);
  639. #endif
  640. return;
  641. }
  642. /*
  643. * Create one or two objects that may result from the memory block
  644. * split. Note that partial freeing is only done by free_bootmem() and
  645. * this happens before kmemleak_init() is called.
  646. */
  647. start = object->pointer;
  648. end = object->pointer + object->size;
  649. if (ptr > start)
  650. create_object(start, ptr - start, object->min_count,
  651. GFP_KERNEL);
  652. if (ptr + size < end)
  653. create_object(ptr + size, end - ptr - size, object->min_count,
  654. GFP_KERNEL);
  655. __delete_object(object);
  656. }
  657. static void __paint_it(struct kmemleak_object *object, int color)
  658. {
  659. object->min_count = color;
  660. if (color == KMEMLEAK_BLACK)
  661. object->flags |= OBJECT_NO_SCAN;
  662. }
  663. static void paint_it(struct kmemleak_object *object, int color)
  664. {
  665. unsigned long flags;
  666. raw_spin_lock_irqsave(&object->lock, flags);
  667. __paint_it(object, color);
  668. raw_spin_unlock_irqrestore(&object->lock, flags);
  669. }
  670. static void paint_ptr(unsigned long ptr, int color)
  671. {
  672. struct kmemleak_object *object;
  673. object = find_and_get_object(ptr, 0);
  674. if (!object) {
  675. kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
  676. ptr,
  677. (color == KMEMLEAK_GREY) ? "Grey" :
  678. (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
  679. return;
  680. }
  681. paint_it(object, color);
  682. put_object(object);
  683. }
  684. /*
  685. * Mark an object permanently as gray-colored so that it can no longer be
  686. * reported as a leak. This is used in general to mark a false positive.
  687. */
  688. static void make_gray_object(unsigned long ptr)
  689. {
  690. paint_ptr(ptr, KMEMLEAK_GREY);
  691. }
  692. /*
  693. * Mark the object as black-colored so that it is ignored from scans and
  694. * reporting.
  695. */
  696. static void make_black_object(unsigned long ptr)
  697. {
  698. paint_ptr(ptr, KMEMLEAK_BLACK);
  699. }
  700. /*
  701. * Add a scanning area to the object. If at least one such area is added,
  702. * kmemleak will only scan these ranges rather than the whole memory block.
  703. */
  704. static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
  705. {
  706. unsigned long flags;
  707. struct kmemleak_object *object;
  708. struct kmemleak_scan_area *area = NULL;
  709. unsigned long untagged_ptr;
  710. unsigned long untagged_objp;
  711. object = find_and_get_object(ptr, 1);
  712. if (!object) {
  713. kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
  714. ptr);
  715. return;
  716. }
  717. untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
  718. untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
  719. if (scan_area_cache)
  720. area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
  721. raw_spin_lock_irqsave(&object->lock, flags);
  722. if (!area) {
  723. pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
  724. /* mark the object for full scan to avoid false positives */
  725. object->flags |= OBJECT_FULL_SCAN;
  726. goto out_unlock;
  727. }
  728. if (size == SIZE_MAX) {
  729. size = untagged_objp + object->size - untagged_ptr;
  730. } else if (untagged_ptr + size > untagged_objp + object->size) {
  731. kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
  732. dump_object_info(object);
  733. kmem_cache_free(scan_area_cache, area);
  734. goto out_unlock;
  735. }
  736. INIT_HLIST_NODE(&area->node);
  737. area->start = ptr;
  738. area->size = size;
  739. hlist_add_head(&area->node, &object->area_list);
  740. out_unlock:
  741. raw_spin_unlock_irqrestore(&object->lock, flags);
  742. put_object(object);
  743. }
  744. /*
  745. * Any surplus references (object already gray) to 'ptr' are passed to
  746. * 'excess_ref'. This is used in the vmalloc() case where a pointer to
  747. * vm_struct may be used as an alternative reference to the vmalloc'ed object
  748. * (see free_thread_stack()).
  749. */
  750. static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
  751. {
  752. unsigned long flags;
  753. struct kmemleak_object *object;
  754. object = find_and_get_object(ptr, 0);
  755. if (!object) {
  756. kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
  757. ptr);
  758. return;
  759. }
  760. raw_spin_lock_irqsave(&object->lock, flags);
  761. object->excess_ref = excess_ref;
  762. raw_spin_unlock_irqrestore(&object->lock, flags);
  763. put_object(object);
  764. }
  765. /*
  766. * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
  767. * pointer. Such object will not be scanned by kmemleak but references to it
  768. * are searched.
  769. */
  770. static void object_no_scan(unsigned long ptr)
  771. {
  772. unsigned long flags;
  773. struct kmemleak_object *object;
  774. object = find_and_get_object(ptr, 0);
  775. if (!object) {
  776. kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
  777. return;
  778. }
  779. raw_spin_lock_irqsave(&object->lock, flags);
  780. object->flags |= OBJECT_NO_SCAN;
  781. raw_spin_unlock_irqrestore(&object->lock, flags);
  782. put_object(object);
  783. }
  784. /**
  785. * kmemleak_alloc - register a newly allocated object
  786. * @ptr: pointer to beginning of the object
  787. * @size: size of the object
  788. * @min_count: minimum number of references to this object. If during memory
  789. * scanning a number of references less than @min_count is found,
  790. * the object is reported as a memory leak. If @min_count is 0,
  791. * the object is never reported as a leak. If @min_count is -1,
  792. * the object is ignored (not scanned and not reported as a leak)
  793. * @gfp: kmalloc() flags used for kmemleak internal memory allocations
  794. *
  795. * This function is called from the kernel allocators when a new object
  796. * (memory block) is allocated (kmem_cache_alloc, kmalloc etc.).
  797. */
  798. void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
  799. gfp_t gfp)
  800. {
  801. pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
  802. if (kmemleak_enabled && ptr && !IS_ERR(ptr))
  803. create_object((unsigned long)ptr, size, min_count, gfp);
  804. }
  805. EXPORT_SYMBOL_GPL(kmemleak_alloc);
  806. /**
  807. * kmemleak_alloc_percpu - register a newly allocated __percpu object
  808. * @ptr: __percpu pointer to beginning of the object
  809. * @size: size of the object
  810. * @gfp: flags used for kmemleak internal memory allocations
  811. *
  812. * This function is called from the kernel percpu allocator when a new object
  813. * (memory block) is allocated (alloc_percpu).
  814. */
  815. void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
  816. gfp_t gfp)
  817. {
  818. unsigned int cpu;
  819. pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
  820. /*
  821. * Percpu allocations are only scanned and not reported as leaks
  822. * (min_count is set to 0).
  823. */
  824. if (kmemleak_enabled && ptr && !IS_ERR(ptr))
  825. for_each_possible_cpu(cpu)
  826. create_object((unsigned long)per_cpu_ptr(ptr, cpu),
  827. size, 0, gfp);
  828. }
  829. EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
  830. /**
  831. * kmemleak_vmalloc - register a newly vmalloc'ed object
  832. * @area: pointer to vm_struct
  833. * @size: size of the object
  834. * @gfp: __vmalloc() flags used for kmemleak internal memory allocations
  835. *
  836. * This function is called from the vmalloc() kernel allocator when a new
  837. * object (memory block) is allocated.
  838. */
  839. void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
  840. {
  841. pr_debug("%s(0x%p, %zu)\n", __func__, area, size);
  842. /*
  843. * A min_count = 2 is needed because vm_struct contains a reference to
  844. * the virtual address of the vmalloc'ed block.
  845. */
  846. if (kmemleak_enabled) {
  847. create_object((unsigned long)area->addr, size, 2, gfp);
  848. object_set_excess_ref((unsigned long)area,
  849. (unsigned long)area->addr);
  850. }
  851. }
  852. EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
  853. /**
  854. * kmemleak_free - unregister a previously registered object
  855. * @ptr: pointer to beginning of the object
  856. *
  857. * This function is called from the kernel allocators when an object (memory
  858. * block) is freed (kmem_cache_free, kfree, vfree etc.).
  859. */
  860. void __ref kmemleak_free(const void *ptr)
  861. {
  862. pr_debug("%s(0x%p)\n", __func__, ptr);
  863. if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
  864. delete_object_full((unsigned long)ptr);
  865. }
  866. EXPORT_SYMBOL_GPL(kmemleak_free);
  867. /**
  868. * kmemleak_free_part - partially unregister a previously registered object
  869. * @ptr: pointer to the beginning or inside the object. This also
  870. * represents the start of the range to be freed
  871. * @size: size to be unregistered
  872. *
  873. * This function is called when only a part of a memory block is freed
  874. * (usually from the bootmem allocator).
  875. */
  876. void __ref kmemleak_free_part(const void *ptr, size_t size)
  877. {
  878. pr_debug("%s(0x%p)\n", __func__, ptr);
  879. if (kmemleak_enabled && ptr && !IS_ERR(ptr))
  880. delete_object_part((unsigned long)ptr, size);
  881. }
  882. EXPORT_SYMBOL_GPL(kmemleak_free_part);
  883. /**
  884. * kmemleak_free_percpu - unregister a previously registered __percpu object
  885. * @ptr: __percpu pointer to beginning of the object
  886. *
  887. * This function is called from the kernel percpu allocator when an object
  888. * (memory block) is freed (free_percpu).
  889. */
  890. void __ref kmemleak_free_percpu(const void __percpu *ptr)
  891. {
  892. unsigned int cpu;
  893. pr_debug("%s(0x%p)\n", __func__, ptr);
  894. if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
  895. for_each_possible_cpu(cpu)
  896. delete_object_full((unsigned long)per_cpu_ptr(ptr,
  897. cpu));
  898. }
  899. EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
  900. /**
  901. * kmemleak_update_trace - update object allocation stack trace
  902. * @ptr: pointer to beginning of the object
  903. *
  904. * Override the object allocation stack trace for cases where the actual
  905. * allocation place is not always useful.
  906. */
  907. void __ref kmemleak_update_trace(const void *ptr)
  908. {
  909. struct kmemleak_object *object;
  910. unsigned long flags;
  911. pr_debug("%s(0x%p)\n", __func__, ptr);
  912. if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
  913. return;
  914. object = find_and_get_object((unsigned long)ptr, 1);
  915. if (!object) {
  916. #ifdef DEBUG
  917. kmemleak_warn("Updating stack trace for unknown object at %p\n",
  918. ptr);
  919. #endif
  920. return;
  921. }
  922. raw_spin_lock_irqsave(&object->lock, flags);
  923. object->trace_len = __save_stack_trace(object->trace);
  924. raw_spin_unlock_irqrestore(&object->lock, flags);
  925. put_object(object);
  926. }
  927. EXPORT_SYMBOL(kmemleak_update_trace);
  928. /**
  929. * kmemleak_not_leak - mark an allocated object as false positive
  930. * @ptr: pointer to beginning of the object
  931. *
  932. * Calling this function on an object will cause the memory block to no longer
  933. * be reported as leak and always be scanned.
  934. */
  935. void __ref kmemleak_not_leak(const void *ptr)
  936. {
  937. pr_debug("%s(0x%p)\n", __func__, ptr);
  938. if (kmemleak_enabled && ptr && !IS_ERR(ptr))
  939. make_gray_object((unsigned long)ptr);
  940. }
  941. EXPORT_SYMBOL(kmemleak_not_leak);
  942. /**
  943. * kmemleak_ignore - ignore an allocated object
  944. * @ptr: pointer to beginning of the object
  945. *
  946. * Calling this function on an object will cause the memory block to be
  947. * ignored (not scanned and not reported as a leak). This is usually done when
  948. * it is known that the corresponding block is not a leak and does not contain
  949. * any references to other allocated memory blocks.
  950. */
  951. void __ref kmemleak_ignore(const void *ptr)
  952. {
  953. pr_debug("%s(0x%p)\n", __func__, ptr);
  954. if (kmemleak_enabled && ptr && !IS_ERR(ptr))
  955. make_black_object((unsigned long)ptr);
  956. }
  957. EXPORT_SYMBOL(kmemleak_ignore);
  958. /**
  959. * kmemleak_scan_area - limit the range to be scanned in an allocated object
  960. * @ptr: pointer to beginning or inside the object. This also
  961. * represents the start of the scan area
  962. * @size: size of the scan area
  963. * @gfp: kmalloc() flags used for kmemleak internal memory allocations
  964. *
  965. * This function is used when it is known that only certain parts of an object
  966. * contain references to other objects. Kmemleak will only scan these areas
  967. * reducing the number false negatives.
  968. */
  969. void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
  970. {
  971. pr_debug("%s(0x%p)\n", __func__, ptr);
  972. if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
  973. add_scan_area((unsigned long)ptr, size, gfp);
  974. }
  975. EXPORT_SYMBOL(kmemleak_scan_area);
  976. /**
  977. * kmemleak_no_scan - do not scan an allocated object
  978. * @ptr: pointer to beginning of the object
  979. *
  980. * This function notifies kmemleak not to scan the given memory block. Useful
  981. * in situations where it is known that the given object does not contain any
  982. * references to other objects. Kmemleak will not scan such objects reducing
  983. * the number of false negatives.
  984. */
  985. void __ref kmemleak_no_scan(const void *ptr)
  986. {
  987. pr_debug("%s(0x%p)\n", __func__, ptr);
  988. if (kmemleak_enabled && ptr && !IS_ERR(ptr))
  989. object_no_scan((unsigned long)ptr);
  990. }
  991. EXPORT_SYMBOL(kmemleak_no_scan);
  992. /**
  993. * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
  994. * address argument
  995. * @phys: physical address of the object
  996. * @size: size of the object
  997. * @min_count: minimum number of references to this object.
  998. * See kmemleak_alloc()
  999. * @gfp: kmalloc() flags used for kmemleak internal memory allocations
  1000. */
  1001. void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
  1002. gfp_t gfp)
  1003. {
  1004. if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
  1005. kmemleak_alloc(__va(phys), size, min_count, gfp);
  1006. }
  1007. EXPORT_SYMBOL(kmemleak_alloc_phys);
  1008. /**
  1009. * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
  1010. * physical address argument
  1011. * @phys: physical address if the beginning or inside an object. This
  1012. * also represents the start of the range to be freed
  1013. * @size: size to be unregistered
  1014. */
  1015. void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
  1016. {
  1017. if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
  1018. kmemleak_free_part(__va(phys), size);
  1019. }
  1020. EXPORT_SYMBOL(kmemleak_free_part_phys);
  1021. /**
  1022. * kmemleak_not_leak_phys - similar to kmemleak_not_leak but taking a physical
  1023. * address argument
  1024. * @phys: physical address of the object
  1025. */
  1026. void __ref kmemleak_not_leak_phys(phys_addr_t phys)
  1027. {
  1028. if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
  1029. kmemleak_not_leak(__va(phys));
  1030. }
  1031. EXPORT_SYMBOL(kmemleak_not_leak_phys);
  1032. /**
  1033. * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
  1034. * address argument
  1035. * @phys: physical address of the object
  1036. */
  1037. void __ref kmemleak_ignore_phys(phys_addr_t phys)
  1038. {
  1039. if (PHYS_PFN(phys) >= min_low_pfn && PHYS_PFN(phys) < max_low_pfn)
  1040. kmemleak_ignore(__va(phys));
  1041. }
  1042. EXPORT_SYMBOL(kmemleak_ignore_phys);
  1043. /*
  1044. * Update an object's checksum and return true if it was modified.
  1045. */
  1046. static bool update_checksum(struct kmemleak_object *object)
  1047. {
  1048. u32 old_csum = object->checksum;
  1049. kasan_disable_current();
  1050. kcsan_disable_current();
  1051. object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
  1052. kasan_enable_current();
  1053. kcsan_enable_current();
  1054. return object->checksum != old_csum;
  1055. }
  1056. /*
  1057. * Update an object's references. object->lock must be held by the caller.
  1058. */
  1059. static void update_refs(struct kmemleak_object *object)
  1060. {
  1061. if (!color_white(object)) {
  1062. /* non-orphan, ignored or new */
  1063. return;
  1064. }
  1065. /*
  1066. * Increase the object's reference count (number of pointers to the
  1067. * memory block). If this count reaches the required minimum, the
  1068. * object's color will become gray and it will be added to the
  1069. * gray_list.
  1070. */
  1071. object->count++;
  1072. if (color_gray(object)) {
  1073. /* put_object() called when removing from gray_list */
  1074. WARN_ON(!get_object(object));
  1075. list_add_tail(&object->gray_list, &gray_list);
  1076. }
  1077. }
  1078. /*
  1079. * Memory scanning is a long process and it needs to be interruptable. This
  1080. * function checks whether such interrupt condition occurred.
  1081. */
  1082. static int scan_should_stop(void)
  1083. {
  1084. if (!kmemleak_enabled)
  1085. return 1;
  1086. /*
  1087. * This function may be called from either process or kthread context,
  1088. * hence the need to check for both stop conditions.
  1089. */
  1090. if (current->mm)
  1091. return signal_pending(current);
  1092. else
  1093. return kthread_should_stop();
  1094. return 0;
  1095. }
  1096. /*
  1097. * Scan a memory block (exclusive range) for valid pointers and add those
  1098. * found to the gray list.
  1099. */
  1100. static void scan_block(void *_start, void *_end,
  1101. struct kmemleak_object *scanned)
  1102. {
  1103. unsigned long *ptr;
  1104. unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
  1105. unsigned long *end = _end - (BYTES_PER_POINTER - 1);
  1106. unsigned long flags;
  1107. unsigned long untagged_ptr;
  1108. raw_spin_lock_irqsave(&kmemleak_lock, flags);
  1109. for (ptr = start; ptr < end; ptr++) {
  1110. struct kmemleak_object *object;
  1111. unsigned long pointer;
  1112. unsigned long excess_ref;
  1113. if (scan_should_stop())
  1114. break;
  1115. kasan_disable_current();
  1116. pointer = *(unsigned long *)kasan_reset_tag((void *)ptr);
  1117. kasan_enable_current();
  1118. untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
  1119. if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
  1120. continue;
  1121. /*
  1122. * No need for get_object() here since we hold kmemleak_lock.
  1123. * object->use_count cannot be dropped to 0 while the object
  1124. * is still present in object_tree_root and object_list
  1125. * (with updates protected by kmemleak_lock).
  1126. */
  1127. object = lookup_object(pointer, 1);
  1128. if (!object)
  1129. continue;
  1130. if (object == scanned)
  1131. /* self referenced, ignore */
  1132. continue;
  1133. /*
  1134. * Avoid the lockdep recursive warning on object->lock being
  1135. * previously acquired in scan_object(). These locks are
  1136. * enclosed by scan_mutex.
  1137. */
  1138. raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
  1139. /* only pass surplus references (object already gray) */
  1140. if (color_gray(object)) {
  1141. excess_ref = object->excess_ref;
  1142. /* no need for update_refs() if object already gray */
  1143. } else {
  1144. excess_ref = 0;
  1145. update_refs(object);
  1146. }
  1147. raw_spin_unlock(&object->lock);
  1148. if (excess_ref) {
  1149. object = lookup_object(excess_ref, 0);
  1150. if (!object)
  1151. continue;
  1152. if (object == scanned)
  1153. /* circular reference, ignore */
  1154. continue;
  1155. raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
  1156. update_refs(object);
  1157. raw_spin_unlock(&object->lock);
  1158. }
  1159. }
  1160. raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
  1161. }
  1162. /*
  1163. * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
  1164. */
  1165. #ifdef CONFIG_SMP
  1166. static void scan_large_block(void *start, void *end)
  1167. {
  1168. void *next;
  1169. while (start < end) {
  1170. next = min(start + MAX_SCAN_SIZE, end);
  1171. scan_block(start, next, NULL);
  1172. start = next;
  1173. cond_resched();
  1174. }
  1175. }
  1176. #endif
  1177. /*
  1178. * Scan a memory block corresponding to a kmemleak_object. A condition is
  1179. * that object->use_count >= 1.
  1180. */
  1181. static void scan_object(struct kmemleak_object *object)
  1182. {
  1183. struct kmemleak_scan_area *area;
  1184. unsigned long flags;
  1185. /*
  1186. * Once the object->lock is acquired, the corresponding memory block
  1187. * cannot be freed (the same lock is acquired in delete_object).
  1188. */
  1189. raw_spin_lock_irqsave(&object->lock, flags);
  1190. if (object->flags & OBJECT_NO_SCAN)
  1191. goto out;
  1192. if (!(object->flags & OBJECT_ALLOCATED))
  1193. /* already freed object */
  1194. goto out;
  1195. if (hlist_empty(&object->area_list) ||
  1196. object->flags & OBJECT_FULL_SCAN) {
  1197. void *start = (void *)object->pointer;
  1198. void *end = (void *)(object->pointer + object->size);
  1199. void *next;
  1200. do {
  1201. next = min(start + MAX_SCAN_SIZE, end);
  1202. scan_block(start, next, object);
  1203. start = next;
  1204. if (start >= end)
  1205. break;
  1206. raw_spin_unlock_irqrestore(&object->lock, flags);
  1207. cond_resched();
  1208. raw_spin_lock_irqsave(&object->lock, flags);
  1209. } while (object->flags & OBJECT_ALLOCATED);
  1210. } else
  1211. hlist_for_each_entry(area, &object->area_list, node)
  1212. scan_block((void *)area->start,
  1213. (void *)(area->start + area->size),
  1214. object);
  1215. out:
  1216. raw_spin_unlock_irqrestore(&object->lock, flags);
  1217. }
  1218. /*
  1219. * Scan the objects already referenced (gray objects). More objects will be
  1220. * referenced and, if there are no memory leaks, all the objects are scanned.
  1221. */
  1222. static void scan_gray_list(void)
  1223. {
  1224. struct kmemleak_object *object, *tmp;
  1225. /*
  1226. * The list traversal is safe for both tail additions and removals
  1227. * from inside the loop. The kmemleak objects cannot be freed from
  1228. * outside the loop because their use_count was incremented.
  1229. */
  1230. object = list_entry(gray_list.next, typeof(*object), gray_list);
  1231. while (&object->gray_list != &gray_list) {
  1232. cond_resched();
  1233. /* may add new objects to the list */
  1234. if (!scan_should_stop())
  1235. scan_object(object);
  1236. tmp = list_entry(object->gray_list.next, typeof(*object),
  1237. gray_list);
  1238. /* remove the object from the list and release it */
  1239. list_del(&object->gray_list);
  1240. put_object(object);
  1241. object = tmp;
  1242. }
  1243. WARN_ON(!list_empty(&gray_list));
  1244. }
  1245. /*
  1246. * Scan data sections and all the referenced memory blocks allocated via the
  1247. * kernel's standard allocators. This function must be called with the
  1248. * scan_mutex held.
  1249. */
  1250. static void kmemleak_scan(void)
  1251. {
  1252. unsigned long flags;
  1253. struct kmemleak_object *object;
  1254. struct zone *zone;
  1255. int __maybe_unused i;
  1256. int new_leaks = 0;
  1257. jiffies_last_scan = jiffies;
  1258. /* prepare the kmemleak_object's */
  1259. rcu_read_lock();
  1260. list_for_each_entry_rcu(object, &object_list, object_list) {
  1261. raw_spin_lock_irqsave(&object->lock, flags);
  1262. #ifdef DEBUG
  1263. /*
  1264. * With a few exceptions there should be a maximum of
  1265. * 1 reference to any object at this point.
  1266. */
  1267. if (atomic_read(&object->use_count) > 1) {
  1268. pr_debug("object->use_count = %d\n",
  1269. atomic_read(&object->use_count));
  1270. dump_object_info(object);
  1271. }
  1272. #endif
  1273. /* reset the reference count (whiten the object) */
  1274. object->count = 0;
  1275. if (color_gray(object) && get_object(object))
  1276. list_add_tail(&object->gray_list, &gray_list);
  1277. raw_spin_unlock_irqrestore(&object->lock, flags);
  1278. }
  1279. rcu_read_unlock();
  1280. #ifdef CONFIG_SMP
  1281. /* per-cpu sections scanning */
  1282. for_each_possible_cpu(i)
  1283. scan_large_block(__per_cpu_start + per_cpu_offset(i),
  1284. __per_cpu_end + per_cpu_offset(i));
  1285. #endif
  1286. /*
  1287. * Struct page scanning for each node.
  1288. */
  1289. get_online_mems();
  1290. for_each_populated_zone(zone) {
  1291. unsigned long start_pfn = zone->zone_start_pfn;
  1292. unsigned long end_pfn = zone_end_pfn(zone);
  1293. unsigned long pfn;
  1294. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  1295. struct page *page = pfn_to_online_page(pfn);
  1296. if (!page)
  1297. continue;
  1298. /* only scan pages belonging to this zone */
  1299. if (page_zone(page) != zone)
  1300. continue;
  1301. /* only scan if page is in use */
  1302. if (page_count(page) == 0)
  1303. continue;
  1304. scan_block(page, page + 1, NULL);
  1305. if (!(pfn & 63))
  1306. cond_resched();
  1307. }
  1308. }
  1309. put_online_mems();
  1310. /*
  1311. * Scanning the task stacks (may introduce false negatives).
  1312. */
  1313. if (kmemleak_stack_scan) {
  1314. struct task_struct *p, *g;
  1315. rcu_read_lock();
  1316. for_each_process_thread(g, p) {
  1317. void *stack = try_get_task_stack(p);
  1318. if (stack) {
  1319. scan_block(stack, stack + THREAD_SIZE, NULL);
  1320. put_task_stack(p);
  1321. }
  1322. }
  1323. rcu_read_unlock();
  1324. }
  1325. /*
  1326. * Scan the objects already referenced from the sections scanned
  1327. * above.
  1328. */
  1329. scan_gray_list();
  1330. /*
  1331. * Check for new or unreferenced objects modified since the previous
  1332. * scan and color them gray until the next scan.
  1333. */
  1334. rcu_read_lock();
  1335. list_for_each_entry_rcu(object, &object_list, object_list) {
  1336. raw_spin_lock_irqsave(&object->lock, flags);
  1337. if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
  1338. && update_checksum(object) && get_object(object)) {
  1339. /* color it gray temporarily */
  1340. object->count = object->min_count;
  1341. list_add_tail(&object->gray_list, &gray_list);
  1342. }
  1343. raw_spin_unlock_irqrestore(&object->lock, flags);
  1344. }
  1345. rcu_read_unlock();
  1346. /*
  1347. * Re-scan the gray list for modified unreferenced objects.
  1348. */
  1349. scan_gray_list();
  1350. /*
  1351. * If scanning was stopped do not report any new unreferenced objects.
  1352. */
  1353. if (scan_should_stop())
  1354. return;
  1355. /*
  1356. * Scanning result reporting.
  1357. */
  1358. rcu_read_lock();
  1359. list_for_each_entry_rcu(object, &object_list, object_list) {
  1360. raw_spin_lock_irqsave(&object->lock, flags);
  1361. if (unreferenced_object(object) &&
  1362. !(object->flags & OBJECT_REPORTED)) {
  1363. object->flags |= OBJECT_REPORTED;
  1364. if (kmemleak_verbose)
  1365. print_unreferenced(NULL, object);
  1366. new_leaks++;
  1367. }
  1368. raw_spin_unlock_irqrestore(&object->lock, flags);
  1369. }
  1370. rcu_read_unlock();
  1371. if (new_leaks) {
  1372. kmemleak_found_leaks = true;
  1373. pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
  1374. new_leaks);
  1375. }
  1376. }
  1377. /*
  1378. * Thread function performing automatic memory scanning. Unreferenced objects
  1379. * at the end of a memory scan are reported but only the first time.
  1380. */
  1381. static int kmemleak_scan_thread(void *arg)
  1382. {
  1383. static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
  1384. pr_info("Automatic memory scanning thread started\n");
  1385. set_user_nice(current, 10);
  1386. /*
  1387. * Wait before the first scan to allow the system to fully initialize.
  1388. */
  1389. if (first_run) {
  1390. signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
  1391. first_run = 0;
  1392. while (timeout && !kthread_should_stop())
  1393. timeout = schedule_timeout_interruptible(timeout);
  1394. }
  1395. while (!kthread_should_stop()) {
  1396. signed long timeout = jiffies_scan_wait;
  1397. mutex_lock(&scan_mutex);
  1398. kmemleak_scan();
  1399. mutex_unlock(&scan_mutex);
  1400. /* wait before the next scan */
  1401. while (timeout && !kthread_should_stop())
  1402. timeout = schedule_timeout_interruptible(timeout);
  1403. }
  1404. pr_info("Automatic memory scanning thread ended\n");
  1405. return 0;
  1406. }
  1407. /*
  1408. * Start the automatic memory scanning thread. This function must be called
  1409. * with the scan_mutex held.
  1410. */
  1411. static void start_scan_thread(void)
  1412. {
  1413. if (scan_thread)
  1414. return;
  1415. scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
  1416. if (IS_ERR(scan_thread)) {
  1417. pr_warn("Failed to create the scan thread\n");
  1418. scan_thread = NULL;
  1419. }
  1420. }
  1421. /*
  1422. * Stop the automatic memory scanning thread.
  1423. */
  1424. static void stop_scan_thread(void)
  1425. {
  1426. if (scan_thread) {
  1427. kthread_stop(scan_thread);
  1428. scan_thread = NULL;
  1429. }
  1430. }
  1431. /*
  1432. * Iterate over the object_list and return the first valid object at or after
  1433. * the required position with its use_count incremented. The function triggers
  1434. * a memory scanning when the pos argument points to the first position.
  1435. */
  1436. static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
  1437. {
  1438. struct kmemleak_object *object;
  1439. loff_t n = *pos;
  1440. int err;
  1441. err = mutex_lock_interruptible(&scan_mutex);
  1442. if (err < 0)
  1443. return ERR_PTR(err);
  1444. rcu_read_lock();
  1445. list_for_each_entry_rcu(object, &object_list, object_list) {
  1446. if (n-- > 0)
  1447. continue;
  1448. if (get_object(object))
  1449. goto out;
  1450. }
  1451. object = NULL;
  1452. out:
  1453. return object;
  1454. }
  1455. /*
  1456. * Return the next object in the object_list. The function decrements the
  1457. * use_count of the previous object and increases that of the next one.
  1458. */
  1459. static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
  1460. {
  1461. struct kmemleak_object *prev_obj = v;
  1462. struct kmemleak_object *next_obj = NULL;
  1463. struct kmemleak_object *obj = prev_obj;
  1464. ++(*pos);
  1465. list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
  1466. if (get_object(obj)) {
  1467. next_obj = obj;
  1468. break;
  1469. }
  1470. }
  1471. put_object(prev_obj);
  1472. return next_obj;
  1473. }
  1474. /*
  1475. * Decrement the use_count of the last object required, if any.
  1476. */
  1477. static void kmemleak_seq_stop(struct seq_file *seq, void *v)
  1478. {
  1479. if (!IS_ERR(v)) {
  1480. /*
  1481. * kmemleak_seq_start may return ERR_PTR if the scan_mutex
  1482. * waiting was interrupted, so only release it if !IS_ERR.
  1483. */
  1484. rcu_read_unlock();
  1485. mutex_unlock(&scan_mutex);
  1486. if (v)
  1487. put_object(v);
  1488. }
  1489. }
  1490. /*
  1491. * Print the information for an unreferenced object to the seq file.
  1492. */
  1493. static int kmemleak_seq_show(struct seq_file *seq, void *v)
  1494. {
  1495. struct kmemleak_object *object = v;
  1496. unsigned long flags;
  1497. raw_spin_lock_irqsave(&object->lock, flags);
  1498. if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
  1499. print_unreferenced(seq, object);
  1500. raw_spin_unlock_irqrestore(&object->lock, flags);
  1501. return 0;
  1502. }
  1503. static const struct seq_operations kmemleak_seq_ops = {
  1504. .start = kmemleak_seq_start,
  1505. .next = kmemleak_seq_next,
  1506. .stop = kmemleak_seq_stop,
  1507. .show = kmemleak_seq_show,
  1508. };
  1509. static int kmemleak_open(struct inode *inode, struct file *file)
  1510. {
  1511. return seq_open(file, &kmemleak_seq_ops);
  1512. }
  1513. static int dump_str_object_info(const char *str)
  1514. {
  1515. unsigned long flags;
  1516. struct kmemleak_object *object;
  1517. unsigned long addr;
  1518. if (kstrtoul(str, 0, &addr))
  1519. return -EINVAL;
  1520. object = find_and_get_object(addr, 0);
  1521. if (!object) {
  1522. pr_info("Unknown object at 0x%08lx\n", addr);
  1523. return -EINVAL;
  1524. }
  1525. raw_spin_lock_irqsave(&object->lock, flags);
  1526. dump_object_info(object);
  1527. raw_spin_unlock_irqrestore(&object->lock, flags);
  1528. put_object(object);
  1529. return 0;
  1530. }
  1531. /*
  1532. * We use grey instead of black to ensure we can do future scans on the same
  1533. * objects. If we did not do future scans these black objects could
  1534. * potentially contain references to newly allocated objects in the future and
  1535. * we'd end up with false positives.
  1536. */
  1537. static void kmemleak_clear(void)
  1538. {
  1539. struct kmemleak_object *object;
  1540. unsigned long flags;
  1541. rcu_read_lock();
  1542. list_for_each_entry_rcu(object, &object_list, object_list) {
  1543. raw_spin_lock_irqsave(&object->lock, flags);
  1544. if ((object->flags & OBJECT_REPORTED) &&
  1545. unreferenced_object(object))
  1546. __paint_it(object, KMEMLEAK_GREY);
  1547. raw_spin_unlock_irqrestore(&object->lock, flags);
  1548. }
  1549. rcu_read_unlock();
  1550. kmemleak_found_leaks = false;
  1551. }
  1552. static void __kmemleak_do_cleanup(void);
  1553. /*
  1554. * File write operation to configure kmemleak at run-time. The following
  1555. * commands can be written to the /sys/kernel/debug/kmemleak file:
  1556. * off - disable kmemleak (irreversible)
  1557. * stack=on - enable the task stacks scanning
  1558. * stack=off - disable the tasks stacks scanning
  1559. * scan=on - start the automatic memory scanning thread
  1560. * scan=off - stop the automatic memory scanning thread
  1561. * scan=... - set the automatic memory scanning period in seconds (0 to
  1562. * disable it)
  1563. * scan - trigger a memory scan
  1564. * clear - mark all current reported unreferenced kmemleak objects as
  1565. * grey to ignore printing them, or free all kmemleak objects
  1566. * if kmemleak has been disabled.
  1567. * dump=... - dump information about the object found at the given address
  1568. */
  1569. static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
  1570. size_t size, loff_t *ppos)
  1571. {
  1572. char buf[64];
  1573. int buf_size;
  1574. int ret;
  1575. buf_size = min(size, (sizeof(buf) - 1));
  1576. if (strncpy_from_user(buf, user_buf, buf_size) < 0)
  1577. return -EFAULT;
  1578. buf[buf_size] = 0;
  1579. ret = mutex_lock_interruptible(&scan_mutex);
  1580. if (ret < 0)
  1581. return ret;
  1582. if (strncmp(buf, "clear", 5) == 0) {
  1583. if (kmemleak_enabled)
  1584. kmemleak_clear();
  1585. else
  1586. __kmemleak_do_cleanup();
  1587. goto out;
  1588. }
  1589. if (!kmemleak_enabled) {
  1590. ret = -EPERM;
  1591. goto out;
  1592. }
  1593. if (strncmp(buf, "off", 3) == 0)
  1594. kmemleak_disable();
  1595. else if (strncmp(buf, "stack=on", 8) == 0)
  1596. kmemleak_stack_scan = 1;
  1597. else if (strncmp(buf, "stack=off", 9) == 0)
  1598. kmemleak_stack_scan = 0;
  1599. else if (strncmp(buf, "scan=on", 7) == 0)
  1600. start_scan_thread();
  1601. else if (strncmp(buf, "scan=off", 8) == 0)
  1602. stop_scan_thread();
  1603. else if (strncmp(buf, "scan=", 5) == 0) {
  1604. unsigned long secs;
  1605. ret = kstrtoul(buf + 5, 0, &secs);
  1606. if (ret < 0)
  1607. goto out;
  1608. stop_scan_thread();
  1609. if (secs) {
  1610. jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
  1611. start_scan_thread();
  1612. }
  1613. } else if (strncmp(buf, "scan", 4) == 0)
  1614. kmemleak_scan();
  1615. else if (strncmp(buf, "dump=", 5) == 0)
  1616. ret = dump_str_object_info(buf + 5);
  1617. else
  1618. ret = -EINVAL;
  1619. out:
  1620. mutex_unlock(&scan_mutex);
  1621. if (ret < 0)
  1622. return ret;
  1623. /* ignore the rest of the buffer, only one command at a time */
  1624. *ppos += size;
  1625. return size;
  1626. }
  1627. static const struct file_operations kmemleak_fops = {
  1628. .owner = THIS_MODULE,
  1629. .open = kmemleak_open,
  1630. .read = seq_read,
  1631. .write = kmemleak_write,
  1632. .llseek = seq_lseek,
  1633. .release = seq_release,
  1634. };
  1635. static void __kmemleak_do_cleanup(void)
  1636. {
  1637. struct kmemleak_object *object, *tmp;
  1638. /*
  1639. * Kmemleak has already been disabled, no need for RCU list traversal
  1640. * or kmemleak_lock held.
  1641. */
  1642. list_for_each_entry_safe(object, tmp, &object_list, object_list) {
  1643. __remove_object(object);
  1644. __delete_object(object);
  1645. }
  1646. }
  1647. /*
  1648. * Stop the memory scanning thread and free the kmemleak internal objects if
  1649. * no previous scan thread (otherwise, kmemleak may still have some useful
  1650. * information on memory leaks).
  1651. */
  1652. static void kmemleak_do_cleanup(struct work_struct *work)
  1653. {
  1654. stop_scan_thread();
  1655. mutex_lock(&scan_mutex);
  1656. /*
  1657. * Once it is made sure that kmemleak_scan has stopped, it is safe to no
  1658. * longer track object freeing. Ordering of the scan thread stopping and
  1659. * the memory accesses below is guaranteed by the kthread_stop()
  1660. * function.
  1661. */
  1662. kmemleak_free_enabled = 0;
  1663. mutex_unlock(&scan_mutex);
  1664. if (!kmemleak_found_leaks)
  1665. __kmemleak_do_cleanup();
  1666. else
  1667. pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
  1668. }
  1669. static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
  1670. /*
  1671. * Disable kmemleak. No memory allocation/freeing will be traced once this
  1672. * function is called. Disabling kmemleak is an irreversible operation.
  1673. */
  1674. static void kmemleak_disable(void)
  1675. {
  1676. /* atomically check whether it was already invoked */
  1677. if (cmpxchg(&kmemleak_error, 0, 1))
  1678. return;
  1679. /* stop any memory operation tracing */
  1680. kmemleak_enabled = 0;
  1681. /* check whether it is too early for a kernel thread */
  1682. if (kmemleak_initialized)
  1683. schedule_work(&cleanup_work);
  1684. else
  1685. kmemleak_free_enabled = 0;
  1686. pr_info("Kernel memory leak detector disabled\n");
  1687. }
  1688. /*
  1689. * Allow boot-time kmemleak disabling (enabled by default).
  1690. */
  1691. static int __init kmemleak_boot_config(char *str)
  1692. {
  1693. if (!str)
  1694. return -EINVAL;
  1695. if (strcmp(str, "off") == 0)
  1696. kmemleak_disable();
  1697. else if (strcmp(str, "on") == 0)
  1698. kmemleak_skip_disable = 1;
  1699. else
  1700. return -EINVAL;
  1701. return 0;
  1702. }
  1703. early_param("kmemleak", kmemleak_boot_config);
  1704. /*
  1705. * Kmemleak initialization.
  1706. */
  1707. void __init kmemleak_init(void)
  1708. {
  1709. #ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
  1710. if (!kmemleak_skip_disable) {
  1711. kmemleak_disable();
  1712. return;
  1713. }
  1714. #endif
  1715. if (kmemleak_error)
  1716. return;
  1717. jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
  1718. jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
  1719. object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
  1720. scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
  1721. /* register the data/bss sections */
  1722. create_object((unsigned long)_sdata, _edata - _sdata,
  1723. KMEMLEAK_GREY, GFP_ATOMIC);
  1724. create_object((unsigned long)__bss_start, __bss_stop - __bss_start,
  1725. KMEMLEAK_GREY, GFP_ATOMIC);
  1726. /* only register .data..ro_after_init if not within .data */
  1727. if (&__start_ro_after_init < &_sdata || &__end_ro_after_init > &_edata)
  1728. create_object((unsigned long)__start_ro_after_init,
  1729. __end_ro_after_init - __start_ro_after_init,
  1730. KMEMLEAK_GREY, GFP_ATOMIC);
  1731. }
  1732. /*
  1733. * Late initialization function.
  1734. */
  1735. static int __init kmemleak_late_init(void)
  1736. {
  1737. kmemleak_initialized = 1;
  1738. debugfs_create_file("kmemleak", 0644, NULL, NULL, &kmemleak_fops);
  1739. if (kmemleak_error) {
  1740. /*
  1741. * Some error occurred and kmemleak was disabled. There is a
  1742. * small chance that kmemleak_disable() was called immediately
  1743. * after setting kmemleak_initialized and we may end up with
  1744. * two clean-up threads but serialized by scan_mutex.
  1745. */
  1746. schedule_work(&cleanup_work);
  1747. return -ENOMEM;
  1748. }
  1749. if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
  1750. mutex_lock(&scan_mutex);
  1751. start_scan_thread();
  1752. mutex_unlock(&scan_mutex);
  1753. }
  1754. pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
  1755. mem_pool_free_count);
  1756. return 0;
  1757. }
  1758. late_initcall(kmemleak_late_init);