lockdep.c 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799
  1. /*
  2. * kernel/lockdep.c
  3. *
  4. * Runtime locking correctness validator
  5. *
  6. * Started by Ingo Molnar:
  7. *
  8. * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  9. *
  10. * this code maps all the lock dependencies as they occur in a live kernel
  11. * and will warn about the following classes of locking bugs:
  12. *
  13. * - lock inversion scenarios
  14. * - circular lock dependencies
  15. * - hardirq/softirq safe/unsafe locking bugs
  16. *
  17. * Bugs are reported even if the current locking scenario does not cause
  18. * any deadlock at this point.
  19. *
  20. * I.e. if anytime in the past two locks were taken in a different order,
  21. * even if it happened for another task, even if those were different
  22. * locks (but of the same class as this lock), this code will detect it.
  23. *
  24. * Thanks to Arjan van de Ven for coming up with the initial idea of
  25. * mapping lock dependencies runtime.
  26. */
  27. #include <linux/mutex.h>
  28. #include <linux/sched.h>
  29. #include <linux/delay.h>
  30. #include <linux/module.h>
  31. #include <linux/proc_fs.h>
  32. #include <linux/seq_file.h>
  33. #include <linux/spinlock.h>
  34. #include <linux/kallsyms.h>
  35. #include <linux/interrupt.h>
  36. #include <linux/stacktrace.h>
  37. #include <linux/debug_locks.h>
  38. #include <linux/irqflags.h>
  39. #include <linux/utsname.h>
  40. #include <asm/sections.h>
  41. #include "lockdep_internals.h"
  42. /*
  43. * lockdep_lock: protects the lockdep graph, the hashes and the
  44. * class/list/hash allocators.
  45. *
  46. * This is one of the rare exceptions where it's justified
  47. * to use a raw spinlock - we really dont want the spinlock
  48. * code to recurse back into the lockdep code...
  49. */
  50. static raw_spinlock_t lockdep_lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
  51. static int graph_lock(void)
  52. {
  53. __raw_spin_lock(&lockdep_lock);
  54. /*
  55. * Make sure that if another CPU detected a bug while
  56. * walking the graph we dont change it (while the other
  57. * CPU is busy printing out stuff with the graph lock
  58. * dropped already)
  59. */
  60. if (!debug_locks) {
  61. __raw_spin_unlock(&lockdep_lock);
  62. return 0;
  63. }
  64. return 1;
  65. }
  66. static inline int graph_unlock(void)
  67. {
  68. if (debug_locks && !__raw_spin_is_locked(&lockdep_lock))
  69. return DEBUG_LOCKS_WARN_ON(1);
  70. __raw_spin_unlock(&lockdep_lock);
  71. return 0;
  72. }
  73. /*
  74. * Turn lock debugging off and return with 0 if it was off already,
  75. * and also release the graph lock:
  76. */
  77. static inline int debug_locks_off_graph_unlock(void)
  78. {
  79. int ret = debug_locks_off();
  80. __raw_spin_unlock(&lockdep_lock);
  81. return ret;
  82. }
  83. static int lockdep_initialized;
  84. unsigned long nr_list_entries;
  85. static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
  86. /*
  87. * Allocate a lockdep entry. (assumes the graph_lock held, returns
  88. * with NULL on failure)
  89. */
  90. static struct lock_list *alloc_list_entry(void)
  91. {
  92. if (nr_list_entries >= MAX_LOCKDEP_ENTRIES) {
  93. if (!debug_locks_off_graph_unlock())
  94. return NULL;
  95. printk("BUG: MAX_LOCKDEP_ENTRIES too low!\n");
  96. printk("turning off the locking correctness validator.\n");
  97. return NULL;
  98. }
  99. return list_entries + nr_list_entries++;
  100. }
  101. /*
  102. * All data structures here are protected by the global debug_lock.
  103. *
  104. * Mutex key structs only get allocated, once during bootup, and never
  105. * get freed - this significantly simplifies the debugging code.
  106. */
  107. unsigned long nr_lock_classes;
  108. static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
  109. /*
  110. * We keep a global list of all lock classes. The list only grows,
  111. * never shrinks. The list is only accessed with the lockdep
  112. * spinlock lock held.
  113. */
  114. LIST_HEAD(all_lock_classes);
  115. /*
  116. * The lockdep classes are in a hash-table as well, for fast lookup:
  117. */
  118. #define CLASSHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)
  119. #define CLASSHASH_SIZE (1UL << CLASSHASH_BITS)
  120. #define CLASSHASH_MASK (CLASSHASH_SIZE - 1)
  121. #define __classhashfn(key) ((((unsigned long)key >> CLASSHASH_BITS) + (unsigned long)key) & CLASSHASH_MASK)
  122. #define classhashentry(key) (classhash_table + __classhashfn((key)))
  123. static struct list_head classhash_table[CLASSHASH_SIZE];
  124. unsigned long nr_lock_chains;
  125. static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
  126. /*
  127. * We put the lock dependency chains into a hash-table as well, to cache
  128. * their existence:
  129. */
  130. #define CHAINHASH_BITS (MAX_LOCKDEP_CHAINS_BITS-1)
  131. #define CHAINHASH_SIZE (1UL << CHAINHASH_BITS)
  132. #define CHAINHASH_MASK (CHAINHASH_SIZE - 1)
  133. #define __chainhashfn(chain) \
  134. (((chain >> CHAINHASH_BITS) + chain) & CHAINHASH_MASK)
  135. #define chainhashentry(chain) (chainhash_table + __chainhashfn((chain)))
  136. static struct list_head chainhash_table[CHAINHASH_SIZE];
  137. /*
  138. * The hash key of the lock dependency chains is a hash itself too:
  139. * it's a hash of all locks taken up to that lock, including that lock.
  140. * It's a 64-bit hash, because it's important for the keys to be
  141. * unique.
  142. */
  143. #define iterate_chain_key(key1, key2) \
  144. (((key1) << MAX_LOCKDEP_KEYS_BITS) ^ \
  145. ((key1) >> (64-MAX_LOCKDEP_KEYS_BITS)) ^ \
  146. (key2))
  147. void lockdep_off(void)
  148. {
  149. current->lockdep_recursion++;
  150. }
  151. EXPORT_SYMBOL(lockdep_off);
  152. void lockdep_on(void)
  153. {
  154. current->lockdep_recursion--;
  155. }
  156. EXPORT_SYMBOL(lockdep_on);
  157. /*
  158. * Debugging switches:
  159. */
  160. #define VERBOSE 0
  161. #define VERY_VERBOSE 0
  162. #if VERBOSE
  163. # define HARDIRQ_VERBOSE 1
  164. # define SOFTIRQ_VERBOSE 1
  165. #else
  166. # define HARDIRQ_VERBOSE 0
  167. # define SOFTIRQ_VERBOSE 0
  168. #endif
  169. #if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE
  170. /*
  171. * Quick filtering for interesting events:
  172. */
  173. static int class_filter(struct lock_class *class)
  174. {
  175. #if 0
  176. /* Example */
  177. if (class->name_version == 1 &&
  178. !strcmp(class->name, "lockname"))
  179. return 1;
  180. if (class->name_version == 1 &&
  181. !strcmp(class->name, "&struct->lockfield"))
  182. return 1;
  183. #endif
  184. /* Filter everything else. 1 would be to allow everything else */
  185. return 0;
  186. }
  187. #endif
  188. static int verbose(struct lock_class *class)
  189. {
  190. #if VERBOSE
  191. return class_filter(class);
  192. #endif
  193. return 0;
  194. }
  195. #ifdef CONFIG_TRACE_IRQFLAGS
  196. static int hardirq_verbose(struct lock_class *class)
  197. {
  198. #if HARDIRQ_VERBOSE
  199. return class_filter(class);
  200. #endif
  201. return 0;
  202. }
  203. static int softirq_verbose(struct lock_class *class)
  204. {
  205. #if SOFTIRQ_VERBOSE
  206. return class_filter(class);
  207. #endif
  208. return 0;
  209. }
  210. #endif
  211. /*
  212. * Stack-trace: tightly packed array of stack backtrace
  213. * addresses. Protected by the graph_lock.
  214. */
  215. unsigned long nr_stack_trace_entries;
  216. static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];
  217. static int save_trace(struct stack_trace *trace)
  218. {
  219. trace->nr_entries = 0;
  220. trace->max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;
  221. trace->entries = stack_trace + nr_stack_trace_entries;
  222. trace->skip = 3;
  223. trace->all_contexts = 0;
  224. save_stack_trace(trace, NULL);
  225. trace->max_entries = trace->nr_entries;
  226. nr_stack_trace_entries += trace->nr_entries;
  227. if (nr_stack_trace_entries == MAX_STACK_TRACE_ENTRIES) {
  228. if (!debug_locks_off_graph_unlock())
  229. return 0;
  230. printk("BUG: MAX_STACK_TRACE_ENTRIES too low!\n");
  231. printk("turning off the locking correctness validator.\n");
  232. dump_stack();
  233. return 0;
  234. }
  235. return 1;
  236. }
  237. unsigned int nr_hardirq_chains;
  238. unsigned int nr_softirq_chains;
  239. unsigned int nr_process_chains;
  240. unsigned int max_lockdep_depth;
  241. unsigned int max_recursion_depth;
  242. #ifdef CONFIG_DEBUG_LOCKDEP
  243. /*
  244. * We cannot printk in early bootup code. Not even early_printk()
  245. * might work. So we mark any initialization errors and printk
  246. * about it later on, in lockdep_info().
  247. */
  248. static int lockdep_init_error;
  249. /*
  250. * Various lockdep statistics:
  251. */
  252. atomic_t chain_lookup_hits;
  253. atomic_t chain_lookup_misses;
  254. atomic_t hardirqs_on_events;
  255. atomic_t hardirqs_off_events;
  256. atomic_t redundant_hardirqs_on;
  257. atomic_t redundant_hardirqs_off;
  258. atomic_t softirqs_on_events;
  259. atomic_t softirqs_off_events;
  260. atomic_t redundant_softirqs_on;
  261. atomic_t redundant_softirqs_off;
  262. atomic_t nr_unused_locks;
  263. atomic_t nr_cyclic_checks;
  264. atomic_t nr_cyclic_check_recursions;
  265. atomic_t nr_find_usage_forwards_checks;
  266. atomic_t nr_find_usage_forwards_recursions;
  267. atomic_t nr_find_usage_backwards_checks;
  268. atomic_t nr_find_usage_backwards_recursions;
  269. # define debug_atomic_inc(ptr) atomic_inc(ptr)
  270. # define debug_atomic_dec(ptr) atomic_dec(ptr)
  271. # define debug_atomic_read(ptr) atomic_read(ptr)
  272. #else
  273. # define debug_atomic_inc(ptr) do { } while (0)
  274. # define debug_atomic_dec(ptr) do { } while (0)
  275. # define debug_atomic_read(ptr) 0
  276. #endif
  277. /*
  278. * Locking printouts:
  279. */
  280. static const char *usage_str[] =
  281. {
  282. [LOCK_USED] = "initial-use ",
  283. [LOCK_USED_IN_HARDIRQ] = "in-hardirq-W",
  284. [LOCK_USED_IN_SOFTIRQ] = "in-softirq-W",
  285. [LOCK_ENABLED_SOFTIRQS] = "softirq-on-W",
  286. [LOCK_ENABLED_HARDIRQS] = "hardirq-on-W",
  287. [LOCK_USED_IN_HARDIRQ_READ] = "in-hardirq-R",
  288. [LOCK_USED_IN_SOFTIRQ_READ] = "in-softirq-R",
  289. [LOCK_ENABLED_SOFTIRQS_READ] = "softirq-on-R",
  290. [LOCK_ENABLED_HARDIRQS_READ] = "hardirq-on-R",
  291. };
  292. const char * __get_key_name(struct lockdep_subclass_key *key, char *str)
  293. {
  294. unsigned long offs, size;
  295. char *modname;
  296. return kallsyms_lookup((unsigned long)key, &size, &offs, &modname, str);
  297. }
  298. void
  299. get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4)
  300. {
  301. *c1 = '.', *c2 = '.', *c3 = '.', *c4 = '.';
  302. if (class->usage_mask & LOCKF_USED_IN_HARDIRQ)
  303. *c1 = '+';
  304. else
  305. if (class->usage_mask & LOCKF_ENABLED_HARDIRQS)
  306. *c1 = '-';
  307. if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ)
  308. *c2 = '+';
  309. else
  310. if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS)
  311. *c2 = '-';
  312. if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ)
  313. *c3 = '-';
  314. if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ) {
  315. *c3 = '+';
  316. if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ)
  317. *c3 = '?';
  318. }
  319. if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ)
  320. *c4 = '-';
  321. if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ) {
  322. *c4 = '+';
  323. if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ)
  324. *c4 = '?';
  325. }
  326. }
  327. static void print_lock_name(struct lock_class *class)
  328. {
  329. char str[KSYM_NAME_LEN + 1], c1, c2, c3, c4;
  330. const char *name;
  331. get_usage_chars(class, &c1, &c2, &c3, &c4);
  332. name = class->name;
  333. if (!name) {
  334. name = __get_key_name(class->key, str);
  335. printk(" (%s", name);
  336. } else {
  337. printk(" (%s", name);
  338. if (class->name_version > 1)
  339. printk("#%d", class->name_version);
  340. if (class->subclass)
  341. printk("/%d", class->subclass);
  342. }
  343. printk("){%c%c%c%c}", c1, c2, c3, c4);
  344. }
  345. static void print_lockdep_cache(struct lockdep_map *lock)
  346. {
  347. const char *name;
  348. char str[KSYM_NAME_LEN + 1];
  349. name = lock->name;
  350. if (!name)
  351. name = __get_key_name(lock->key->subkeys, str);
  352. printk("%s", name);
  353. }
  354. static void print_lock(struct held_lock *hlock)
  355. {
  356. print_lock_name(hlock->class);
  357. printk(", at: ");
  358. print_ip_sym(hlock->acquire_ip);
  359. }
  360. static void lockdep_print_held_locks(struct task_struct *curr)
  361. {
  362. int i, depth = curr->lockdep_depth;
  363. if (!depth) {
  364. printk("no locks held by %s/%d.\n", curr->comm, curr->pid);
  365. return;
  366. }
  367. printk("%d lock%s held by %s/%d:\n",
  368. depth, depth > 1 ? "s" : "", curr->comm, curr->pid);
  369. for (i = 0; i < depth; i++) {
  370. printk(" #%d: ", i);
  371. print_lock(curr->held_locks + i);
  372. }
  373. }
  374. static void print_lock_class_header(struct lock_class *class, int depth)
  375. {
  376. int bit;
  377. printk("%*s->", depth, "");
  378. print_lock_name(class);
  379. printk(" ops: %lu", class->ops);
  380. printk(" {\n");
  381. for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
  382. if (class->usage_mask & (1 << bit)) {
  383. int len = depth;
  384. len += printk("%*s %s", depth, "", usage_str[bit]);
  385. len += printk(" at:\n");
  386. print_stack_trace(class->usage_traces + bit, len);
  387. }
  388. }
  389. printk("%*s }\n", depth, "");
  390. printk("%*s ... key at: ",depth,"");
  391. print_ip_sym((unsigned long)class->key);
  392. }
  393. /*
  394. * printk all lock dependencies starting at <entry>:
  395. */
  396. static void print_lock_dependencies(struct lock_class *class, int depth)
  397. {
  398. struct lock_list *entry;
  399. if (DEBUG_LOCKS_WARN_ON(depth >= 20))
  400. return;
  401. print_lock_class_header(class, depth);
  402. list_for_each_entry(entry, &class->locks_after, entry) {
  403. if (DEBUG_LOCKS_WARN_ON(!entry->class))
  404. return;
  405. print_lock_dependencies(entry->class, depth + 1);
  406. printk("%*s ... acquired at:\n",depth,"");
  407. print_stack_trace(&entry->trace, 2);
  408. printk("\n");
  409. }
  410. }
  411. /*
  412. * Add a new dependency to the head of the list:
  413. */
  414. static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
  415. struct list_head *head, unsigned long ip, int distance)
  416. {
  417. struct lock_list *entry;
  418. /*
  419. * Lock not present yet - get a new dependency struct and
  420. * add it to the list:
  421. */
  422. entry = alloc_list_entry();
  423. if (!entry)
  424. return 0;
  425. entry->class = this;
  426. entry->distance = distance;
  427. if (!save_trace(&entry->trace))
  428. return 0;
  429. /*
  430. * Since we never remove from the dependency list, the list can
  431. * be walked lockless by other CPUs, it's only allocation
  432. * that must be protected by the spinlock. But this also means
  433. * we must make new entries visible only once writes to the
  434. * entry become visible - hence the RCU op:
  435. */
  436. list_add_tail_rcu(&entry->entry, head);
  437. return 1;
  438. }
  439. /*
  440. * Recursive, forwards-direction lock-dependency checking, used for
  441. * both noncyclic checking and for hardirq-unsafe/softirq-unsafe
  442. * checking.
  443. *
  444. * (to keep the stackframe of the recursive functions small we
  445. * use these global variables, and we also mark various helper
  446. * functions as noinline.)
  447. */
  448. static struct held_lock *check_source, *check_target;
  449. /*
  450. * Print a dependency chain entry (this is only done when a deadlock
  451. * has been detected):
  452. */
  453. static noinline int
  454. print_circular_bug_entry(struct lock_list *target, unsigned int depth)
  455. {
  456. if (debug_locks_silent)
  457. return 0;
  458. printk("\n-> #%u", depth);
  459. print_lock_name(target->class);
  460. printk(":\n");
  461. print_stack_trace(&target->trace, 6);
  462. return 0;
  463. }
  464. static void print_kernel_version(void)
  465. {
  466. printk("%s %.*s\n", init_utsname()->release,
  467. (int)strcspn(init_utsname()->version, " "),
  468. init_utsname()->version);
  469. }
  470. /*
  471. * When a circular dependency is detected, print the
  472. * header first:
  473. */
  474. static noinline int
  475. print_circular_bug_header(struct lock_list *entry, unsigned int depth)
  476. {
  477. struct task_struct *curr = current;
  478. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  479. return 0;
  480. printk("\n=======================================================\n");
  481. printk( "[ INFO: possible circular locking dependency detected ]\n");
  482. print_kernel_version();
  483. printk( "-------------------------------------------------------\n");
  484. printk("%s/%d is trying to acquire lock:\n",
  485. curr->comm, curr->pid);
  486. print_lock(check_source);
  487. printk("\nbut task is already holding lock:\n");
  488. print_lock(check_target);
  489. printk("\nwhich lock already depends on the new lock.\n\n");
  490. printk("\nthe existing dependency chain (in reverse order) is:\n");
  491. print_circular_bug_entry(entry, depth);
  492. return 0;
  493. }
  494. static noinline int print_circular_bug_tail(void)
  495. {
  496. struct task_struct *curr = current;
  497. struct lock_list this;
  498. if (debug_locks_silent)
  499. return 0;
  500. this.class = check_source->class;
  501. if (!save_trace(&this.trace))
  502. return 0;
  503. print_circular_bug_entry(&this, 0);
  504. printk("\nother info that might help us debug this:\n\n");
  505. lockdep_print_held_locks(curr);
  506. printk("\nstack backtrace:\n");
  507. dump_stack();
  508. return 0;
  509. }
  510. #define RECURSION_LIMIT 40
  511. static int noinline print_infinite_recursion_bug(void)
  512. {
  513. if (!debug_locks_off_graph_unlock())
  514. return 0;
  515. WARN_ON(1);
  516. return 0;
  517. }
  518. /*
  519. * Prove that the dependency graph starting at <entry> can not
  520. * lead to <target>. Print an error and return 0 if it does.
  521. */
  522. static noinline int
  523. check_noncircular(struct lock_class *source, unsigned int depth)
  524. {
  525. struct lock_list *entry;
  526. debug_atomic_inc(&nr_cyclic_check_recursions);
  527. if (depth > max_recursion_depth)
  528. max_recursion_depth = depth;
  529. if (depth >= RECURSION_LIMIT)
  530. return print_infinite_recursion_bug();
  531. /*
  532. * Check this lock's dependency list:
  533. */
  534. list_for_each_entry(entry, &source->locks_after, entry) {
  535. if (entry->class == check_target->class)
  536. return print_circular_bug_header(entry, depth+1);
  537. debug_atomic_inc(&nr_cyclic_checks);
  538. if (!check_noncircular(entry->class, depth+1))
  539. return print_circular_bug_entry(entry, depth+1);
  540. }
  541. return 1;
  542. }
  543. static int very_verbose(struct lock_class *class)
  544. {
  545. #if VERY_VERBOSE
  546. return class_filter(class);
  547. #endif
  548. return 0;
  549. }
  550. #ifdef CONFIG_TRACE_IRQFLAGS
  551. /*
  552. * Forwards and backwards subgraph searching, for the purposes of
  553. * proving that two subgraphs can be connected by a new dependency
  554. * without creating any illegal irq-safe -> irq-unsafe lock dependency.
  555. */
  556. static enum lock_usage_bit find_usage_bit;
  557. static struct lock_class *forwards_match, *backwards_match;
  558. /*
  559. * Find a node in the forwards-direction dependency sub-graph starting
  560. * at <source> that matches <find_usage_bit>.
  561. *
  562. * Return 2 if such a node exists in the subgraph, and put that node
  563. * into <forwards_match>.
  564. *
  565. * Return 1 otherwise and keep <forwards_match> unchanged.
  566. * Return 0 on error.
  567. */
  568. static noinline int
  569. find_usage_forwards(struct lock_class *source, unsigned int depth)
  570. {
  571. struct lock_list *entry;
  572. int ret;
  573. if (depth > max_recursion_depth)
  574. max_recursion_depth = depth;
  575. if (depth >= RECURSION_LIMIT)
  576. return print_infinite_recursion_bug();
  577. debug_atomic_inc(&nr_find_usage_forwards_checks);
  578. if (source->usage_mask & (1 << find_usage_bit)) {
  579. forwards_match = source;
  580. return 2;
  581. }
  582. /*
  583. * Check this lock's dependency list:
  584. */
  585. list_for_each_entry(entry, &source->locks_after, entry) {
  586. debug_atomic_inc(&nr_find_usage_forwards_recursions);
  587. ret = find_usage_forwards(entry->class, depth+1);
  588. if (ret == 2 || ret == 0)
  589. return ret;
  590. }
  591. return 1;
  592. }
  593. /*
  594. * Find a node in the backwards-direction dependency sub-graph starting
  595. * at <source> that matches <find_usage_bit>.
  596. *
  597. * Return 2 if such a node exists in the subgraph, and put that node
  598. * into <backwards_match>.
  599. *
  600. * Return 1 otherwise and keep <backwards_match> unchanged.
  601. * Return 0 on error.
  602. */
  603. static noinline int
  604. find_usage_backwards(struct lock_class *source, unsigned int depth)
  605. {
  606. struct lock_list *entry;
  607. int ret;
  608. if (!__raw_spin_is_locked(&lockdep_lock))
  609. return DEBUG_LOCKS_WARN_ON(1);
  610. if (depth > max_recursion_depth)
  611. max_recursion_depth = depth;
  612. if (depth >= RECURSION_LIMIT)
  613. return print_infinite_recursion_bug();
  614. debug_atomic_inc(&nr_find_usage_backwards_checks);
  615. if (source->usage_mask & (1 << find_usage_bit)) {
  616. backwards_match = source;
  617. return 2;
  618. }
  619. /*
  620. * Check this lock's dependency list:
  621. */
  622. list_for_each_entry(entry, &source->locks_before, entry) {
  623. debug_atomic_inc(&nr_find_usage_backwards_recursions);
  624. ret = find_usage_backwards(entry->class, depth+1);
  625. if (ret == 2 || ret == 0)
  626. return ret;
  627. }
  628. return 1;
  629. }
  630. static int
  631. print_bad_irq_dependency(struct task_struct *curr,
  632. struct held_lock *prev,
  633. struct held_lock *next,
  634. enum lock_usage_bit bit1,
  635. enum lock_usage_bit bit2,
  636. const char *irqclass)
  637. {
  638. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  639. return 0;
  640. printk("\n======================================================\n");
  641. printk( "[ INFO: %s-safe -> %s-unsafe lock order detected ]\n",
  642. irqclass, irqclass);
  643. print_kernel_version();
  644. printk( "------------------------------------------------------\n");
  645. printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] is trying to acquire:\n",
  646. curr->comm, curr->pid,
  647. curr->hardirq_context, hardirq_count() >> HARDIRQ_SHIFT,
  648. curr->softirq_context, softirq_count() >> SOFTIRQ_SHIFT,
  649. curr->hardirqs_enabled,
  650. curr->softirqs_enabled);
  651. print_lock(next);
  652. printk("\nand this task is already holding:\n");
  653. print_lock(prev);
  654. printk("which would create a new lock dependency:\n");
  655. print_lock_name(prev->class);
  656. printk(" ->");
  657. print_lock_name(next->class);
  658. printk("\n");
  659. printk("\nbut this new dependency connects a %s-irq-safe lock:\n",
  660. irqclass);
  661. print_lock_name(backwards_match);
  662. printk("\n... which became %s-irq-safe at:\n", irqclass);
  663. print_stack_trace(backwards_match->usage_traces + bit1, 1);
  664. printk("\nto a %s-irq-unsafe lock:\n", irqclass);
  665. print_lock_name(forwards_match);
  666. printk("\n... which became %s-irq-unsafe at:\n", irqclass);
  667. printk("...");
  668. print_stack_trace(forwards_match->usage_traces + bit2, 1);
  669. printk("\nother info that might help us debug this:\n\n");
  670. lockdep_print_held_locks(curr);
  671. printk("\nthe %s-irq-safe lock's dependencies:\n", irqclass);
  672. print_lock_dependencies(backwards_match, 0);
  673. printk("\nthe %s-irq-unsafe lock's dependencies:\n", irqclass);
  674. print_lock_dependencies(forwards_match, 0);
  675. printk("\nstack backtrace:\n");
  676. dump_stack();
  677. return 0;
  678. }
  679. static int
  680. check_usage(struct task_struct *curr, struct held_lock *prev,
  681. struct held_lock *next, enum lock_usage_bit bit_backwards,
  682. enum lock_usage_bit bit_forwards, const char *irqclass)
  683. {
  684. int ret;
  685. find_usage_bit = bit_backwards;
  686. /* fills in <backwards_match> */
  687. ret = find_usage_backwards(prev->class, 0);
  688. if (!ret || ret == 1)
  689. return ret;
  690. find_usage_bit = bit_forwards;
  691. ret = find_usage_forwards(next->class, 0);
  692. if (!ret || ret == 1)
  693. return ret;
  694. /* ret == 2 */
  695. return print_bad_irq_dependency(curr, prev, next,
  696. bit_backwards, bit_forwards, irqclass);
  697. }
  698. #endif
  699. static int
  700. print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,
  701. struct held_lock *next)
  702. {
  703. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  704. return 0;
  705. printk("\n=============================================\n");
  706. printk( "[ INFO: possible recursive locking detected ]\n");
  707. print_kernel_version();
  708. printk( "---------------------------------------------\n");
  709. printk("%s/%d is trying to acquire lock:\n",
  710. curr->comm, curr->pid);
  711. print_lock(next);
  712. printk("\nbut task is already holding lock:\n");
  713. print_lock(prev);
  714. printk("\nother info that might help us debug this:\n");
  715. lockdep_print_held_locks(curr);
  716. printk("\nstack backtrace:\n");
  717. dump_stack();
  718. return 0;
  719. }
  720. /*
  721. * Check whether we are holding such a class already.
  722. *
  723. * (Note that this has to be done separately, because the graph cannot
  724. * detect such classes of deadlocks.)
  725. *
  726. * Returns: 0 on deadlock detected, 1 on OK, 2 on recursive read
  727. */
  728. static int
  729. check_deadlock(struct task_struct *curr, struct held_lock *next,
  730. struct lockdep_map *next_instance, int read)
  731. {
  732. struct held_lock *prev;
  733. int i;
  734. for (i = 0; i < curr->lockdep_depth; i++) {
  735. prev = curr->held_locks + i;
  736. if (prev->class != next->class)
  737. continue;
  738. /*
  739. * Allow read-after-read recursion of the same
  740. * lock class (i.e. read_lock(lock)+read_lock(lock)):
  741. */
  742. if ((read == 2) && prev->read)
  743. return 2;
  744. return print_deadlock_bug(curr, prev, next);
  745. }
  746. return 1;
  747. }
  748. /*
  749. * There was a chain-cache miss, and we are about to add a new dependency
  750. * to a previous lock. We recursively validate the following rules:
  751. *
  752. * - would the adding of the <prev> -> <next> dependency create a
  753. * circular dependency in the graph? [== circular deadlock]
  754. *
  755. * - does the new prev->next dependency connect any hardirq-safe lock
  756. * (in the full backwards-subgraph starting at <prev>) with any
  757. * hardirq-unsafe lock (in the full forwards-subgraph starting at
  758. * <next>)? [== illegal lock inversion with hardirq contexts]
  759. *
  760. * - does the new prev->next dependency connect any softirq-safe lock
  761. * (in the full backwards-subgraph starting at <prev>) with any
  762. * softirq-unsafe lock (in the full forwards-subgraph starting at
  763. * <next>)? [== illegal lock inversion with softirq contexts]
  764. *
  765. * any of these scenarios could lead to a deadlock.
  766. *
  767. * Then if all the validations pass, we add the forwards and backwards
  768. * dependency.
  769. */
  770. static int
  771. check_prev_add(struct task_struct *curr, struct held_lock *prev,
  772. struct held_lock *next, int distance)
  773. {
  774. struct lock_list *entry;
  775. int ret;
  776. /*
  777. * Prove that the new <prev> -> <next> dependency would not
  778. * create a circular dependency in the graph. (We do this by
  779. * forward-recursing into the graph starting at <next>, and
  780. * checking whether we can reach <prev>.)
  781. *
  782. * We are using global variables to control the recursion, to
  783. * keep the stackframe size of the recursive functions low:
  784. */
  785. check_source = next;
  786. check_target = prev;
  787. if (!(check_noncircular(next->class, 0)))
  788. return print_circular_bug_tail();
  789. #ifdef CONFIG_TRACE_IRQFLAGS
  790. /*
  791. * Prove that the new dependency does not connect a hardirq-safe
  792. * lock with a hardirq-unsafe lock - to achieve this we search
  793. * the backwards-subgraph starting at <prev>, and the
  794. * forwards-subgraph starting at <next>:
  795. */
  796. if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ,
  797. LOCK_ENABLED_HARDIRQS, "hard"))
  798. return 0;
  799. /*
  800. * Prove that the new dependency does not connect a hardirq-safe-read
  801. * lock with a hardirq-unsafe lock - to achieve this we search
  802. * the backwards-subgraph starting at <prev>, and the
  803. * forwards-subgraph starting at <next>:
  804. */
  805. if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ_READ,
  806. LOCK_ENABLED_HARDIRQS, "hard-read"))
  807. return 0;
  808. /*
  809. * Prove that the new dependency does not connect a softirq-safe
  810. * lock with a softirq-unsafe lock - to achieve this we search
  811. * the backwards-subgraph starting at <prev>, and the
  812. * forwards-subgraph starting at <next>:
  813. */
  814. if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ,
  815. LOCK_ENABLED_SOFTIRQS, "soft"))
  816. return 0;
  817. /*
  818. * Prove that the new dependency does not connect a softirq-safe-read
  819. * lock with a softirq-unsafe lock - to achieve this we search
  820. * the backwards-subgraph starting at <prev>, and the
  821. * forwards-subgraph starting at <next>:
  822. */
  823. if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ_READ,
  824. LOCK_ENABLED_SOFTIRQS, "soft"))
  825. return 0;
  826. #endif
  827. /*
  828. * For recursive read-locks we do all the dependency checks,
  829. * but we dont store read-triggered dependencies (only
  830. * write-triggered dependencies). This ensures that only the
  831. * write-side dependencies matter, and that if for example a
  832. * write-lock never takes any other locks, then the reads are
  833. * equivalent to a NOP.
  834. */
  835. if (next->read == 2 || prev->read == 2)
  836. return 1;
  837. /*
  838. * Is the <prev> -> <next> dependency already present?
  839. *
  840. * (this may occur even though this is a new chain: consider
  841. * e.g. the L1 -> L2 -> L3 -> L4 and the L5 -> L1 -> L2 -> L3
  842. * chains - the second one will be new, but L1 already has
  843. * L2 added to its dependency list, due to the first chain.)
  844. */
  845. list_for_each_entry(entry, &prev->class->locks_after, entry) {
  846. if (entry->class == next->class) {
  847. if (distance == 1)
  848. entry->distance = 1;
  849. return 2;
  850. }
  851. }
  852. /*
  853. * Ok, all validations passed, add the new lock
  854. * to the previous lock's dependency list:
  855. */
  856. ret = add_lock_to_list(prev->class, next->class,
  857. &prev->class->locks_after, next->acquire_ip, distance);
  858. if (!ret)
  859. return 0;
  860. ret = add_lock_to_list(next->class, prev->class,
  861. &next->class->locks_before, next->acquire_ip, distance);
  862. if (!ret)
  863. return 0;
  864. /*
  865. * Debugging printouts:
  866. */
  867. if (verbose(prev->class) || verbose(next->class)) {
  868. graph_unlock();
  869. printk("\n new dependency: ");
  870. print_lock_name(prev->class);
  871. printk(" => ");
  872. print_lock_name(next->class);
  873. printk("\n");
  874. dump_stack();
  875. return graph_lock();
  876. }
  877. return 1;
  878. }
  879. /*
  880. * Add the dependency to all directly-previous locks that are 'relevant'.
  881. * The ones that are relevant are (in increasing distance from curr):
  882. * all consecutive trylock entries and the final non-trylock entry - or
  883. * the end of this context's lock-chain - whichever comes first.
  884. */
  885. static int
  886. check_prevs_add(struct task_struct *curr, struct held_lock *next)
  887. {
  888. int depth = curr->lockdep_depth;
  889. struct held_lock *hlock;
  890. /*
  891. * Debugging checks.
  892. *
  893. * Depth must not be zero for a non-head lock:
  894. */
  895. if (!depth)
  896. goto out_bug;
  897. /*
  898. * At least two relevant locks must exist for this
  899. * to be a head:
  900. */
  901. if (curr->held_locks[depth].irq_context !=
  902. curr->held_locks[depth-1].irq_context)
  903. goto out_bug;
  904. for (;;) {
  905. int distance = curr->lockdep_depth - depth + 1;
  906. hlock = curr->held_locks + depth-1;
  907. /*
  908. * Only non-recursive-read entries get new dependencies
  909. * added:
  910. */
  911. if (hlock->read != 2) {
  912. if (!check_prev_add(curr, hlock, next, distance))
  913. return 0;
  914. /*
  915. * Stop after the first non-trylock entry,
  916. * as non-trylock entries have added their
  917. * own direct dependencies already, so this
  918. * lock is connected to them indirectly:
  919. */
  920. if (!hlock->trylock)
  921. break;
  922. }
  923. depth--;
  924. /*
  925. * End of lock-stack?
  926. */
  927. if (!depth)
  928. break;
  929. /*
  930. * Stop the search if we cross into another context:
  931. */
  932. if (curr->held_locks[depth].irq_context !=
  933. curr->held_locks[depth-1].irq_context)
  934. break;
  935. }
  936. return 1;
  937. out_bug:
  938. if (!debug_locks_off_graph_unlock())
  939. return 0;
  940. WARN_ON(1);
  941. return 0;
  942. }
  943. /*
  944. * Is this the address of a static object:
  945. */
  946. static int static_obj(void *obj)
  947. {
  948. unsigned long start = (unsigned long) &_stext,
  949. end = (unsigned long) &_end,
  950. addr = (unsigned long) obj;
  951. #ifdef CONFIG_SMP
  952. int i;
  953. #endif
  954. /*
  955. * static variable?
  956. */
  957. if ((addr >= start) && (addr < end))
  958. return 1;
  959. #ifdef CONFIG_SMP
  960. /*
  961. * percpu var?
  962. */
  963. for_each_possible_cpu(i) {
  964. start = (unsigned long) &__per_cpu_start + per_cpu_offset(i);
  965. end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
  966. + per_cpu_offset(i);
  967. if ((addr >= start) && (addr < end))
  968. return 1;
  969. }
  970. #endif
  971. /*
  972. * module var?
  973. */
  974. return is_module_address(addr);
  975. }
  976. /*
  977. * To make lock name printouts unique, we calculate a unique
  978. * class->name_version generation counter:
  979. */
  980. static int count_matching_names(struct lock_class *new_class)
  981. {
  982. struct lock_class *class;
  983. int count = 0;
  984. if (!new_class->name)
  985. return 0;
  986. list_for_each_entry(class, &all_lock_classes, lock_entry) {
  987. if (new_class->key - new_class->subclass == class->key)
  988. return class->name_version;
  989. if (class->name && !strcmp(class->name, new_class->name))
  990. count = max(count, class->name_version);
  991. }
  992. return count + 1;
  993. }
  994. /*
  995. * Register a lock's class in the hash-table, if the class is not present
  996. * yet. Otherwise we look it up. We cache the result in the lock object
  997. * itself, so actual lookup of the hash should be once per lock object.
  998. */
  999. static inline struct lock_class *
  1000. look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
  1001. {
  1002. struct lockdep_subclass_key *key;
  1003. struct list_head *hash_head;
  1004. struct lock_class *class;
  1005. #ifdef CONFIG_DEBUG_LOCKDEP
  1006. /*
  1007. * If the architecture calls into lockdep before initializing
  1008. * the hashes then we'll warn about it later. (we cannot printk
  1009. * right now)
  1010. */
  1011. if (unlikely(!lockdep_initialized)) {
  1012. lockdep_init();
  1013. lockdep_init_error = 1;
  1014. }
  1015. #endif
  1016. /*
  1017. * Static locks do not have their class-keys yet - for them the key
  1018. * is the lock object itself:
  1019. */
  1020. if (unlikely(!lock->key))
  1021. lock->key = (void *)lock;
  1022. /*
  1023. * NOTE: the class-key must be unique. For dynamic locks, a static
  1024. * lock_class_key variable is passed in through the mutex_init()
  1025. * (or spin_lock_init()) call - which acts as the key. For static
  1026. * locks we use the lock object itself as the key.
  1027. */
  1028. BUILD_BUG_ON(sizeof(struct lock_class_key) > sizeof(struct lock_class));
  1029. key = lock->key->subkeys + subclass;
  1030. hash_head = classhashentry(key);
  1031. /*
  1032. * We can walk the hash lockfree, because the hash only
  1033. * grows, and we are careful when adding entries to the end:
  1034. */
  1035. list_for_each_entry(class, hash_head, hash_entry)
  1036. if (class->key == key)
  1037. return class;
  1038. return NULL;
  1039. }
  1040. /*
  1041. * Register a lock's class in the hash-table, if the class is not present
  1042. * yet. Otherwise we look it up. We cache the result in the lock object
  1043. * itself, so actual lookup of the hash should be once per lock object.
  1044. */
  1045. static inline struct lock_class *
  1046. register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
  1047. {
  1048. struct lockdep_subclass_key *key;
  1049. struct list_head *hash_head;
  1050. struct lock_class *class;
  1051. unsigned long flags;
  1052. class = look_up_lock_class(lock, subclass);
  1053. if (likely(class))
  1054. return class;
  1055. /*
  1056. * Debug-check: all keys must be persistent!
  1057. */
  1058. if (!static_obj(lock->key)) {
  1059. debug_locks_off();
  1060. printk("INFO: trying to register non-static key.\n");
  1061. printk("the code is fine but needs lockdep annotation.\n");
  1062. printk("turning off the locking correctness validator.\n");
  1063. dump_stack();
  1064. return NULL;
  1065. }
  1066. key = lock->key->subkeys + subclass;
  1067. hash_head = classhashentry(key);
  1068. raw_local_irq_save(flags);
  1069. if (!graph_lock()) {
  1070. raw_local_irq_restore(flags);
  1071. return NULL;
  1072. }
  1073. /*
  1074. * We have to do the hash-walk again, to avoid races
  1075. * with another CPU:
  1076. */
  1077. list_for_each_entry(class, hash_head, hash_entry)
  1078. if (class->key == key)
  1079. goto out_unlock_set;
  1080. /*
  1081. * Allocate a new key from the static array, and add it to
  1082. * the hash:
  1083. */
  1084. if (nr_lock_classes >= MAX_LOCKDEP_KEYS) {
  1085. if (!debug_locks_off_graph_unlock()) {
  1086. raw_local_irq_restore(flags);
  1087. return NULL;
  1088. }
  1089. raw_local_irq_restore(flags);
  1090. printk("BUG: MAX_LOCKDEP_KEYS too low!\n");
  1091. printk("turning off the locking correctness validator.\n");
  1092. return NULL;
  1093. }
  1094. class = lock_classes + nr_lock_classes++;
  1095. debug_atomic_inc(&nr_unused_locks);
  1096. class->key = key;
  1097. class->name = lock->name;
  1098. class->subclass = subclass;
  1099. INIT_LIST_HEAD(&class->lock_entry);
  1100. INIT_LIST_HEAD(&class->locks_before);
  1101. INIT_LIST_HEAD(&class->locks_after);
  1102. class->name_version = count_matching_names(class);
  1103. /*
  1104. * We use RCU's safe list-add method to make
  1105. * parallel walking of the hash-list safe:
  1106. */
  1107. list_add_tail_rcu(&class->hash_entry, hash_head);
  1108. if (verbose(class)) {
  1109. graph_unlock();
  1110. raw_local_irq_restore(flags);
  1111. printk("\nnew class %p: %s", class->key, class->name);
  1112. if (class->name_version > 1)
  1113. printk("#%d", class->name_version);
  1114. printk("\n");
  1115. dump_stack();
  1116. raw_local_irq_save(flags);
  1117. if (!graph_lock()) {
  1118. raw_local_irq_restore(flags);
  1119. return NULL;
  1120. }
  1121. }
  1122. out_unlock_set:
  1123. graph_unlock();
  1124. raw_local_irq_restore(flags);
  1125. if (!subclass || force)
  1126. lock->class_cache = class;
  1127. if (DEBUG_LOCKS_WARN_ON(class->subclass != subclass))
  1128. return NULL;
  1129. return class;
  1130. }
  1131. /*
  1132. * Look up a dependency chain. If the key is not present yet then
  1133. * add it and return 0 - in this case the new dependency chain is
  1134. * validated. If the key is already hashed, return 1.
  1135. */
  1136. static inline int lookup_chain_cache(u64 chain_key, struct lock_class *class)
  1137. {
  1138. struct list_head *hash_head = chainhashentry(chain_key);
  1139. struct lock_chain *chain;
  1140. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1141. return 0;
  1142. /*
  1143. * We can walk it lock-free, because entries only get added
  1144. * to the hash:
  1145. */
  1146. list_for_each_entry(chain, hash_head, entry) {
  1147. if (chain->chain_key == chain_key) {
  1148. cache_hit:
  1149. debug_atomic_inc(&chain_lookup_hits);
  1150. if (very_verbose(class))
  1151. printk("\nhash chain already cached, key: "
  1152. "%016Lx tail class: [%p] %s\n",
  1153. (unsigned long long)chain_key,
  1154. class->key, class->name);
  1155. return 0;
  1156. }
  1157. }
  1158. if (very_verbose(class))
  1159. printk("\nnew hash chain, key: %016Lx tail class: [%p] %s\n",
  1160. (unsigned long long)chain_key, class->key, class->name);
  1161. /*
  1162. * Allocate a new chain entry from the static array, and add
  1163. * it to the hash:
  1164. */
  1165. if (!graph_lock())
  1166. return 0;
  1167. /*
  1168. * We have to walk the chain again locked - to avoid duplicates:
  1169. */
  1170. list_for_each_entry(chain, hash_head, entry) {
  1171. if (chain->chain_key == chain_key) {
  1172. graph_unlock();
  1173. goto cache_hit;
  1174. }
  1175. }
  1176. if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {
  1177. if (!debug_locks_off_graph_unlock())
  1178. return 0;
  1179. printk("BUG: MAX_LOCKDEP_CHAINS too low!\n");
  1180. printk("turning off the locking correctness validator.\n");
  1181. return 0;
  1182. }
  1183. chain = lock_chains + nr_lock_chains++;
  1184. chain->chain_key = chain_key;
  1185. list_add_tail_rcu(&chain->entry, hash_head);
  1186. debug_atomic_inc(&chain_lookup_misses);
  1187. #ifdef CONFIG_TRACE_IRQFLAGS
  1188. if (current->hardirq_context)
  1189. nr_hardirq_chains++;
  1190. else {
  1191. if (current->softirq_context)
  1192. nr_softirq_chains++;
  1193. else
  1194. nr_process_chains++;
  1195. }
  1196. #else
  1197. nr_process_chains++;
  1198. #endif
  1199. return 1;
  1200. }
  1201. /*
  1202. * We are building curr_chain_key incrementally, so double-check
  1203. * it from scratch, to make sure that it's done correctly:
  1204. */
  1205. static void check_chain_key(struct task_struct *curr)
  1206. {
  1207. #ifdef CONFIG_DEBUG_LOCKDEP
  1208. struct held_lock *hlock, *prev_hlock = NULL;
  1209. unsigned int i, id;
  1210. u64 chain_key = 0;
  1211. for (i = 0; i < curr->lockdep_depth; i++) {
  1212. hlock = curr->held_locks + i;
  1213. if (chain_key != hlock->prev_chain_key) {
  1214. debug_locks_off();
  1215. printk("hm#1, depth: %u [%u], %016Lx != %016Lx\n",
  1216. curr->lockdep_depth, i,
  1217. (unsigned long long)chain_key,
  1218. (unsigned long long)hlock->prev_chain_key);
  1219. WARN_ON(1);
  1220. return;
  1221. }
  1222. id = hlock->class - lock_classes;
  1223. if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))
  1224. return;
  1225. if (prev_hlock && (prev_hlock->irq_context !=
  1226. hlock->irq_context))
  1227. chain_key = 0;
  1228. chain_key = iterate_chain_key(chain_key, id);
  1229. prev_hlock = hlock;
  1230. }
  1231. if (chain_key != curr->curr_chain_key) {
  1232. debug_locks_off();
  1233. printk("hm#2, depth: %u [%u], %016Lx != %016Lx\n",
  1234. curr->lockdep_depth, i,
  1235. (unsigned long long)chain_key,
  1236. (unsigned long long)curr->curr_chain_key);
  1237. WARN_ON(1);
  1238. }
  1239. #endif
  1240. }
  1241. #ifdef CONFIG_TRACE_IRQFLAGS
  1242. /*
  1243. * print irq inversion bug:
  1244. */
  1245. static int
  1246. print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other,
  1247. struct held_lock *this, int forwards,
  1248. const char *irqclass)
  1249. {
  1250. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  1251. return 0;
  1252. printk("\n=========================================================\n");
  1253. printk( "[ INFO: possible irq lock inversion dependency detected ]\n");
  1254. print_kernel_version();
  1255. printk( "---------------------------------------------------------\n");
  1256. printk("%s/%d just changed the state of lock:\n",
  1257. curr->comm, curr->pid);
  1258. print_lock(this);
  1259. if (forwards)
  1260. printk("but this lock took another, %s-irq-unsafe lock in the past:\n", irqclass);
  1261. else
  1262. printk("but this lock was taken by another, %s-irq-safe lock in the past:\n", irqclass);
  1263. print_lock_name(other);
  1264. printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
  1265. printk("\nother info that might help us debug this:\n");
  1266. lockdep_print_held_locks(curr);
  1267. printk("\nthe first lock's dependencies:\n");
  1268. print_lock_dependencies(this->class, 0);
  1269. printk("\nthe second lock's dependencies:\n");
  1270. print_lock_dependencies(other, 0);
  1271. printk("\nstack backtrace:\n");
  1272. dump_stack();
  1273. return 0;
  1274. }
  1275. /*
  1276. * Prove that in the forwards-direction subgraph starting at <this>
  1277. * there is no lock matching <mask>:
  1278. */
  1279. static int
  1280. check_usage_forwards(struct task_struct *curr, struct held_lock *this,
  1281. enum lock_usage_bit bit, const char *irqclass)
  1282. {
  1283. int ret;
  1284. find_usage_bit = bit;
  1285. /* fills in <forwards_match> */
  1286. ret = find_usage_forwards(this->class, 0);
  1287. if (!ret || ret == 1)
  1288. return ret;
  1289. return print_irq_inversion_bug(curr, forwards_match, this, 1, irqclass);
  1290. }
  1291. /*
  1292. * Prove that in the backwards-direction subgraph starting at <this>
  1293. * there is no lock matching <mask>:
  1294. */
  1295. static int
  1296. check_usage_backwards(struct task_struct *curr, struct held_lock *this,
  1297. enum lock_usage_bit bit, const char *irqclass)
  1298. {
  1299. int ret;
  1300. find_usage_bit = bit;
  1301. /* fills in <backwards_match> */
  1302. ret = find_usage_backwards(this->class, 0);
  1303. if (!ret || ret == 1)
  1304. return ret;
  1305. return print_irq_inversion_bug(curr, backwards_match, this, 0, irqclass);
  1306. }
  1307. void print_irqtrace_events(struct task_struct *curr)
  1308. {
  1309. printk("irq event stamp: %u\n", curr->irq_events);
  1310. printk("hardirqs last enabled at (%u): ", curr->hardirq_enable_event);
  1311. print_ip_sym(curr->hardirq_enable_ip);
  1312. printk("hardirqs last disabled at (%u): ", curr->hardirq_disable_event);
  1313. print_ip_sym(curr->hardirq_disable_ip);
  1314. printk("softirqs last enabled at (%u): ", curr->softirq_enable_event);
  1315. print_ip_sym(curr->softirq_enable_ip);
  1316. printk("softirqs last disabled at (%u): ", curr->softirq_disable_event);
  1317. print_ip_sym(curr->softirq_disable_ip);
  1318. }
  1319. #endif
  1320. static int
  1321. print_usage_bug(struct task_struct *curr, struct held_lock *this,
  1322. enum lock_usage_bit prev_bit, enum lock_usage_bit new_bit)
  1323. {
  1324. if (!debug_locks_off_graph_unlock() || debug_locks_silent)
  1325. return 0;
  1326. printk("\n=================================\n");
  1327. printk( "[ INFO: inconsistent lock state ]\n");
  1328. print_kernel_version();
  1329. printk( "---------------------------------\n");
  1330. printk("inconsistent {%s} -> {%s} usage.\n",
  1331. usage_str[prev_bit], usage_str[new_bit]);
  1332. printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] takes:\n",
  1333. curr->comm, curr->pid,
  1334. trace_hardirq_context(curr), hardirq_count() >> HARDIRQ_SHIFT,
  1335. trace_softirq_context(curr), softirq_count() >> SOFTIRQ_SHIFT,
  1336. trace_hardirqs_enabled(curr),
  1337. trace_softirqs_enabled(curr));
  1338. print_lock(this);
  1339. printk("{%s} state was registered at:\n", usage_str[prev_bit]);
  1340. print_stack_trace(this->class->usage_traces + prev_bit, 1);
  1341. print_irqtrace_events(curr);
  1342. printk("\nother info that might help us debug this:\n");
  1343. lockdep_print_held_locks(curr);
  1344. printk("\nstack backtrace:\n");
  1345. dump_stack();
  1346. return 0;
  1347. }
  1348. /*
  1349. * Print out an error if an invalid bit is set:
  1350. */
  1351. static inline int
  1352. valid_state(struct task_struct *curr, struct held_lock *this,
  1353. enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit)
  1354. {
  1355. if (unlikely(this->class->usage_mask & (1 << bad_bit)))
  1356. return print_usage_bug(curr, this, bad_bit, new_bit);
  1357. return 1;
  1358. }
  1359. #define STRICT_READ_CHECKS 1
  1360. /*
  1361. * Mark a lock with a usage bit, and validate the state transition:
  1362. */
  1363. static int mark_lock(struct task_struct *curr, struct held_lock *this,
  1364. enum lock_usage_bit new_bit, unsigned long ip)
  1365. {
  1366. unsigned int new_mask = 1 << new_bit, ret = 1;
  1367. /*
  1368. * If already set then do not dirty the cacheline,
  1369. * nor do any checks:
  1370. */
  1371. if (likely(this->class->usage_mask & new_mask))
  1372. return 1;
  1373. if (!graph_lock())
  1374. return 0;
  1375. /*
  1376. * Make sure we didnt race:
  1377. */
  1378. if (unlikely(this->class->usage_mask & new_mask)) {
  1379. graph_unlock();
  1380. return 1;
  1381. }
  1382. this->class->usage_mask |= new_mask;
  1383. #ifdef CONFIG_TRACE_IRQFLAGS
  1384. if (new_bit == LOCK_ENABLED_HARDIRQS ||
  1385. new_bit == LOCK_ENABLED_HARDIRQS_READ)
  1386. ip = curr->hardirq_enable_ip;
  1387. else if (new_bit == LOCK_ENABLED_SOFTIRQS ||
  1388. new_bit == LOCK_ENABLED_SOFTIRQS_READ)
  1389. ip = curr->softirq_enable_ip;
  1390. #endif
  1391. if (!save_trace(this->class->usage_traces + new_bit))
  1392. return 0;
  1393. switch (new_bit) {
  1394. #ifdef CONFIG_TRACE_IRQFLAGS
  1395. case LOCK_USED_IN_HARDIRQ:
  1396. if (!valid_state(curr, this, new_bit, LOCK_ENABLED_HARDIRQS))
  1397. return 0;
  1398. if (!valid_state(curr, this, new_bit,
  1399. LOCK_ENABLED_HARDIRQS_READ))
  1400. return 0;
  1401. /*
  1402. * just marked it hardirq-safe, check that this lock
  1403. * took no hardirq-unsafe lock in the past:
  1404. */
  1405. if (!check_usage_forwards(curr, this,
  1406. LOCK_ENABLED_HARDIRQS, "hard"))
  1407. return 0;
  1408. #if STRICT_READ_CHECKS
  1409. /*
  1410. * just marked it hardirq-safe, check that this lock
  1411. * took no hardirq-unsafe-read lock in the past:
  1412. */
  1413. if (!check_usage_forwards(curr, this,
  1414. LOCK_ENABLED_HARDIRQS_READ, "hard-read"))
  1415. return 0;
  1416. #endif
  1417. if (hardirq_verbose(this->class))
  1418. ret = 2;
  1419. break;
  1420. case LOCK_USED_IN_SOFTIRQ:
  1421. if (!valid_state(curr, this, new_bit, LOCK_ENABLED_SOFTIRQS))
  1422. return 0;
  1423. if (!valid_state(curr, this, new_bit,
  1424. LOCK_ENABLED_SOFTIRQS_READ))
  1425. return 0;
  1426. /*
  1427. * just marked it softirq-safe, check that this lock
  1428. * took no softirq-unsafe lock in the past:
  1429. */
  1430. if (!check_usage_forwards(curr, this,
  1431. LOCK_ENABLED_SOFTIRQS, "soft"))
  1432. return 0;
  1433. #if STRICT_READ_CHECKS
  1434. /*
  1435. * just marked it softirq-safe, check that this lock
  1436. * took no softirq-unsafe-read lock in the past:
  1437. */
  1438. if (!check_usage_forwards(curr, this,
  1439. LOCK_ENABLED_SOFTIRQS_READ, "soft-read"))
  1440. return 0;
  1441. #endif
  1442. if (softirq_verbose(this->class))
  1443. ret = 2;
  1444. break;
  1445. case LOCK_USED_IN_HARDIRQ_READ:
  1446. if (!valid_state(curr, this, new_bit, LOCK_ENABLED_HARDIRQS))
  1447. return 0;
  1448. /*
  1449. * just marked it hardirq-read-safe, check that this lock
  1450. * took no hardirq-unsafe lock in the past:
  1451. */
  1452. if (!check_usage_forwards(curr, this,
  1453. LOCK_ENABLED_HARDIRQS, "hard"))
  1454. return 0;
  1455. if (hardirq_verbose(this->class))
  1456. ret = 2;
  1457. break;
  1458. case LOCK_USED_IN_SOFTIRQ_READ:
  1459. if (!valid_state(curr, this, new_bit, LOCK_ENABLED_SOFTIRQS))
  1460. return 0;
  1461. /*
  1462. * just marked it softirq-read-safe, check that this lock
  1463. * took no softirq-unsafe lock in the past:
  1464. */
  1465. if (!check_usage_forwards(curr, this,
  1466. LOCK_ENABLED_SOFTIRQS, "soft"))
  1467. return 0;
  1468. if (softirq_verbose(this->class))
  1469. ret = 2;
  1470. break;
  1471. case LOCK_ENABLED_HARDIRQS:
  1472. if (!valid_state(curr, this, new_bit, LOCK_USED_IN_HARDIRQ))
  1473. return 0;
  1474. if (!valid_state(curr, this, new_bit,
  1475. LOCK_USED_IN_HARDIRQ_READ))
  1476. return 0;
  1477. /*
  1478. * just marked it hardirq-unsafe, check that no hardirq-safe
  1479. * lock in the system ever took it in the past:
  1480. */
  1481. if (!check_usage_backwards(curr, this,
  1482. LOCK_USED_IN_HARDIRQ, "hard"))
  1483. return 0;
  1484. #if STRICT_READ_CHECKS
  1485. /*
  1486. * just marked it hardirq-unsafe, check that no
  1487. * hardirq-safe-read lock in the system ever took
  1488. * it in the past:
  1489. */
  1490. if (!check_usage_backwards(curr, this,
  1491. LOCK_USED_IN_HARDIRQ_READ, "hard-read"))
  1492. return 0;
  1493. #endif
  1494. if (hardirq_verbose(this->class))
  1495. ret = 2;
  1496. break;
  1497. case LOCK_ENABLED_SOFTIRQS:
  1498. if (!valid_state(curr, this, new_bit, LOCK_USED_IN_SOFTIRQ))
  1499. return 0;
  1500. if (!valid_state(curr, this, new_bit,
  1501. LOCK_USED_IN_SOFTIRQ_READ))
  1502. return 0;
  1503. /*
  1504. * just marked it softirq-unsafe, check that no softirq-safe
  1505. * lock in the system ever took it in the past:
  1506. */
  1507. if (!check_usage_backwards(curr, this,
  1508. LOCK_USED_IN_SOFTIRQ, "soft"))
  1509. return 0;
  1510. #if STRICT_READ_CHECKS
  1511. /*
  1512. * just marked it softirq-unsafe, check that no
  1513. * softirq-safe-read lock in the system ever took
  1514. * it in the past:
  1515. */
  1516. if (!check_usage_backwards(curr, this,
  1517. LOCK_USED_IN_SOFTIRQ_READ, "soft-read"))
  1518. return 0;
  1519. #endif
  1520. if (softirq_verbose(this->class))
  1521. ret = 2;
  1522. break;
  1523. case LOCK_ENABLED_HARDIRQS_READ:
  1524. if (!valid_state(curr, this, new_bit, LOCK_USED_IN_HARDIRQ))
  1525. return 0;
  1526. #if STRICT_READ_CHECKS
  1527. /*
  1528. * just marked it hardirq-read-unsafe, check that no
  1529. * hardirq-safe lock in the system ever took it in the past:
  1530. */
  1531. if (!check_usage_backwards(curr, this,
  1532. LOCK_USED_IN_HARDIRQ, "hard"))
  1533. return 0;
  1534. #endif
  1535. if (hardirq_verbose(this->class))
  1536. ret = 2;
  1537. break;
  1538. case LOCK_ENABLED_SOFTIRQS_READ:
  1539. if (!valid_state(curr, this, new_bit, LOCK_USED_IN_SOFTIRQ))
  1540. return 0;
  1541. #if STRICT_READ_CHECKS
  1542. /*
  1543. * just marked it softirq-read-unsafe, check that no
  1544. * softirq-safe lock in the system ever took it in the past:
  1545. */
  1546. if (!check_usage_backwards(curr, this,
  1547. LOCK_USED_IN_SOFTIRQ, "soft"))
  1548. return 0;
  1549. #endif
  1550. if (softirq_verbose(this->class))
  1551. ret = 2;
  1552. break;
  1553. #endif
  1554. case LOCK_USED:
  1555. /*
  1556. * Add it to the global list of classes:
  1557. */
  1558. list_add_tail_rcu(&this->class->lock_entry, &all_lock_classes);
  1559. debug_atomic_dec(&nr_unused_locks);
  1560. break;
  1561. default:
  1562. if (!debug_locks_off_graph_unlock())
  1563. return 0;
  1564. WARN_ON(1);
  1565. return 0;
  1566. }
  1567. graph_unlock();
  1568. /*
  1569. * We must printk outside of the graph_lock:
  1570. */
  1571. if (ret == 2) {
  1572. printk("\nmarked lock as {%s}:\n", usage_str[new_bit]);
  1573. print_lock(this);
  1574. print_irqtrace_events(curr);
  1575. dump_stack();
  1576. }
  1577. return ret;
  1578. }
  1579. #ifdef CONFIG_TRACE_IRQFLAGS
  1580. /*
  1581. * Mark all held locks with a usage bit:
  1582. */
  1583. static int
  1584. mark_held_locks(struct task_struct *curr, int hardirq, unsigned long ip)
  1585. {
  1586. enum lock_usage_bit usage_bit;
  1587. struct held_lock *hlock;
  1588. int i;
  1589. for (i = 0; i < curr->lockdep_depth; i++) {
  1590. hlock = curr->held_locks + i;
  1591. if (hardirq) {
  1592. if (hlock->read)
  1593. usage_bit = LOCK_ENABLED_HARDIRQS_READ;
  1594. else
  1595. usage_bit = LOCK_ENABLED_HARDIRQS;
  1596. } else {
  1597. if (hlock->read)
  1598. usage_bit = LOCK_ENABLED_SOFTIRQS_READ;
  1599. else
  1600. usage_bit = LOCK_ENABLED_SOFTIRQS;
  1601. }
  1602. if (!mark_lock(curr, hlock, usage_bit, ip))
  1603. return 0;
  1604. }
  1605. return 1;
  1606. }
  1607. /*
  1608. * Debugging helper: via this flag we know that we are in
  1609. * 'early bootup code', and will warn about any invalid irqs-on event:
  1610. */
  1611. static int early_boot_irqs_enabled;
  1612. void early_boot_irqs_off(void)
  1613. {
  1614. early_boot_irqs_enabled = 0;
  1615. }
  1616. void early_boot_irqs_on(void)
  1617. {
  1618. early_boot_irqs_enabled = 1;
  1619. }
  1620. /*
  1621. * Hardirqs will be enabled:
  1622. */
  1623. void trace_hardirqs_on(void)
  1624. {
  1625. struct task_struct *curr = current;
  1626. unsigned long ip;
  1627. if (unlikely(!debug_locks || current->lockdep_recursion))
  1628. return;
  1629. if (DEBUG_LOCKS_WARN_ON(unlikely(!early_boot_irqs_enabled)))
  1630. return;
  1631. if (unlikely(curr->hardirqs_enabled)) {
  1632. debug_atomic_inc(&redundant_hardirqs_on);
  1633. return;
  1634. }
  1635. /* we'll do an OFF -> ON transition: */
  1636. curr->hardirqs_enabled = 1;
  1637. ip = (unsigned long) __builtin_return_address(0);
  1638. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1639. return;
  1640. if (DEBUG_LOCKS_WARN_ON(current->hardirq_context))
  1641. return;
  1642. /*
  1643. * We are going to turn hardirqs on, so set the
  1644. * usage bit for all held locks:
  1645. */
  1646. if (!mark_held_locks(curr, 1, ip))
  1647. return;
  1648. /*
  1649. * If we have softirqs enabled, then set the usage
  1650. * bit for all held locks. (disabled hardirqs prevented
  1651. * this bit from being set before)
  1652. */
  1653. if (curr->softirqs_enabled)
  1654. if (!mark_held_locks(curr, 0, ip))
  1655. return;
  1656. curr->hardirq_enable_ip = ip;
  1657. curr->hardirq_enable_event = ++curr->irq_events;
  1658. debug_atomic_inc(&hardirqs_on_events);
  1659. }
  1660. EXPORT_SYMBOL(trace_hardirqs_on);
  1661. /*
  1662. * Hardirqs were disabled:
  1663. */
  1664. void trace_hardirqs_off(void)
  1665. {
  1666. struct task_struct *curr = current;
  1667. if (unlikely(!debug_locks || current->lockdep_recursion))
  1668. return;
  1669. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1670. return;
  1671. if (curr->hardirqs_enabled) {
  1672. /*
  1673. * We have done an ON -> OFF transition:
  1674. */
  1675. curr->hardirqs_enabled = 0;
  1676. curr->hardirq_disable_ip = _RET_IP_;
  1677. curr->hardirq_disable_event = ++curr->irq_events;
  1678. debug_atomic_inc(&hardirqs_off_events);
  1679. } else
  1680. debug_atomic_inc(&redundant_hardirqs_off);
  1681. }
  1682. EXPORT_SYMBOL(trace_hardirqs_off);
  1683. /*
  1684. * Softirqs will be enabled:
  1685. */
  1686. void trace_softirqs_on(unsigned long ip)
  1687. {
  1688. struct task_struct *curr = current;
  1689. if (unlikely(!debug_locks))
  1690. return;
  1691. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1692. return;
  1693. if (curr->softirqs_enabled) {
  1694. debug_atomic_inc(&redundant_softirqs_on);
  1695. return;
  1696. }
  1697. /*
  1698. * We'll do an OFF -> ON transition:
  1699. */
  1700. curr->softirqs_enabled = 1;
  1701. curr->softirq_enable_ip = ip;
  1702. curr->softirq_enable_event = ++curr->irq_events;
  1703. debug_atomic_inc(&softirqs_on_events);
  1704. /*
  1705. * We are going to turn softirqs on, so set the
  1706. * usage bit for all held locks, if hardirqs are
  1707. * enabled too:
  1708. */
  1709. if (curr->hardirqs_enabled)
  1710. mark_held_locks(curr, 0, ip);
  1711. }
  1712. /*
  1713. * Softirqs were disabled:
  1714. */
  1715. void trace_softirqs_off(unsigned long ip)
  1716. {
  1717. struct task_struct *curr = current;
  1718. if (unlikely(!debug_locks))
  1719. return;
  1720. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1721. return;
  1722. if (curr->softirqs_enabled) {
  1723. /*
  1724. * We have done an ON -> OFF transition:
  1725. */
  1726. curr->softirqs_enabled = 0;
  1727. curr->softirq_disable_ip = ip;
  1728. curr->softirq_disable_event = ++curr->irq_events;
  1729. debug_atomic_inc(&softirqs_off_events);
  1730. DEBUG_LOCKS_WARN_ON(!softirq_count());
  1731. } else
  1732. debug_atomic_inc(&redundant_softirqs_off);
  1733. }
  1734. #endif
  1735. /*
  1736. * Initialize a lock instance's lock-class mapping info:
  1737. */
  1738. void lockdep_init_map(struct lockdep_map *lock, const char *name,
  1739. struct lock_class_key *key, int subclass)
  1740. {
  1741. if (unlikely(!debug_locks))
  1742. return;
  1743. if (DEBUG_LOCKS_WARN_ON(!key))
  1744. return;
  1745. if (DEBUG_LOCKS_WARN_ON(!name))
  1746. return;
  1747. /*
  1748. * Sanity check, the lock-class key must be persistent:
  1749. */
  1750. if (!static_obj(key)) {
  1751. printk("BUG: key %p not in .data!\n", key);
  1752. DEBUG_LOCKS_WARN_ON(1);
  1753. return;
  1754. }
  1755. lock->name = name;
  1756. lock->key = key;
  1757. lock->class_cache = NULL;
  1758. if (subclass)
  1759. register_lock_class(lock, subclass, 1);
  1760. }
  1761. EXPORT_SYMBOL_GPL(lockdep_init_map);
  1762. /*
  1763. * This gets called for every mutex_lock*()/spin_lock*() operation.
  1764. * We maintain the dependency maps and validate the locking attempt:
  1765. */
  1766. static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
  1767. int trylock, int read, int check, int hardirqs_off,
  1768. unsigned long ip)
  1769. {
  1770. struct task_struct *curr = current;
  1771. struct lock_class *class = NULL;
  1772. struct held_lock *hlock;
  1773. unsigned int depth, id;
  1774. int chain_head = 0;
  1775. u64 chain_key;
  1776. if (unlikely(!debug_locks))
  1777. return 0;
  1778. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  1779. return 0;
  1780. if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
  1781. debug_locks_off();
  1782. printk("BUG: MAX_LOCKDEP_SUBCLASSES too low!\n");
  1783. printk("turning off the locking correctness validator.\n");
  1784. return 0;
  1785. }
  1786. if (!subclass)
  1787. class = lock->class_cache;
  1788. /*
  1789. * Not cached yet or subclass?
  1790. */
  1791. if (unlikely(!class)) {
  1792. class = register_lock_class(lock, subclass, 0);
  1793. if (!class)
  1794. return 0;
  1795. }
  1796. debug_atomic_inc((atomic_t *)&class->ops);
  1797. if (very_verbose(class)) {
  1798. printk("\nacquire class [%p] %s", class->key, class->name);
  1799. if (class->name_version > 1)
  1800. printk("#%d", class->name_version);
  1801. printk("\n");
  1802. dump_stack();
  1803. }
  1804. /*
  1805. * Add the lock to the list of currently held locks.
  1806. * (we dont increase the depth just yet, up until the
  1807. * dependency checks are done)
  1808. */
  1809. depth = curr->lockdep_depth;
  1810. if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
  1811. return 0;
  1812. hlock = curr->held_locks + depth;
  1813. hlock->class = class;
  1814. hlock->acquire_ip = ip;
  1815. hlock->instance = lock;
  1816. hlock->trylock = trylock;
  1817. hlock->read = read;
  1818. hlock->check = check;
  1819. hlock->hardirqs_off = hardirqs_off;
  1820. if (check != 2)
  1821. goto out_calc_hash;
  1822. #ifdef CONFIG_TRACE_IRQFLAGS
  1823. /*
  1824. * If non-trylock use in a hardirq or softirq context, then
  1825. * mark the lock as used in these contexts:
  1826. */
  1827. if (!trylock) {
  1828. if (read) {
  1829. if (curr->hardirq_context)
  1830. if (!mark_lock(curr, hlock,
  1831. LOCK_USED_IN_HARDIRQ_READ, ip))
  1832. return 0;
  1833. if (curr->softirq_context)
  1834. if (!mark_lock(curr, hlock,
  1835. LOCK_USED_IN_SOFTIRQ_READ, ip))
  1836. return 0;
  1837. } else {
  1838. if (curr->hardirq_context)
  1839. if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ, ip))
  1840. return 0;
  1841. if (curr->softirq_context)
  1842. if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ, ip))
  1843. return 0;
  1844. }
  1845. }
  1846. if (!hardirqs_off) {
  1847. if (read) {
  1848. if (!mark_lock(curr, hlock,
  1849. LOCK_ENABLED_HARDIRQS_READ, ip))
  1850. return 0;
  1851. if (curr->softirqs_enabled)
  1852. if (!mark_lock(curr, hlock,
  1853. LOCK_ENABLED_SOFTIRQS_READ, ip))
  1854. return 0;
  1855. } else {
  1856. if (!mark_lock(curr, hlock,
  1857. LOCK_ENABLED_HARDIRQS, ip))
  1858. return 0;
  1859. if (curr->softirqs_enabled)
  1860. if (!mark_lock(curr, hlock,
  1861. LOCK_ENABLED_SOFTIRQS, ip))
  1862. return 0;
  1863. }
  1864. }
  1865. #endif
  1866. /* mark it as used: */
  1867. if (!mark_lock(curr, hlock, LOCK_USED, ip))
  1868. return 0;
  1869. out_calc_hash:
  1870. /*
  1871. * Calculate the chain hash: it's the combined has of all the
  1872. * lock keys along the dependency chain. We save the hash value
  1873. * at every step so that we can get the current hash easily
  1874. * after unlock. The chain hash is then used to cache dependency
  1875. * results.
  1876. *
  1877. * The 'key ID' is what is the most compact key value to drive
  1878. * the hash, not class->key.
  1879. */
  1880. id = class - lock_classes;
  1881. if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))
  1882. return 0;
  1883. chain_key = curr->curr_chain_key;
  1884. if (!depth) {
  1885. if (DEBUG_LOCKS_WARN_ON(chain_key != 0))
  1886. return 0;
  1887. chain_head = 1;
  1888. }
  1889. hlock->prev_chain_key = chain_key;
  1890. #ifdef CONFIG_TRACE_IRQFLAGS
  1891. /*
  1892. * Keep track of points where we cross into an interrupt context:
  1893. */
  1894. hlock->irq_context = 2*(curr->hardirq_context ? 1 : 0) +
  1895. curr->softirq_context;
  1896. if (depth) {
  1897. struct held_lock *prev_hlock;
  1898. prev_hlock = curr->held_locks + depth-1;
  1899. /*
  1900. * If we cross into another context, reset the
  1901. * hash key (this also prevents the checking and the
  1902. * adding of the dependency to 'prev'):
  1903. */
  1904. if (prev_hlock->irq_context != hlock->irq_context) {
  1905. chain_key = 0;
  1906. chain_head = 1;
  1907. }
  1908. }
  1909. #endif
  1910. chain_key = iterate_chain_key(chain_key, id);
  1911. curr->curr_chain_key = chain_key;
  1912. /*
  1913. * Trylock needs to maintain the stack of held locks, but it
  1914. * does not add new dependencies, because trylock can be done
  1915. * in any order.
  1916. *
  1917. * We look up the chain_key and do the O(N^2) check and update of
  1918. * the dependencies only if this is a new dependency chain.
  1919. * (If lookup_chain_cache() returns with 1 it acquires
  1920. * graph_lock for us)
  1921. */
  1922. if (!trylock && (check == 2) && lookup_chain_cache(chain_key, class)) {
  1923. /*
  1924. * Check whether last held lock:
  1925. *
  1926. * - is irq-safe, if this lock is irq-unsafe
  1927. * - is softirq-safe, if this lock is hardirq-unsafe
  1928. *
  1929. * And check whether the new lock's dependency graph
  1930. * could lead back to the previous lock.
  1931. *
  1932. * any of these scenarios could lead to a deadlock. If
  1933. * All validations
  1934. */
  1935. int ret = check_deadlock(curr, hlock, lock, read);
  1936. if (!ret)
  1937. return 0;
  1938. /*
  1939. * Mark recursive read, as we jump over it when
  1940. * building dependencies (just like we jump over
  1941. * trylock entries):
  1942. */
  1943. if (ret == 2)
  1944. hlock->read = 2;
  1945. /*
  1946. * Add dependency only if this lock is not the head
  1947. * of the chain, and if it's not a secondary read-lock:
  1948. */
  1949. if (!chain_head && ret != 2)
  1950. if (!check_prevs_add(curr, hlock))
  1951. return 0;
  1952. graph_unlock();
  1953. } else
  1954. /* after lookup_chain_cache(): */
  1955. if (unlikely(!debug_locks))
  1956. return 0;
  1957. curr->lockdep_depth++;
  1958. check_chain_key(curr);
  1959. #ifdef CONFIG_DEBUG_LOCKDEP
  1960. if (unlikely(!debug_locks))
  1961. return 0;
  1962. #endif
  1963. if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) {
  1964. debug_locks_off();
  1965. printk("BUG: MAX_LOCK_DEPTH too low!\n");
  1966. printk("turning off the locking correctness validator.\n");
  1967. return 0;
  1968. }
  1969. if (unlikely(curr->lockdep_depth > max_lockdep_depth))
  1970. max_lockdep_depth = curr->lockdep_depth;
  1971. return 1;
  1972. }
  1973. static int
  1974. print_unlock_inbalance_bug(struct task_struct *curr, struct lockdep_map *lock,
  1975. unsigned long ip)
  1976. {
  1977. if (!debug_locks_off())
  1978. return 0;
  1979. if (debug_locks_silent)
  1980. return 0;
  1981. printk("\n=====================================\n");
  1982. printk( "[ BUG: bad unlock balance detected! ]\n");
  1983. printk( "-------------------------------------\n");
  1984. printk("%s/%d is trying to release lock (",
  1985. curr->comm, curr->pid);
  1986. print_lockdep_cache(lock);
  1987. printk(") at:\n");
  1988. print_ip_sym(ip);
  1989. printk("but there are no more locks to release!\n");
  1990. printk("\nother info that might help us debug this:\n");
  1991. lockdep_print_held_locks(curr);
  1992. printk("\nstack backtrace:\n");
  1993. dump_stack();
  1994. return 0;
  1995. }
  1996. /*
  1997. * Common debugging checks for both nested and non-nested unlock:
  1998. */
  1999. static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
  2000. unsigned long ip)
  2001. {
  2002. if (unlikely(!debug_locks))
  2003. return 0;
  2004. if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
  2005. return 0;
  2006. if (curr->lockdep_depth <= 0)
  2007. return print_unlock_inbalance_bug(curr, lock, ip);
  2008. return 1;
  2009. }
  2010. /*
  2011. * Remove the lock to the list of currently held locks in a
  2012. * potentially non-nested (out of order) manner. This is a
  2013. * relatively rare operation, as all the unlock APIs default
  2014. * to nested mode (which uses lock_release()):
  2015. */
  2016. static int
  2017. lock_release_non_nested(struct task_struct *curr,
  2018. struct lockdep_map *lock, unsigned long ip)
  2019. {
  2020. struct held_lock *hlock, *prev_hlock;
  2021. unsigned int depth;
  2022. int i;
  2023. /*
  2024. * Check whether the lock exists in the current stack
  2025. * of held locks:
  2026. */
  2027. depth = curr->lockdep_depth;
  2028. if (DEBUG_LOCKS_WARN_ON(!depth))
  2029. return 0;
  2030. prev_hlock = NULL;
  2031. for (i = depth-1; i >= 0; i--) {
  2032. hlock = curr->held_locks + i;
  2033. /*
  2034. * We must not cross into another context:
  2035. */
  2036. if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
  2037. break;
  2038. if (hlock->instance == lock)
  2039. goto found_it;
  2040. prev_hlock = hlock;
  2041. }
  2042. return print_unlock_inbalance_bug(curr, lock, ip);
  2043. found_it:
  2044. /*
  2045. * We have the right lock to unlock, 'hlock' points to it.
  2046. * Now we remove it from the stack, and add back the other
  2047. * entries (if any), recalculating the hash along the way:
  2048. */
  2049. curr->lockdep_depth = i;
  2050. curr->curr_chain_key = hlock->prev_chain_key;
  2051. for (i++; i < depth; i++) {
  2052. hlock = curr->held_locks + i;
  2053. if (!__lock_acquire(hlock->instance,
  2054. hlock->class->subclass, hlock->trylock,
  2055. hlock->read, hlock->check, hlock->hardirqs_off,
  2056. hlock->acquire_ip))
  2057. return 0;
  2058. }
  2059. if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))
  2060. return 0;
  2061. return 1;
  2062. }
  2063. /*
  2064. * Remove the lock to the list of currently held locks - this gets
  2065. * called on mutex_unlock()/spin_unlock*() (or on a failed
  2066. * mutex_lock_interruptible()). This is done for unlocks that nest
  2067. * perfectly. (i.e. the current top of the lock-stack is unlocked)
  2068. */
  2069. static int lock_release_nested(struct task_struct *curr,
  2070. struct lockdep_map *lock, unsigned long ip)
  2071. {
  2072. struct held_lock *hlock;
  2073. unsigned int depth;
  2074. /*
  2075. * Pop off the top of the lock stack:
  2076. */
  2077. depth = curr->lockdep_depth - 1;
  2078. hlock = curr->held_locks + depth;
  2079. /*
  2080. * Is the unlock non-nested:
  2081. */
  2082. if (hlock->instance != lock)
  2083. return lock_release_non_nested(curr, lock, ip);
  2084. curr->lockdep_depth--;
  2085. if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0)))
  2086. return 0;
  2087. curr->curr_chain_key = hlock->prev_chain_key;
  2088. #ifdef CONFIG_DEBUG_LOCKDEP
  2089. hlock->prev_chain_key = 0;
  2090. hlock->class = NULL;
  2091. hlock->acquire_ip = 0;
  2092. hlock->irq_context = 0;
  2093. #endif
  2094. return 1;
  2095. }
  2096. /*
  2097. * Remove the lock to the list of currently held locks - this gets
  2098. * called on mutex_unlock()/spin_unlock*() (or on a failed
  2099. * mutex_lock_interruptible()). This is done for unlocks that nest
  2100. * perfectly. (i.e. the current top of the lock-stack is unlocked)
  2101. */
  2102. static void
  2103. __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
  2104. {
  2105. struct task_struct *curr = current;
  2106. if (!check_unlock(curr, lock, ip))
  2107. return;
  2108. if (nested) {
  2109. if (!lock_release_nested(curr, lock, ip))
  2110. return;
  2111. } else {
  2112. if (!lock_release_non_nested(curr, lock, ip))
  2113. return;
  2114. }
  2115. check_chain_key(curr);
  2116. }
  2117. /*
  2118. * Check whether we follow the irq-flags state precisely:
  2119. */
  2120. static void check_flags(unsigned long flags)
  2121. {
  2122. #if defined(CONFIG_DEBUG_LOCKDEP) && defined(CONFIG_TRACE_IRQFLAGS)
  2123. if (!debug_locks)
  2124. return;
  2125. if (irqs_disabled_flags(flags))
  2126. DEBUG_LOCKS_WARN_ON(current->hardirqs_enabled);
  2127. else
  2128. DEBUG_LOCKS_WARN_ON(!current->hardirqs_enabled);
  2129. /*
  2130. * We dont accurately track softirq state in e.g.
  2131. * hardirq contexts (such as on 4KSTACKS), so only
  2132. * check if not in hardirq contexts:
  2133. */
  2134. if (!hardirq_count()) {
  2135. if (softirq_count())
  2136. DEBUG_LOCKS_WARN_ON(current->softirqs_enabled);
  2137. else
  2138. DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
  2139. }
  2140. if (!debug_locks)
  2141. print_irqtrace_events(current);
  2142. #endif
  2143. }
  2144. /*
  2145. * We are not always called with irqs disabled - do that here,
  2146. * and also avoid lockdep recursion:
  2147. */
  2148. void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
  2149. int trylock, int read, int check, unsigned long ip)
  2150. {
  2151. unsigned long flags;
  2152. if (unlikely(current->lockdep_recursion))
  2153. return;
  2154. raw_local_irq_save(flags);
  2155. check_flags(flags);
  2156. current->lockdep_recursion = 1;
  2157. __lock_acquire(lock, subclass, trylock, read, check,
  2158. irqs_disabled_flags(flags), ip);
  2159. current->lockdep_recursion = 0;
  2160. raw_local_irq_restore(flags);
  2161. }
  2162. EXPORT_SYMBOL_GPL(lock_acquire);
  2163. void lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
  2164. {
  2165. unsigned long flags;
  2166. if (unlikely(current->lockdep_recursion))
  2167. return;
  2168. raw_local_irq_save(flags);
  2169. check_flags(flags);
  2170. current->lockdep_recursion = 1;
  2171. __lock_release(lock, nested, ip);
  2172. current->lockdep_recursion = 0;
  2173. raw_local_irq_restore(flags);
  2174. }
  2175. EXPORT_SYMBOL_GPL(lock_release);
  2176. /*
  2177. * Used by the testsuite, sanitize the validator state
  2178. * after a simulated failure:
  2179. */
  2180. void lockdep_reset(void)
  2181. {
  2182. unsigned long flags;
  2183. int i;
  2184. raw_local_irq_save(flags);
  2185. current->curr_chain_key = 0;
  2186. current->lockdep_depth = 0;
  2187. current->lockdep_recursion = 0;
  2188. memset(current->held_locks, 0, MAX_LOCK_DEPTH*sizeof(struct held_lock));
  2189. nr_hardirq_chains = 0;
  2190. nr_softirq_chains = 0;
  2191. nr_process_chains = 0;
  2192. debug_locks = 1;
  2193. for (i = 0; i < CHAINHASH_SIZE; i++)
  2194. INIT_LIST_HEAD(chainhash_table + i);
  2195. raw_local_irq_restore(flags);
  2196. }
  2197. static void zap_class(struct lock_class *class)
  2198. {
  2199. int i;
  2200. /*
  2201. * Remove all dependencies this lock is
  2202. * involved in:
  2203. */
  2204. for (i = 0; i < nr_list_entries; i++) {
  2205. if (list_entries[i].class == class)
  2206. list_del_rcu(&list_entries[i].entry);
  2207. }
  2208. /*
  2209. * Unhash the class and remove it from the all_lock_classes list:
  2210. */
  2211. list_del_rcu(&class->hash_entry);
  2212. list_del_rcu(&class->lock_entry);
  2213. }
  2214. static inline int within(void *addr, void *start, unsigned long size)
  2215. {
  2216. return addr >= start && addr < start + size;
  2217. }
  2218. void lockdep_free_key_range(void *start, unsigned long size)
  2219. {
  2220. struct lock_class *class, *next;
  2221. struct list_head *head;
  2222. unsigned long flags;
  2223. int i;
  2224. raw_local_irq_save(flags);
  2225. graph_lock();
  2226. /*
  2227. * Unhash all classes that were created by this module:
  2228. */
  2229. for (i = 0; i < CLASSHASH_SIZE; i++) {
  2230. head = classhash_table + i;
  2231. if (list_empty(head))
  2232. continue;
  2233. list_for_each_entry_safe(class, next, head, hash_entry)
  2234. if (within(class->key, start, size))
  2235. zap_class(class);
  2236. }
  2237. graph_unlock();
  2238. raw_local_irq_restore(flags);
  2239. }
  2240. void lockdep_reset_lock(struct lockdep_map *lock)
  2241. {
  2242. struct lock_class *class, *next;
  2243. struct list_head *head;
  2244. unsigned long flags;
  2245. int i, j;
  2246. raw_local_irq_save(flags);
  2247. /*
  2248. * Remove all classes this lock might have:
  2249. */
  2250. for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
  2251. /*
  2252. * If the class exists we look it up and zap it:
  2253. */
  2254. class = look_up_lock_class(lock, j);
  2255. if (class)
  2256. zap_class(class);
  2257. }
  2258. /*
  2259. * Debug check: in the end all mapped classes should
  2260. * be gone.
  2261. */
  2262. graph_lock();
  2263. for (i = 0; i < CLASSHASH_SIZE; i++) {
  2264. head = classhash_table + i;
  2265. if (list_empty(head))
  2266. continue;
  2267. list_for_each_entry_safe(class, next, head, hash_entry) {
  2268. if (unlikely(class == lock->class_cache)) {
  2269. if (debug_locks_off_graph_unlock())
  2270. WARN_ON(1);
  2271. goto out_restore;
  2272. }
  2273. }
  2274. }
  2275. graph_unlock();
  2276. out_restore:
  2277. raw_local_irq_restore(flags);
  2278. }
  2279. void lockdep_init(void)
  2280. {
  2281. int i;
  2282. /*
  2283. * Some architectures have their own start_kernel()
  2284. * code which calls lockdep_init(), while we also
  2285. * call lockdep_init() from the start_kernel() itself,
  2286. * and we want to initialize the hashes only once:
  2287. */
  2288. if (lockdep_initialized)
  2289. return;
  2290. for (i = 0; i < CLASSHASH_SIZE; i++)
  2291. INIT_LIST_HEAD(classhash_table + i);
  2292. for (i = 0; i < CHAINHASH_SIZE; i++)
  2293. INIT_LIST_HEAD(chainhash_table + i);
  2294. lockdep_initialized = 1;
  2295. }
  2296. void __init lockdep_info(void)
  2297. {
  2298. printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n");
  2299. printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES);
  2300. printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH);
  2301. printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS);
  2302. printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE);
  2303. printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES);
  2304. printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS);
  2305. printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE);
  2306. printk(" memory used by lock dependency info: %lu kB\n",
  2307. (sizeof(struct lock_class) * MAX_LOCKDEP_KEYS +
  2308. sizeof(struct list_head) * CLASSHASH_SIZE +
  2309. sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES +
  2310. sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS +
  2311. sizeof(struct list_head) * CHAINHASH_SIZE) / 1024);
  2312. printk(" per task-struct memory footprint: %lu bytes\n",
  2313. sizeof(struct held_lock) * MAX_LOCK_DEPTH);
  2314. #ifdef CONFIG_DEBUG_LOCKDEP
  2315. if (lockdep_init_error)
  2316. printk("WARNING: lockdep init error! Arch code didnt call lockdep_init() early enough?\n");
  2317. #endif
  2318. }
  2319. static inline int in_range(const void *start, const void *addr, const void *end)
  2320. {
  2321. return addr >= start && addr <= end;
  2322. }
  2323. static void
  2324. print_freed_lock_bug(struct task_struct *curr, const void *mem_from,
  2325. const void *mem_to, struct held_lock *hlock)
  2326. {
  2327. if (!debug_locks_off())
  2328. return;
  2329. if (debug_locks_silent)
  2330. return;
  2331. printk("\n=========================\n");
  2332. printk( "[ BUG: held lock freed! ]\n");
  2333. printk( "-------------------------\n");
  2334. printk("%s/%d is freeing memory %p-%p, with a lock still held there!\n",
  2335. curr->comm, curr->pid, mem_from, mem_to-1);
  2336. print_lock(hlock);
  2337. lockdep_print_held_locks(curr);
  2338. printk("\nstack backtrace:\n");
  2339. dump_stack();
  2340. }
  2341. /*
  2342. * Called when kernel memory is freed (or unmapped), or if a lock
  2343. * is destroyed or reinitialized - this code checks whether there is
  2344. * any held lock in the memory range of <from> to <to>:
  2345. */
  2346. void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len)
  2347. {
  2348. const void *mem_to = mem_from + mem_len, *lock_from, *lock_to;
  2349. struct task_struct *curr = current;
  2350. struct held_lock *hlock;
  2351. unsigned long flags;
  2352. int i;
  2353. if (unlikely(!debug_locks))
  2354. return;
  2355. local_irq_save(flags);
  2356. for (i = 0; i < curr->lockdep_depth; i++) {
  2357. hlock = curr->held_locks + i;
  2358. lock_from = (void *)hlock->instance;
  2359. lock_to = (void *)(hlock->instance + 1);
  2360. if (!in_range(mem_from, lock_from, mem_to) &&
  2361. !in_range(mem_from, lock_to, mem_to))
  2362. continue;
  2363. print_freed_lock_bug(curr, mem_from, mem_to, hlock);
  2364. break;
  2365. }
  2366. local_irq_restore(flags);
  2367. }
  2368. EXPORT_SYMBOL_GPL(debug_check_no_locks_freed);
  2369. static void print_held_locks_bug(struct task_struct *curr)
  2370. {
  2371. if (!debug_locks_off())
  2372. return;
  2373. if (debug_locks_silent)
  2374. return;
  2375. printk("\n=====================================\n");
  2376. printk( "[ BUG: lock held at task exit time! ]\n");
  2377. printk( "-------------------------------------\n");
  2378. printk("%s/%d is exiting with locks still held!\n",
  2379. curr->comm, curr->pid);
  2380. lockdep_print_held_locks(curr);
  2381. printk("\nstack backtrace:\n");
  2382. dump_stack();
  2383. }
  2384. void debug_check_no_locks_held(struct task_struct *task)
  2385. {
  2386. if (unlikely(task->lockdep_depth > 0))
  2387. print_held_locks_bug(task);
  2388. }
  2389. void debug_show_all_locks(void)
  2390. {
  2391. struct task_struct *g, *p;
  2392. int count = 10;
  2393. int unlock = 1;
  2394. if (unlikely(!debug_locks)) {
  2395. printk("INFO: lockdep is turned off.\n");
  2396. return;
  2397. }
  2398. printk("\nShowing all locks held in the system:\n");
  2399. /*
  2400. * Here we try to get the tasklist_lock as hard as possible,
  2401. * if not successful after 2 seconds we ignore it (but keep
  2402. * trying). This is to enable a debug printout even if a
  2403. * tasklist_lock-holding task deadlocks or crashes.
  2404. */
  2405. retry:
  2406. if (!read_trylock(&tasklist_lock)) {
  2407. if (count == 10)
  2408. printk("hm, tasklist_lock locked, retrying... ");
  2409. if (count) {
  2410. count--;
  2411. printk(" #%d", 10-count);
  2412. mdelay(200);
  2413. goto retry;
  2414. }
  2415. printk(" ignoring it.\n");
  2416. unlock = 0;
  2417. }
  2418. if (count != 10)
  2419. printk(" locked it.\n");
  2420. do_each_thread(g, p) {
  2421. if (p->lockdep_depth)
  2422. lockdep_print_held_locks(p);
  2423. if (!unlock)
  2424. if (read_trylock(&tasklist_lock))
  2425. unlock = 1;
  2426. } while_each_thread(g, p);
  2427. printk("\n");
  2428. printk("=============================================\n\n");
  2429. if (unlock)
  2430. read_unlock(&tasklist_lock);
  2431. }
  2432. EXPORT_SYMBOL_GPL(debug_show_all_locks);
  2433. void debug_show_held_locks(struct task_struct *task)
  2434. {
  2435. if (unlikely(!debug_locks)) {
  2436. printk("INFO: lockdep is turned off.\n");
  2437. return;
  2438. }
  2439. lockdep_print_held_locks(task);
  2440. }
  2441. EXPORT_SYMBOL_GPL(debug_show_held_locks);