module.c 123 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. Copyright (C) 2002 Richard Henderson
  4. Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
  5. */
  6. #define INCLUDE_VERMAGIC
  7. #include <linux/export.h>
  8. #include <linux/extable.h>
  9. #include <linux/moduleloader.h>
  10. #include <linux/module_signature.h>
  11. #include <linux/trace_events.h>
  12. #include <linux/init.h>
  13. #include <linux/kallsyms.h>
  14. #include <linux/file.h>
  15. #include <linux/fs.h>
  16. #include <linux/sysfs.h>
  17. #include <linux/kernel.h>
  18. #include <linux/kernel_read_file.h>
  19. #include <linux/slab.h>
  20. #include <linux/vmalloc.h>
  21. #include <linux/elf.h>
  22. #include <linux/proc_fs.h>
  23. #include <linux/security.h>
  24. #include <linux/seq_file.h>
  25. #include <linux/syscalls.h>
  26. #include <linux/fcntl.h>
  27. #include <linux/rcupdate.h>
  28. #include <linux/capability.h>
  29. #include <linux/cpu.h>
  30. #include <linux/moduleparam.h>
  31. #include <linux/errno.h>
  32. #include <linux/err.h>
  33. #include <linux/vermagic.h>
  34. #include <linux/notifier.h>
  35. #include <linux/sched.h>
  36. #include <linux/device.h>
  37. #include <linux/string.h>
  38. #include <linux/mutex.h>
  39. #include <linux/rculist.h>
  40. #include <linux/uaccess.h>
  41. #include <asm/cacheflush.h>
  42. #include <linux/set_memory.h>
  43. #include <asm/mmu_context.h>
  44. #include <linux/license.h>
  45. #include <asm/sections.h>
  46. #include <linux/tracepoint.h>
  47. #include <linux/ftrace.h>
  48. #include <linux/livepatch.h>
  49. #include <linux/async.h>
  50. #include <linux/percpu.h>
  51. #include <linux/kmemleak.h>
  52. #include <linux/jump_label.h>
  53. #include <linux/pfn.h>
  54. #include <linux/bsearch.h>
  55. #include <linux/dynamic_debug.h>
  56. #include <linux/audit.h>
  57. #include <uapi/linux/module.h>
  58. #include "module-internal.h"
  59. #define CREATE_TRACE_POINTS
  60. #include <trace/events/module.h>
  61. #undef CREATE_TRACE_POINTS
  62. #include <trace/hooks/module.h>
  63. #include <trace/hooks/memory.h>
  64. #ifndef ARCH_SHF_SMALL
  65. #define ARCH_SHF_SMALL 0
  66. #endif
  67. /*
  68. * Modules' sections will be aligned on page boundaries
  69. * to ensure complete separation of code and data, but
  70. * only when CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y
  71. */
  72. #ifdef CONFIG_ARCH_HAS_STRICT_MODULE_RWX
  73. # define debug_align(X) ALIGN(X, PAGE_SIZE)
  74. #else
  75. # define debug_align(X) (X)
  76. #endif
  77. /* If this is set, the section belongs in the init part of the module */
  78. #define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))
  79. /*
  80. * Mutex protects:
  81. * 1) List of modules (also safely readable with preempt_disable),
  82. * 2) module_use links,
  83. * 3) module_addr_min/module_addr_max.
  84. * (delete and add uses RCU list operations). */
  85. DEFINE_MUTEX(module_mutex);
  86. static LIST_HEAD(modules);
  87. /* Work queue for freeing init sections in success case */
  88. static void do_free_init(struct work_struct *w);
  89. static DECLARE_WORK(init_free_wq, do_free_init);
  90. static LLIST_HEAD(init_free_list);
  91. #ifdef CONFIG_MODULES_TREE_LOOKUP
  92. /*
  93. * Use a latched RB-tree for __module_address(); this allows us to use
  94. * RCU-sched lookups of the address from any context.
  95. *
  96. * This is conditional on PERF_EVENTS || TRACING because those can really hit
  97. * __module_address() hard by doing a lot of stack unwinding; potentially from
  98. * NMI context.
  99. */
  100. static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
  101. {
  102. struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
  103. return (unsigned long)layout->base;
  104. }
  105. static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
  106. {
  107. struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
  108. return (unsigned long)layout->size;
  109. }
  110. static __always_inline bool
  111. mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)
  112. {
  113. return __mod_tree_val(a) < __mod_tree_val(b);
  114. }
  115. static __always_inline int
  116. mod_tree_comp(void *key, struct latch_tree_node *n)
  117. {
  118. unsigned long val = (unsigned long)key;
  119. unsigned long start, end;
  120. start = __mod_tree_val(n);
  121. if (val < start)
  122. return -1;
  123. end = start + __mod_tree_size(n);
  124. if (val >= end)
  125. return 1;
  126. return 0;
  127. }
  128. static const struct latch_tree_ops mod_tree_ops = {
  129. .less = mod_tree_less,
  130. .comp = mod_tree_comp,
  131. };
  132. static struct mod_tree_root {
  133. struct latch_tree_root root;
  134. unsigned long addr_min;
  135. unsigned long addr_max;
  136. } mod_tree __cacheline_aligned = {
  137. .addr_min = -1UL,
  138. };
  139. #define module_addr_min mod_tree.addr_min
  140. #define module_addr_max mod_tree.addr_max
  141. static noinline void __mod_tree_insert(struct mod_tree_node *node)
  142. {
  143. latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);
  144. }
  145. static void __mod_tree_remove(struct mod_tree_node *node)
  146. {
  147. latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);
  148. }
  149. /*
  150. * These modifications: insert, remove_init and remove; are serialized by the
  151. * module_mutex.
  152. */
  153. static void mod_tree_insert(struct module *mod)
  154. {
  155. mod->core_layout.mtn.mod = mod;
  156. mod->init_layout.mtn.mod = mod;
  157. __mod_tree_insert(&mod->core_layout.mtn);
  158. if (mod->init_layout.size)
  159. __mod_tree_insert(&mod->init_layout.mtn);
  160. }
  161. static void mod_tree_remove_init(struct module *mod)
  162. {
  163. if (mod->init_layout.size)
  164. __mod_tree_remove(&mod->init_layout.mtn);
  165. }
  166. static void mod_tree_remove(struct module *mod)
  167. {
  168. __mod_tree_remove(&mod->core_layout.mtn);
  169. mod_tree_remove_init(mod);
  170. }
  171. static struct module *mod_find(unsigned long addr)
  172. {
  173. struct latch_tree_node *ltn;
  174. ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);
  175. if (!ltn)
  176. return NULL;
  177. return container_of(ltn, struct mod_tree_node, node)->mod;
  178. }
  179. #else /* MODULES_TREE_LOOKUP */
  180. static unsigned long module_addr_min = -1UL, module_addr_max = 0;
  181. static void mod_tree_insert(struct module *mod) { }
  182. static void mod_tree_remove_init(struct module *mod) { }
  183. static void mod_tree_remove(struct module *mod) { }
  184. static struct module *mod_find(unsigned long addr)
  185. {
  186. struct module *mod;
  187. list_for_each_entry_rcu(mod, &modules, list,
  188. lockdep_is_held(&module_mutex)) {
  189. if (within_module(addr, mod))
  190. return mod;
  191. }
  192. return NULL;
  193. }
  194. #endif /* MODULES_TREE_LOOKUP */
  195. /*
  196. * Bounds of module text, for speeding up __module_address.
  197. * Protected by module_mutex.
  198. */
  199. static void __mod_update_bounds(void *base, unsigned int size)
  200. {
  201. unsigned long min = (unsigned long)base;
  202. unsigned long max = min + size;
  203. if (min < module_addr_min)
  204. module_addr_min = min;
  205. if (max > module_addr_max)
  206. module_addr_max = max;
  207. }
  208. static void mod_update_bounds(struct module *mod)
  209. {
  210. __mod_update_bounds(mod->core_layout.base, mod->core_layout.size);
  211. if (mod->init_layout.size)
  212. __mod_update_bounds(mod->init_layout.base, mod->init_layout.size);
  213. }
  214. #ifdef CONFIG_KGDB_KDB
  215. struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
  216. #endif /* CONFIG_KGDB_KDB */
  217. static void module_assert_mutex(void)
  218. {
  219. lockdep_assert_held(&module_mutex);
  220. }
  221. static void module_assert_mutex_or_preempt(void)
  222. {
  223. #ifdef CONFIG_LOCKDEP
  224. if (unlikely(!debug_locks))
  225. return;
  226. WARN_ON_ONCE(!rcu_read_lock_sched_held() &&
  227. !lockdep_is_held(&module_mutex));
  228. #endif
  229. }
  230. #ifdef CONFIG_MODULE_SIG
  231. static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
  232. module_param(sig_enforce, bool_enable_only, 0644);
  233. void set_module_sig_enforced(void)
  234. {
  235. sig_enforce = true;
  236. }
  237. #else
  238. #define sig_enforce false
  239. #endif
  240. /*
  241. * Export sig_enforce kernel cmdline parameter to allow other subsystems rely
  242. * on that instead of directly to CONFIG_MODULE_SIG_FORCE config.
  243. */
  244. bool is_module_sig_enforced(void)
  245. {
  246. return sig_enforce;
  247. }
  248. EXPORT_SYMBOL(is_module_sig_enforced);
  249. /* Block module loading/unloading? */
  250. int modules_disabled = 0;
  251. core_param(nomodule, modules_disabled, bint, 0);
  252. /* Waiting for a module to finish initializing? */
  253. static DECLARE_WAIT_QUEUE_HEAD(module_wq);
  254. static BLOCKING_NOTIFIER_HEAD(module_notify_list);
  255. int register_module_notifier(struct notifier_block *nb)
  256. {
  257. return blocking_notifier_chain_register(&module_notify_list, nb);
  258. }
  259. EXPORT_SYMBOL(register_module_notifier);
  260. int unregister_module_notifier(struct notifier_block *nb)
  261. {
  262. return blocking_notifier_chain_unregister(&module_notify_list, nb);
  263. }
  264. EXPORT_SYMBOL(unregister_module_notifier);
  265. /*
  266. * We require a truly strong try_module_get(): 0 means success.
  267. * Otherwise an error is returned due to ongoing or failed
  268. * initialization etc.
  269. */
  270. static inline int strong_try_module_get(struct module *mod)
  271. {
  272. BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
  273. if (mod && mod->state == MODULE_STATE_COMING)
  274. return -EBUSY;
  275. if (try_module_get(mod))
  276. return 0;
  277. else
  278. return -ENOENT;
  279. }
  280. static inline void add_taint_module(struct module *mod, unsigned flag,
  281. enum lockdep_ok lockdep_ok)
  282. {
  283. add_taint(flag, lockdep_ok);
  284. set_bit(flag, &mod->taints);
  285. }
  286. /*
  287. * A thread that wants to hold a reference to a module only while it
  288. * is running can call this to safely exit. nfsd and lockd use this.
  289. */
  290. void __noreturn __module_put_and_exit(struct module *mod, long code)
  291. {
  292. module_put(mod);
  293. do_exit(code);
  294. }
  295. EXPORT_SYMBOL(__module_put_and_exit);
  296. /* Find a module section: 0 means not found. */
  297. static unsigned int find_sec(const struct load_info *info, const char *name)
  298. {
  299. unsigned int i;
  300. for (i = 1; i < info->hdr->e_shnum; i++) {
  301. Elf_Shdr *shdr = &info->sechdrs[i];
  302. /* Alloc bit cleared means "ignore it." */
  303. if ((shdr->sh_flags & SHF_ALLOC)
  304. && strcmp(info->secstrings + shdr->sh_name, name) == 0)
  305. return i;
  306. }
  307. return 0;
  308. }
  309. /* Find a module section, or NULL. */
  310. static void *section_addr(const struct load_info *info, const char *name)
  311. {
  312. /* Section 0 has sh_addr 0. */
  313. return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
  314. }
  315. /* Find a module section, or NULL. Fill in number of "objects" in section. */
  316. static void *section_objs(const struct load_info *info,
  317. const char *name,
  318. size_t object_size,
  319. unsigned int *num)
  320. {
  321. unsigned int sec = find_sec(info, name);
  322. /* Section 0 has sh_addr 0 and sh_size 0. */
  323. *num = info->sechdrs[sec].sh_size / object_size;
  324. return (void *)info->sechdrs[sec].sh_addr;
  325. }
  326. /* Provided by the linker */
  327. extern const struct kernel_symbol __start___ksymtab[];
  328. extern const struct kernel_symbol __stop___ksymtab[];
  329. extern const struct kernel_symbol __start___ksymtab_gpl[];
  330. extern const struct kernel_symbol __stop___ksymtab_gpl[];
  331. extern const struct kernel_symbol __start___ksymtab_gpl_future[];
  332. extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
  333. extern const s32 __start___kcrctab[];
  334. extern const s32 __start___kcrctab_gpl[];
  335. extern const s32 __start___kcrctab_gpl_future[];
  336. #ifdef CONFIG_UNUSED_SYMBOLS
  337. extern const struct kernel_symbol __start___ksymtab_unused[];
  338. extern const struct kernel_symbol __stop___ksymtab_unused[];
  339. extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
  340. extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
  341. extern const s32 __start___kcrctab_unused[];
  342. extern const s32 __start___kcrctab_unused_gpl[];
  343. #endif
  344. #ifndef CONFIG_MODVERSIONS
  345. #define symversion(base, idx) NULL
  346. #else
  347. #define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
  348. #endif
  349. static bool each_symbol_in_section(const struct symsearch *arr,
  350. unsigned int arrsize,
  351. struct module *owner,
  352. bool (*fn)(const struct symsearch *syms,
  353. struct module *owner,
  354. void *data),
  355. void *data)
  356. {
  357. unsigned int j;
  358. for (j = 0; j < arrsize; j++) {
  359. if (fn(&arr[j], owner, data))
  360. return true;
  361. }
  362. return false;
  363. }
  364. /* Returns true as soon as fn returns true, otherwise false. */
  365. static bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
  366. struct module *owner,
  367. void *data),
  368. void *data)
  369. {
  370. struct module *mod;
  371. static const struct symsearch arr[] = {
  372. { __start___ksymtab, __stop___ksymtab, __start___kcrctab,
  373. NOT_GPL_ONLY, false },
  374. { __start___ksymtab_gpl, __stop___ksymtab_gpl,
  375. __start___kcrctab_gpl,
  376. GPL_ONLY, false },
  377. { __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
  378. __start___kcrctab_gpl_future,
  379. WILL_BE_GPL_ONLY, false },
  380. #ifdef CONFIG_UNUSED_SYMBOLS
  381. { __start___ksymtab_unused, __stop___ksymtab_unused,
  382. __start___kcrctab_unused,
  383. NOT_GPL_ONLY, true },
  384. { __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
  385. __start___kcrctab_unused_gpl,
  386. GPL_ONLY, true },
  387. #endif
  388. };
  389. module_assert_mutex_or_preempt();
  390. if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
  391. return true;
  392. list_for_each_entry_rcu(mod, &modules, list,
  393. lockdep_is_held(&module_mutex)) {
  394. struct symsearch arr[] = {
  395. { mod->syms, mod->syms + mod->num_syms, mod->crcs,
  396. NOT_GPL_ONLY, false },
  397. { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
  398. mod->gpl_crcs,
  399. GPL_ONLY, false },
  400. { mod->gpl_future_syms,
  401. mod->gpl_future_syms + mod->num_gpl_future_syms,
  402. mod->gpl_future_crcs,
  403. WILL_BE_GPL_ONLY, false },
  404. #ifdef CONFIG_UNUSED_SYMBOLS
  405. { mod->unused_syms,
  406. mod->unused_syms + mod->num_unused_syms,
  407. mod->unused_crcs,
  408. NOT_GPL_ONLY, true },
  409. { mod->unused_gpl_syms,
  410. mod->unused_gpl_syms + mod->num_unused_gpl_syms,
  411. mod->unused_gpl_crcs,
  412. GPL_ONLY, true },
  413. #endif
  414. };
  415. if (mod->state == MODULE_STATE_UNFORMED)
  416. continue;
  417. if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
  418. return true;
  419. }
  420. return false;
  421. }
  422. struct find_symbol_arg {
  423. /* Input */
  424. const char *name;
  425. bool gplok;
  426. bool warn;
  427. /* Output */
  428. struct module *owner;
  429. const s32 *crc;
  430. const struct kernel_symbol *sym;
  431. enum mod_license license;
  432. };
  433. static bool check_exported_symbol(const struct symsearch *syms,
  434. struct module *owner,
  435. unsigned int symnum, void *data)
  436. {
  437. struct find_symbol_arg *fsa = data;
  438. if (!fsa->gplok) {
  439. if (syms->license == GPL_ONLY)
  440. return false;
  441. if (syms->license == WILL_BE_GPL_ONLY && fsa->warn) {
  442. pr_warn("Symbol %s is being used by a non-GPL module, "
  443. "which will not be allowed in the future\n",
  444. fsa->name);
  445. }
  446. }
  447. #ifdef CONFIG_UNUSED_SYMBOLS
  448. if (syms->unused && fsa->warn) {
  449. pr_warn("Symbol %s is marked as UNUSED, however this module is "
  450. "using it.\n", fsa->name);
  451. pr_warn("This symbol will go away in the future.\n");
  452. pr_warn("Please evaluate if this is the right api to use and "
  453. "if it really is, submit a report to the linux kernel "
  454. "mailing list together with submitting your code for "
  455. "inclusion.\n");
  456. }
  457. #endif
  458. fsa->owner = owner;
  459. fsa->crc = symversion(syms->crcs, symnum);
  460. fsa->sym = &syms->start[symnum];
  461. fsa->license = syms->license;
  462. return true;
  463. }
  464. static unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
  465. {
  466. #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
  467. return (unsigned long)offset_to_ptr(&sym->value_offset);
  468. #else
  469. return sym->value;
  470. #endif
  471. }
  472. static const char *kernel_symbol_name(const struct kernel_symbol *sym)
  473. {
  474. #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
  475. return offset_to_ptr(&sym->name_offset);
  476. #else
  477. return sym->name;
  478. #endif
  479. }
  480. static const char *kernel_symbol_namespace(const struct kernel_symbol *sym)
  481. {
  482. #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
  483. if (!sym->namespace_offset)
  484. return NULL;
  485. return offset_to_ptr(&sym->namespace_offset);
  486. #else
  487. return sym->namespace;
  488. #endif
  489. }
  490. static int cmp_name(const void *name, const void *sym)
  491. {
  492. return strcmp(name, kernel_symbol_name(sym));
  493. }
  494. static bool find_exported_symbol_in_section(const struct symsearch *syms,
  495. struct module *owner,
  496. void *data)
  497. {
  498. struct find_symbol_arg *fsa = data;
  499. struct kernel_symbol *sym;
  500. sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
  501. sizeof(struct kernel_symbol), cmp_name);
  502. if (sym != NULL && check_exported_symbol(syms, owner,
  503. sym - syms->start, data))
  504. return true;
  505. return false;
  506. }
  507. /* Find an exported symbol and return it, along with, (optional) crc and
  508. * (optional) module which owns it. Needs preempt disabled or module_mutex. */
  509. static const struct kernel_symbol *find_symbol(const char *name,
  510. struct module **owner,
  511. const s32 **crc,
  512. enum mod_license *license,
  513. bool gplok,
  514. bool warn)
  515. {
  516. struct find_symbol_arg fsa;
  517. fsa.name = name;
  518. fsa.gplok = gplok;
  519. fsa.warn = warn;
  520. if (each_symbol_section(find_exported_symbol_in_section, &fsa)) {
  521. if (owner)
  522. *owner = fsa.owner;
  523. if (crc)
  524. *crc = fsa.crc;
  525. if (license)
  526. *license = fsa.license;
  527. return fsa.sym;
  528. }
  529. pr_debug("Failed to find symbol %s\n", name);
  530. return NULL;
  531. }
  532. /*
  533. * Search for module by name: must hold module_mutex (or preempt disabled
  534. * for read-only access).
  535. */
  536. static struct module *find_module_all(const char *name, size_t len,
  537. bool even_unformed)
  538. {
  539. struct module *mod;
  540. module_assert_mutex_or_preempt();
  541. list_for_each_entry_rcu(mod, &modules, list,
  542. lockdep_is_held(&module_mutex)) {
  543. if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
  544. continue;
  545. if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
  546. return mod;
  547. }
  548. return NULL;
  549. }
  550. struct module *find_module(const char *name)
  551. {
  552. module_assert_mutex();
  553. return find_module_all(name, strlen(name), false);
  554. }
  555. #ifdef CONFIG_SMP
  556. static inline void __percpu *mod_percpu(struct module *mod)
  557. {
  558. return mod->percpu;
  559. }
  560. static int percpu_modalloc(struct module *mod, struct load_info *info)
  561. {
  562. Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
  563. unsigned long align = pcpusec->sh_addralign;
  564. if (!pcpusec->sh_size)
  565. return 0;
  566. if (align > PAGE_SIZE) {
  567. pr_warn("%s: per-cpu alignment %li > %li\n",
  568. mod->name, align, PAGE_SIZE);
  569. align = PAGE_SIZE;
  570. }
  571. mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
  572. if (!mod->percpu) {
  573. pr_warn("%s: Could not allocate %lu bytes percpu data\n",
  574. mod->name, (unsigned long)pcpusec->sh_size);
  575. return -ENOMEM;
  576. }
  577. mod->percpu_size = pcpusec->sh_size;
  578. return 0;
  579. }
  580. static void percpu_modfree(struct module *mod)
  581. {
  582. free_percpu(mod->percpu);
  583. }
  584. static unsigned int find_pcpusec(struct load_info *info)
  585. {
  586. return find_sec(info, ".data..percpu");
  587. }
  588. static void percpu_modcopy(struct module *mod,
  589. const void *from, unsigned long size)
  590. {
  591. int cpu;
  592. for_each_possible_cpu(cpu)
  593. memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
  594. }
  595. bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
  596. {
  597. struct module *mod;
  598. unsigned int cpu;
  599. preempt_disable();
  600. list_for_each_entry_rcu(mod, &modules, list) {
  601. if (mod->state == MODULE_STATE_UNFORMED)
  602. continue;
  603. if (!mod->percpu_size)
  604. continue;
  605. for_each_possible_cpu(cpu) {
  606. void *start = per_cpu_ptr(mod->percpu, cpu);
  607. void *va = (void *)addr;
  608. if (va >= start && va < start + mod->percpu_size) {
  609. if (can_addr) {
  610. *can_addr = (unsigned long) (va - start);
  611. *can_addr += (unsigned long)
  612. per_cpu_ptr(mod->percpu,
  613. get_boot_cpu_id());
  614. }
  615. preempt_enable();
  616. return true;
  617. }
  618. }
  619. }
  620. preempt_enable();
  621. return false;
  622. }
  623. /**
  624. * is_module_percpu_address - test whether address is from module static percpu
  625. * @addr: address to test
  626. *
  627. * Test whether @addr belongs to module static percpu area.
  628. *
  629. * RETURNS:
  630. * %true if @addr is from module static percpu area
  631. */
  632. bool is_module_percpu_address(unsigned long addr)
  633. {
  634. return __is_module_percpu_address(addr, NULL);
  635. }
  636. #else /* ... !CONFIG_SMP */
  637. static inline void __percpu *mod_percpu(struct module *mod)
  638. {
  639. return NULL;
  640. }
  641. static int percpu_modalloc(struct module *mod, struct load_info *info)
  642. {
  643. /* UP modules shouldn't have this section: ENOMEM isn't quite right */
  644. if (info->sechdrs[info->index.pcpu].sh_size != 0)
  645. return -ENOMEM;
  646. return 0;
  647. }
  648. static inline void percpu_modfree(struct module *mod)
  649. {
  650. }
  651. static unsigned int find_pcpusec(struct load_info *info)
  652. {
  653. return 0;
  654. }
  655. static inline void percpu_modcopy(struct module *mod,
  656. const void *from, unsigned long size)
  657. {
  658. /* pcpusec should be 0, and size of that section should be 0. */
  659. BUG_ON(size != 0);
  660. }
  661. bool is_module_percpu_address(unsigned long addr)
  662. {
  663. return false;
  664. }
  665. bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
  666. {
  667. return false;
  668. }
  669. #endif /* CONFIG_SMP */
  670. #define MODINFO_ATTR(field) \
  671. static void setup_modinfo_##field(struct module *mod, const char *s) \
  672. { \
  673. mod->field = kstrdup(s, GFP_KERNEL); \
  674. } \
  675. static ssize_t show_modinfo_##field(struct module_attribute *mattr, \
  676. struct module_kobject *mk, char *buffer) \
  677. { \
  678. return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
  679. } \
  680. static int modinfo_##field##_exists(struct module *mod) \
  681. { \
  682. return mod->field != NULL; \
  683. } \
  684. static void free_modinfo_##field(struct module *mod) \
  685. { \
  686. kfree(mod->field); \
  687. mod->field = NULL; \
  688. } \
  689. static struct module_attribute modinfo_##field = { \
  690. .attr = { .name = __stringify(field), .mode = 0444 }, \
  691. .show = show_modinfo_##field, \
  692. .setup = setup_modinfo_##field, \
  693. .test = modinfo_##field##_exists, \
  694. .free = free_modinfo_##field, \
  695. };
  696. MODINFO_ATTR(version);
  697. MODINFO_ATTR(srcversion);
  698. MODINFO_ATTR(scmversion);
  699. static char last_unloaded_module[MODULE_NAME_LEN+1];
  700. #ifdef CONFIG_MODULE_UNLOAD
  701. EXPORT_TRACEPOINT_SYMBOL(module_get);
  702. /* MODULE_REF_BASE is the base reference count by kmodule loader. */
  703. #define MODULE_REF_BASE 1
  704. /* Init the unload section of the module. */
  705. static int module_unload_init(struct module *mod)
  706. {
  707. /*
  708. * Initialize reference counter to MODULE_REF_BASE.
  709. * refcnt == 0 means module is going.
  710. */
  711. atomic_set(&mod->refcnt, MODULE_REF_BASE);
  712. INIT_LIST_HEAD(&mod->source_list);
  713. INIT_LIST_HEAD(&mod->target_list);
  714. /* Hold reference count during initialization. */
  715. atomic_inc(&mod->refcnt);
  716. return 0;
  717. }
  718. /* Does a already use b? */
  719. static int already_uses(struct module *a, struct module *b)
  720. {
  721. struct module_use *use;
  722. list_for_each_entry(use, &b->source_list, source_list) {
  723. if (use->source == a) {
  724. pr_debug("%s uses %s!\n", a->name, b->name);
  725. return 1;
  726. }
  727. }
  728. pr_debug("%s does not use %s!\n", a->name, b->name);
  729. return 0;
  730. }
  731. /*
  732. * Module a uses b
  733. * - we add 'a' as a "source", 'b' as a "target" of module use
  734. * - the module_use is added to the list of 'b' sources (so
  735. * 'b' can walk the list to see who sourced them), and of 'a'
  736. * targets (so 'a' can see what modules it targets).
  737. */
  738. static int add_module_usage(struct module *a, struct module *b)
  739. {
  740. struct module_use *use;
  741. pr_debug("Allocating new usage for %s.\n", a->name);
  742. use = kmalloc(sizeof(*use), GFP_ATOMIC);
  743. if (!use)
  744. return -ENOMEM;
  745. use->source = a;
  746. use->target = b;
  747. list_add(&use->source_list, &b->source_list);
  748. list_add(&use->target_list, &a->target_list);
  749. return 0;
  750. }
  751. /* Module a uses b: caller needs module_mutex() */
  752. static int ref_module(struct module *a, struct module *b)
  753. {
  754. int err;
  755. if (b == NULL || already_uses(a, b))
  756. return 0;
  757. /* If module isn't available, we fail. */
  758. err = strong_try_module_get(b);
  759. if (err)
  760. return err;
  761. err = add_module_usage(a, b);
  762. if (err) {
  763. module_put(b);
  764. return err;
  765. }
  766. return 0;
  767. }
  768. /* Clear the unload stuff of the module. */
  769. static void module_unload_free(struct module *mod)
  770. {
  771. struct module_use *use, *tmp;
  772. mutex_lock(&module_mutex);
  773. list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
  774. struct module *i = use->target;
  775. pr_debug("%s unusing %s\n", mod->name, i->name);
  776. module_put(i);
  777. list_del(&use->source_list);
  778. list_del(&use->target_list);
  779. kfree(use);
  780. }
  781. mutex_unlock(&module_mutex);
  782. }
  783. #ifdef CONFIG_MODULE_FORCE_UNLOAD
  784. static inline int try_force_unload(unsigned int flags)
  785. {
  786. int ret = (flags & O_TRUNC);
  787. if (ret)
  788. add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
  789. return ret;
  790. }
  791. #else
  792. static inline int try_force_unload(unsigned int flags)
  793. {
  794. return 0;
  795. }
  796. #endif /* CONFIG_MODULE_FORCE_UNLOAD */
  797. /* Try to release refcount of module, 0 means success. */
  798. static int try_release_module_ref(struct module *mod)
  799. {
  800. int ret;
  801. /* Try to decrement refcnt which we set at loading */
  802. ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
  803. BUG_ON(ret < 0);
  804. if (ret)
  805. /* Someone can put this right now, recover with checking */
  806. ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
  807. return ret;
  808. }
  809. static int try_stop_module(struct module *mod, int flags, int *forced)
  810. {
  811. /* If it's not unused, quit unless we're forcing. */
  812. if (try_release_module_ref(mod) != 0) {
  813. *forced = try_force_unload(flags);
  814. if (!(*forced))
  815. return -EWOULDBLOCK;
  816. }
  817. /* Mark it as dying. */
  818. mod->state = MODULE_STATE_GOING;
  819. return 0;
  820. }
  821. /**
  822. * module_refcount - return the refcount or -1 if unloading
  823. *
  824. * @mod: the module we're checking
  825. *
  826. * Returns:
  827. * -1 if the module is in the process of unloading
  828. * otherwise the number of references in the kernel to the module
  829. */
  830. int module_refcount(struct module *mod)
  831. {
  832. return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
  833. }
  834. EXPORT_SYMBOL(module_refcount);
  835. /* This exists whether we can unload or not */
  836. static void free_module(struct module *mod);
  837. SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
  838. unsigned int, flags)
  839. {
  840. struct module *mod;
  841. char name[MODULE_NAME_LEN];
  842. int ret, forced = 0;
  843. if (!capable(CAP_SYS_MODULE) || modules_disabled)
  844. return -EPERM;
  845. if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
  846. return -EFAULT;
  847. name[MODULE_NAME_LEN-1] = '\0';
  848. audit_log_kern_module(name);
  849. if (mutex_lock_interruptible(&module_mutex) != 0)
  850. return -EINTR;
  851. mod = find_module(name);
  852. if (!mod) {
  853. ret = -ENOENT;
  854. goto out;
  855. }
  856. if (!list_empty(&mod->source_list)) {
  857. /* Other modules depend on us: get rid of them first. */
  858. ret = -EWOULDBLOCK;
  859. goto out;
  860. }
  861. /* Doing init or already dying? */
  862. if (mod->state != MODULE_STATE_LIVE) {
  863. /* FIXME: if (force), slam module count damn the torpedoes */
  864. pr_debug("%s already dying\n", mod->name);
  865. ret = -EBUSY;
  866. goto out;
  867. }
  868. /* If it has an init func, it must have an exit func to unload */
  869. if (mod->init && !mod->exit) {
  870. forced = try_force_unload(flags);
  871. if (!forced) {
  872. /* This module can't be removed */
  873. ret = -EBUSY;
  874. goto out;
  875. }
  876. }
  877. /* Stop the machine so refcounts can't move and disable module. */
  878. ret = try_stop_module(mod, flags, &forced);
  879. if (ret != 0)
  880. goto out;
  881. mutex_unlock(&module_mutex);
  882. /* Final destruction now no one is using it. */
  883. if (mod->exit != NULL)
  884. mod->exit();
  885. blocking_notifier_call_chain(&module_notify_list,
  886. MODULE_STATE_GOING, mod);
  887. klp_module_going(mod);
  888. ftrace_release_mod(mod);
  889. async_synchronize_full();
  890. /* Store the name of the last unloaded module for diagnostic purposes */
  891. strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
  892. free_module(mod);
  893. /* someone could wait for the module in add_unformed_module() */
  894. wake_up_all(&module_wq);
  895. return 0;
  896. out:
  897. mutex_unlock(&module_mutex);
  898. return ret;
  899. }
  900. static inline void print_unload_info(struct seq_file *m, struct module *mod)
  901. {
  902. struct module_use *use;
  903. int printed_something = 0;
  904. seq_printf(m, " %i ", module_refcount(mod));
  905. /*
  906. * Always include a trailing , so userspace can differentiate
  907. * between this and the old multi-field proc format.
  908. */
  909. list_for_each_entry(use, &mod->source_list, source_list) {
  910. printed_something = 1;
  911. seq_printf(m, "%s,", use->source->name);
  912. }
  913. if (mod->init != NULL && mod->exit == NULL) {
  914. printed_something = 1;
  915. seq_puts(m, "[permanent],");
  916. }
  917. if (!printed_something)
  918. seq_puts(m, "-");
  919. }
  920. void __symbol_put(const char *symbol)
  921. {
  922. struct module *owner;
  923. preempt_disable();
  924. if (!find_symbol(symbol, &owner, NULL, NULL, true, false))
  925. BUG();
  926. module_put(owner);
  927. preempt_enable();
  928. }
  929. EXPORT_SYMBOL(__symbol_put);
  930. /* Note this assumes addr is a function, which it currently always is. */
  931. void symbol_put_addr(void *addr)
  932. {
  933. struct module *modaddr;
  934. unsigned long a = (unsigned long)dereference_function_descriptor(addr);
  935. if (core_kernel_text(a))
  936. return;
  937. /*
  938. * Even though we hold a reference on the module; we still need to
  939. * disable preemption in order to safely traverse the data structure.
  940. */
  941. preempt_disable();
  942. modaddr = __module_text_address(a);
  943. BUG_ON(!modaddr);
  944. module_put(modaddr);
  945. preempt_enable();
  946. }
  947. EXPORT_SYMBOL_GPL(symbol_put_addr);
  948. static ssize_t show_refcnt(struct module_attribute *mattr,
  949. struct module_kobject *mk, char *buffer)
  950. {
  951. return sprintf(buffer, "%i\n", module_refcount(mk->mod));
  952. }
  953. static struct module_attribute modinfo_refcnt =
  954. __ATTR(refcnt, 0444, show_refcnt, NULL);
  955. void __module_get(struct module *module)
  956. {
  957. if (module) {
  958. preempt_disable();
  959. atomic_inc(&module->refcnt);
  960. trace_module_get(module, _RET_IP_);
  961. preempt_enable();
  962. }
  963. }
  964. EXPORT_SYMBOL(__module_get);
  965. bool try_module_get(struct module *module)
  966. {
  967. bool ret = true;
  968. if (module) {
  969. preempt_disable();
  970. /* Note: here, we can fail to get a reference */
  971. if (likely(module_is_live(module) &&
  972. atomic_inc_not_zero(&module->refcnt) != 0))
  973. trace_module_get(module, _RET_IP_);
  974. else
  975. ret = false;
  976. preempt_enable();
  977. }
  978. return ret;
  979. }
  980. EXPORT_SYMBOL(try_module_get);
  981. void module_put(struct module *module)
  982. {
  983. int ret;
  984. if (module) {
  985. preempt_disable();
  986. ret = atomic_dec_if_positive(&module->refcnt);
  987. WARN_ON(ret < 0); /* Failed to put refcount */
  988. trace_module_put(module, _RET_IP_);
  989. preempt_enable();
  990. }
  991. }
  992. EXPORT_SYMBOL(module_put);
  993. #else /* !CONFIG_MODULE_UNLOAD */
  994. static inline void print_unload_info(struct seq_file *m, struct module *mod)
  995. {
  996. /* We don't know the usage count, or what modules are using. */
  997. seq_puts(m, " - -");
  998. }
  999. static inline void module_unload_free(struct module *mod)
  1000. {
  1001. }
  1002. static int ref_module(struct module *a, struct module *b)
  1003. {
  1004. return strong_try_module_get(b);
  1005. }
  1006. static inline int module_unload_init(struct module *mod)
  1007. {
  1008. return 0;
  1009. }
  1010. #endif /* CONFIG_MODULE_UNLOAD */
  1011. static size_t module_flags_taint(struct module *mod, char *buf)
  1012. {
  1013. size_t l = 0;
  1014. int i;
  1015. for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
  1016. if (taint_flags[i].module && test_bit(i, &mod->taints))
  1017. buf[l++] = taint_flags[i].c_true;
  1018. }
  1019. return l;
  1020. }
  1021. static ssize_t show_initstate(struct module_attribute *mattr,
  1022. struct module_kobject *mk, char *buffer)
  1023. {
  1024. const char *state = "unknown";
  1025. switch (mk->mod->state) {
  1026. case MODULE_STATE_LIVE:
  1027. state = "live";
  1028. break;
  1029. case MODULE_STATE_COMING:
  1030. state = "coming";
  1031. break;
  1032. case MODULE_STATE_GOING:
  1033. state = "going";
  1034. break;
  1035. default:
  1036. BUG();
  1037. }
  1038. return sprintf(buffer, "%s\n", state);
  1039. }
  1040. static struct module_attribute modinfo_initstate =
  1041. __ATTR(initstate, 0444, show_initstate, NULL);
  1042. static ssize_t store_uevent(struct module_attribute *mattr,
  1043. struct module_kobject *mk,
  1044. const char *buffer, size_t count)
  1045. {
  1046. int rc;
  1047. rc = kobject_synth_uevent(&mk->kobj, buffer, count);
  1048. return rc ? rc : count;
  1049. }
  1050. struct module_attribute module_uevent =
  1051. __ATTR(uevent, 0200, NULL, store_uevent);
  1052. static ssize_t show_coresize(struct module_attribute *mattr,
  1053. struct module_kobject *mk, char *buffer)
  1054. {
  1055. return sprintf(buffer, "%u\n", mk->mod->core_layout.size);
  1056. }
  1057. static struct module_attribute modinfo_coresize =
  1058. __ATTR(coresize, 0444, show_coresize, NULL);
  1059. static ssize_t show_initsize(struct module_attribute *mattr,
  1060. struct module_kobject *mk, char *buffer)
  1061. {
  1062. return sprintf(buffer, "%u\n", mk->mod->init_layout.size);
  1063. }
  1064. static struct module_attribute modinfo_initsize =
  1065. __ATTR(initsize, 0444, show_initsize, NULL);
  1066. static ssize_t show_taint(struct module_attribute *mattr,
  1067. struct module_kobject *mk, char *buffer)
  1068. {
  1069. size_t l;
  1070. l = module_flags_taint(mk->mod, buffer);
  1071. buffer[l++] = '\n';
  1072. return l;
  1073. }
  1074. static struct module_attribute modinfo_taint =
  1075. __ATTR(taint, 0444, show_taint, NULL);
  1076. static struct module_attribute *modinfo_attrs[] = {
  1077. &module_uevent,
  1078. &modinfo_version,
  1079. &modinfo_srcversion,
  1080. &modinfo_scmversion,
  1081. &modinfo_initstate,
  1082. &modinfo_coresize,
  1083. &modinfo_initsize,
  1084. &modinfo_taint,
  1085. #ifdef CONFIG_MODULE_UNLOAD
  1086. &modinfo_refcnt,
  1087. #endif
  1088. NULL,
  1089. };
  1090. static const char vermagic[] = VERMAGIC_STRING;
  1091. static int try_to_force_load(struct module *mod, const char *reason)
  1092. {
  1093. #ifdef CONFIG_MODULE_FORCE_LOAD
  1094. if (!test_taint(TAINT_FORCED_MODULE))
  1095. pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
  1096. add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
  1097. return 0;
  1098. #else
  1099. return -ENOEXEC;
  1100. #endif
  1101. }
  1102. #ifdef CONFIG_MODVERSIONS
  1103. static u32 resolve_rel_crc(const s32 *crc)
  1104. {
  1105. return *(u32 *)((void *)crc + *crc);
  1106. }
  1107. static int check_version(const struct load_info *info,
  1108. const char *symname,
  1109. struct module *mod,
  1110. const s32 *crc)
  1111. {
  1112. Elf_Shdr *sechdrs = info->sechdrs;
  1113. unsigned int versindex = info->index.vers;
  1114. unsigned int i, num_versions;
  1115. struct modversion_info *versions;
  1116. /* Exporting module didn't supply crcs? OK, we're already tainted. */
  1117. if (!crc)
  1118. return 1;
  1119. /* No versions at all? modprobe --force does this. */
  1120. if (versindex == 0)
  1121. return try_to_force_load(mod, symname) == 0;
  1122. versions = (void *) sechdrs[versindex].sh_addr;
  1123. num_versions = sechdrs[versindex].sh_size
  1124. / sizeof(struct modversion_info);
  1125. for (i = 0; i < num_versions; i++) {
  1126. u32 crcval;
  1127. if (strcmp(versions[i].name, symname) != 0)
  1128. continue;
  1129. if (IS_ENABLED(CONFIG_MODULE_REL_CRCS))
  1130. crcval = resolve_rel_crc(crc);
  1131. else
  1132. crcval = *crc;
  1133. if (versions[i].crc == crcval)
  1134. return 1;
  1135. pr_debug("Found checksum %X vs module %lX\n",
  1136. crcval, versions[i].crc);
  1137. goto bad_version;
  1138. }
  1139. /* Broken toolchain. Warn once, then let it go.. */
  1140. pr_warn_once("%s: no symbol version for %s\n", info->name, symname);
  1141. return 1;
  1142. bad_version:
  1143. pr_warn("%s: disagrees about version of symbol %s\n",
  1144. info->name, symname);
  1145. return 0;
  1146. }
  1147. static inline int check_modstruct_version(const struct load_info *info,
  1148. struct module *mod)
  1149. {
  1150. const s32 *crc;
  1151. /*
  1152. * Since this should be found in kernel (which can't be removed), no
  1153. * locking is necessary -- use preempt_disable() to placate lockdep.
  1154. */
  1155. preempt_disable();
  1156. if (!find_symbol("module_layout", NULL, &crc, NULL, true, false)) {
  1157. preempt_enable();
  1158. BUG();
  1159. }
  1160. preempt_enable();
  1161. return check_version(info, "module_layout", mod, crc);
  1162. }
  1163. /* First part is kernel version, which we ignore if module has crcs. */
  1164. static inline int same_magic(const char *amagic, const char *bmagic,
  1165. bool has_crcs)
  1166. {
  1167. if (has_crcs) {
  1168. amagic += strcspn(amagic, " ");
  1169. bmagic += strcspn(bmagic, " ");
  1170. }
  1171. return strcmp(amagic, bmagic) == 0;
  1172. }
  1173. #else
  1174. static inline int check_version(const struct load_info *info,
  1175. const char *symname,
  1176. struct module *mod,
  1177. const s32 *crc)
  1178. {
  1179. return 1;
  1180. }
  1181. static inline int check_modstruct_version(const struct load_info *info,
  1182. struct module *mod)
  1183. {
  1184. return 1;
  1185. }
  1186. static inline int same_magic(const char *amagic, const char *bmagic,
  1187. bool has_crcs)
  1188. {
  1189. return strcmp(amagic, bmagic) == 0;
  1190. }
  1191. #endif /* CONFIG_MODVERSIONS */
  1192. static char *get_modinfo(const struct load_info *info, const char *tag);
  1193. static char *get_next_modinfo(const struct load_info *info, const char *tag,
  1194. char *prev);
  1195. static int verify_namespace_is_imported(const struct load_info *info,
  1196. const struct kernel_symbol *sym,
  1197. struct module *mod)
  1198. {
  1199. const char *namespace;
  1200. char *imported_namespace;
  1201. namespace = kernel_symbol_namespace(sym);
  1202. if (namespace && namespace[0]) {
  1203. imported_namespace = get_modinfo(info, "import_ns");
  1204. while (imported_namespace) {
  1205. if (strcmp(namespace, imported_namespace) == 0)
  1206. return 0;
  1207. imported_namespace = get_next_modinfo(
  1208. info, "import_ns", imported_namespace);
  1209. }
  1210. #ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
  1211. pr_warn(
  1212. #else
  1213. pr_err(
  1214. #endif
  1215. "%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
  1216. mod->name, kernel_symbol_name(sym), namespace);
  1217. #ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
  1218. return -EINVAL;
  1219. #endif
  1220. }
  1221. return 0;
  1222. }
  1223. static bool inherit_taint(struct module *mod, struct module *owner)
  1224. {
  1225. if (!owner || !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
  1226. return true;
  1227. if (mod->using_gplonly_symbols) {
  1228. pr_err("%s: module using GPL-only symbols uses symbols from proprietary module %s.\n",
  1229. mod->name, owner->name);
  1230. return false;
  1231. }
  1232. if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
  1233. pr_warn("%s: module uses symbols from proprietary module %s, inheriting taint.\n",
  1234. mod->name, owner->name);
  1235. set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
  1236. }
  1237. return true;
  1238. }
  1239. /* Resolve a symbol for this module. I.e. if we find one, record usage. */
  1240. static const struct kernel_symbol *resolve_symbol(struct module *mod,
  1241. const struct load_info *info,
  1242. const char *name,
  1243. char ownername[])
  1244. {
  1245. struct module *owner;
  1246. const struct kernel_symbol *sym;
  1247. const s32 *crc;
  1248. enum mod_license license;
  1249. int err;
  1250. /*
  1251. * The module_mutex should not be a heavily contended lock;
  1252. * if we get the occasional sleep here, we'll go an extra iteration
  1253. * in the wait_event_interruptible(), which is harmless.
  1254. */
  1255. sched_annotate_sleep();
  1256. mutex_lock(&module_mutex);
  1257. sym = find_symbol(name, &owner, &crc, &license,
  1258. !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
  1259. if (!sym)
  1260. goto unlock;
  1261. if (license == GPL_ONLY)
  1262. mod->using_gplonly_symbols = true;
  1263. if (!inherit_taint(mod, owner)) {
  1264. sym = NULL;
  1265. goto getname;
  1266. }
  1267. if (!check_version(info, name, mod, crc)) {
  1268. sym = ERR_PTR(-EINVAL);
  1269. goto getname;
  1270. }
  1271. err = verify_namespace_is_imported(info, sym, mod);
  1272. if (err) {
  1273. sym = ERR_PTR(err);
  1274. goto getname;
  1275. }
  1276. err = ref_module(mod, owner);
  1277. if (err) {
  1278. sym = ERR_PTR(err);
  1279. goto getname;
  1280. }
  1281. getname:
  1282. /* We must make copy under the lock if we failed to get ref. */
  1283. strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
  1284. unlock:
  1285. mutex_unlock(&module_mutex);
  1286. return sym;
  1287. }
  1288. static const struct kernel_symbol *
  1289. resolve_symbol_wait(struct module *mod,
  1290. const struct load_info *info,
  1291. const char *name)
  1292. {
  1293. const struct kernel_symbol *ksym;
  1294. char owner[MODULE_NAME_LEN];
  1295. if (wait_event_interruptible_timeout(module_wq,
  1296. !IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
  1297. || PTR_ERR(ksym) != -EBUSY,
  1298. 30 * HZ) <= 0) {
  1299. pr_warn("%s: gave up waiting for init of module %s.\n",
  1300. mod->name, owner);
  1301. }
  1302. return ksym;
  1303. }
  1304. /*
  1305. * /sys/module/foo/sections stuff
  1306. * J. Corbet <corbet@lwn.net>
  1307. */
  1308. #ifdef CONFIG_SYSFS
  1309. #ifdef CONFIG_KALLSYMS
  1310. static inline bool sect_empty(const Elf_Shdr *sect)
  1311. {
  1312. return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
  1313. }
  1314. struct module_sect_attr {
  1315. struct bin_attribute battr;
  1316. unsigned long address;
  1317. };
  1318. struct module_sect_attrs {
  1319. struct attribute_group grp;
  1320. unsigned int nsections;
  1321. struct module_sect_attr attrs[];
  1322. };
  1323. #define MODULE_SECT_READ_SIZE (3 /* "0x", "\n" */ + (BITS_PER_LONG / 4))
  1324. static ssize_t module_sect_read(struct file *file, struct kobject *kobj,
  1325. struct bin_attribute *battr,
  1326. char *buf, loff_t pos, size_t count)
  1327. {
  1328. struct module_sect_attr *sattr =
  1329. container_of(battr, struct module_sect_attr, battr);
  1330. char bounce[MODULE_SECT_READ_SIZE + 1];
  1331. size_t wrote;
  1332. if (pos != 0)
  1333. return -EINVAL;
  1334. /*
  1335. * Since we're a binary read handler, we must account for the
  1336. * trailing NUL byte that sprintf will write: if "buf" is
  1337. * too small to hold the NUL, or the NUL is exactly the last
  1338. * byte, the read will look like it got truncated by one byte.
  1339. * Since there is no way to ask sprintf nicely to not write
  1340. * the NUL, we have to use a bounce buffer.
  1341. */
  1342. wrote = scnprintf(bounce, sizeof(bounce), "0x%px\n",
  1343. kallsyms_show_value(file->f_cred)
  1344. ? (void *)sattr->address : NULL);
  1345. count = min(count, wrote);
  1346. memcpy(buf, bounce, count);
  1347. return count;
  1348. }
  1349. static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
  1350. {
  1351. unsigned int section;
  1352. for (section = 0; section < sect_attrs->nsections; section++)
  1353. kfree(sect_attrs->attrs[section].battr.attr.name);
  1354. kfree(sect_attrs);
  1355. }
  1356. static void add_sect_attrs(struct module *mod, const struct load_info *info)
  1357. {
  1358. unsigned int nloaded = 0, i, size[2];
  1359. struct module_sect_attrs *sect_attrs;
  1360. struct module_sect_attr *sattr;
  1361. struct bin_attribute **gattr;
  1362. /* Count loaded sections and allocate structures */
  1363. for (i = 0; i < info->hdr->e_shnum; i++)
  1364. if (!sect_empty(&info->sechdrs[i]))
  1365. nloaded++;
  1366. size[0] = ALIGN(struct_size(sect_attrs, attrs, nloaded),
  1367. sizeof(sect_attrs->grp.bin_attrs[0]));
  1368. size[1] = (nloaded + 1) * sizeof(sect_attrs->grp.bin_attrs[0]);
  1369. sect_attrs = kzalloc(size[0] + size[1], GFP_KERNEL);
  1370. if (sect_attrs == NULL)
  1371. return;
  1372. /* Setup section attributes. */
  1373. sect_attrs->grp.name = "sections";
  1374. sect_attrs->grp.bin_attrs = (void *)sect_attrs + size[0];
  1375. sect_attrs->nsections = 0;
  1376. sattr = &sect_attrs->attrs[0];
  1377. gattr = &sect_attrs->grp.bin_attrs[0];
  1378. for (i = 0; i < info->hdr->e_shnum; i++) {
  1379. Elf_Shdr *sec = &info->sechdrs[i];
  1380. if (sect_empty(sec))
  1381. continue;
  1382. sysfs_bin_attr_init(&sattr->battr);
  1383. sattr->address = sec->sh_addr;
  1384. sattr->battr.attr.name =
  1385. kstrdup(info->secstrings + sec->sh_name, GFP_KERNEL);
  1386. if (sattr->battr.attr.name == NULL)
  1387. goto out;
  1388. sect_attrs->nsections++;
  1389. sattr->battr.read = module_sect_read;
  1390. sattr->battr.size = MODULE_SECT_READ_SIZE;
  1391. sattr->battr.attr.mode = 0400;
  1392. *(gattr++) = &(sattr++)->battr;
  1393. }
  1394. *gattr = NULL;
  1395. if (sysfs_create_group(&mod->mkobj.kobj, &sect_attrs->grp))
  1396. goto out;
  1397. mod->sect_attrs = sect_attrs;
  1398. return;
  1399. out:
  1400. free_sect_attrs(sect_attrs);
  1401. }
  1402. static void remove_sect_attrs(struct module *mod)
  1403. {
  1404. if (mod->sect_attrs) {
  1405. sysfs_remove_group(&mod->mkobj.kobj,
  1406. &mod->sect_attrs->grp);
  1407. /* We are positive that no one is using any sect attrs
  1408. * at this point. Deallocate immediately. */
  1409. free_sect_attrs(mod->sect_attrs);
  1410. mod->sect_attrs = NULL;
  1411. }
  1412. }
  1413. /*
  1414. * /sys/module/foo/notes/.section.name gives contents of SHT_NOTE sections.
  1415. */
  1416. struct module_notes_attrs {
  1417. struct kobject *dir;
  1418. unsigned int notes;
  1419. struct bin_attribute attrs[];
  1420. };
  1421. static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
  1422. struct bin_attribute *bin_attr,
  1423. char *buf, loff_t pos, size_t count)
  1424. {
  1425. /*
  1426. * The caller checked the pos and count against our size.
  1427. */
  1428. memcpy(buf, bin_attr->private + pos, count);
  1429. return count;
  1430. }
  1431. static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
  1432. unsigned int i)
  1433. {
  1434. if (notes_attrs->dir) {
  1435. while (i-- > 0)
  1436. sysfs_remove_bin_file(notes_attrs->dir,
  1437. &notes_attrs->attrs[i]);
  1438. kobject_put(notes_attrs->dir);
  1439. }
  1440. kfree(notes_attrs);
  1441. }
  1442. static void add_notes_attrs(struct module *mod, const struct load_info *info)
  1443. {
  1444. unsigned int notes, loaded, i;
  1445. struct module_notes_attrs *notes_attrs;
  1446. struct bin_attribute *nattr;
  1447. /* failed to create section attributes, so can't create notes */
  1448. if (!mod->sect_attrs)
  1449. return;
  1450. /* Count notes sections and allocate structures. */
  1451. notes = 0;
  1452. for (i = 0; i < info->hdr->e_shnum; i++)
  1453. if (!sect_empty(&info->sechdrs[i]) &&
  1454. (info->sechdrs[i].sh_type == SHT_NOTE))
  1455. ++notes;
  1456. if (notes == 0)
  1457. return;
  1458. notes_attrs = kzalloc(struct_size(notes_attrs, attrs, notes),
  1459. GFP_KERNEL);
  1460. if (notes_attrs == NULL)
  1461. return;
  1462. notes_attrs->notes = notes;
  1463. nattr = &notes_attrs->attrs[0];
  1464. for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
  1465. if (sect_empty(&info->sechdrs[i]))
  1466. continue;
  1467. if (info->sechdrs[i].sh_type == SHT_NOTE) {
  1468. sysfs_bin_attr_init(nattr);
  1469. nattr->attr.name = mod->sect_attrs->attrs[loaded].battr.attr.name;
  1470. nattr->attr.mode = S_IRUGO;
  1471. nattr->size = info->sechdrs[i].sh_size;
  1472. nattr->private = (void *) info->sechdrs[i].sh_addr;
  1473. nattr->read = module_notes_read;
  1474. ++nattr;
  1475. }
  1476. ++loaded;
  1477. }
  1478. notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj);
  1479. if (!notes_attrs->dir)
  1480. goto out;
  1481. for (i = 0; i < notes; ++i)
  1482. if (sysfs_create_bin_file(notes_attrs->dir,
  1483. &notes_attrs->attrs[i]))
  1484. goto out;
  1485. mod->notes_attrs = notes_attrs;
  1486. return;
  1487. out:
  1488. free_notes_attrs(notes_attrs, i);
  1489. }
  1490. static void remove_notes_attrs(struct module *mod)
  1491. {
  1492. if (mod->notes_attrs)
  1493. free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes);
  1494. }
  1495. #else
  1496. static inline void add_sect_attrs(struct module *mod,
  1497. const struct load_info *info)
  1498. {
  1499. }
  1500. static inline void remove_sect_attrs(struct module *mod)
  1501. {
  1502. }
  1503. static inline void add_notes_attrs(struct module *mod,
  1504. const struct load_info *info)
  1505. {
  1506. }
  1507. static inline void remove_notes_attrs(struct module *mod)
  1508. {
  1509. }
  1510. #endif /* CONFIG_KALLSYMS */
  1511. static void del_usage_links(struct module *mod)
  1512. {
  1513. #ifdef CONFIG_MODULE_UNLOAD
  1514. struct module_use *use;
  1515. mutex_lock(&module_mutex);
  1516. list_for_each_entry(use, &mod->target_list, target_list)
  1517. sysfs_remove_link(use->target->holders_dir, mod->name);
  1518. mutex_unlock(&module_mutex);
  1519. #endif
  1520. }
  1521. static int add_usage_links(struct module *mod)
  1522. {
  1523. int ret = 0;
  1524. #ifdef CONFIG_MODULE_UNLOAD
  1525. struct module_use *use;
  1526. mutex_lock(&module_mutex);
  1527. list_for_each_entry(use, &mod->target_list, target_list) {
  1528. ret = sysfs_create_link(use->target->holders_dir,
  1529. &mod->mkobj.kobj, mod->name);
  1530. if (ret)
  1531. break;
  1532. }
  1533. mutex_unlock(&module_mutex);
  1534. if (ret)
  1535. del_usage_links(mod);
  1536. #endif
  1537. return ret;
  1538. }
  1539. static void module_remove_modinfo_attrs(struct module *mod, int end);
  1540. static int module_add_modinfo_attrs(struct module *mod)
  1541. {
  1542. struct module_attribute *attr;
  1543. struct module_attribute *temp_attr;
  1544. int error = 0;
  1545. int i;
  1546. mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) *
  1547. (ARRAY_SIZE(modinfo_attrs) + 1)),
  1548. GFP_KERNEL);
  1549. if (!mod->modinfo_attrs)
  1550. return -ENOMEM;
  1551. temp_attr = mod->modinfo_attrs;
  1552. for (i = 0; (attr = modinfo_attrs[i]); i++) {
  1553. if (!attr->test || attr->test(mod)) {
  1554. memcpy(temp_attr, attr, sizeof(*temp_attr));
  1555. sysfs_attr_init(&temp_attr->attr);
  1556. error = sysfs_create_file(&mod->mkobj.kobj,
  1557. &temp_attr->attr);
  1558. if (error)
  1559. goto error_out;
  1560. ++temp_attr;
  1561. }
  1562. }
  1563. return 0;
  1564. error_out:
  1565. if (i > 0)
  1566. module_remove_modinfo_attrs(mod, --i);
  1567. else
  1568. kfree(mod->modinfo_attrs);
  1569. return error;
  1570. }
  1571. static void module_remove_modinfo_attrs(struct module *mod, int end)
  1572. {
  1573. struct module_attribute *attr;
  1574. int i;
  1575. for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) {
  1576. if (end >= 0 && i > end)
  1577. break;
  1578. /* pick a field to test for end of list */
  1579. if (!attr->attr.name)
  1580. break;
  1581. sysfs_remove_file(&mod->mkobj.kobj, &attr->attr);
  1582. if (attr->free)
  1583. attr->free(mod);
  1584. }
  1585. kfree(mod->modinfo_attrs);
  1586. }
  1587. static void mod_kobject_put(struct module *mod)
  1588. {
  1589. DECLARE_COMPLETION_ONSTACK(c);
  1590. mod->mkobj.kobj_completion = &c;
  1591. kobject_put(&mod->mkobj.kobj);
  1592. wait_for_completion(&c);
  1593. }
  1594. static int mod_sysfs_init(struct module *mod)
  1595. {
  1596. int err;
  1597. struct kobject *kobj;
  1598. if (!module_sysfs_initialized) {
  1599. pr_err("%s: module sysfs not initialized\n", mod->name);
  1600. err = -EINVAL;
  1601. goto out;
  1602. }
  1603. kobj = kset_find_obj(module_kset, mod->name);
  1604. if (kobj) {
  1605. pr_err("%s: module is already loaded\n", mod->name);
  1606. kobject_put(kobj);
  1607. err = -EINVAL;
  1608. goto out;
  1609. }
  1610. mod->mkobj.mod = mod;
  1611. memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj));
  1612. mod->mkobj.kobj.kset = module_kset;
  1613. err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL,
  1614. "%s", mod->name);
  1615. if (err)
  1616. mod_kobject_put(mod);
  1617. out:
  1618. return err;
  1619. }
  1620. static int mod_sysfs_setup(struct module *mod,
  1621. const struct load_info *info,
  1622. struct kernel_param *kparam,
  1623. unsigned int num_params)
  1624. {
  1625. int err;
  1626. err = mod_sysfs_init(mod);
  1627. if (err)
  1628. goto out;
  1629. mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
  1630. if (!mod->holders_dir) {
  1631. err = -ENOMEM;
  1632. goto out_unreg;
  1633. }
  1634. err = module_param_sysfs_setup(mod, kparam, num_params);
  1635. if (err)
  1636. goto out_unreg_holders;
  1637. err = module_add_modinfo_attrs(mod);
  1638. if (err)
  1639. goto out_unreg_param;
  1640. err = add_usage_links(mod);
  1641. if (err)
  1642. goto out_unreg_modinfo_attrs;
  1643. add_sect_attrs(mod, info);
  1644. add_notes_attrs(mod, info);
  1645. return 0;
  1646. out_unreg_modinfo_attrs:
  1647. module_remove_modinfo_attrs(mod, -1);
  1648. out_unreg_param:
  1649. module_param_sysfs_remove(mod);
  1650. out_unreg_holders:
  1651. kobject_put(mod->holders_dir);
  1652. out_unreg:
  1653. mod_kobject_put(mod);
  1654. out:
  1655. return err;
  1656. }
  1657. static void mod_sysfs_fini(struct module *mod)
  1658. {
  1659. remove_notes_attrs(mod);
  1660. remove_sect_attrs(mod);
  1661. mod_kobject_put(mod);
  1662. }
  1663. static void init_param_lock(struct module *mod)
  1664. {
  1665. mutex_init(&mod->param_lock);
  1666. }
  1667. #else /* !CONFIG_SYSFS */
  1668. static int mod_sysfs_setup(struct module *mod,
  1669. const struct load_info *info,
  1670. struct kernel_param *kparam,
  1671. unsigned int num_params)
  1672. {
  1673. return 0;
  1674. }
  1675. static void mod_sysfs_fini(struct module *mod)
  1676. {
  1677. }
  1678. static void module_remove_modinfo_attrs(struct module *mod, int end)
  1679. {
  1680. }
  1681. static void del_usage_links(struct module *mod)
  1682. {
  1683. }
  1684. static void init_param_lock(struct module *mod)
  1685. {
  1686. }
  1687. #endif /* CONFIG_SYSFS */
  1688. static void mod_sysfs_teardown(struct module *mod)
  1689. {
  1690. del_usage_links(mod);
  1691. module_remove_modinfo_attrs(mod, -1);
  1692. module_param_sysfs_remove(mod);
  1693. kobject_put(mod->mkobj.drivers_dir);
  1694. kobject_put(mod->holders_dir);
  1695. mod_sysfs_fini(mod);
  1696. }
  1697. /*
  1698. * LKM RO/NX protection: protect module's text/ro-data
  1699. * from modification and any data from execution.
  1700. *
  1701. * General layout of module is:
  1702. * [text] [read-only-data] [ro-after-init] [writable data]
  1703. * text_size -----^ ^ ^ ^
  1704. * ro_size ------------------------| | |
  1705. * ro_after_init_size -----------------------------| |
  1706. * size -----------------------------------------------------------|
  1707. *
  1708. * These values are always page-aligned (as is base)
  1709. */
  1710. /*
  1711. * Since some arches are moving towards PAGE_KERNEL module allocations instead
  1712. * of PAGE_KERNEL_EXEC, keep frob_text() and module_enable_x() outside of the
  1713. * CONFIG_STRICT_MODULE_RWX block below because they are needed regardless of
  1714. * whether we are strict.
  1715. */
  1716. #ifdef CONFIG_ARCH_HAS_STRICT_MODULE_RWX
  1717. static void frob_text(const struct module_layout *layout,
  1718. int (*set_memory)(unsigned long start, int num_pages))
  1719. {
  1720. BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
  1721. BUG_ON((unsigned long)layout->text_size & (PAGE_SIZE-1));
  1722. set_memory((unsigned long)layout->base,
  1723. layout->text_size >> PAGE_SHIFT);
  1724. }
  1725. static void module_enable_x(const struct module *mod)
  1726. {
  1727. frob_text(&mod->core_layout, set_memory_x);
  1728. frob_text(&mod->init_layout, set_memory_x);
  1729. }
  1730. #else /* !CONFIG_ARCH_HAS_STRICT_MODULE_RWX */
  1731. static void module_enable_x(const struct module *mod) { }
  1732. #endif /* CONFIG_ARCH_HAS_STRICT_MODULE_RWX */
  1733. #ifdef CONFIG_STRICT_MODULE_RWX
  1734. static void frob_rodata(const struct module_layout *layout,
  1735. int (*set_memory)(unsigned long start, int num_pages))
  1736. {
  1737. BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
  1738. BUG_ON((unsigned long)layout->text_size & (PAGE_SIZE-1));
  1739. BUG_ON((unsigned long)layout->ro_size & (PAGE_SIZE-1));
  1740. set_memory((unsigned long)layout->base + layout->text_size,
  1741. (layout->ro_size - layout->text_size) >> PAGE_SHIFT);
  1742. }
  1743. static void frob_ro_after_init(const struct module_layout *layout,
  1744. int (*set_memory)(unsigned long start, int num_pages))
  1745. {
  1746. BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
  1747. BUG_ON((unsigned long)layout->ro_size & (PAGE_SIZE-1));
  1748. BUG_ON((unsigned long)layout->ro_after_init_size & (PAGE_SIZE-1));
  1749. set_memory((unsigned long)layout->base + layout->ro_size,
  1750. (layout->ro_after_init_size - layout->ro_size) >> PAGE_SHIFT);
  1751. }
  1752. static void frob_writable_data(const struct module_layout *layout,
  1753. int (*set_memory)(unsigned long start, int num_pages))
  1754. {
  1755. BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
  1756. BUG_ON((unsigned long)layout->ro_after_init_size & (PAGE_SIZE-1));
  1757. BUG_ON((unsigned long)layout->size & (PAGE_SIZE-1));
  1758. set_memory((unsigned long)layout->base + layout->ro_after_init_size,
  1759. (layout->size - layout->ro_after_init_size) >> PAGE_SHIFT);
  1760. }
  1761. static void module_enable_ro(const struct module *mod, bool after_init)
  1762. {
  1763. if (!rodata_enabled)
  1764. return;
  1765. set_vm_flush_reset_perms(mod->core_layout.base);
  1766. set_vm_flush_reset_perms(mod->init_layout.base);
  1767. frob_text(&mod->core_layout, set_memory_ro);
  1768. frob_rodata(&mod->core_layout, set_memory_ro);
  1769. frob_text(&mod->init_layout, set_memory_ro);
  1770. frob_rodata(&mod->init_layout, set_memory_ro);
  1771. if (after_init)
  1772. frob_ro_after_init(&mod->core_layout, set_memory_ro);
  1773. }
  1774. static void module_enable_nx(const struct module *mod)
  1775. {
  1776. frob_rodata(&mod->core_layout, set_memory_nx);
  1777. frob_ro_after_init(&mod->core_layout, set_memory_nx);
  1778. frob_writable_data(&mod->core_layout, set_memory_nx);
  1779. frob_rodata(&mod->init_layout, set_memory_nx);
  1780. frob_writable_data(&mod->init_layout, set_memory_nx);
  1781. }
  1782. static int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
  1783. char *secstrings, struct module *mod)
  1784. {
  1785. const unsigned long shf_wx = SHF_WRITE|SHF_EXECINSTR;
  1786. int i;
  1787. for (i = 0; i < hdr->e_shnum; i++) {
  1788. if ((sechdrs[i].sh_flags & shf_wx) == shf_wx) {
  1789. pr_err("%s: section %s (index %d) has invalid WRITE|EXEC flags\n",
  1790. mod->name, secstrings + sechdrs[i].sh_name, i);
  1791. return -ENOEXEC;
  1792. }
  1793. }
  1794. return 0;
  1795. }
  1796. #else /* !CONFIG_STRICT_MODULE_RWX */
  1797. static void module_enable_nx(const struct module *mod) { }
  1798. static void module_enable_ro(const struct module *mod, bool after_init) {}
  1799. static int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
  1800. char *secstrings, struct module *mod)
  1801. {
  1802. return 0;
  1803. }
  1804. #endif /* CONFIG_STRICT_MODULE_RWX */
  1805. #ifdef CONFIG_LIVEPATCH
  1806. /*
  1807. * Persist Elf information about a module. Copy the Elf header,
  1808. * section header table, section string table, and symtab section
  1809. * index from info to mod->klp_info.
  1810. */
  1811. static int copy_module_elf(struct module *mod, struct load_info *info)
  1812. {
  1813. unsigned int size, symndx;
  1814. int ret;
  1815. size = sizeof(*mod->klp_info);
  1816. mod->klp_info = kmalloc(size, GFP_KERNEL);
  1817. if (mod->klp_info == NULL)
  1818. return -ENOMEM;
  1819. /* Elf header */
  1820. size = sizeof(mod->klp_info->hdr);
  1821. memcpy(&mod->klp_info->hdr, info->hdr, size);
  1822. /* Elf section header table */
  1823. size = sizeof(*info->sechdrs) * info->hdr->e_shnum;
  1824. mod->klp_info->sechdrs = kmemdup(info->sechdrs, size, GFP_KERNEL);
  1825. if (mod->klp_info->sechdrs == NULL) {
  1826. ret = -ENOMEM;
  1827. goto free_info;
  1828. }
  1829. /* Elf section name string table */
  1830. size = info->sechdrs[info->hdr->e_shstrndx].sh_size;
  1831. mod->klp_info->secstrings = kmemdup(info->secstrings, size, GFP_KERNEL);
  1832. if (mod->klp_info->secstrings == NULL) {
  1833. ret = -ENOMEM;
  1834. goto free_sechdrs;
  1835. }
  1836. /* Elf symbol section index */
  1837. symndx = info->index.sym;
  1838. mod->klp_info->symndx = symndx;
  1839. /*
  1840. * For livepatch modules, core_kallsyms.symtab is a complete
  1841. * copy of the original symbol table. Adjust sh_addr to point
  1842. * to core_kallsyms.symtab since the copy of the symtab in module
  1843. * init memory is freed at the end of do_init_module().
  1844. */
  1845. mod->klp_info->sechdrs[symndx].sh_addr = \
  1846. (unsigned long) mod->core_kallsyms.symtab;
  1847. return 0;
  1848. free_sechdrs:
  1849. kfree(mod->klp_info->sechdrs);
  1850. free_info:
  1851. kfree(mod->klp_info);
  1852. return ret;
  1853. }
  1854. static void free_module_elf(struct module *mod)
  1855. {
  1856. kfree(mod->klp_info->sechdrs);
  1857. kfree(mod->klp_info->secstrings);
  1858. kfree(mod->klp_info);
  1859. }
  1860. #else /* !CONFIG_LIVEPATCH */
  1861. static int copy_module_elf(struct module *mod, struct load_info *info)
  1862. {
  1863. return 0;
  1864. }
  1865. static void free_module_elf(struct module *mod)
  1866. {
  1867. }
  1868. #endif /* CONFIG_LIVEPATCH */
  1869. void __weak module_memfree(void *module_region)
  1870. {
  1871. /*
  1872. * This memory may be RO, and freeing RO memory in an interrupt is not
  1873. * supported by vmalloc.
  1874. */
  1875. WARN_ON(in_interrupt());
  1876. vfree(module_region);
  1877. }
  1878. void __weak module_arch_cleanup(struct module *mod)
  1879. {
  1880. }
  1881. void __weak module_arch_freeing_init(struct module *mod)
  1882. {
  1883. }
  1884. static void cfi_cleanup(struct module *mod);
  1885. /* Free a module, remove from lists, etc. */
  1886. static void free_module(struct module *mod)
  1887. {
  1888. trace_module_free(mod);
  1889. mod_sysfs_teardown(mod);
  1890. /* We leave it in list to prevent duplicate loads, but make sure
  1891. * that noone uses it while it's being deconstructed. */
  1892. mutex_lock(&module_mutex);
  1893. mod->state = MODULE_STATE_UNFORMED;
  1894. mutex_unlock(&module_mutex);
  1895. /* Remove dynamic debug info */
  1896. ddebug_remove_module(mod->name);
  1897. /* Arch-specific cleanup. */
  1898. module_arch_cleanup(mod);
  1899. /* Module unload stuff */
  1900. module_unload_free(mod);
  1901. /* Free any allocated parameters. */
  1902. destroy_params(mod->kp, mod->num_kp);
  1903. if (is_livepatch_module(mod))
  1904. free_module_elf(mod);
  1905. /* Now we can delete it from the lists */
  1906. mutex_lock(&module_mutex);
  1907. /* Unlink carefully: kallsyms could be walking list. */
  1908. list_del_rcu(&mod->list);
  1909. mod_tree_remove(mod);
  1910. /* Remove this module from bug list, this uses list_del_rcu */
  1911. module_bug_cleanup(mod);
  1912. /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
  1913. synchronize_rcu();
  1914. mutex_unlock(&module_mutex);
  1915. /* Clean up CFI for the module. */
  1916. cfi_cleanup(mod);
  1917. /* This may be empty, but that's OK */
  1918. module_arch_freeing_init(mod);
  1919. trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
  1920. (mod->init_layout.size)>>PAGE_SHIFT);
  1921. trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
  1922. (mod->init_layout.size)>>PAGE_SHIFT);
  1923. module_memfree(mod->init_layout.base);
  1924. kfree(mod->args);
  1925. percpu_modfree(mod);
  1926. /* Free lock-classes; relies on the preceding sync_rcu(). */
  1927. lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
  1928. /* Finally, free the core (containing the module structure) */
  1929. trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base,
  1930. (mod->core_layout.size)>>PAGE_SHIFT);
  1931. trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base,
  1932. (mod->core_layout.size)>>PAGE_SHIFT);
  1933. module_memfree(mod->core_layout.base);
  1934. }
  1935. void *__symbol_get(const char *symbol)
  1936. {
  1937. struct module *owner;
  1938. const struct kernel_symbol *sym;
  1939. preempt_disable();
  1940. sym = find_symbol(symbol, &owner, NULL, NULL, true, true);
  1941. if (sym && strong_try_module_get(owner))
  1942. sym = NULL;
  1943. preempt_enable();
  1944. return sym ? (void *)kernel_symbol_value(sym) : NULL;
  1945. }
  1946. EXPORT_SYMBOL_GPL(__symbol_get);
  1947. /*
  1948. * Ensure that an exported symbol [global namespace] does not already exist
  1949. * in the kernel or in some other module's exported symbol table.
  1950. *
  1951. * You must hold the module_mutex.
  1952. */
  1953. static int verify_exported_symbols(struct module *mod)
  1954. {
  1955. unsigned int i;
  1956. struct module *owner;
  1957. const struct kernel_symbol *s;
  1958. struct {
  1959. const struct kernel_symbol *sym;
  1960. unsigned int num;
  1961. } arr[] = {
  1962. { mod->syms, mod->num_syms },
  1963. { mod->gpl_syms, mod->num_gpl_syms },
  1964. { mod->gpl_future_syms, mod->num_gpl_future_syms },
  1965. #ifdef CONFIG_UNUSED_SYMBOLS
  1966. { mod->unused_syms, mod->num_unused_syms },
  1967. { mod->unused_gpl_syms, mod->num_unused_gpl_syms },
  1968. #endif
  1969. };
  1970. for (i = 0; i < ARRAY_SIZE(arr); i++) {
  1971. for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
  1972. if (find_symbol(kernel_symbol_name(s), &owner, NULL,
  1973. NULL, true, false)) {
  1974. pr_err("%s: exports duplicate symbol %s"
  1975. " (owned by %s)\n",
  1976. mod->name, kernel_symbol_name(s),
  1977. module_name(owner));
  1978. return -ENOEXEC;
  1979. }
  1980. }
  1981. }
  1982. return 0;
  1983. }
  1984. static bool ignore_undef_symbol(Elf_Half emachine, const char *name)
  1985. {
  1986. /*
  1987. * On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as
  1988. * before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64.
  1989. * i386 has a similar problem but may not deserve a fix.
  1990. *
  1991. * If we ever have to ignore many symbols, consider refactoring the code to
  1992. * only warn if referenced by a relocation.
  1993. */
  1994. if (emachine == EM_386 || emachine == EM_X86_64)
  1995. return !strcmp(name, "_GLOBAL_OFFSET_TABLE_");
  1996. return false;
  1997. }
  1998. /* Change all symbols so that st_value encodes the pointer directly. */
  1999. static int simplify_symbols(struct module *mod, const struct load_info *info)
  2000. {
  2001. Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
  2002. Elf_Sym *sym = (void *)symsec->sh_addr;
  2003. unsigned long secbase;
  2004. unsigned int i;
  2005. int ret = 0;
  2006. const struct kernel_symbol *ksym;
  2007. for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
  2008. const char *name = info->strtab + sym[i].st_name;
  2009. switch (sym[i].st_shndx) {
  2010. case SHN_COMMON:
  2011. /* Ignore common symbols */
  2012. if (!strncmp(name, "__gnu_lto", 9))
  2013. break;
  2014. /* We compiled with -fno-common. These are not
  2015. supposed to happen. */
  2016. pr_debug("Common symbol: %s\n", name);
  2017. pr_warn("%s: please compile with -fno-common\n",
  2018. mod->name);
  2019. ret = -ENOEXEC;
  2020. break;
  2021. case SHN_ABS:
  2022. /* Don't need to do anything */
  2023. pr_debug("Absolute symbol: 0x%08lx\n",
  2024. (long)sym[i].st_value);
  2025. break;
  2026. case SHN_LIVEPATCH:
  2027. /* Livepatch symbols are resolved by livepatch */
  2028. break;
  2029. case SHN_UNDEF:
  2030. ksym = resolve_symbol_wait(mod, info, name);
  2031. /* Ok if resolved. */
  2032. if (ksym && !IS_ERR(ksym)) {
  2033. sym[i].st_value = kernel_symbol_value(ksym);
  2034. break;
  2035. }
  2036. /* Ok if weak or ignored. */
  2037. if (!ksym &&
  2038. (ELF_ST_BIND(sym[i].st_info) == STB_WEAK ||
  2039. ignore_undef_symbol(info->hdr->e_machine, name)))
  2040. break;
  2041. ret = PTR_ERR(ksym) ?: -ENOENT;
  2042. pr_warn("%s: Unknown symbol %s (err %d)\n",
  2043. mod->name, name, ret);
  2044. break;
  2045. default:
  2046. /* Divert to percpu allocation if a percpu var. */
  2047. if (sym[i].st_shndx == info->index.pcpu)
  2048. secbase = (unsigned long)mod_percpu(mod);
  2049. else
  2050. secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
  2051. sym[i].st_value += secbase;
  2052. break;
  2053. }
  2054. }
  2055. return ret;
  2056. }
  2057. static int apply_relocations(struct module *mod, const struct load_info *info)
  2058. {
  2059. unsigned int i;
  2060. int err = 0;
  2061. /* Now do relocations. */
  2062. for (i = 1; i < info->hdr->e_shnum; i++) {
  2063. unsigned int infosec = info->sechdrs[i].sh_info;
  2064. /* Not a valid relocation section? */
  2065. if (infosec >= info->hdr->e_shnum)
  2066. continue;
  2067. /* Don't bother with non-allocated sections */
  2068. if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
  2069. continue;
  2070. if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
  2071. err = klp_apply_section_relocs(mod, info->sechdrs,
  2072. info->secstrings,
  2073. info->strtab,
  2074. info->index.sym, i,
  2075. NULL);
  2076. else if (info->sechdrs[i].sh_type == SHT_REL)
  2077. err = apply_relocate(info->sechdrs, info->strtab,
  2078. info->index.sym, i, mod);
  2079. else if (info->sechdrs[i].sh_type == SHT_RELA)
  2080. err = apply_relocate_add(info->sechdrs, info->strtab,
  2081. info->index.sym, i, mod);
  2082. if (err < 0)
  2083. break;
  2084. }
  2085. return err;
  2086. }
  2087. /* Additional bytes needed by arch in front of individual sections */
  2088. unsigned int __weak arch_mod_section_prepend(struct module *mod,
  2089. unsigned int section)
  2090. {
  2091. /* default implementation just returns zero */
  2092. return 0;
  2093. }
  2094. /* Update size with this section: return offset. */
  2095. static long get_offset(struct module *mod, unsigned int *size,
  2096. Elf_Shdr *sechdr, unsigned int section)
  2097. {
  2098. long ret;
  2099. *size += arch_mod_section_prepend(mod, section);
  2100. ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
  2101. *size = ret + sechdr->sh_size;
  2102. return ret;
  2103. }
  2104. /* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
  2105. might -- code, read-only data, read-write data, small data. Tally
  2106. sizes, and place the offsets into sh_entsize fields: high bit means it
  2107. belongs in init. */
  2108. static void layout_sections(struct module *mod, struct load_info *info)
  2109. {
  2110. static unsigned long const masks[][2] = {
  2111. /* NOTE: all executable code must be the first section
  2112. * in this array; otherwise modify the text_size
  2113. * finder in the two loops below */
  2114. { SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
  2115. { SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
  2116. { SHF_RO_AFTER_INIT | SHF_ALLOC, ARCH_SHF_SMALL },
  2117. { SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
  2118. { ARCH_SHF_SMALL | SHF_ALLOC, 0 }
  2119. };
  2120. unsigned int m, i;
  2121. for (i = 0; i < info->hdr->e_shnum; i++)
  2122. info->sechdrs[i].sh_entsize = ~0UL;
  2123. pr_debug("Core section allocation order:\n");
  2124. for (m = 0; m < ARRAY_SIZE(masks); ++m) {
  2125. for (i = 0; i < info->hdr->e_shnum; ++i) {
  2126. Elf_Shdr *s = &info->sechdrs[i];
  2127. const char *sname = info->secstrings + s->sh_name;
  2128. if ((s->sh_flags & masks[m][0]) != masks[m][0]
  2129. || (s->sh_flags & masks[m][1])
  2130. || s->sh_entsize != ~0UL
  2131. || module_init_section(sname))
  2132. continue;
  2133. s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i);
  2134. pr_debug("\t%s\n", sname);
  2135. }
  2136. switch (m) {
  2137. case 0: /* executable */
  2138. mod->core_layout.size = debug_align(mod->core_layout.size);
  2139. mod->core_layout.text_size = mod->core_layout.size;
  2140. break;
  2141. case 1: /* RO: text and ro-data */
  2142. mod->core_layout.size = debug_align(mod->core_layout.size);
  2143. mod->core_layout.ro_size = mod->core_layout.size;
  2144. break;
  2145. case 2: /* RO after init */
  2146. mod->core_layout.size = debug_align(mod->core_layout.size);
  2147. mod->core_layout.ro_after_init_size = mod->core_layout.size;
  2148. break;
  2149. case 4: /* whole core */
  2150. mod->core_layout.size = debug_align(mod->core_layout.size);
  2151. break;
  2152. }
  2153. }
  2154. pr_debug("Init section allocation order:\n");
  2155. for (m = 0; m < ARRAY_SIZE(masks); ++m) {
  2156. for (i = 0; i < info->hdr->e_shnum; ++i) {
  2157. Elf_Shdr *s = &info->sechdrs[i];
  2158. const char *sname = info->secstrings + s->sh_name;
  2159. if ((s->sh_flags & masks[m][0]) != masks[m][0]
  2160. || (s->sh_flags & masks[m][1])
  2161. || s->sh_entsize != ~0UL
  2162. || !module_init_section(sname))
  2163. continue;
  2164. s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i)
  2165. | INIT_OFFSET_MASK);
  2166. pr_debug("\t%s\n", sname);
  2167. }
  2168. switch (m) {
  2169. case 0: /* executable */
  2170. mod->init_layout.size = debug_align(mod->init_layout.size);
  2171. mod->init_layout.text_size = mod->init_layout.size;
  2172. break;
  2173. case 1: /* RO: text and ro-data */
  2174. mod->init_layout.size = debug_align(mod->init_layout.size);
  2175. mod->init_layout.ro_size = mod->init_layout.size;
  2176. break;
  2177. case 2:
  2178. /*
  2179. * RO after init doesn't apply to init_layout (only
  2180. * core_layout), so it just takes the value of ro_size.
  2181. */
  2182. mod->init_layout.ro_after_init_size = mod->init_layout.ro_size;
  2183. break;
  2184. case 4: /* whole init */
  2185. mod->init_layout.size = debug_align(mod->init_layout.size);
  2186. break;
  2187. }
  2188. }
  2189. }
  2190. static void set_license(struct module *mod, const char *license)
  2191. {
  2192. if (!license)
  2193. license = "unspecified";
  2194. if (!license_is_gpl_compatible(license)) {
  2195. if (!test_taint(TAINT_PROPRIETARY_MODULE))
  2196. pr_warn("%s: module license '%s' taints kernel.\n",
  2197. mod->name, license);
  2198. add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
  2199. LOCKDEP_NOW_UNRELIABLE);
  2200. }
  2201. }
  2202. /* Parse tag=value strings from .modinfo section */
  2203. static char *next_string(char *string, unsigned long *secsize)
  2204. {
  2205. /* Skip non-zero chars */
  2206. while (string[0]) {
  2207. string++;
  2208. if ((*secsize)-- <= 1)
  2209. return NULL;
  2210. }
  2211. /* Skip any zero padding. */
  2212. while (!string[0]) {
  2213. string++;
  2214. if ((*secsize)-- <= 1)
  2215. return NULL;
  2216. }
  2217. return string;
  2218. }
  2219. static char *get_next_modinfo(const struct load_info *info, const char *tag,
  2220. char *prev)
  2221. {
  2222. char *p;
  2223. unsigned int taglen = strlen(tag);
  2224. Elf_Shdr *infosec = &info->sechdrs[info->index.info];
  2225. unsigned long size = infosec->sh_size;
  2226. /*
  2227. * get_modinfo() calls made before rewrite_section_headers()
  2228. * must use sh_offset, as sh_addr isn't set!
  2229. */
  2230. char *modinfo = (char *)info->hdr + infosec->sh_offset;
  2231. if (prev) {
  2232. size -= prev - modinfo;
  2233. modinfo = next_string(prev, &size);
  2234. }
  2235. for (p = modinfo; p; p = next_string(p, &size)) {
  2236. if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
  2237. return p + taglen + 1;
  2238. }
  2239. return NULL;
  2240. }
  2241. static char *get_modinfo(const struct load_info *info, const char *tag)
  2242. {
  2243. return get_next_modinfo(info, tag, NULL);
  2244. }
  2245. static void setup_modinfo(struct module *mod, struct load_info *info)
  2246. {
  2247. struct module_attribute *attr;
  2248. int i;
  2249. for (i = 0; (attr = modinfo_attrs[i]); i++) {
  2250. if (attr->setup)
  2251. attr->setup(mod, get_modinfo(info, attr->attr.name));
  2252. }
  2253. }
  2254. static void free_modinfo(struct module *mod)
  2255. {
  2256. struct module_attribute *attr;
  2257. int i;
  2258. for (i = 0; (attr = modinfo_attrs[i]); i++) {
  2259. if (attr->free)
  2260. attr->free(mod);
  2261. }
  2262. }
  2263. #ifdef CONFIG_KALLSYMS
  2264. /* Lookup exported symbol in given range of kernel_symbols */
  2265. static const struct kernel_symbol *lookup_exported_symbol(const char *name,
  2266. const struct kernel_symbol *start,
  2267. const struct kernel_symbol *stop)
  2268. {
  2269. return bsearch(name, start, stop - start,
  2270. sizeof(struct kernel_symbol), cmp_name);
  2271. }
  2272. static int is_exported(const char *name, unsigned long value,
  2273. const struct module *mod)
  2274. {
  2275. const struct kernel_symbol *ks;
  2276. if (!mod)
  2277. ks = lookup_exported_symbol(name, __start___ksymtab, __stop___ksymtab);
  2278. else
  2279. ks = lookup_exported_symbol(name, mod->syms, mod->syms + mod->num_syms);
  2280. return ks != NULL && kernel_symbol_value(ks) == value;
  2281. }
  2282. /* As per nm */
  2283. static char elf_type(const Elf_Sym *sym, const struct load_info *info)
  2284. {
  2285. const Elf_Shdr *sechdrs = info->sechdrs;
  2286. if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
  2287. if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
  2288. return 'v';
  2289. else
  2290. return 'w';
  2291. }
  2292. if (sym->st_shndx == SHN_UNDEF)
  2293. return 'U';
  2294. if (sym->st_shndx == SHN_ABS || sym->st_shndx == info->index.pcpu)
  2295. return 'a';
  2296. if (sym->st_shndx >= SHN_LORESERVE)
  2297. return '?';
  2298. if (sechdrs[sym->st_shndx].sh_flags & SHF_EXECINSTR)
  2299. return 't';
  2300. if (sechdrs[sym->st_shndx].sh_flags & SHF_ALLOC
  2301. && sechdrs[sym->st_shndx].sh_type != SHT_NOBITS) {
  2302. if (!(sechdrs[sym->st_shndx].sh_flags & SHF_WRITE))
  2303. return 'r';
  2304. else if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
  2305. return 'g';
  2306. else
  2307. return 'd';
  2308. }
  2309. if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
  2310. if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
  2311. return 's';
  2312. else
  2313. return 'b';
  2314. }
  2315. if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
  2316. ".debug")) {
  2317. return 'n';
  2318. }
  2319. return '?';
  2320. }
  2321. static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
  2322. unsigned int shnum, unsigned int pcpundx)
  2323. {
  2324. const Elf_Shdr *sec;
  2325. if (src->st_shndx == SHN_UNDEF
  2326. || src->st_shndx >= shnum
  2327. || !src->st_name)
  2328. return false;
  2329. #ifdef CONFIG_KALLSYMS_ALL
  2330. if (src->st_shndx == pcpundx)
  2331. return true;
  2332. #endif
  2333. sec = sechdrs + src->st_shndx;
  2334. if (!(sec->sh_flags & SHF_ALLOC)
  2335. #ifndef CONFIG_KALLSYMS_ALL
  2336. || !(sec->sh_flags & SHF_EXECINSTR)
  2337. #endif
  2338. || (sec->sh_entsize & INIT_OFFSET_MASK))
  2339. return false;
  2340. return true;
  2341. }
  2342. /*
  2343. * We only allocate and copy the strings needed by the parts of symtab
  2344. * we keep. This is simple, but has the effect of making multiple
  2345. * copies of duplicates. We could be more sophisticated, see
  2346. * linux-kernel thread starting with
  2347. * <73defb5e4bca04a6431392cc341112b1@localhost>.
  2348. */
  2349. static void layout_symtab(struct module *mod, struct load_info *info)
  2350. {
  2351. Elf_Shdr *symsect = info->sechdrs + info->index.sym;
  2352. Elf_Shdr *strsect = info->sechdrs + info->index.str;
  2353. const Elf_Sym *src;
  2354. unsigned int i, nsrc, ndst, strtab_size = 0;
  2355. /* Put symbol section at end of init part of module. */
  2356. symsect->sh_flags |= SHF_ALLOC;
  2357. symsect->sh_entsize = get_offset(mod, &mod->init_layout.size, symsect,
  2358. info->index.sym) | INIT_OFFSET_MASK;
  2359. pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
  2360. src = (void *)info->hdr + symsect->sh_offset;
  2361. nsrc = symsect->sh_size / sizeof(*src);
  2362. /* Compute total space required for the core symbols' strtab. */
  2363. for (ndst = i = 0; i < nsrc; i++) {
  2364. if (i == 0 || is_livepatch_module(mod) ||
  2365. is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
  2366. info->index.pcpu)) {
  2367. strtab_size += strlen(&info->strtab[src[i].st_name])+1;
  2368. ndst++;
  2369. }
  2370. }
  2371. /* Append room for core symbols at end of core part. */
  2372. info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1);
  2373. info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
  2374. mod->core_layout.size += strtab_size;
  2375. info->core_typeoffs = mod->core_layout.size;
  2376. mod->core_layout.size += ndst * sizeof(char);
  2377. mod->core_layout.size = debug_align(mod->core_layout.size);
  2378. /* Put string table section at end of init part of module. */
  2379. strsect->sh_flags |= SHF_ALLOC;
  2380. strsect->sh_entsize = get_offset(mod, &mod->init_layout.size, strsect,
  2381. info->index.str) | INIT_OFFSET_MASK;
  2382. pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
  2383. /* We'll tack temporary mod_kallsyms on the end. */
  2384. mod->init_layout.size = ALIGN(mod->init_layout.size,
  2385. __alignof__(struct mod_kallsyms));
  2386. info->mod_kallsyms_init_off = mod->init_layout.size;
  2387. mod->init_layout.size += sizeof(struct mod_kallsyms);
  2388. info->init_typeoffs = mod->init_layout.size;
  2389. mod->init_layout.size += nsrc * sizeof(char);
  2390. mod->init_layout.size = debug_align(mod->init_layout.size);
  2391. }
  2392. /*
  2393. * We use the full symtab and strtab which layout_symtab arranged to
  2394. * be appended to the init section. Later we switch to the cut-down
  2395. * core-only ones.
  2396. */
  2397. static void add_kallsyms(struct module *mod, const struct load_info *info)
  2398. {
  2399. unsigned int i, ndst;
  2400. const Elf_Sym *src;
  2401. Elf_Sym *dst;
  2402. char *s;
  2403. Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
  2404. /* Set up to point into init section. */
  2405. mod->kallsyms = mod->init_layout.base + info->mod_kallsyms_init_off;
  2406. mod->kallsyms->symtab = (void *)symsec->sh_addr;
  2407. mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
  2408. /* Make sure we get permanent strtab: don't use info->strtab. */
  2409. mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
  2410. mod->kallsyms->typetab = mod->init_layout.base + info->init_typeoffs;
  2411. /*
  2412. * Now populate the cut down core kallsyms for after init
  2413. * and set types up while we still have access to sections.
  2414. */
  2415. mod->core_kallsyms.symtab = dst = mod->core_layout.base + info->symoffs;
  2416. mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs;
  2417. mod->core_kallsyms.typetab = mod->core_layout.base + info->core_typeoffs;
  2418. src = mod->kallsyms->symtab;
  2419. for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
  2420. mod->kallsyms->typetab[i] = elf_type(src + i, info);
  2421. if (i == 0 || is_livepatch_module(mod) ||
  2422. is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
  2423. info->index.pcpu)) {
  2424. mod->core_kallsyms.typetab[ndst] =
  2425. mod->kallsyms->typetab[i];
  2426. dst[ndst] = src[i];
  2427. dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
  2428. s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name],
  2429. KSYM_NAME_LEN) + 1;
  2430. }
  2431. }
  2432. mod->core_kallsyms.num_symtab = ndst;
  2433. }
  2434. #else
  2435. static inline void layout_symtab(struct module *mod, struct load_info *info)
  2436. {
  2437. }
  2438. static void add_kallsyms(struct module *mod, const struct load_info *info)
  2439. {
  2440. }
  2441. #endif /* CONFIG_KALLSYMS */
  2442. static void dynamic_debug_setup(struct module *mod, struct _ddebug *debug, unsigned int num)
  2443. {
  2444. if (!debug)
  2445. return;
  2446. ddebug_add_module(debug, num, mod->name);
  2447. }
  2448. static void dynamic_debug_remove(struct module *mod, struct _ddebug *debug)
  2449. {
  2450. if (debug)
  2451. ddebug_remove_module(mod->name);
  2452. }
  2453. void * __weak module_alloc(unsigned long size)
  2454. {
  2455. return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
  2456. GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
  2457. NUMA_NO_NODE, __builtin_return_address(0));
  2458. }
  2459. bool __weak module_init_section(const char *name)
  2460. {
  2461. return strstarts(name, ".init");
  2462. }
  2463. bool __weak module_exit_section(const char *name)
  2464. {
  2465. return strstarts(name, ".exit");
  2466. }
  2467. #ifdef CONFIG_DEBUG_KMEMLEAK
  2468. static void kmemleak_load_module(const struct module *mod,
  2469. const struct load_info *info)
  2470. {
  2471. unsigned int i;
  2472. /* only scan the sections containing data */
  2473. kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
  2474. for (i = 1; i < info->hdr->e_shnum; i++) {
  2475. /* Scan all writable sections that's not executable */
  2476. if (!(info->sechdrs[i].sh_flags & SHF_ALLOC) ||
  2477. !(info->sechdrs[i].sh_flags & SHF_WRITE) ||
  2478. (info->sechdrs[i].sh_flags & SHF_EXECINSTR))
  2479. continue;
  2480. kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
  2481. info->sechdrs[i].sh_size, GFP_KERNEL);
  2482. }
  2483. }
  2484. #else
  2485. static inline void kmemleak_load_module(const struct module *mod,
  2486. const struct load_info *info)
  2487. {
  2488. }
  2489. #endif
  2490. #ifdef CONFIG_MODULE_SIG
  2491. static int module_sig_check(struct load_info *info, int flags)
  2492. {
  2493. int err = -ENODATA;
  2494. const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
  2495. const char *reason;
  2496. const void *mod = info->hdr;
  2497. /*
  2498. * Require flags == 0, as a module with version information
  2499. * removed is no longer the module that was signed
  2500. */
  2501. if (flags == 0 &&
  2502. info->len > markerlen &&
  2503. memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
  2504. /* We truncate the module to discard the signature */
  2505. info->len -= markerlen;
  2506. err = mod_verify_sig(mod, info);
  2507. }
  2508. switch (err) {
  2509. case 0:
  2510. info->sig_ok = true;
  2511. return 0;
  2512. /* We don't permit modules to be loaded into trusted kernels
  2513. * without a valid signature on them, but if we're not
  2514. * enforcing, certain errors are non-fatal.
  2515. */
  2516. case -ENODATA:
  2517. reason = "unsigned module";
  2518. break;
  2519. case -ENOPKG:
  2520. reason = "module with unsupported crypto";
  2521. break;
  2522. case -ENOKEY:
  2523. reason = "module with unavailable key";
  2524. break;
  2525. /* All other errors are fatal, including nomem, unparseable
  2526. * signatures and signature check failures - even if signatures
  2527. * aren't required.
  2528. */
  2529. default:
  2530. return err;
  2531. }
  2532. if (is_module_sig_enforced()) {
  2533. pr_notice("Loading of %s is rejected\n", reason);
  2534. return -EKEYREJECTED;
  2535. }
  2536. return security_locked_down(LOCKDOWN_MODULE_SIGNATURE);
  2537. }
  2538. #else /* !CONFIG_MODULE_SIG */
  2539. static int module_sig_check(struct load_info *info, int flags)
  2540. {
  2541. return 0;
  2542. }
  2543. #endif /* !CONFIG_MODULE_SIG */
  2544. static int validate_section_offset(struct load_info *info, Elf_Shdr *shdr)
  2545. {
  2546. unsigned long secend;
  2547. /*
  2548. * Check for both overflow and offset/size being
  2549. * too large.
  2550. */
  2551. secend = shdr->sh_offset + shdr->sh_size;
  2552. if (secend < shdr->sh_offset || secend > info->len)
  2553. return -ENOEXEC;
  2554. return 0;
  2555. }
  2556. /*
  2557. * Sanity checks against invalid binaries, wrong arch, weird elf version.
  2558. *
  2559. * Also do basic validity checks against section offsets and sizes, the
  2560. * section name string table, and the indices used for it (sh_name).
  2561. */
  2562. static int elf_validity_check(struct load_info *info)
  2563. {
  2564. unsigned int i;
  2565. Elf_Shdr *shdr, *strhdr;
  2566. int err;
  2567. if (info->len < sizeof(*(info->hdr)))
  2568. return -ENOEXEC;
  2569. if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0
  2570. || info->hdr->e_type != ET_REL
  2571. || !elf_check_arch(info->hdr)
  2572. || info->hdr->e_shentsize != sizeof(Elf_Shdr))
  2573. return -ENOEXEC;
  2574. /*
  2575. * e_shnum is 16 bits, and sizeof(Elf_Shdr) is
  2576. * known and small. So e_shnum * sizeof(Elf_Shdr)
  2577. * will not overflow unsigned long on any platform.
  2578. */
  2579. if (info->hdr->e_shoff >= info->len
  2580. || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
  2581. info->len - info->hdr->e_shoff))
  2582. return -ENOEXEC;
  2583. info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
  2584. /*
  2585. * Verify if the section name table index is valid.
  2586. */
  2587. if (info->hdr->e_shstrndx == SHN_UNDEF
  2588. || info->hdr->e_shstrndx >= info->hdr->e_shnum)
  2589. return -ENOEXEC;
  2590. strhdr = &info->sechdrs[info->hdr->e_shstrndx];
  2591. err = validate_section_offset(info, strhdr);
  2592. if (err < 0)
  2593. return err;
  2594. /*
  2595. * The section name table must be NUL-terminated, as required
  2596. * by the spec. This makes strcmp and pr_* calls that access
  2597. * strings in the section safe.
  2598. */
  2599. info->secstrings = (void *)info->hdr + strhdr->sh_offset;
  2600. if (info->secstrings[strhdr->sh_size - 1] != '\0')
  2601. return -ENOEXEC;
  2602. /*
  2603. * The code assumes that section 0 has a length of zero and
  2604. * an addr of zero, so check for it.
  2605. */
  2606. if (info->sechdrs[0].sh_type != SHT_NULL
  2607. || info->sechdrs[0].sh_size != 0
  2608. || info->sechdrs[0].sh_addr != 0)
  2609. return -ENOEXEC;
  2610. for (i = 1; i < info->hdr->e_shnum; i++) {
  2611. shdr = &info->sechdrs[i];
  2612. switch (shdr->sh_type) {
  2613. case SHT_NULL:
  2614. case SHT_NOBITS:
  2615. continue;
  2616. case SHT_SYMTAB:
  2617. if (shdr->sh_link == SHN_UNDEF
  2618. || shdr->sh_link >= info->hdr->e_shnum)
  2619. return -ENOEXEC;
  2620. fallthrough;
  2621. default:
  2622. err = validate_section_offset(info, shdr);
  2623. if (err < 0) {
  2624. pr_err("Invalid ELF section in module (section %u type %u)\n",
  2625. i, shdr->sh_type);
  2626. return err;
  2627. }
  2628. if (shdr->sh_flags & SHF_ALLOC) {
  2629. if (shdr->sh_name >= strhdr->sh_size) {
  2630. pr_err("Invalid ELF section name in module (section %u type %u)\n",
  2631. i, shdr->sh_type);
  2632. return -ENOEXEC;
  2633. }
  2634. }
  2635. break;
  2636. }
  2637. }
  2638. return 0;
  2639. }
  2640. #define COPY_CHUNK_SIZE (16*PAGE_SIZE)
  2641. static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len)
  2642. {
  2643. do {
  2644. unsigned long n = min(len, COPY_CHUNK_SIZE);
  2645. if (copy_from_user(dst, usrc, n) != 0)
  2646. return -EFAULT;
  2647. cond_resched();
  2648. dst += n;
  2649. usrc += n;
  2650. len -= n;
  2651. } while (len);
  2652. return 0;
  2653. }
  2654. #ifdef CONFIG_LIVEPATCH
  2655. static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
  2656. {
  2657. if (get_modinfo(info, "livepatch")) {
  2658. mod->klp = true;
  2659. add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
  2660. pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
  2661. mod->name);
  2662. }
  2663. return 0;
  2664. }
  2665. #else /* !CONFIG_LIVEPATCH */
  2666. static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
  2667. {
  2668. if (get_modinfo(info, "livepatch")) {
  2669. pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
  2670. mod->name);
  2671. return -ENOEXEC;
  2672. }
  2673. return 0;
  2674. }
  2675. #endif /* CONFIG_LIVEPATCH */
  2676. static void check_modinfo_retpoline(struct module *mod, struct load_info *info)
  2677. {
  2678. if (retpoline_module_ok(get_modinfo(info, "retpoline")))
  2679. return;
  2680. pr_warn("%s: loading module not compiled with retpoline compiler.\n",
  2681. mod->name);
  2682. }
  2683. /* Sets info->hdr and info->len. */
  2684. static int copy_module_from_user(const void __user *umod, unsigned long len,
  2685. struct load_info *info)
  2686. {
  2687. int err;
  2688. info->len = len;
  2689. if (info->len < sizeof(*(info->hdr)))
  2690. return -ENOEXEC;
  2691. err = security_kernel_load_data(LOADING_MODULE, true);
  2692. if (err)
  2693. return err;
  2694. /* Suck in entire file: we'll want most of it. */
  2695. info->hdr = __vmalloc(info->len, GFP_KERNEL | __GFP_NOWARN);
  2696. if (!info->hdr)
  2697. return -ENOMEM;
  2698. if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
  2699. err = -EFAULT;
  2700. goto out;
  2701. }
  2702. err = security_kernel_post_load_data((char *)info->hdr, info->len,
  2703. LOADING_MODULE, "init_module");
  2704. out:
  2705. if (err)
  2706. vfree(info->hdr);
  2707. return err;
  2708. }
  2709. static void free_copy(struct load_info *info)
  2710. {
  2711. vfree(info->hdr);
  2712. }
  2713. static int rewrite_section_headers(struct load_info *info, int flags)
  2714. {
  2715. unsigned int i;
  2716. /* This should always be true, but let's be sure. */
  2717. info->sechdrs[0].sh_addr = 0;
  2718. for (i = 1; i < info->hdr->e_shnum; i++) {
  2719. Elf_Shdr *shdr = &info->sechdrs[i];
  2720. /* Mark all sections sh_addr with their address in the
  2721. temporary image. */
  2722. shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
  2723. #ifndef CONFIG_MODULE_UNLOAD
  2724. /* Don't load .exit sections */
  2725. if (module_exit_section(info->secstrings+shdr->sh_name))
  2726. shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
  2727. #endif
  2728. }
  2729. /* Track but don't keep modinfo and version sections. */
  2730. info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
  2731. info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
  2732. return 0;
  2733. }
  2734. /*
  2735. * Set up our basic convenience variables (pointers to section headers,
  2736. * search for module section index etc), and do some basic section
  2737. * verification.
  2738. *
  2739. * Set info->mod to the temporary copy of the module in info->hdr. The final one
  2740. * will be allocated in move_module().
  2741. */
  2742. static int setup_load_info(struct load_info *info, int flags)
  2743. {
  2744. unsigned int i;
  2745. /* Try to find a name early so we can log errors with a module name */
  2746. info->index.info = find_sec(info, ".modinfo");
  2747. if (info->index.info)
  2748. info->name = get_modinfo(info, "name");
  2749. /* Find internal symbols and strings. */
  2750. for (i = 1; i < info->hdr->e_shnum; i++) {
  2751. if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
  2752. info->index.sym = i;
  2753. info->index.str = info->sechdrs[i].sh_link;
  2754. info->strtab = (char *)info->hdr
  2755. + info->sechdrs[info->index.str].sh_offset;
  2756. break;
  2757. }
  2758. }
  2759. if (info->index.sym == 0) {
  2760. pr_warn("%s: module has no symbols (stripped?)\n",
  2761. info->name ?: "(missing .modinfo section or name field)");
  2762. return -ENOEXEC;
  2763. }
  2764. info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
  2765. if (!info->index.mod) {
  2766. pr_warn("%s: No module found in object\n",
  2767. info->name ?: "(missing .modinfo section or name field)");
  2768. return -ENOEXEC;
  2769. }
  2770. /* This is temporary: point mod into copy of data. */
  2771. info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset;
  2772. /*
  2773. * If we didn't load the .modinfo 'name' field earlier, fall back to
  2774. * on-disk struct mod 'name' field.
  2775. */
  2776. if (!info->name)
  2777. info->name = info->mod->name;
  2778. if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
  2779. info->index.vers = 0; /* Pretend no __versions section! */
  2780. else
  2781. info->index.vers = find_sec(info, "__versions");
  2782. info->index.pcpu = find_pcpusec(info);
  2783. return 0;
  2784. }
  2785. static int check_modinfo(struct module *mod, struct load_info *info, int flags)
  2786. {
  2787. const char *modmagic = get_modinfo(info, "vermagic");
  2788. int err;
  2789. if (flags & MODULE_INIT_IGNORE_VERMAGIC)
  2790. modmagic = NULL;
  2791. /* This is allowed: modprobe --force will invalidate it. */
  2792. if (!modmagic) {
  2793. err = try_to_force_load(mod, "bad vermagic");
  2794. if (err)
  2795. return err;
  2796. } else if (!same_magic(modmagic, vermagic, info->index.vers)) {
  2797. pr_err("%s: version magic '%s' should be '%s'\n",
  2798. info->name, modmagic, vermagic);
  2799. return -ENOEXEC;
  2800. }
  2801. if (!get_modinfo(info, "intree")) {
  2802. if (!test_taint(TAINT_OOT_MODULE))
  2803. pr_warn("%s: loading out-of-tree module taints kernel.\n",
  2804. mod->name);
  2805. add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
  2806. }
  2807. check_modinfo_retpoline(mod, info);
  2808. if (get_modinfo(info, "staging")) {
  2809. add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
  2810. pr_warn("%s: module is from the staging directory, the quality "
  2811. "is unknown, you have been warned.\n", mod->name);
  2812. }
  2813. err = check_modinfo_livepatch(mod, info);
  2814. if (err)
  2815. return err;
  2816. /* Set up license info based on the info section */
  2817. set_license(mod, get_modinfo(info, "license"));
  2818. return 0;
  2819. }
  2820. static int find_module_sections(struct module *mod, struct load_info *info)
  2821. {
  2822. mod->kp = section_objs(info, "__param",
  2823. sizeof(*mod->kp), &mod->num_kp);
  2824. mod->syms = section_objs(info, "__ksymtab",
  2825. sizeof(*mod->syms), &mod->num_syms);
  2826. mod->crcs = section_addr(info, "__kcrctab");
  2827. mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
  2828. sizeof(*mod->gpl_syms),
  2829. &mod->num_gpl_syms);
  2830. mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
  2831. mod->gpl_future_syms = section_objs(info,
  2832. "__ksymtab_gpl_future",
  2833. sizeof(*mod->gpl_future_syms),
  2834. &mod->num_gpl_future_syms);
  2835. mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
  2836. #ifdef CONFIG_UNUSED_SYMBOLS
  2837. mod->unused_syms = section_objs(info, "__ksymtab_unused",
  2838. sizeof(*mod->unused_syms),
  2839. &mod->num_unused_syms);
  2840. mod->unused_crcs = section_addr(info, "__kcrctab_unused");
  2841. mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
  2842. sizeof(*mod->unused_gpl_syms),
  2843. &mod->num_unused_gpl_syms);
  2844. mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
  2845. #endif
  2846. #ifdef CONFIG_CONSTRUCTORS
  2847. mod->ctors = section_objs(info, ".ctors",
  2848. sizeof(*mod->ctors), &mod->num_ctors);
  2849. if (!mod->ctors)
  2850. mod->ctors = section_objs(info, ".init_array",
  2851. sizeof(*mod->ctors), &mod->num_ctors);
  2852. else if (find_sec(info, ".init_array")) {
  2853. /*
  2854. * This shouldn't happen with same compiler and binutils
  2855. * building all parts of the module.
  2856. */
  2857. pr_warn("%s: has both .ctors and .init_array.\n",
  2858. mod->name);
  2859. return -EINVAL;
  2860. }
  2861. #endif
  2862. mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
  2863. &mod->noinstr_text_size);
  2864. #ifdef CONFIG_TRACEPOINTS
  2865. mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
  2866. sizeof(*mod->tracepoints_ptrs),
  2867. &mod->num_tracepoints);
  2868. #endif
  2869. #ifdef CONFIG_TREE_SRCU
  2870. mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
  2871. sizeof(*mod->srcu_struct_ptrs),
  2872. &mod->num_srcu_structs);
  2873. #endif
  2874. #ifdef CONFIG_BPF_EVENTS
  2875. mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
  2876. sizeof(*mod->bpf_raw_events),
  2877. &mod->num_bpf_raw_events);
  2878. #endif
  2879. #ifdef CONFIG_JUMP_LABEL
  2880. mod->jump_entries = section_objs(info, "__jump_table",
  2881. sizeof(*mod->jump_entries),
  2882. &mod->num_jump_entries);
  2883. #endif
  2884. #ifdef CONFIG_EVENT_TRACING
  2885. mod->trace_events = section_objs(info, "_ftrace_events",
  2886. sizeof(*mod->trace_events),
  2887. &mod->num_trace_events);
  2888. mod->trace_evals = section_objs(info, "_ftrace_eval_map",
  2889. sizeof(*mod->trace_evals),
  2890. &mod->num_trace_evals);
  2891. #endif
  2892. #ifdef CONFIG_TRACING
  2893. mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
  2894. sizeof(*mod->trace_bprintk_fmt_start),
  2895. &mod->num_trace_bprintk_fmt);
  2896. #endif
  2897. #ifdef CONFIG_FTRACE_MCOUNT_RECORD
  2898. /* sechdrs[0].sh_size is always zero */
  2899. mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
  2900. sizeof(*mod->ftrace_callsites),
  2901. &mod->num_ftrace_callsites);
  2902. #endif
  2903. #ifdef CONFIG_FUNCTION_ERROR_INJECTION
  2904. mod->ei_funcs = section_objs(info, "_error_injection_whitelist",
  2905. sizeof(*mod->ei_funcs),
  2906. &mod->num_ei_funcs);
  2907. #endif
  2908. #ifdef CONFIG_KPROBES
  2909. mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
  2910. &mod->kprobes_text_size);
  2911. mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
  2912. sizeof(unsigned long),
  2913. &mod->num_kprobe_blacklist);
  2914. #endif
  2915. #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
  2916. mod->static_call_sites = section_objs(info, ".static_call_sites",
  2917. sizeof(*mod->static_call_sites),
  2918. &mod->num_static_call_sites);
  2919. #endif
  2920. mod->extable = section_objs(info, "__ex_table",
  2921. sizeof(*mod->extable), &mod->num_exentries);
  2922. if (section_addr(info, "__obsparm"))
  2923. pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
  2924. info->debug = section_objs(info, "__dyndbg",
  2925. sizeof(*info->debug), &info->num_debug);
  2926. return 0;
  2927. }
  2928. static int move_module(struct module *mod, struct load_info *info)
  2929. {
  2930. int i;
  2931. void *ptr;
  2932. /* Do the allocs. */
  2933. ptr = module_alloc(mod->core_layout.size);
  2934. /*
  2935. * The pointer to this block is stored in the module structure
  2936. * which is inside the block. Just mark it as not being a
  2937. * leak.
  2938. */
  2939. kmemleak_not_leak(ptr);
  2940. if (!ptr)
  2941. return -ENOMEM;
  2942. memset(ptr, 0, mod->core_layout.size);
  2943. mod->core_layout.base = ptr;
  2944. if (mod->init_layout.size) {
  2945. ptr = module_alloc(mod->init_layout.size);
  2946. /*
  2947. * The pointer to this block is stored in the module structure
  2948. * which is inside the block. This block doesn't need to be
  2949. * scanned as it contains data and code that will be freed
  2950. * after the module is initialized.
  2951. */
  2952. kmemleak_ignore(ptr);
  2953. if (!ptr) {
  2954. module_memfree(mod->core_layout.base);
  2955. return -ENOMEM;
  2956. }
  2957. memset(ptr, 0, mod->init_layout.size);
  2958. mod->init_layout.base = ptr;
  2959. } else
  2960. mod->init_layout.base = NULL;
  2961. /* Transfer each section which specifies SHF_ALLOC */
  2962. pr_debug("final section addresses:\n");
  2963. for (i = 0; i < info->hdr->e_shnum; i++) {
  2964. void *dest;
  2965. Elf_Shdr *shdr = &info->sechdrs[i];
  2966. if (!(shdr->sh_flags & SHF_ALLOC))
  2967. continue;
  2968. if (shdr->sh_entsize & INIT_OFFSET_MASK)
  2969. dest = mod->init_layout.base
  2970. + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
  2971. else
  2972. dest = mod->core_layout.base + shdr->sh_entsize;
  2973. if (shdr->sh_type != SHT_NOBITS)
  2974. memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
  2975. /* Update sh_addr to point to copy in image. */
  2976. shdr->sh_addr = (unsigned long)dest;
  2977. pr_debug("\t0x%lx %s\n",
  2978. (long)shdr->sh_addr, info->secstrings + shdr->sh_name);
  2979. }
  2980. return 0;
  2981. }
  2982. static int check_module_license_and_versions(struct module *mod)
  2983. {
  2984. int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);
  2985. /*
  2986. * ndiswrapper is under GPL by itself, but loads proprietary modules.
  2987. * Don't use add_taint_module(), as it would prevent ndiswrapper from
  2988. * using GPL-only symbols it needs.
  2989. */
  2990. if (strcmp(mod->name, "ndiswrapper") == 0)
  2991. add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
  2992. /* driverloader was caught wrongly pretending to be under GPL */
  2993. if (strcmp(mod->name, "driverloader") == 0)
  2994. add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
  2995. LOCKDEP_NOW_UNRELIABLE);
  2996. /* lve claims to be GPL but upstream won't provide source */
  2997. if (strcmp(mod->name, "lve") == 0)
  2998. add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
  2999. LOCKDEP_NOW_UNRELIABLE);
  3000. if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
  3001. pr_warn("%s: module license taints kernel.\n", mod->name);
  3002. #ifdef CONFIG_MODVERSIONS
  3003. if ((mod->num_syms && !mod->crcs)
  3004. || (mod->num_gpl_syms && !mod->gpl_crcs)
  3005. || (mod->num_gpl_future_syms && !mod->gpl_future_crcs)
  3006. #ifdef CONFIG_UNUSED_SYMBOLS
  3007. || (mod->num_unused_syms && !mod->unused_crcs)
  3008. || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
  3009. #endif
  3010. ) {
  3011. return try_to_force_load(mod,
  3012. "no versions for exported symbols");
  3013. }
  3014. #endif
  3015. return 0;
  3016. }
  3017. static void flush_module_icache(const struct module *mod)
  3018. {
  3019. /*
  3020. * Flush the instruction cache, since we've played with text.
  3021. * Do it before processing of module parameters, so the module
  3022. * can provide parameter accessor functions of its own.
  3023. */
  3024. if (mod->init_layout.base)
  3025. flush_icache_range((unsigned long)mod->init_layout.base,
  3026. (unsigned long)mod->init_layout.base
  3027. + mod->init_layout.size);
  3028. flush_icache_range((unsigned long)mod->core_layout.base,
  3029. (unsigned long)mod->core_layout.base + mod->core_layout.size);
  3030. }
  3031. int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
  3032. Elf_Shdr *sechdrs,
  3033. char *secstrings,
  3034. struct module *mod)
  3035. {
  3036. return 0;
  3037. }
  3038. /* module_blacklist is a comma-separated list of module names */
  3039. static char *module_blacklist;
  3040. static bool blacklisted(const char *module_name)
  3041. {
  3042. const char *p;
  3043. size_t len;
  3044. if (!module_blacklist)
  3045. return false;
  3046. for (p = module_blacklist; *p; p += len) {
  3047. len = strcspn(p, ",");
  3048. if (strlen(module_name) == len && !memcmp(module_name, p, len))
  3049. return true;
  3050. if (p[len] == ',')
  3051. len++;
  3052. }
  3053. return false;
  3054. }
  3055. core_param(module_blacklist, module_blacklist, charp, 0400);
  3056. static struct module *layout_and_allocate(struct load_info *info, int flags)
  3057. {
  3058. struct module *mod;
  3059. unsigned int ndx;
  3060. int err;
  3061. err = check_modinfo(info->mod, info, flags);
  3062. if (err)
  3063. return ERR_PTR(err);
  3064. /* Allow arches to frob section contents and sizes. */
  3065. err = module_frob_arch_sections(info->hdr, info->sechdrs,
  3066. info->secstrings, info->mod);
  3067. if (err < 0)
  3068. return ERR_PTR(err);
  3069. err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
  3070. info->secstrings, info->mod);
  3071. if (err < 0)
  3072. return ERR_PTR(err);
  3073. /* We will do a special allocation for per-cpu sections later. */
  3074. info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
  3075. /*
  3076. * Mark ro_after_init section with SHF_RO_AFTER_INIT so that
  3077. * layout_sections() can put it in the right place.
  3078. * Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
  3079. */
  3080. ndx = find_sec(info, ".data..ro_after_init");
  3081. if (ndx)
  3082. info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
  3083. /*
  3084. * Mark the __jump_table section as ro_after_init as well: these data
  3085. * structures are never modified, with the exception of entries that
  3086. * refer to code in the __init section, which are annotated as such
  3087. * at module load time.
  3088. */
  3089. ndx = find_sec(info, "__jump_table");
  3090. if (ndx)
  3091. info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;
  3092. /* Determine total sizes, and put offsets in sh_entsize. For now
  3093. this is done generically; there doesn't appear to be any
  3094. special cases for the architectures. */
  3095. layout_sections(info->mod, info);
  3096. layout_symtab(info->mod, info);
  3097. /* Allocate and move to the final place */
  3098. err = move_module(info->mod, info);
  3099. if (err)
  3100. return ERR_PTR(err);
  3101. /* Module has been copied to its final place now: return it. */
  3102. mod = (void *)info->sechdrs[info->index.mod].sh_addr;
  3103. kmemleak_load_module(mod, info);
  3104. return mod;
  3105. }
  3106. /* mod is no longer valid after this! */
  3107. static void module_deallocate(struct module *mod, struct load_info *info)
  3108. {
  3109. percpu_modfree(mod);
  3110. module_arch_freeing_init(mod);
  3111. trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
  3112. (mod->init_layout.size)>>PAGE_SHIFT);
  3113. trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
  3114. (mod->init_layout.size)>>PAGE_SHIFT);
  3115. module_memfree(mod->init_layout.base);
  3116. trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base,
  3117. (mod->core_layout.size)>>PAGE_SHIFT);
  3118. trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base,
  3119. (mod->core_layout.size)>>PAGE_SHIFT);
  3120. module_memfree(mod->core_layout.base);
  3121. }
  3122. int __weak module_finalize(const Elf_Ehdr *hdr,
  3123. const Elf_Shdr *sechdrs,
  3124. struct module *me)
  3125. {
  3126. return 0;
  3127. }
  3128. static int post_relocation(struct module *mod, const struct load_info *info)
  3129. {
  3130. /* Sort exception table now relocations are done. */
  3131. sort_extable(mod->extable, mod->extable + mod->num_exentries);
  3132. /* Copy relocated percpu area over. */
  3133. percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
  3134. info->sechdrs[info->index.pcpu].sh_size);
  3135. /* Setup kallsyms-specific fields. */
  3136. add_kallsyms(mod, info);
  3137. /* Arch-specific module finalizing. */
  3138. return module_finalize(info->hdr, info->sechdrs, mod);
  3139. }
  3140. /* Is this module of this name done loading? No locks held. */
  3141. static bool finished_loading(const char *name)
  3142. {
  3143. struct module *mod;
  3144. bool ret;
  3145. /*
  3146. * The module_mutex should not be a heavily contended lock;
  3147. * if we get the occasional sleep here, we'll go an extra iteration
  3148. * in the wait_event_interruptible(), which is harmless.
  3149. */
  3150. sched_annotate_sleep();
  3151. mutex_lock(&module_mutex);
  3152. mod = find_module_all(name, strlen(name), true);
  3153. ret = !mod || mod->state == MODULE_STATE_LIVE;
  3154. mutex_unlock(&module_mutex);
  3155. return ret;
  3156. }
  3157. /* Call module constructors. */
  3158. static void do_mod_ctors(struct module *mod)
  3159. {
  3160. #ifdef CONFIG_CONSTRUCTORS
  3161. unsigned long i;
  3162. for (i = 0; i < mod->num_ctors; i++)
  3163. mod->ctors[i]();
  3164. #endif
  3165. }
  3166. /* For freeing module_init on success, in case kallsyms traversing */
  3167. struct mod_initfree {
  3168. struct llist_node node;
  3169. void *module_init;
  3170. };
  3171. static void do_free_init(struct work_struct *w)
  3172. {
  3173. struct llist_node *pos, *n, *list;
  3174. struct mod_initfree *initfree;
  3175. list = llist_del_all(&init_free_list);
  3176. synchronize_rcu();
  3177. llist_for_each_safe(pos, n, list) {
  3178. initfree = container_of(pos, struct mod_initfree, node);
  3179. module_memfree(initfree->module_init);
  3180. kfree(initfree);
  3181. }
  3182. }
  3183. /*
  3184. * This is where the real work happens.
  3185. *
  3186. * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
  3187. * helper command 'lx-symbols'.
  3188. */
  3189. static noinline int do_init_module(struct module *mod)
  3190. {
  3191. int ret = 0;
  3192. struct mod_initfree *freeinit;
  3193. freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
  3194. if (!freeinit) {
  3195. ret = -ENOMEM;
  3196. goto fail;
  3197. }
  3198. freeinit->module_init = mod->init_layout.base;
  3199. do_mod_ctors(mod);
  3200. /* Start the module */
  3201. if (mod->init != NULL)
  3202. ret = do_one_initcall(mod->init);
  3203. if (ret < 0) {
  3204. goto fail_free_freeinit;
  3205. }
  3206. if (ret > 0) {
  3207. pr_warn("%s: '%s'->init suspiciously returned %d, it should "
  3208. "follow 0/-E convention\n"
  3209. "%s: loading module anyway...\n",
  3210. __func__, mod->name, ret, __func__);
  3211. dump_stack();
  3212. }
  3213. /* Now it's a first class citizen! */
  3214. mod->state = MODULE_STATE_LIVE;
  3215. blocking_notifier_call_chain(&module_notify_list,
  3216. MODULE_STATE_LIVE, mod);
  3217. /* Delay uevent until module has finished its init routine */
  3218. kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
  3219. /*
  3220. * We need to finish all async code before the module init sequence
  3221. * is done. This has potential to deadlock if synchronous module
  3222. * loading is requested from async (which is not allowed!).
  3223. *
  3224. * See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous
  3225. * request_module() from async workers") for more details.
  3226. */
  3227. if (!mod->async_probe_requested)
  3228. async_synchronize_full();
  3229. ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base +
  3230. mod->init_layout.size);
  3231. mutex_lock(&module_mutex);
  3232. /* Drop initial reference. */
  3233. module_put(mod);
  3234. trim_init_extable(mod);
  3235. #ifdef CONFIG_KALLSYMS
  3236. /* Switch to core kallsyms now init is done: kallsyms may be walking! */
  3237. rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
  3238. #endif
  3239. module_enable_ro(mod, true);
  3240. trace_android_vh_set_module_permit_after_init(mod);
  3241. mod_tree_remove_init(mod);
  3242. module_arch_freeing_init(mod);
  3243. trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
  3244. (mod->init_layout.size)>>PAGE_SHIFT);
  3245. trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
  3246. (mod->init_layout.size)>>PAGE_SHIFT);
  3247. mod->init_layout.base = NULL;
  3248. mod->init_layout.size = 0;
  3249. mod->init_layout.ro_size = 0;
  3250. mod->init_layout.ro_after_init_size = 0;
  3251. mod->init_layout.text_size = 0;
  3252. /*
  3253. * We want to free module_init, but be aware that kallsyms may be
  3254. * walking this with preempt disabled. In all the failure paths, we
  3255. * call synchronize_rcu(), but we don't want to slow down the success
  3256. * path. module_memfree() cannot be called in an interrupt, so do the
  3257. * work and call synchronize_rcu() in a work queue.
  3258. *
  3259. * Note that module_alloc() on most architectures creates W+X page
  3260. * mappings which won't be cleaned up until do_free_init() runs. Any
  3261. * code such as mark_rodata_ro() which depends on those mappings to
  3262. * be cleaned up needs to sync with the queued work - ie
  3263. * rcu_barrier()
  3264. */
  3265. if (llist_add(&freeinit->node, &init_free_list))
  3266. schedule_work(&init_free_wq);
  3267. mutex_unlock(&module_mutex);
  3268. wake_up_all(&module_wq);
  3269. return 0;
  3270. fail_free_freeinit:
  3271. kfree(freeinit);
  3272. fail:
  3273. /* Try to protect us from buggy refcounters. */
  3274. mod->state = MODULE_STATE_GOING;
  3275. synchronize_rcu();
  3276. module_put(mod);
  3277. blocking_notifier_call_chain(&module_notify_list,
  3278. MODULE_STATE_GOING, mod);
  3279. klp_module_going(mod);
  3280. ftrace_release_mod(mod);
  3281. free_module(mod);
  3282. wake_up_all(&module_wq);
  3283. return ret;
  3284. }
  3285. static int may_init_module(void)
  3286. {
  3287. if (!capable(CAP_SYS_MODULE) || modules_disabled)
  3288. return -EPERM;
  3289. return 0;
  3290. }
  3291. /*
  3292. * We try to place it in the list now to make sure it's unique before
  3293. * we dedicate too many resources. In particular, temporary percpu
  3294. * memory exhaustion.
  3295. */
  3296. static int add_unformed_module(struct module *mod)
  3297. {
  3298. int err;
  3299. struct module *old;
  3300. mod->state = MODULE_STATE_UNFORMED;
  3301. again:
  3302. mutex_lock(&module_mutex);
  3303. old = find_module_all(mod->name, strlen(mod->name), true);
  3304. if (old != NULL) {
  3305. if (old->state != MODULE_STATE_LIVE) {
  3306. /* Wait in case it fails to load. */
  3307. mutex_unlock(&module_mutex);
  3308. err = wait_event_interruptible(module_wq,
  3309. finished_loading(mod->name));
  3310. if (err)
  3311. goto out_unlocked;
  3312. goto again;
  3313. }
  3314. err = -EEXIST;
  3315. goto out;
  3316. }
  3317. mod_update_bounds(mod);
  3318. list_add_rcu(&mod->list, &modules);
  3319. mod_tree_insert(mod);
  3320. err = 0;
  3321. out:
  3322. mutex_unlock(&module_mutex);
  3323. out_unlocked:
  3324. return err;
  3325. }
  3326. static int complete_formation(struct module *mod, struct load_info *info)
  3327. {
  3328. int err;
  3329. mutex_lock(&module_mutex);
  3330. /* Find duplicate symbols (must be called under lock). */
  3331. err = verify_exported_symbols(mod);
  3332. if (err < 0)
  3333. goto out;
  3334. /* This relies on module_mutex for list integrity. */
  3335. module_bug_finalize(info->hdr, info->sechdrs, mod);
  3336. module_enable_ro(mod, false);
  3337. module_enable_nx(mod);
  3338. module_enable_x(mod);
  3339. trace_android_vh_set_module_permit_before_init(mod);
  3340. /* Mark state as coming so strong_try_module_get() ignores us,
  3341. * but kallsyms etc. can see us. */
  3342. mod->state = MODULE_STATE_COMING;
  3343. mutex_unlock(&module_mutex);
  3344. return 0;
  3345. out:
  3346. mutex_unlock(&module_mutex);
  3347. return err;
  3348. }
  3349. static int prepare_coming_module(struct module *mod)
  3350. {
  3351. int err;
  3352. ftrace_module_enable(mod);
  3353. err = klp_module_coming(mod);
  3354. if (err)
  3355. return err;
  3356. err = blocking_notifier_call_chain_robust(&module_notify_list,
  3357. MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
  3358. err = notifier_to_errno(err);
  3359. if (err)
  3360. klp_module_going(mod);
  3361. return err;
  3362. }
  3363. static int unknown_module_param_cb(char *param, char *val, const char *modname,
  3364. void *arg)
  3365. {
  3366. struct module *mod = arg;
  3367. int ret;
  3368. if (strcmp(param, "async_probe") == 0) {
  3369. mod->async_probe_requested = true;
  3370. return 0;
  3371. }
  3372. /* Check for magic 'dyndbg' arg */
  3373. ret = ddebug_dyndbg_module_param_cb(param, val, modname);
  3374. if (ret != 0)
  3375. pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
  3376. return 0;
  3377. }
  3378. static void cfi_init(struct module *mod);
  3379. /* Allocate and load the module: note that size of section 0 is always
  3380. zero, and we rely on this for optional sections. */
  3381. static int load_module(struct load_info *info, const char __user *uargs,
  3382. int flags)
  3383. {
  3384. struct module *mod;
  3385. long err = 0;
  3386. char *after_dashes;
  3387. /*
  3388. * Do the signature check (if any) first. All that
  3389. * the signature check needs is info->len, it does
  3390. * not need any of the section info. That can be
  3391. * set up later. This will minimize the chances
  3392. * of a corrupt module causing problems before
  3393. * we even get to the signature check.
  3394. *
  3395. * The check will also adjust info->len by stripping
  3396. * off the sig length at the end of the module, making
  3397. * checks against info->len more correct.
  3398. */
  3399. err = module_sig_check(info, flags);
  3400. if (err)
  3401. goto free_copy;
  3402. /*
  3403. * Do basic sanity checks against the ELF header and
  3404. * sections.
  3405. */
  3406. err = elf_validity_check(info);
  3407. if (err) {
  3408. pr_err("Module has invalid ELF structures\n");
  3409. goto free_copy;
  3410. }
  3411. /*
  3412. * Everything checks out, so set up the section info
  3413. * in the info structure.
  3414. */
  3415. err = setup_load_info(info, flags);
  3416. if (err)
  3417. goto free_copy;
  3418. /*
  3419. * Now that we know we have the correct module name, check
  3420. * if it's blacklisted.
  3421. */
  3422. if (blacklisted(info->name)) {
  3423. err = -EPERM;
  3424. pr_err("Module %s is blacklisted\n", info->name);
  3425. goto free_copy;
  3426. }
  3427. err = rewrite_section_headers(info, flags);
  3428. if (err)
  3429. goto free_copy;
  3430. /* Check module struct version now, before we try to use module. */
  3431. if (!check_modstruct_version(info, info->mod)) {
  3432. err = -ENOEXEC;
  3433. goto free_copy;
  3434. }
  3435. /* Figure out module layout, and allocate all the memory. */
  3436. mod = layout_and_allocate(info, flags);
  3437. if (IS_ERR(mod)) {
  3438. err = PTR_ERR(mod);
  3439. goto free_copy;
  3440. }
  3441. audit_log_kern_module(mod->name);
  3442. /* Reserve our place in the list. */
  3443. err = add_unformed_module(mod);
  3444. if (err)
  3445. goto free_module;
  3446. #ifdef CONFIG_MODULE_SIG
  3447. mod->sig_ok = info->sig_ok;
  3448. if (!mod->sig_ok) {
  3449. pr_notice_once("%s: module verification failed: signature "
  3450. "and/or required key missing - tainting "
  3451. "kernel\n", mod->name);
  3452. add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
  3453. }
  3454. #endif
  3455. /* To avoid stressing percpu allocator, do this once we're unique. */
  3456. err = percpu_modalloc(mod, info);
  3457. if (err)
  3458. goto unlink_mod;
  3459. /* Now module is in final location, initialize linked lists, etc. */
  3460. err = module_unload_init(mod);
  3461. if (err)
  3462. goto unlink_mod;
  3463. init_param_lock(mod);
  3464. /* Now we've got everything in the final locations, we can
  3465. * find optional sections. */
  3466. err = find_module_sections(mod, info);
  3467. if (err)
  3468. goto free_unload;
  3469. err = check_module_license_and_versions(mod);
  3470. if (err)
  3471. goto free_unload;
  3472. /* Set up MODINFO_ATTR fields */
  3473. setup_modinfo(mod, info);
  3474. /* Fix up syms, so that st_value is a pointer to location. */
  3475. err = simplify_symbols(mod, info);
  3476. if (err < 0)
  3477. goto free_modinfo;
  3478. err = apply_relocations(mod, info);
  3479. if (err < 0)
  3480. goto free_modinfo;
  3481. err = post_relocation(mod, info);
  3482. if (err < 0)
  3483. goto free_modinfo;
  3484. flush_module_icache(mod);
  3485. /* Setup CFI for the module. */
  3486. cfi_init(mod);
  3487. /* Now copy in args */
  3488. mod->args = strndup_user(uargs, ~0UL >> 1);
  3489. if (IS_ERR(mod->args)) {
  3490. err = PTR_ERR(mod->args);
  3491. goto free_arch_cleanup;
  3492. }
  3493. dynamic_debug_setup(mod, info->debug, info->num_debug);
  3494. /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
  3495. ftrace_module_init(mod);
  3496. /* Finally it's fully formed, ready to start executing. */
  3497. err = complete_formation(mod, info);
  3498. if (err)
  3499. goto ddebug_cleanup;
  3500. err = prepare_coming_module(mod);
  3501. if (err)
  3502. goto bug_cleanup;
  3503. /* Module is ready to execute: parsing args may do that. */
  3504. after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
  3505. -32768, 32767, mod,
  3506. unknown_module_param_cb);
  3507. if (IS_ERR(after_dashes)) {
  3508. err = PTR_ERR(after_dashes);
  3509. goto coming_cleanup;
  3510. } else if (after_dashes) {
  3511. pr_warn("%s: parameters '%s' after `--' ignored\n",
  3512. mod->name, after_dashes);
  3513. }
  3514. /* Link in to sysfs. */
  3515. err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
  3516. if (err < 0)
  3517. goto coming_cleanup;
  3518. if (is_livepatch_module(mod)) {
  3519. err = copy_module_elf(mod, info);
  3520. if (err < 0)
  3521. goto sysfs_cleanup;
  3522. }
  3523. /* Get rid of temporary copy. */
  3524. free_copy(info);
  3525. /* Done! */
  3526. trace_module_load(mod);
  3527. return do_init_module(mod);
  3528. sysfs_cleanup:
  3529. mod_sysfs_teardown(mod);
  3530. coming_cleanup:
  3531. mod->state = MODULE_STATE_GOING;
  3532. destroy_params(mod->kp, mod->num_kp);
  3533. blocking_notifier_call_chain(&module_notify_list,
  3534. MODULE_STATE_GOING, mod);
  3535. klp_module_going(mod);
  3536. bug_cleanup:
  3537. mod->state = MODULE_STATE_GOING;
  3538. /* module_bug_cleanup needs module_mutex protection */
  3539. mutex_lock(&module_mutex);
  3540. module_bug_cleanup(mod);
  3541. mutex_unlock(&module_mutex);
  3542. ddebug_cleanup:
  3543. ftrace_release_mod(mod);
  3544. dynamic_debug_remove(mod, info->debug);
  3545. synchronize_rcu();
  3546. kfree(mod->args);
  3547. free_arch_cleanup:
  3548. cfi_cleanup(mod);
  3549. module_arch_cleanup(mod);
  3550. free_modinfo:
  3551. free_modinfo(mod);
  3552. free_unload:
  3553. module_unload_free(mod);
  3554. unlink_mod:
  3555. mutex_lock(&module_mutex);
  3556. /* Unlink carefully: kallsyms could be walking list. */
  3557. list_del_rcu(&mod->list);
  3558. mod_tree_remove(mod);
  3559. wake_up_all(&module_wq);
  3560. /* Wait for RCU-sched synchronizing before releasing mod->list. */
  3561. synchronize_rcu();
  3562. mutex_unlock(&module_mutex);
  3563. free_module:
  3564. /* Free lock-classes; relies on the preceding sync_rcu() */
  3565. lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
  3566. module_deallocate(mod, info);
  3567. free_copy:
  3568. free_copy(info);
  3569. return err;
  3570. }
  3571. SYSCALL_DEFINE3(init_module, void __user *, umod,
  3572. unsigned long, len, const char __user *, uargs)
  3573. {
  3574. int err;
  3575. struct load_info info = { };
  3576. err = may_init_module();
  3577. if (err)
  3578. return err;
  3579. pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
  3580. umod, len, uargs);
  3581. err = copy_module_from_user(umod, len, &info);
  3582. if (err)
  3583. return err;
  3584. return load_module(&info, uargs, 0);
  3585. }
  3586. SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
  3587. {
  3588. struct load_info info = { };
  3589. void *hdr = NULL;
  3590. int err;
  3591. err = may_init_module();
  3592. if (err)
  3593. return err;
  3594. pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
  3595. if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
  3596. |MODULE_INIT_IGNORE_VERMAGIC))
  3597. return -EINVAL;
  3598. err = kernel_read_file_from_fd(fd, 0, &hdr, INT_MAX, NULL,
  3599. READING_MODULE);
  3600. if (err < 0)
  3601. return err;
  3602. info.hdr = hdr;
  3603. info.len = err;
  3604. return load_module(&info, uargs, flags);
  3605. }
  3606. static inline int within(unsigned long addr, void *start, unsigned long size)
  3607. {
  3608. return ((void *)addr >= start && (void *)addr < start + size);
  3609. }
  3610. #ifdef CONFIG_KALLSYMS
  3611. /*
  3612. * This ignores the intensely annoying "mapping symbols" found
  3613. * in ARM ELF files: $a, $t and $d.
  3614. */
  3615. static inline int is_arm_mapping_symbol(const char *str)
  3616. {
  3617. if (str[0] == '.' && str[1] == 'L')
  3618. return true;
  3619. return str[0] == '$' && strchr("axtd", str[1])
  3620. && (str[2] == '\0' || str[2] == '.');
  3621. }
  3622. static inline int is_cfi_typeid_symbol(const char *str)
  3623. {
  3624. return !strncmp(str, "__typeid__", 10);
  3625. }
  3626. static const char *kallsyms_symbol_name(struct mod_kallsyms *kallsyms, unsigned int symnum)
  3627. {
  3628. return kallsyms->strtab + kallsyms->symtab[symnum].st_name;
  3629. }
  3630. /*
  3631. * Given a module and address, find the corresponding symbol and return its name
  3632. * while providing its size and offset if needed.
  3633. */
  3634. static const char *find_kallsyms_symbol(struct module *mod,
  3635. unsigned long addr,
  3636. unsigned long *size,
  3637. unsigned long *offset)
  3638. {
  3639. unsigned int i, best = 0;
  3640. unsigned long nextval, bestval;
  3641. struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
  3642. /* At worse, next value is at end of module */
  3643. if (within_module_init(addr, mod))
  3644. nextval = (unsigned long)mod->init_layout.base+mod->init_layout.text_size;
  3645. else
  3646. nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size;
  3647. bestval = kallsyms_symbol_value(&kallsyms->symtab[best]);
  3648. /* Scan for closest preceding symbol, and next symbol. (ELF
  3649. starts real symbols at 1). */
  3650. for (i = 1; i < kallsyms->num_symtab; i++) {
  3651. const Elf_Sym *sym = &kallsyms->symtab[i];
  3652. unsigned long thisval = kallsyms_symbol_value(sym);
  3653. if (sym->st_shndx == SHN_UNDEF)
  3654. continue;
  3655. /* We ignore unnamed symbols: they're uninformative
  3656. * and inserted at a whim. */
  3657. if (*kallsyms_symbol_name(kallsyms, i) == '\0'
  3658. || is_arm_mapping_symbol(kallsyms_symbol_name(kallsyms, i))
  3659. || is_cfi_typeid_symbol(kallsyms_symbol_name(kallsyms, i)))
  3660. continue;
  3661. if (thisval <= addr && thisval > bestval) {
  3662. best = i;
  3663. bestval = thisval;
  3664. }
  3665. if (thisval > addr && thisval < nextval)
  3666. nextval = thisval;
  3667. }
  3668. if (!best)
  3669. return NULL;
  3670. if (size)
  3671. *size = nextval - bestval;
  3672. if (offset)
  3673. *offset = addr - bestval;
  3674. return kallsyms_symbol_name(kallsyms, best);
  3675. }
  3676. void * __weak dereference_module_function_descriptor(struct module *mod,
  3677. void *ptr)
  3678. {
  3679. return ptr;
  3680. }
  3681. /* For kallsyms to ask for address resolution. NULL means not found. Careful
  3682. * not to lock to avoid deadlock on oopses, simply disable preemption. */
  3683. const char *module_address_lookup(unsigned long addr,
  3684. unsigned long *size,
  3685. unsigned long *offset,
  3686. char **modname,
  3687. char *namebuf)
  3688. {
  3689. const char *ret = NULL;
  3690. struct module *mod;
  3691. preempt_disable();
  3692. mod = __module_address(addr);
  3693. if (mod) {
  3694. if (modname)
  3695. *modname = mod->name;
  3696. ret = find_kallsyms_symbol(mod, addr, size, offset);
  3697. }
  3698. /* Make a copy in here where it's safe */
  3699. if (ret) {
  3700. strncpy(namebuf, ret, KSYM_NAME_LEN - 1);
  3701. ret = namebuf;
  3702. }
  3703. preempt_enable();
  3704. return ret;
  3705. }
  3706. int lookup_module_symbol_name(unsigned long addr, char *symname)
  3707. {
  3708. struct module *mod;
  3709. preempt_disable();
  3710. list_for_each_entry_rcu(mod, &modules, list) {
  3711. if (mod->state == MODULE_STATE_UNFORMED)
  3712. continue;
  3713. if (within_module(addr, mod)) {
  3714. const char *sym;
  3715. sym = find_kallsyms_symbol(mod, addr, NULL, NULL);
  3716. if (!sym)
  3717. goto out;
  3718. strlcpy(symname, sym, KSYM_NAME_LEN);
  3719. preempt_enable();
  3720. return 0;
  3721. }
  3722. }
  3723. out:
  3724. preempt_enable();
  3725. return -ERANGE;
  3726. }
  3727. int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size,
  3728. unsigned long *offset, char *modname, char *name)
  3729. {
  3730. struct module *mod;
  3731. preempt_disable();
  3732. list_for_each_entry_rcu(mod, &modules, list) {
  3733. if (mod->state == MODULE_STATE_UNFORMED)
  3734. continue;
  3735. if (within_module(addr, mod)) {
  3736. const char *sym;
  3737. sym = find_kallsyms_symbol(mod, addr, size, offset);
  3738. if (!sym)
  3739. goto out;
  3740. if (modname)
  3741. strlcpy(modname, mod->name, MODULE_NAME_LEN);
  3742. if (name)
  3743. strlcpy(name, sym, KSYM_NAME_LEN);
  3744. preempt_enable();
  3745. return 0;
  3746. }
  3747. }
  3748. out:
  3749. preempt_enable();
  3750. return -ERANGE;
  3751. }
  3752. int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
  3753. char *name, char *module_name, int *exported)
  3754. {
  3755. struct module *mod;
  3756. preempt_disable();
  3757. list_for_each_entry_rcu(mod, &modules, list) {
  3758. struct mod_kallsyms *kallsyms;
  3759. if (mod->state == MODULE_STATE_UNFORMED)
  3760. continue;
  3761. kallsyms = rcu_dereference_sched(mod->kallsyms);
  3762. if (symnum < kallsyms->num_symtab) {
  3763. const Elf_Sym *sym = &kallsyms->symtab[symnum];
  3764. *value = kallsyms_symbol_value(sym);
  3765. *type = kallsyms->typetab[symnum];
  3766. strlcpy(name, kallsyms_symbol_name(kallsyms, symnum), KSYM_NAME_LEN);
  3767. strlcpy(module_name, mod->name, MODULE_NAME_LEN);
  3768. *exported = is_exported(name, *value, mod);
  3769. preempt_enable();
  3770. return 0;
  3771. }
  3772. symnum -= kallsyms->num_symtab;
  3773. }
  3774. preempt_enable();
  3775. return -ERANGE;
  3776. }
  3777. /* Given a module and name of symbol, find and return the symbol's value */
  3778. static unsigned long find_kallsyms_symbol_value(struct module *mod, const char *name)
  3779. {
  3780. unsigned int i;
  3781. struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
  3782. for (i = 0; i < kallsyms->num_symtab; i++) {
  3783. const Elf_Sym *sym = &kallsyms->symtab[i];
  3784. if (strcmp(name, kallsyms_symbol_name(kallsyms, i)) == 0 &&
  3785. sym->st_shndx != SHN_UNDEF)
  3786. return kallsyms_symbol_value(sym);
  3787. }
  3788. return 0;
  3789. }
  3790. /* Look for this name: can be of form module:name. */
  3791. unsigned long module_kallsyms_lookup_name(const char *name)
  3792. {
  3793. struct module *mod;
  3794. char *colon;
  3795. unsigned long ret = 0;
  3796. /* Don't lock: we're in enough trouble already. */
  3797. preempt_disable();
  3798. if ((colon = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
  3799. if ((mod = find_module_all(name, colon - name, false)) != NULL)
  3800. ret = find_kallsyms_symbol_value(mod, colon+1);
  3801. } else {
  3802. list_for_each_entry_rcu(mod, &modules, list) {
  3803. if (mod->state == MODULE_STATE_UNFORMED)
  3804. continue;
  3805. if ((ret = find_kallsyms_symbol_value(mod, name)) != 0)
  3806. break;
  3807. }
  3808. }
  3809. preempt_enable();
  3810. return ret;
  3811. }
  3812. int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
  3813. struct module *, unsigned long),
  3814. void *data)
  3815. {
  3816. struct module *mod;
  3817. unsigned int i;
  3818. int ret;
  3819. module_assert_mutex();
  3820. list_for_each_entry(mod, &modules, list) {
  3821. /* We hold module_mutex: no need for rcu_dereference_sched */
  3822. struct mod_kallsyms *kallsyms = mod->kallsyms;
  3823. if (mod->state == MODULE_STATE_UNFORMED)
  3824. continue;
  3825. for (i = 0; i < kallsyms->num_symtab; i++) {
  3826. const Elf_Sym *sym = &kallsyms->symtab[i];
  3827. if (sym->st_shndx == SHN_UNDEF)
  3828. continue;
  3829. ret = fn(data, kallsyms_symbol_name(kallsyms, i),
  3830. mod, kallsyms_symbol_value(sym));
  3831. if (ret != 0)
  3832. return ret;
  3833. }
  3834. }
  3835. return 0;
  3836. }
  3837. #endif /* CONFIG_KALLSYMS */
  3838. static void cfi_init(struct module *mod)
  3839. {
  3840. #ifdef CONFIG_CFI_CLANG
  3841. initcall_t *init;
  3842. exitcall_t *exit;
  3843. rcu_read_lock_sched();
  3844. mod->cfi_check = (cfi_check_fn)
  3845. find_kallsyms_symbol_value(mod, "__cfi_check");
  3846. init = (initcall_t *)
  3847. find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
  3848. exit = (exitcall_t *)
  3849. find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
  3850. rcu_read_unlock_sched();
  3851. /* Fix init/exit functions to point to the CFI jump table */
  3852. if (init) mod->init = *init;
  3853. if (exit) mod->exit = *exit;
  3854. cfi_module_add(mod, module_addr_min);
  3855. #endif
  3856. }
  3857. static void cfi_cleanup(struct module *mod)
  3858. {
  3859. #ifdef CONFIG_CFI_CLANG
  3860. cfi_module_remove(mod, module_addr_min);
  3861. #endif
  3862. }
  3863. /* Maximum number of characters written by module_flags() */
  3864. #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
  3865. /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
  3866. static char *module_flags(struct module *mod, char *buf)
  3867. {
  3868. int bx = 0;
  3869. BUG_ON(mod->state == MODULE_STATE_UNFORMED);
  3870. if (mod->taints ||
  3871. mod->state == MODULE_STATE_GOING ||
  3872. mod->state == MODULE_STATE_COMING) {
  3873. buf[bx++] = '(';
  3874. bx += module_flags_taint(mod, buf + bx);
  3875. /* Show a - for module-is-being-unloaded */
  3876. if (mod->state == MODULE_STATE_GOING)
  3877. buf[bx++] = '-';
  3878. /* Show a + for module-is-being-loaded */
  3879. if (mod->state == MODULE_STATE_COMING)
  3880. buf[bx++] = '+';
  3881. buf[bx++] = ')';
  3882. }
  3883. buf[bx] = '\0';
  3884. return buf;
  3885. }
  3886. #ifdef CONFIG_PROC_FS
  3887. /* Called by the /proc file system to return a list of modules. */
  3888. static void *m_start(struct seq_file *m, loff_t *pos)
  3889. {
  3890. mutex_lock(&module_mutex);
  3891. return seq_list_start(&modules, *pos);
  3892. }
  3893. static void *m_next(struct seq_file *m, void *p, loff_t *pos)
  3894. {
  3895. return seq_list_next(p, &modules, pos);
  3896. }
  3897. static void m_stop(struct seq_file *m, void *p)
  3898. {
  3899. mutex_unlock(&module_mutex);
  3900. }
  3901. static int m_show(struct seq_file *m, void *p)
  3902. {
  3903. struct module *mod = list_entry(p, struct module, list);
  3904. char buf[MODULE_FLAGS_BUF_SIZE];
  3905. void *value;
  3906. /* We always ignore unformed modules. */
  3907. if (mod->state == MODULE_STATE_UNFORMED)
  3908. return 0;
  3909. seq_printf(m, "%s %u",
  3910. mod->name, mod->init_layout.size + mod->core_layout.size);
  3911. print_unload_info(m, mod);
  3912. /* Informative for users. */
  3913. seq_printf(m, " %s",
  3914. mod->state == MODULE_STATE_GOING ? "Unloading" :
  3915. mod->state == MODULE_STATE_COMING ? "Loading" :
  3916. "Live");
  3917. /* Used by oprofile and other similar tools. */
  3918. value = m->private ? NULL : mod->core_layout.base;
  3919. seq_printf(m, " 0x%px", value);
  3920. /* Taints info */
  3921. if (mod->taints)
  3922. seq_printf(m, " %s", module_flags(mod, buf));
  3923. seq_puts(m, "\n");
  3924. return 0;
  3925. }
  3926. /* Format: modulename size refcount deps address
  3927. Where refcount is a number or -, and deps is a comma-separated list
  3928. of depends or -.
  3929. */
  3930. static const struct seq_operations modules_op = {
  3931. .start = m_start,
  3932. .next = m_next,
  3933. .stop = m_stop,
  3934. .show = m_show
  3935. };
  3936. /*
  3937. * This also sets the "private" pointer to non-NULL if the
  3938. * kernel pointers should be hidden (so you can just test
  3939. * "m->private" to see if you should keep the values private).
  3940. *
  3941. * We use the same logic as for /proc/kallsyms.
  3942. */
  3943. static int modules_open(struct inode *inode, struct file *file)
  3944. {
  3945. int err = seq_open(file, &modules_op);
  3946. if (!err) {
  3947. struct seq_file *m = file->private_data;
  3948. m->private = kallsyms_show_value(file->f_cred) ? NULL : (void *)8ul;
  3949. }
  3950. return err;
  3951. }
  3952. static const struct proc_ops modules_proc_ops = {
  3953. .proc_flags = PROC_ENTRY_PERMANENT,
  3954. .proc_open = modules_open,
  3955. .proc_read = seq_read,
  3956. .proc_lseek = seq_lseek,
  3957. .proc_release = seq_release,
  3958. };
  3959. static int __init proc_modules_init(void)
  3960. {
  3961. proc_create("modules", 0, NULL, &modules_proc_ops);
  3962. return 0;
  3963. }
  3964. module_init(proc_modules_init);
  3965. #endif
  3966. /* Given an address, look for it in the module exception tables. */
  3967. const struct exception_table_entry *search_module_extables(unsigned long addr)
  3968. {
  3969. const struct exception_table_entry *e = NULL;
  3970. struct module *mod;
  3971. preempt_disable();
  3972. mod = __module_address(addr);
  3973. if (!mod)
  3974. goto out;
  3975. if (!mod->num_exentries)
  3976. goto out;
  3977. e = search_extable(mod->extable,
  3978. mod->num_exentries,
  3979. addr);
  3980. out:
  3981. preempt_enable();
  3982. /*
  3983. * Now, if we found one, we are running inside it now, hence
  3984. * we cannot unload the module, hence no refcnt needed.
  3985. */
  3986. return e;
  3987. }
  3988. /*
  3989. * is_module_address - is this address inside a module?
  3990. * @addr: the address to check.
  3991. *
  3992. * See is_module_text_address() if you simply want to see if the address
  3993. * is code (not data).
  3994. */
  3995. bool is_module_address(unsigned long addr)
  3996. {
  3997. bool ret;
  3998. preempt_disable();
  3999. ret = __module_address(addr) != NULL;
  4000. preempt_enable();
  4001. return ret;
  4002. }
  4003. /*
  4004. * __module_address - get the module which contains an address.
  4005. * @addr: the address.
  4006. *
  4007. * Must be called with preempt disabled or module mutex held so that
  4008. * module doesn't get freed during this.
  4009. */
  4010. struct module *__module_address(unsigned long addr)
  4011. {
  4012. struct module *mod;
  4013. if (addr < module_addr_min || addr > module_addr_max)
  4014. return NULL;
  4015. module_assert_mutex_or_preempt();
  4016. mod = mod_find(addr);
  4017. if (mod) {
  4018. BUG_ON(!within_module(addr, mod));
  4019. if (mod->state == MODULE_STATE_UNFORMED)
  4020. mod = NULL;
  4021. }
  4022. return mod;
  4023. }
  4024. /*
  4025. * is_module_text_address - is this address inside module code?
  4026. * @addr: the address to check.
  4027. *
  4028. * See is_module_address() if you simply want to see if the address is
  4029. * anywhere in a module. See kernel_text_address() for testing if an
  4030. * address corresponds to kernel or module code.
  4031. */
  4032. bool is_module_text_address(unsigned long addr)
  4033. {
  4034. bool ret;
  4035. preempt_disable();
  4036. ret = __module_text_address(addr) != NULL;
  4037. preempt_enable();
  4038. return ret;
  4039. }
  4040. /*
  4041. * __module_text_address - get the module whose code contains an address.
  4042. * @addr: the address.
  4043. *
  4044. * Must be called with preempt disabled or module mutex held so that
  4045. * module doesn't get freed during this.
  4046. */
  4047. struct module *__module_text_address(unsigned long addr)
  4048. {
  4049. struct module *mod = __module_address(addr);
  4050. if (mod) {
  4051. /* Make sure it's within the text section. */
  4052. if (!within(addr, mod->init_layout.base, mod->init_layout.text_size)
  4053. && !within(addr, mod->core_layout.base, mod->core_layout.text_size))
  4054. mod = NULL;
  4055. }
  4056. return mod;
  4057. }
  4058. /* Don't grab lock, we're oopsing. */
  4059. void print_modules(void)
  4060. {
  4061. struct module *mod;
  4062. char buf[MODULE_FLAGS_BUF_SIZE];
  4063. printk(KERN_DEFAULT "Modules linked in:");
  4064. /* Most callers should already have preempt disabled, but make sure */
  4065. preempt_disable();
  4066. list_for_each_entry_rcu(mod, &modules, list) {
  4067. if (mod->state == MODULE_STATE_UNFORMED)
  4068. continue;
  4069. pr_cont(" %s%s", mod->name, module_flags(mod, buf));
  4070. }
  4071. preempt_enable();
  4072. if (last_unloaded_module[0])
  4073. pr_cont(" [last unloaded: %s]", last_unloaded_module);
  4074. pr_cont("\n");
  4075. }
  4076. #ifdef CONFIG_ANDROID_DEBUG_SYMBOLS
  4077. void android_debug_for_each_module(int (*fn)(const char *mod_name, void *mod_addr, void *data),
  4078. void *data)
  4079. {
  4080. struct module *module;
  4081. preempt_disable();
  4082. list_for_each_entry_rcu(module, &modules, list) {
  4083. if (fn(module->name, module->core_layout.base, data))
  4084. goto out;
  4085. }
  4086. out:
  4087. preempt_enable();
  4088. }
  4089. EXPORT_SYMBOL_GPL(android_debug_for_each_module);
  4090. #endif
  4091. #ifdef CONFIG_MODVERSIONS
  4092. /* Generate the signature for all relevant module structures here.
  4093. * If these change, we don't want to try to parse the module. */
  4094. void module_layout(struct module *mod,
  4095. struct modversion_info *ver,
  4096. struct kernel_param *kp,
  4097. struct kernel_symbol *ks,
  4098. struct tracepoint * const *tp)
  4099. {
  4100. }
  4101. EXPORT_SYMBOL(module_layout);
  4102. #endif