dev.c 283 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * NET3 Protocol independent device support routines.
  4. *
  5. * Derived from the non IP parts of dev.c 1.0.19
  6. * Authors: Ross Biro
  7. * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  8. * Mark Evans, <evansmp@uhura.aston.ac.uk>
  9. *
  10. * Additional Authors:
  11. * Florian la Roche <rzsfl@rz.uni-sb.de>
  12. * Alan Cox <gw4pts@gw4pts.ampr.org>
  13. * David Hinds <dahinds@users.sourceforge.net>
  14. * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  15. * Adam Sulmicki <adam@cfar.umd.edu>
  16. * Pekka Riikonen <priikone@poesidon.pspt.fi>
  17. *
  18. * Changes:
  19. * D.J. Barrow : Fixed bug where dev->refcnt gets set
  20. * to 2 if register_netdev gets called
  21. * before net_dev_init & also removed a
  22. * few lines of code in the process.
  23. * Alan Cox : device private ioctl copies fields back.
  24. * Alan Cox : Transmit queue code does relevant
  25. * stunts to keep the queue safe.
  26. * Alan Cox : Fixed double lock.
  27. * Alan Cox : Fixed promisc NULL pointer trap
  28. * ???????? : Support the full private ioctl range
  29. * Alan Cox : Moved ioctl permission check into
  30. * drivers
  31. * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
  32. * Alan Cox : 100 backlog just doesn't cut it when
  33. * you start doing multicast video 8)
  34. * Alan Cox : Rewrote net_bh and list manager.
  35. * Alan Cox : Fix ETH_P_ALL echoback lengths.
  36. * Alan Cox : Took out transmit every packet pass
  37. * Saved a few bytes in the ioctl handler
  38. * Alan Cox : Network driver sets packet type before
  39. * calling netif_rx. Saves a function
  40. * call a packet.
  41. * Alan Cox : Hashed net_bh()
  42. * Richard Kooijman: Timestamp fixes.
  43. * Alan Cox : Wrong field in SIOCGIFDSTADDR
  44. * Alan Cox : Device lock protection.
  45. * Alan Cox : Fixed nasty side effect of device close
  46. * changes.
  47. * Rudi Cilibrasi : Pass the right thing to
  48. * set_mac_address()
  49. * Dave Miller : 32bit quantity for the device lock to
  50. * make it work out on a Sparc.
  51. * Bjorn Ekwall : Added KERNELD hack.
  52. * Alan Cox : Cleaned up the backlog initialise.
  53. * Craig Metz : SIOCGIFCONF fix if space for under
  54. * 1 device.
  55. * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
  56. * is no device open function.
  57. * Andi Kleen : Fix error reporting for SIOCGIFCONF
  58. * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
  59. * Cyrus Durgin : Cleaned for KMOD
  60. * Adam Sulmicki : Bug Fix : Network Device Unload
  61. * A network device unload needs to purge
  62. * the backlog queue.
  63. * Paul Rusty Russell : SIOCSIFNAME
  64. * Pekka Riikonen : Netdev boot-time settings code
  65. * Andrew Morton : Make unregister_netdevice wait
  66. * indefinitely on dev->refcnt
  67. * J Hadi Salim : - Backlog queue sampling
  68. * - netif_rx() feedback
  69. */
  70. #include <linux/uaccess.h>
  71. #include <linux/bitops.h>
  72. #include <linux/capability.h>
  73. #include <linux/cpu.h>
  74. #include <linux/types.h>
  75. #include <linux/kernel.h>
  76. #include <linux/hash.h>
  77. #include <linux/slab.h>
  78. #include <linux/sched.h>
  79. #include <linux/sched/mm.h>
  80. #include <linux/mutex.h>
  81. #include <linux/rwsem.h>
  82. #include <linux/string.h>
  83. #include <linux/mm.h>
  84. #include <linux/socket.h>
  85. #include <linux/sockios.h>
  86. #include <linux/errno.h>
  87. #include <linux/interrupt.h>
  88. #include <linux/if_ether.h>
  89. #include <linux/netdevice.h>
  90. #include <linux/etherdevice.h>
  91. #include <linux/ethtool.h>
  92. #include <linux/skbuff.h>
  93. #include <linux/bpf.h>
  94. #include <linux/bpf_trace.h>
  95. #include <net/net_namespace.h>
  96. #include <net/sock.h>
  97. #include <net/busy_poll.h>
  98. #include <linux/rtnetlink.h>
  99. #include <linux/stat.h>
  100. #include <net/dsa.h>
  101. #include <net/dst.h>
  102. #include <net/dst_metadata.h>
  103. #include <net/pkt_sched.h>
  104. #include <net/pkt_cls.h>
  105. #include <net/checksum.h>
  106. #include <net/xfrm.h>
  107. #include <linux/highmem.h>
  108. #include <linux/init.h>
  109. #include <linux/module.h>
  110. #include <linux/netpoll.h>
  111. #include <linux/rcupdate.h>
  112. #include <linux/delay.h>
  113. #include <net/iw_handler.h>
  114. #include <asm/current.h>
  115. #include <linux/audit.h>
  116. #include <linux/dmaengine.h>
  117. #include <linux/err.h>
  118. #include <linux/ctype.h>
  119. #include <linux/if_arp.h>
  120. #include <linux/if_vlan.h>
  121. #include <linux/ip.h>
  122. #include <net/ip.h>
  123. #include <net/mpls.h>
  124. #include <linux/ipv6.h>
  125. #include <linux/in.h>
  126. #include <linux/jhash.h>
  127. #include <linux/random.h>
  128. #include <trace/events/napi.h>
  129. #include <trace/events/net.h>
  130. #include <trace/events/skb.h>
  131. #include <linux/inetdevice.h>
  132. #include <linux/cpu_rmap.h>
  133. #include <linux/static_key.h>
  134. #include <linux/hashtable.h>
  135. #include <linux/vmalloc.h>
  136. #include <linux/if_macvlan.h>
  137. #include <linux/errqueue.h>
  138. #include <linux/hrtimer.h>
  139. #include <linux/netfilter_ingress.h>
  140. #include <linux/crash_dump.h>
  141. #include <linux/sctp.h>
  142. #include <net/udp_tunnel.h>
  143. #include <linux/net_namespace.h>
  144. #include <linux/indirect_call_wrapper.h>
  145. #include <net/devlink.h>
  146. #include <linux/pm_runtime.h>
  147. #include <linux/prandom.h>
  148. #include <trace/hooks/net.h>
  149. #include "net-sysfs.h"
  150. #define MAX_GRO_SKBS 8
  151. /* This should be increased if a protocol with a bigger head is added. */
  152. #define GRO_MAX_HEAD (MAX_HEADER + 128)
  153. static DEFINE_SPINLOCK(ptype_lock);
  154. static DEFINE_SPINLOCK(offload_lock);
  155. struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
  156. struct list_head ptype_all __read_mostly; /* Taps */
  157. static struct list_head offload_base __read_mostly;
  158. static int netif_rx_internal(struct sk_buff *skb);
  159. static int call_netdevice_notifiers_info(unsigned long val,
  160. struct netdev_notifier_info *info);
  161. static int call_netdevice_notifiers_extack(unsigned long val,
  162. struct net_device *dev,
  163. struct netlink_ext_ack *extack);
  164. static struct napi_struct *napi_by_id(unsigned int napi_id);
  165. /*
  166. * The @dev_base_head list is protected by @dev_base_lock and the rtnl
  167. * semaphore.
  168. *
  169. * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
  170. *
  171. * Writers must hold the rtnl semaphore while they loop through the
  172. * dev_base_head list, and hold dev_base_lock for writing when they do the
  173. * actual updates. This allows pure readers to access the list even
  174. * while a writer is preparing to update it.
  175. *
  176. * To put it another way, dev_base_lock is held for writing only to
  177. * protect against pure readers; the rtnl semaphore provides the
  178. * protection against other writers.
  179. *
  180. * See, for example usages, register_netdevice() and
  181. * unregister_netdevice(), which must be called with the rtnl
  182. * semaphore held.
  183. */
  184. DEFINE_RWLOCK(dev_base_lock);
  185. EXPORT_SYMBOL(dev_base_lock);
  186. static DEFINE_MUTEX(ifalias_mutex);
  187. /* protects napi_hash addition/deletion and napi_gen_id */
  188. static DEFINE_SPINLOCK(napi_hash_lock);
  189. static unsigned int napi_gen_id = NR_CPUS;
  190. static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
  191. static DECLARE_RWSEM(devnet_rename_sem);
  192. static inline void dev_base_seq_inc(struct net *net)
  193. {
  194. while (++net->dev_base_seq == 0)
  195. ;
  196. }
  197. static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
  198. {
  199. unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
  200. return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
  201. }
  202. static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
  203. {
  204. return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
  205. }
  206. static inline void rps_lock(struct softnet_data *sd)
  207. {
  208. #ifdef CONFIG_RPS
  209. spin_lock(&sd->input_pkt_queue.lock);
  210. #endif
  211. }
  212. static inline void rps_unlock(struct softnet_data *sd)
  213. {
  214. #ifdef CONFIG_RPS
  215. spin_unlock(&sd->input_pkt_queue.lock);
  216. #endif
  217. }
  218. static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
  219. const char *name)
  220. {
  221. struct netdev_name_node *name_node;
  222. name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
  223. if (!name_node)
  224. return NULL;
  225. INIT_HLIST_NODE(&name_node->hlist);
  226. name_node->dev = dev;
  227. name_node->name = name;
  228. return name_node;
  229. }
  230. static struct netdev_name_node *
  231. netdev_name_node_head_alloc(struct net_device *dev)
  232. {
  233. struct netdev_name_node *name_node;
  234. name_node = netdev_name_node_alloc(dev, dev->name);
  235. if (!name_node)
  236. return NULL;
  237. INIT_LIST_HEAD(&name_node->list);
  238. return name_node;
  239. }
  240. static void netdev_name_node_free(struct netdev_name_node *name_node)
  241. {
  242. kfree(name_node);
  243. }
  244. static void netdev_name_node_add(struct net *net,
  245. struct netdev_name_node *name_node)
  246. {
  247. hlist_add_head_rcu(&name_node->hlist,
  248. dev_name_hash(net, name_node->name));
  249. }
  250. static void netdev_name_node_del(struct netdev_name_node *name_node)
  251. {
  252. hlist_del_rcu(&name_node->hlist);
  253. }
  254. static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
  255. const char *name)
  256. {
  257. struct hlist_head *head = dev_name_hash(net, name);
  258. struct netdev_name_node *name_node;
  259. hlist_for_each_entry(name_node, head, hlist)
  260. if (!strcmp(name_node->name, name))
  261. return name_node;
  262. return NULL;
  263. }
  264. static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
  265. const char *name)
  266. {
  267. struct hlist_head *head = dev_name_hash(net, name);
  268. struct netdev_name_node *name_node;
  269. hlist_for_each_entry_rcu(name_node, head, hlist)
  270. if (!strcmp(name_node->name, name))
  271. return name_node;
  272. return NULL;
  273. }
  274. int netdev_name_node_alt_create(struct net_device *dev, const char *name)
  275. {
  276. struct netdev_name_node *name_node;
  277. struct net *net = dev_net(dev);
  278. name_node = netdev_name_node_lookup(net, name);
  279. if (name_node)
  280. return -EEXIST;
  281. name_node = netdev_name_node_alloc(dev, name);
  282. if (!name_node)
  283. return -ENOMEM;
  284. netdev_name_node_add(net, name_node);
  285. /* The node that holds dev->name acts as a head of per-device list. */
  286. list_add_tail(&name_node->list, &dev->name_node->list);
  287. return 0;
  288. }
  289. EXPORT_SYMBOL(netdev_name_node_alt_create);
  290. static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
  291. {
  292. list_del(&name_node->list);
  293. netdev_name_node_del(name_node);
  294. kfree(name_node->name);
  295. netdev_name_node_free(name_node);
  296. }
  297. int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
  298. {
  299. struct netdev_name_node *name_node;
  300. struct net *net = dev_net(dev);
  301. name_node = netdev_name_node_lookup(net, name);
  302. if (!name_node)
  303. return -ENOENT;
  304. /* lookup might have found our primary name or a name belonging
  305. * to another device.
  306. */
  307. if (name_node == dev->name_node || name_node->dev != dev)
  308. return -EINVAL;
  309. __netdev_name_node_alt_destroy(name_node);
  310. return 0;
  311. }
  312. EXPORT_SYMBOL(netdev_name_node_alt_destroy);
  313. static void netdev_name_node_alt_flush(struct net_device *dev)
  314. {
  315. struct netdev_name_node *name_node, *tmp;
  316. list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
  317. __netdev_name_node_alt_destroy(name_node);
  318. }
  319. /* Device list insertion */
  320. static void list_netdevice(struct net_device *dev)
  321. {
  322. struct net *net = dev_net(dev);
  323. ASSERT_RTNL();
  324. write_lock_bh(&dev_base_lock);
  325. list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
  326. netdev_name_node_add(net, dev->name_node);
  327. hlist_add_head_rcu(&dev->index_hlist,
  328. dev_index_hash(net, dev->ifindex));
  329. write_unlock_bh(&dev_base_lock);
  330. dev_base_seq_inc(net);
  331. }
  332. /* Device list removal
  333. * caller must respect a RCU grace period before freeing/reusing dev
  334. */
  335. static void unlist_netdevice(struct net_device *dev)
  336. {
  337. ASSERT_RTNL();
  338. /* Unlink dev from the device chain */
  339. write_lock_bh(&dev_base_lock);
  340. list_del_rcu(&dev->dev_list);
  341. netdev_name_node_del(dev->name_node);
  342. hlist_del_rcu(&dev->index_hlist);
  343. write_unlock_bh(&dev_base_lock);
  344. dev_base_seq_inc(dev_net(dev));
  345. }
  346. /*
  347. * Our notifier list
  348. */
  349. static RAW_NOTIFIER_HEAD(netdev_chain);
  350. /*
  351. * Device drivers call our routines to queue packets here. We empty the
  352. * queue in the local softnet handler.
  353. */
  354. DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
  355. EXPORT_PER_CPU_SYMBOL(softnet_data);
  356. #ifdef CONFIG_LOCKDEP
  357. /*
  358. * register_netdevice() inits txq->_xmit_lock and sets lockdep class
  359. * according to dev->type
  360. */
  361. static const unsigned short netdev_lock_type[] = {
  362. ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
  363. ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
  364. ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
  365. ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
  366. ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
  367. ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
  368. ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
  369. ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
  370. ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
  371. ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
  372. ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
  373. ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
  374. ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
  375. ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
  376. ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
  377. static const char *const netdev_lock_name[] = {
  378. "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
  379. "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
  380. "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
  381. "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
  382. "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
  383. "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
  384. "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
  385. "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
  386. "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
  387. "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
  388. "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
  389. "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
  390. "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
  391. "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
  392. "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
  393. static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
  394. static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
  395. static inline unsigned short netdev_lock_pos(unsigned short dev_type)
  396. {
  397. int i;
  398. for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
  399. if (netdev_lock_type[i] == dev_type)
  400. return i;
  401. /* the last key is used by default */
  402. return ARRAY_SIZE(netdev_lock_type) - 1;
  403. }
  404. static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
  405. unsigned short dev_type)
  406. {
  407. int i;
  408. i = netdev_lock_pos(dev_type);
  409. lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
  410. netdev_lock_name[i]);
  411. }
  412. static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
  413. {
  414. int i;
  415. i = netdev_lock_pos(dev->type);
  416. lockdep_set_class_and_name(&dev->addr_list_lock,
  417. &netdev_addr_lock_key[i],
  418. netdev_lock_name[i]);
  419. }
  420. #else
  421. static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
  422. unsigned short dev_type)
  423. {
  424. }
  425. static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
  426. {
  427. }
  428. #endif
  429. /*******************************************************************************
  430. *
  431. * Protocol management and registration routines
  432. *
  433. *******************************************************************************/
  434. /*
  435. * Add a protocol ID to the list. Now that the input handler is
  436. * smarter we can dispense with all the messy stuff that used to be
  437. * here.
  438. *
  439. * BEWARE!!! Protocol handlers, mangling input packets,
  440. * MUST BE last in hash buckets and checking protocol handlers
  441. * MUST start from promiscuous ptype_all chain in net_bh.
  442. * It is true now, do not change it.
  443. * Explanation follows: if protocol handler, mangling packet, will
  444. * be the first on list, it is not able to sense, that packet
  445. * is cloned and should be copied-on-write, so that it will
  446. * change it and subsequent readers will get broken packet.
  447. * --ANK (980803)
  448. */
  449. static inline struct list_head *ptype_head(const struct packet_type *pt)
  450. {
  451. struct list_head vendor_pt = { .next = NULL, };
  452. trace_android_vh_ptype_head(pt, &vendor_pt);
  453. if (vendor_pt.next)
  454. return vendor_pt.next;
  455. if (pt->type == htons(ETH_P_ALL))
  456. return pt->dev ? &pt->dev->ptype_all : &ptype_all;
  457. else
  458. return pt->dev ? &pt->dev->ptype_specific :
  459. &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
  460. }
  461. /**
  462. * dev_add_pack - add packet handler
  463. * @pt: packet type declaration
  464. *
  465. * Add a protocol handler to the networking stack. The passed &packet_type
  466. * is linked into kernel lists and may not be freed until it has been
  467. * removed from the kernel lists.
  468. *
  469. * This call does not sleep therefore it can not
  470. * guarantee all CPU's that are in middle of receiving packets
  471. * will see the new packet type (until the next received packet).
  472. */
  473. void dev_add_pack(struct packet_type *pt)
  474. {
  475. struct list_head *head = ptype_head(pt);
  476. spin_lock(&ptype_lock);
  477. list_add_rcu(&pt->list, head);
  478. spin_unlock(&ptype_lock);
  479. }
  480. EXPORT_SYMBOL(dev_add_pack);
  481. /**
  482. * __dev_remove_pack - remove packet handler
  483. * @pt: packet type declaration
  484. *
  485. * Remove a protocol handler that was previously added to the kernel
  486. * protocol handlers by dev_add_pack(). The passed &packet_type is removed
  487. * from the kernel lists and can be freed or reused once this function
  488. * returns.
  489. *
  490. * The packet type might still be in use by receivers
  491. * and must not be freed until after all the CPU's have gone
  492. * through a quiescent state.
  493. */
  494. void __dev_remove_pack(struct packet_type *pt)
  495. {
  496. struct list_head *head = ptype_head(pt);
  497. struct packet_type *pt1;
  498. spin_lock(&ptype_lock);
  499. list_for_each_entry(pt1, head, list) {
  500. if (pt == pt1) {
  501. list_del_rcu(&pt->list);
  502. goto out;
  503. }
  504. }
  505. pr_warn("dev_remove_pack: %p not found\n", pt);
  506. out:
  507. spin_unlock(&ptype_lock);
  508. }
  509. EXPORT_SYMBOL(__dev_remove_pack);
  510. /**
  511. * dev_remove_pack - remove packet handler
  512. * @pt: packet type declaration
  513. *
  514. * Remove a protocol handler that was previously added to the kernel
  515. * protocol handlers by dev_add_pack(). The passed &packet_type is removed
  516. * from the kernel lists and can be freed or reused once this function
  517. * returns.
  518. *
  519. * This call sleeps to guarantee that no CPU is looking at the packet
  520. * type after return.
  521. */
  522. void dev_remove_pack(struct packet_type *pt)
  523. {
  524. __dev_remove_pack(pt);
  525. synchronize_net();
  526. }
  527. EXPORT_SYMBOL(dev_remove_pack);
  528. /**
  529. * dev_add_offload - register offload handlers
  530. * @po: protocol offload declaration
  531. *
  532. * Add protocol offload handlers to the networking stack. The passed
  533. * &proto_offload is linked into kernel lists and may not be freed until
  534. * it has been removed from the kernel lists.
  535. *
  536. * This call does not sleep therefore it can not
  537. * guarantee all CPU's that are in middle of receiving packets
  538. * will see the new offload handlers (until the next received packet).
  539. */
  540. void dev_add_offload(struct packet_offload *po)
  541. {
  542. struct packet_offload *elem;
  543. spin_lock(&offload_lock);
  544. list_for_each_entry(elem, &offload_base, list) {
  545. if (po->priority < elem->priority)
  546. break;
  547. }
  548. list_add_rcu(&po->list, elem->list.prev);
  549. spin_unlock(&offload_lock);
  550. }
  551. EXPORT_SYMBOL(dev_add_offload);
  552. /**
  553. * __dev_remove_offload - remove offload handler
  554. * @po: packet offload declaration
  555. *
  556. * Remove a protocol offload handler that was previously added to the
  557. * kernel offload handlers by dev_add_offload(). The passed &offload_type
  558. * is removed from the kernel lists and can be freed or reused once this
  559. * function returns.
  560. *
  561. * The packet type might still be in use by receivers
  562. * and must not be freed until after all the CPU's have gone
  563. * through a quiescent state.
  564. */
  565. static void __dev_remove_offload(struct packet_offload *po)
  566. {
  567. struct list_head *head = &offload_base;
  568. struct packet_offload *po1;
  569. spin_lock(&offload_lock);
  570. list_for_each_entry(po1, head, list) {
  571. if (po == po1) {
  572. list_del_rcu(&po->list);
  573. goto out;
  574. }
  575. }
  576. pr_warn("dev_remove_offload: %p not found\n", po);
  577. out:
  578. spin_unlock(&offload_lock);
  579. }
  580. /**
  581. * dev_remove_offload - remove packet offload handler
  582. * @po: packet offload declaration
  583. *
  584. * Remove a packet offload handler that was previously added to the kernel
  585. * offload handlers by dev_add_offload(). The passed &offload_type is
  586. * removed from the kernel lists and can be freed or reused once this
  587. * function returns.
  588. *
  589. * This call sleeps to guarantee that no CPU is looking at the packet
  590. * type after return.
  591. */
  592. void dev_remove_offload(struct packet_offload *po)
  593. {
  594. __dev_remove_offload(po);
  595. synchronize_net();
  596. }
  597. EXPORT_SYMBOL(dev_remove_offload);
  598. /******************************************************************************
  599. *
  600. * Device Boot-time Settings Routines
  601. *
  602. ******************************************************************************/
  603. /* Boot time configuration table */
  604. static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
  605. /**
  606. * netdev_boot_setup_add - add new setup entry
  607. * @name: name of the device
  608. * @map: configured settings for the device
  609. *
  610. * Adds new setup entry to the dev_boot_setup list. The function
  611. * returns 0 on error and 1 on success. This is a generic routine to
  612. * all netdevices.
  613. */
  614. static int netdev_boot_setup_add(char *name, struct ifmap *map)
  615. {
  616. struct netdev_boot_setup *s;
  617. int i;
  618. s = dev_boot_setup;
  619. for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
  620. if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
  621. memset(s[i].name, 0, sizeof(s[i].name));
  622. strlcpy(s[i].name, name, IFNAMSIZ);
  623. memcpy(&s[i].map, map, sizeof(s[i].map));
  624. break;
  625. }
  626. }
  627. return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
  628. }
  629. /**
  630. * netdev_boot_setup_check - check boot time settings
  631. * @dev: the netdevice
  632. *
  633. * Check boot time settings for the device.
  634. * The found settings are set for the device to be used
  635. * later in the device probing.
  636. * Returns 0 if no settings found, 1 if they are.
  637. */
  638. int netdev_boot_setup_check(struct net_device *dev)
  639. {
  640. struct netdev_boot_setup *s = dev_boot_setup;
  641. int i;
  642. for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
  643. if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
  644. !strcmp(dev->name, s[i].name)) {
  645. dev->irq = s[i].map.irq;
  646. dev->base_addr = s[i].map.base_addr;
  647. dev->mem_start = s[i].map.mem_start;
  648. dev->mem_end = s[i].map.mem_end;
  649. return 1;
  650. }
  651. }
  652. return 0;
  653. }
  654. EXPORT_SYMBOL(netdev_boot_setup_check);
  655. /**
  656. * netdev_boot_base - get address from boot time settings
  657. * @prefix: prefix for network device
  658. * @unit: id for network device
  659. *
  660. * Check boot time settings for the base address of device.
  661. * The found settings are set for the device to be used
  662. * later in the device probing.
  663. * Returns 0 if no settings found.
  664. */
  665. unsigned long netdev_boot_base(const char *prefix, int unit)
  666. {
  667. const struct netdev_boot_setup *s = dev_boot_setup;
  668. char name[IFNAMSIZ];
  669. int i;
  670. sprintf(name, "%s%d", prefix, unit);
  671. /*
  672. * If device already registered then return base of 1
  673. * to indicate not to probe for this interface
  674. */
  675. if (__dev_get_by_name(&init_net, name))
  676. return 1;
  677. for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
  678. if (!strcmp(name, s[i].name))
  679. return s[i].map.base_addr;
  680. return 0;
  681. }
  682. /*
  683. * Saves at boot time configured settings for any netdevice.
  684. */
  685. int __init netdev_boot_setup(char *str)
  686. {
  687. int ints[5];
  688. struct ifmap map;
  689. str = get_options(str, ARRAY_SIZE(ints), ints);
  690. if (!str || !*str)
  691. return 0;
  692. /* Save settings */
  693. memset(&map, 0, sizeof(map));
  694. if (ints[0] > 0)
  695. map.irq = ints[1];
  696. if (ints[0] > 1)
  697. map.base_addr = ints[2];
  698. if (ints[0] > 2)
  699. map.mem_start = ints[3];
  700. if (ints[0] > 3)
  701. map.mem_end = ints[4];
  702. /* Add new entry to the list */
  703. return netdev_boot_setup_add(str, &map);
  704. }
  705. __setup("netdev=", netdev_boot_setup);
  706. /*******************************************************************************
  707. *
  708. * Device Interface Subroutines
  709. *
  710. *******************************************************************************/
  711. /**
  712. * dev_get_iflink - get 'iflink' value of a interface
  713. * @dev: targeted interface
  714. *
  715. * Indicates the ifindex the interface is linked to.
  716. * Physical interfaces have the same 'ifindex' and 'iflink' values.
  717. */
  718. int dev_get_iflink(const struct net_device *dev)
  719. {
  720. if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
  721. return dev->netdev_ops->ndo_get_iflink(dev);
  722. return dev->ifindex;
  723. }
  724. EXPORT_SYMBOL(dev_get_iflink);
  725. /**
  726. * dev_fill_metadata_dst - Retrieve tunnel egress information.
  727. * @dev: targeted interface
  728. * @skb: The packet.
  729. *
  730. * For better visibility of tunnel traffic OVS needs to retrieve
  731. * egress tunnel information for a packet. Following API allows
  732. * user to get this info.
  733. */
  734. int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
  735. {
  736. struct ip_tunnel_info *info;
  737. if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
  738. return -EINVAL;
  739. info = skb_tunnel_info_unclone(skb);
  740. if (!info)
  741. return -ENOMEM;
  742. if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
  743. return -EINVAL;
  744. return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
  745. }
  746. EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
  747. /**
  748. * __dev_get_by_name - find a device by its name
  749. * @net: the applicable net namespace
  750. * @name: name to find
  751. *
  752. * Find an interface by name. Must be called under RTNL semaphore
  753. * or @dev_base_lock. If the name is found a pointer to the device
  754. * is returned. If the name is not found then %NULL is returned. The
  755. * reference counters are not incremented so the caller must be
  756. * careful with locks.
  757. */
  758. struct net_device *__dev_get_by_name(struct net *net, const char *name)
  759. {
  760. struct netdev_name_node *node_name;
  761. node_name = netdev_name_node_lookup(net, name);
  762. return node_name ? node_name->dev : NULL;
  763. }
  764. EXPORT_SYMBOL(__dev_get_by_name);
  765. /**
  766. * dev_get_by_name_rcu - find a device by its name
  767. * @net: the applicable net namespace
  768. * @name: name to find
  769. *
  770. * Find an interface by name.
  771. * If the name is found a pointer to the device is returned.
  772. * If the name is not found then %NULL is returned.
  773. * The reference counters are not incremented so the caller must be
  774. * careful with locks. The caller must hold RCU lock.
  775. */
  776. struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
  777. {
  778. struct netdev_name_node *node_name;
  779. node_name = netdev_name_node_lookup_rcu(net, name);
  780. return node_name ? node_name->dev : NULL;
  781. }
  782. EXPORT_SYMBOL(dev_get_by_name_rcu);
  783. /**
  784. * dev_get_by_name - find a device by its name
  785. * @net: the applicable net namespace
  786. * @name: name to find
  787. *
  788. * Find an interface by name. This can be called from any
  789. * context and does its own locking. The returned handle has
  790. * the usage count incremented and the caller must use dev_put() to
  791. * release it when it is no longer needed. %NULL is returned if no
  792. * matching device is found.
  793. */
  794. struct net_device *dev_get_by_name(struct net *net, const char *name)
  795. {
  796. struct net_device *dev;
  797. rcu_read_lock();
  798. dev = dev_get_by_name_rcu(net, name);
  799. if (dev)
  800. dev_hold(dev);
  801. rcu_read_unlock();
  802. return dev;
  803. }
  804. EXPORT_SYMBOL(dev_get_by_name);
  805. /**
  806. * __dev_get_by_index - find a device by its ifindex
  807. * @net: the applicable net namespace
  808. * @ifindex: index of device
  809. *
  810. * Search for an interface by index. Returns %NULL if the device
  811. * is not found or a pointer to the device. The device has not
  812. * had its reference counter increased so the caller must be careful
  813. * about locking. The caller must hold either the RTNL semaphore
  814. * or @dev_base_lock.
  815. */
  816. struct net_device *__dev_get_by_index(struct net *net, int ifindex)
  817. {
  818. struct net_device *dev;
  819. struct hlist_head *head = dev_index_hash(net, ifindex);
  820. hlist_for_each_entry(dev, head, index_hlist)
  821. if (dev->ifindex == ifindex)
  822. return dev;
  823. return NULL;
  824. }
  825. EXPORT_SYMBOL(__dev_get_by_index);
  826. /**
  827. * dev_get_by_index_rcu - find a device by its ifindex
  828. * @net: the applicable net namespace
  829. * @ifindex: index of device
  830. *
  831. * Search for an interface by index. Returns %NULL if the device
  832. * is not found or a pointer to the device. The device has not
  833. * had its reference counter increased so the caller must be careful
  834. * about locking. The caller must hold RCU lock.
  835. */
  836. struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
  837. {
  838. struct net_device *dev;
  839. struct hlist_head *head = dev_index_hash(net, ifindex);
  840. hlist_for_each_entry_rcu(dev, head, index_hlist)
  841. if (dev->ifindex == ifindex)
  842. return dev;
  843. return NULL;
  844. }
  845. EXPORT_SYMBOL(dev_get_by_index_rcu);
  846. /**
  847. * dev_get_by_index - find a device by its ifindex
  848. * @net: the applicable net namespace
  849. * @ifindex: index of device
  850. *
  851. * Search for an interface by index. Returns NULL if the device
  852. * is not found or a pointer to the device. The device returned has
  853. * had a reference added and the pointer is safe until the user calls
  854. * dev_put to indicate they have finished with it.
  855. */
  856. struct net_device *dev_get_by_index(struct net *net, int ifindex)
  857. {
  858. struct net_device *dev;
  859. rcu_read_lock();
  860. dev = dev_get_by_index_rcu(net, ifindex);
  861. if (dev)
  862. dev_hold(dev);
  863. rcu_read_unlock();
  864. return dev;
  865. }
  866. EXPORT_SYMBOL(dev_get_by_index);
  867. /**
  868. * dev_get_by_napi_id - find a device by napi_id
  869. * @napi_id: ID of the NAPI struct
  870. *
  871. * Search for an interface by NAPI ID. Returns %NULL if the device
  872. * is not found or a pointer to the device. The device has not had
  873. * its reference counter increased so the caller must be careful
  874. * about locking. The caller must hold RCU lock.
  875. */
  876. struct net_device *dev_get_by_napi_id(unsigned int napi_id)
  877. {
  878. struct napi_struct *napi;
  879. WARN_ON_ONCE(!rcu_read_lock_held());
  880. if (napi_id < MIN_NAPI_ID)
  881. return NULL;
  882. napi = napi_by_id(napi_id);
  883. return napi ? napi->dev : NULL;
  884. }
  885. EXPORT_SYMBOL(dev_get_by_napi_id);
  886. /**
  887. * netdev_get_name - get a netdevice name, knowing its ifindex.
  888. * @net: network namespace
  889. * @name: a pointer to the buffer where the name will be stored.
  890. * @ifindex: the ifindex of the interface to get the name from.
  891. */
  892. int netdev_get_name(struct net *net, char *name, int ifindex)
  893. {
  894. struct net_device *dev;
  895. int ret;
  896. down_read(&devnet_rename_sem);
  897. rcu_read_lock();
  898. dev = dev_get_by_index_rcu(net, ifindex);
  899. if (!dev) {
  900. ret = -ENODEV;
  901. goto out;
  902. }
  903. strcpy(name, dev->name);
  904. ret = 0;
  905. out:
  906. rcu_read_unlock();
  907. up_read(&devnet_rename_sem);
  908. return ret;
  909. }
  910. /**
  911. * dev_getbyhwaddr_rcu - find a device by its hardware address
  912. * @net: the applicable net namespace
  913. * @type: media type of device
  914. * @ha: hardware address
  915. *
  916. * Search for an interface by MAC address. Returns NULL if the device
  917. * is not found or a pointer to the device.
  918. * The caller must hold RCU or RTNL.
  919. * The returned device has not had its ref count increased
  920. * and the caller must therefore be careful about locking
  921. *
  922. */
  923. struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
  924. const char *ha)
  925. {
  926. struct net_device *dev;
  927. for_each_netdev_rcu(net, dev)
  928. if (dev->type == type &&
  929. !memcmp(dev->dev_addr, ha, dev->addr_len))
  930. return dev;
  931. return NULL;
  932. }
  933. EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
  934. struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
  935. {
  936. struct net_device *dev;
  937. ASSERT_RTNL();
  938. for_each_netdev(net, dev)
  939. if (dev->type == type)
  940. return dev;
  941. return NULL;
  942. }
  943. EXPORT_SYMBOL(__dev_getfirstbyhwtype);
  944. struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
  945. {
  946. struct net_device *dev, *ret = NULL;
  947. rcu_read_lock();
  948. for_each_netdev_rcu(net, dev)
  949. if (dev->type == type) {
  950. dev_hold(dev);
  951. ret = dev;
  952. break;
  953. }
  954. rcu_read_unlock();
  955. return ret;
  956. }
  957. EXPORT_SYMBOL(dev_getfirstbyhwtype);
  958. /**
  959. * __dev_get_by_flags - find any device with given flags
  960. * @net: the applicable net namespace
  961. * @if_flags: IFF_* values
  962. * @mask: bitmask of bits in if_flags to check
  963. *
  964. * Search for any interface with the given flags. Returns NULL if a device
  965. * is not found or a pointer to the device. Must be called inside
  966. * rtnl_lock(), and result refcount is unchanged.
  967. */
  968. struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
  969. unsigned short mask)
  970. {
  971. struct net_device *dev, *ret;
  972. ASSERT_RTNL();
  973. ret = NULL;
  974. for_each_netdev(net, dev) {
  975. if (((dev->flags ^ if_flags) & mask) == 0) {
  976. ret = dev;
  977. break;
  978. }
  979. }
  980. return ret;
  981. }
  982. EXPORT_SYMBOL(__dev_get_by_flags);
  983. /**
  984. * dev_valid_name - check if name is okay for network device
  985. * @name: name string
  986. *
  987. * Network device names need to be valid file names to
  988. * allow sysfs to work. We also disallow any kind of
  989. * whitespace.
  990. */
  991. bool dev_valid_name(const char *name)
  992. {
  993. if (*name == '\0')
  994. return false;
  995. if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
  996. return false;
  997. if (!strcmp(name, ".") || !strcmp(name, ".."))
  998. return false;
  999. while (*name) {
  1000. if (*name == '/' || *name == ':' || isspace(*name))
  1001. return false;
  1002. name++;
  1003. }
  1004. return true;
  1005. }
  1006. EXPORT_SYMBOL(dev_valid_name);
  1007. /**
  1008. * __dev_alloc_name - allocate a name for a device
  1009. * @net: network namespace to allocate the device name in
  1010. * @name: name format string
  1011. * @buf: scratch buffer and result name string
  1012. *
  1013. * Passed a format string - eg "lt%d" it will try and find a suitable
  1014. * id. It scans list of devices to build up a free map, then chooses
  1015. * the first empty slot. The caller must hold the dev_base or rtnl lock
  1016. * while allocating the name and adding the device in order to avoid
  1017. * duplicates.
  1018. * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
  1019. * Returns the number of the unit assigned or a negative errno code.
  1020. */
  1021. static int __dev_alloc_name(struct net *net, const char *name, char *buf)
  1022. {
  1023. int i = 0;
  1024. const char *p;
  1025. const int max_netdevices = 8*PAGE_SIZE;
  1026. unsigned long *inuse;
  1027. struct net_device *d;
  1028. if (!dev_valid_name(name))
  1029. return -EINVAL;
  1030. p = strchr(name, '%');
  1031. if (p) {
  1032. /*
  1033. * Verify the string as this thing may have come from
  1034. * the user. There must be either one "%d" and no other "%"
  1035. * characters.
  1036. */
  1037. if (p[1] != 'd' || strchr(p + 2, '%'))
  1038. return -EINVAL;
  1039. /* Use one page as a bit array of possible slots */
  1040. inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
  1041. if (!inuse)
  1042. return -ENOMEM;
  1043. for_each_netdev(net, d) {
  1044. struct netdev_name_node *name_node;
  1045. list_for_each_entry(name_node, &d->name_node->list, list) {
  1046. if (!sscanf(name_node->name, name, &i))
  1047. continue;
  1048. if (i < 0 || i >= max_netdevices)
  1049. continue;
  1050. /* avoid cases where sscanf is not exact inverse of printf */
  1051. snprintf(buf, IFNAMSIZ, name, i);
  1052. if (!strncmp(buf, name_node->name, IFNAMSIZ))
  1053. set_bit(i, inuse);
  1054. }
  1055. if (!sscanf(d->name, name, &i))
  1056. continue;
  1057. if (i < 0 || i >= max_netdevices)
  1058. continue;
  1059. /* avoid cases where sscanf is not exact inverse of printf */
  1060. snprintf(buf, IFNAMSIZ, name, i);
  1061. if (!strncmp(buf, d->name, IFNAMSIZ))
  1062. set_bit(i, inuse);
  1063. }
  1064. i = find_first_zero_bit(inuse, max_netdevices);
  1065. free_page((unsigned long) inuse);
  1066. }
  1067. snprintf(buf, IFNAMSIZ, name, i);
  1068. if (!__dev_get_by_name(net, buf))
  1069. return i;
  1070. /* It is possible to run out of possible slots
  1071. * when the name is long and there isn't enough space left
  1072. * for the digits, or if all bits are used.
  1073. */
  1074. return -ENFILE;
  1075. }
  1076. static int dev_alloc_name_ns(struct net *net,
  1077. struct net_device *dev,
  1078. const char *name)
  1079. {
  1080. char buf[IFNAMSIZ];
  1081. int ret;
  1082. BUG_ON(!net);
  1083. ret = __dev_alloc_name(net, name, buf);
  1084. if (ret >= 0)
  1085. strlcpy(dev->name, buf, IFNAMSIZ);
  1086. return ret;
  1087. }
  1088. /**
  1089. * dev_alloc_name - allocate a name for a device
  1090. * @dev: device
  1091. * @name: name format string
  1092. *
  1093. * Passed a format string - eg "lt%d" it will try and find a suitable
  1094. * id. It scans list of devices to build up a free map, then chooses
  1095. * the first empty slot. The caller must hold the dev_base or rtnl lock
  1096. * while allocating the name and adding the device in order to avoid
  1097. * duplicates.
  1098. * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
  1099. * Returns the number of the unit assigned or a negative errno code.
  1100. */
  1101. int dev_alloc_name(struct net_device *dev, const char *name)
  1102. {
  1103. return dev_alloc_name_ns(dev_net(dev), dev, name);
  1104. }
  1105. EXPORT_SYMBOL(dev_alloc_name);
  1106. static int dev_get_valid_name(struct net *net, struct net_device *dev,
  1107. const char *name)
  1108. {
  1109. BUG_ON(!net);
  1110. if (!dev_valid_name(name))
  1111. return -EINVAL;
  1112. if (strchr(name, '%'))
  1113. return dev_alloc_name_ns(net, dev, name);
  1114. else if (__dev_get_by_name(net, name))
  1115. return -EEXIST;
  1116. else if (dev->name != name)
  1117. strlcpy(dev->name, name, IFNAMSIZ);
  1118. return 0;
  1119. }
  1120. /**
  1121. * dev_change_name - change name of a device
  1122. * @dev: device
  1123. * @newname: name (or format string) must be at least IFNAMSIZ
  1124. *
  1125. * Change name of a device, can pass format strings "eth%d".
  1126. * for wildcarding.
  1127. */
  1128. int dev_change_name(struct net_device *dev, const char *newname)
  1129. {
  1130. unsigned char old_assign_type;
  1131. char oldname[IFNAMSIZ];
  1132. int err = 0;
  1133. int ret;
  1134. struct net *net;
  1135. ASSERT_RTNL();
  1136. BUG_ON(!dev_net(dev));
  1137. net = dev_net(dev);
  1138. /* Some auto-enslaved devices e.g. failover slaves are
  1139. * special, as userspace might rename the device after
  1140. * the interface had been brought up and running since
  1141. * the point kernel initiated auto-enslavement. Allow
  1142. * live name change even when these slave devices are
  1143. * up and running.
  1144. *
  1145. * Typically, users of these auto-enslaving devices
  1146. * don't actually care about slave name change, as
  1147. * they are supposed to operate on master interface
  1148. * directly.
  1149. */
  1150. if (dev->flags & IFF_UP &&
  1151. likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
  1152. return -EBUSY;
  1153. down_write(&devnet_rename_sem);
  1154. if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
  1155. up_write(&devnet_rename_sem);
  1156. return 0;
  1157. }
  1158. memcpy(oldname, dev->name, IFNAMSIZ);
  1159. err = dev_get_valid_name(net, dev, newname);
  1160. if (err < 0) {
  1161. up_write(&devnet_rename_sem);
  1162. return err;
  1163. }
  1164. if (oldname[0] && !strchr(oldname, '%'))
  1165. netdev_info(dev, "renamed from %s\n", oldname);
  1166. old_assign_type = dev->name_assign_type;
  1167. dev->name_assign_type = NET_NAME_RENAMED;
  1168. rollback:
  1169. ret = device_rename(&dev->dev, dev->name);
  1170. if (ret) {
  1171. memcpy(dev->name, oldname, IFNAMSIZ);
  1172. dev->name_assign_type = old_assign_type;
  1173. up_write(&devnet_rename_sem);
  1174. return ret;
  1175. }
  1176. up_write(&devnet_rename_sem);
  1177. netdev_adjacent_rename_links(dev, oldname);
  1178. write_lock_bh(&dev_base_lock);
  1179. netdev_name_node_del(dev->name_node);
  1180. write_unlock_bh(&dev_base_lock);
  1181. synchronize_rcu();
  1182. write_lock_bh(&dev_base_lock);
  1183. netdev_name_node_add(net, dev->name_node);
  1184. write_unlock_bh(&dev_base_lock);
  1185. ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
  1186. ret = notifier_to_errno(ret);
  1187. if (ret) {
  1188. /* err >= 0 after dev_alloc_name() or stores the first errno */
  1189. if (err >= 0) {
  1190. err = ret;
  1191. down_write(&devnet_rename_sem);
  1192. memcpy(dev->name, oldname, IFNAMSIZ);
  1193. memcpy(oldname, newname, IFNAMSIZ);
  1194. dev->name_assign_type = old_assign_type;
  1195. old_assign_type = NET_NAME_RENAMED;
  1196. goto rollback;
  1197. } else {
  1198. pr_err("%s: name change rollback failed: %d\n",
  1199. dev->name, ret);
  1200. }
  1201. }
  1202. return err;
  1203. }
  1204. /**
  1205. * dev_set_alias - change ifalias of a device
  1206. * @dev: device
  1207. * @alias: name up to IFALIASZ
  1208. * @len: limit of bytes to copy from info
  1209. *
  1210. * Set ifalias for a device,
  1211. */
  1212. int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
  1213. {
  1214. struct dev_ifalias *new_alias = NULL;
  1215. if (len >= IFALIASZ)
  1216. return -EINVAL;
  1217. if (len) {
  1218. new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
  1219. if (!new_alias)
  1220. return -ENOMEM;
  1221. memcpy(new_alias->ifalias, alias, len);
  1222. new_alias->ifalias[len] = 0;
  1223. }
  1224. mutex_lock(&ifalias_mutex);
  1225. new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
  1226. mutex_is_locked(&ifalias_mutex));
  1227. mutex_unlock(&ifalias_mutex);
  1228. if (new_alias)
  1229. kfree_rcu(new_alias, rcuhead);
  1230. return len;
  1231. }
  1232. EXPORT_SYMBOL(dev_set_alias);
  1233. /**
  1234. * dev_get_alias - get ifalias of a device
  1235. * @dev: device
  1236. * @name: buffer to store name of ifalias
  1237. * @len: size of buffer
  1238. *
  1239. * get ifalias for a device. Caller must make sure dev cannot go
  1240. * away, e.g. rcu read lock or own a reference count to device.
  1241. */
  1242. int dev_get_alias(const struct net_device *dev, char *name, size_t len)
  1243. {
  1244. const struct dev_ifalias *alias;
  1245. int ret = 0;
  1246. rcu_read_lock();
  1247. alias = rcu_dereference(dev->ifalias);
  1248. if (alias)
  1249. ret = snprintf(name, len, "%s", alias->ifalias);
  1250. rcu_read_unlock();
  1251. return ret;
  1252. }
  1253. /**
  1254. * netdev_features_change - device changes features
  1255. * @dev: device to cause notification
  1256. *
  1257. * Called to indicate a device has changed features.
  1258. */
  1259. void netdev_features_change(struct net_device *dev)
  1260. {
  1261. call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
  1262. }
  1263. EXPORT_SYMBOL(netdev_features_change);
  1264. /**
  1265. * netdev_state_change - device changes state
  1266. * @dev: device to cause notification
  1267. *
  1268. * Called to indicate a device has changed state. This function calls
  1269. * the notifier chains for netdev_chain and sends a NEWLINK message
  1270. * to the routing socket.
  1271. */
  1272. void netdev_state_change(struct net_device *dev)
  1273. {
  1274. if (dev->flags & IFF_UP) {
  1275. struct netdev_notifier_change_info change_info = {
  1276. .info.dev = dev,
  1277. };
  1278. call_netdevice_notifiers_info(NETDEV_CHANGE,
  1279. &change_info.info);
  1280. rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
  1281. }
  1282. }
  1283. EXPORT_SYMBOL(netdev_state_change);
  1284. /**
  1285. * netdev_notify_peers - notify network peers about existence of @dev
  1286. * @dev: network device
  1287. *
  1288. * Generate traffic such that interested network peers are aware of
  1289. * @dev, such as by generating a gratuitous ARP. This may be used when
  1290. * a device wants to inform the rest of the network about some sort of
  1291. * reconfiguration such as a failover event or virtual machine
  1292. * migration.
  1293. */
  1294. void netdev_notify_peers(struct net_device *dev)
  1295. {
  1296. rtnl_lock();
  1297. call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
  1298. call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
  1299. rtnl_unlock();
  1300. }
  1301. EXPORT_SYMBOL(netdev_notify_peers);
  1302. static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
  1303. {
  1304. const struct net_device_ops *ops = dev->netdev_ops;
  1305. int ret;
  1306. ASSERT_RTNL();
  1307. if (!netif_device_present(dev)) {
  1308. /* may be detached because parent is runtime-suspended */
  1309. if (dev->dev.parent)
  1310. pm_runtime_resume(dev->dev.parent);
  1311. if (!netif_device_present(dev))
  1312. return -ENODEV;
  1313. }
  1314. /* Block netpoll from trying to do any rx path servicing.
  1315. * If we don't do this there is a chance ndo_poll_controller
  1316. * or ndo_poll may be running while we open the device
  1317. */
  1318. netpoll_poll_disable(dev);
  1319. ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
  1320. ret = notifier_to_errno(ret);
  1321. if (ret)
  1322. return ret;
  1323. set_bit(__LINK_STATE_START, &dev->state);
  1324. if (ops->ndo_validate_addr)
  1325. ret = ops->ndo_validate_addr(dev);
  1326. if (!ret && ops->ndo_open)
  1327. ret = ops->ndo_open(dev);
  1328. netpoll_poll_enable(dev);
  1329. if (ret)
  1330. clear_bit(__LINK_STATE_START, &dev->state);
  1331. else {
  1332. dev->flags |= IFF_UP;
  1333. dev_set_rx_mode(dev);
  1334. dev_activate(dev);
  1335. add_device_randomness(dev->dev_addr, dev->addr_len);
  1336. }
  1337. return ret;
  1338. }
  1339. /**
  1340. * dev_open - prepare an interface for use.
  1341. * @dev: device to open
  1342. * @extack: netlink extended ack
  1343. *
  1344. * Takes a device from down to up state. The device's private open
  1345. * function is invoked and then the multicast lists are loaded. Finally
  1346. * the device is moved into the up state and a %NETDEV_UP message is
  1347. * sent to the netdev notifier chain.
  1348. *
  1349. * Calling this function on an active interface is a nop. On a failure
  1350. * a negative errno code is returned.
  1351. */
  1352. int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
  1353. {
  1354. int ret;
  1355. if (dev->flags & IFF_UP)
  1356. return 0;
  1357. ret = __dev_open(dev, extack);
  1358. if (ret < 0)
  1359. return ret;
  1360. rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
  1361. call_netdevice_notifiers(NETDEV_UP, dev);
  1362. return ret;
  1363. }
  1364. EXPORT_SYMBOL(dev_open);
  1365. static void __dev_close_many(struct list_head *head)
  1366. {
  1367. struct net_device *dev;
  1368. ASSERT_RTNL();
  1369. might_sleep();
  1370. list_for_each_entry(dev, head, close_list) {
  1371. /* Temporarily disable netpoll until the interface is down */
  1372. netpoll_poll_disable(dev);
  1373. call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
  1374. clear_bit(__LINK_STATE_START, &dev->state);
  1375. /* Synchronize to scheduled poll. We cannot touch poll list, it
  1376. * can be even on different cpu. So just clear netif_running().
  1377. *
  1378. * dev->stop() will invoke napi_disable() on all of it's
  1379. * napi_struct instances on this device.
  1380. */
  1381. smp_mb__after_atomic(); /* Commit netif_running(). */
  1382. }
  1383. dev_deactivate_many(head);
  1384. list_for_each_entry(dev, head, close_list) {
  1385. const struct net_device_ops *ops = dev->netdev_ops;
  1386. /*
  1387. * Call the device specific close. This cannot fail.
  1388. * Only if device is UP
  1389. *
  1390. * We allow it to be called even after a DETACH hot-plug
  1391. * event.
  1392. */
  1393. if (ops->ndo_stop)
  1394. ops->ndo_stop(dev);
  1395. dev->flags &= ~IFF_UP;
  1396. netpoll_poll_enable(dev);
  1397. }
  1398. }
  1399. static void __dev_close(struct net_device *dev)
  1400. {
  1401. LIST_HEAD(single);
  1402. list_add(&dev->close_list, &single);
  1403. __dev_close_many(&single);
  1404. list_del(&single);
  1405. }
  1406. void dev_close_many(struct list_head *head, bool unlink)
  1407. {
  1408. struct net_device *dev, *tmp;
  1409. /* Remove the devices that don't need to be closed */
  1410. list_for_each_entry_safe(dev, tmp, head, close_list)
  1411. if (!(dev->flags & IFF_UP))
  1412. list_del_init(&dev->close_list);
  1413. __dev_close_many(head);
  1414. list_for_each_entry_safe(dev, tmp, head, close_list) {
  1415. rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
  1416. call_netdevice_notifiers(NETDEV_DOWN, dev);
  1417. if (unlink)
  1418. list_del_init(&dev->close_list);
  1419. }
  1420. }
  1421. EXPORT_SYMBOL(dev_close_many);
  1422. /**
  1423. * dev_close - shutdown an interface.
  1424. * @dev: device to shutdown
  1425. *
  1426. * This function moves an active device into down state. A
  1427. * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
  1428. * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
  1429. * chain.
  1430. */
  1431. void dev_close(struct net_device *dev)
  1432. {
  1433. if (dev->flags & IFF_UP) {
  1434. LIST_HEAD(single);
  1435. list_add(&dev->close_list, &single);
  1436. dev_close_many(&single, true);
  1437. list_del(&single);
  1438. }
  1439. }
  1440. EXPORT_SYMBOL(dev_close);
  1441. /**
  1442. * dev_disable_lro - disable Large Receive Offload on a device
  1443. * @dev: device
  1444. *
  1445. * Disable Large Receive Offload (LRO) on a net device. Must be
  1446. * called under RTNL. This is needed if received packets may be
  1447. * forwarded to another interface.
  1448. */
  1449. void dev_disable_lro(struct net_device *dev)
  1450. {
  1451. struct net_device *lower_dev;
  1452. struct list_head *iter;
  1453. dev->wanted_features &= ~NETIF_F_LRO;
  1454. netdev_update_features(dev);
  1455. if (unlikely(dev->features & NETIF_F_LRO))
  1456. netdev_WARN(dev, "failed to disable LRO!\n");
  1457. netdev_for_each_lower_dev(dev, lower_dev, iter)
  1458. dev_disable_lro(lower_dev);
  1459. }
  1460. EXPORT_SYMBOL(dev_disable_lro);
  1461. /**
  1462. * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
  1463. * @dev: device
  1464. *
  1465. * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
  1466. * called under RTNL. This is needed if Generic XDP is installed on
  1467. * the device.
  1468. */
  1469. static void dev_disable_gro_hw(struct net_device *dev)
  1470. {
  1471. dev->wanted_features &= ~NETIF_F_GRO_HW;
  1472. netdev_update_features(dev);
  1473. if (unlikely(dev->features & NETIF_F_GRO_HW))
  1474. netdev_WARN(dev, "failed to disable GRO_HW!\n");
  1475. }
  1476. const char *netdev_cmd_to_name(enum netdev_cmd cmd)
  1477. {
  1478. #define N(val) \
  1479. case NETDEV_##val: \
  1480. return "NETDEV_" __stringify(val);
  1481. switch (cmd) {
  1482. N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
  1483. N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
  1484. N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
  1485. N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
  1486. N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
  1487. N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
  1488. N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
  1489. N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
  1490. N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
  1491. N(PRE_CHANGEADDR)
  1492. }
  1493. #undef N
  1494. return "UNKNOWN_NETDEV_EVENT";
  1495. }
  1496. EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
  1497. static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
  1498. struct net_device *dev)
  1499. {
  1500. struct netdev_notifier_info info = {
  1501. .dev = dev,
  1502. };
  1503. return nb->notifier_call(nb, val, &info);
  1504. }
  1505. static int call_netdevice_register_notifiers(struct notifier_block *nb,
  1506. struct net_device *dev)
  1507. {
  1508. int err;
  1509. err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
  1510. err = notifier_to_errno(err);
  1511. if (err)
  1512. return err;
  1513. if (!(dev->flags & IFF_UP))
  1514. return 0;
  1515. call_netdevice_notifier(nb, NETDEV_UP, dev);
  1516. return 0;
  1517. }
  1518. static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
  1519. struct net_device *dev)
  1520. {
  1521. if (dev->flags & IFF_UP) {
  1522. call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
  1523. dev);
  1524. call_netdevice_notifier(nb, NETDEV_DOWN, dev);
  1525. }
  1526. call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
  1527. }
  1528. static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
  1529. struct net *net)
  1530. {
  1531. struct net_device *dev;
  1532. int err;
  1533. for_each_netdev(net, dev) {
  1534. err = call_netdevice_register_notifiers(nb, dev);
  1535. if (err)
  1536. goto rollback;
  1537. }
  1538. return 0;
  1539. rollback:
  1540. for_each_netdev_continue_reverse(net, dev)
  1541. call_netdevice_unregister_notifiers(nb, dev);
  1542. return err;
  1543. }
  1544. static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
  1545. struct net *net)
  1546. {
  1547. struct net_device *dev;
  1548. for_each_netdev(net, dev)
  1549. call_netdevice_unregister_notifiers(nb, dev);
  1550. }
  1551. static int dev_boot_phase = 1;
  1552. /**
  1553. * register_netdevice_notifier - register a network notifier block
  1554. * @nb: notifier
  1555. *
  1556. * Register a notifier to be called when network device events occur.
  1557. * The notifier passed is linked into the kernel structures and must
  1558. * not be reused until it has been unregistered. A negative errno code
  1559. * is returned on a failure.
  1560. *
  1561. * When registered all registration and up events are replayed
  1562. * to the new notifier to allow device to have a race free
  1563. * view of the network device list.
  1564. */
  1565. int register_netdevice_notifier(struct notifier_block *nb)
  1566. {
  1567. struct net *net;
  1568. int err;
  1569. /* Close race with setup_net() and cleanup_net() */
  1570. down_write(&pernet_ops_rwsem);
  1571. rtnl_lock();
  1572. err = raw_notifier_chain_register(&netdev_chain, nb);
  1573. if (err)
  1574. goto unlock;
  1575. if (dev_boot_phase)
  1576. goto unlock;
  1577. for_each_net(net) {
  1578. err = call_netdevice_register_net_notifiers(nb, net);
  1579. if (err)
  1580. goto rollback;
  1581. }
  1582. unlock:
  1583. rtnl_unlock();
  1584. up_write(&pernet_ops_rwsem);
  1585. return err;
  1586. rollback:
  1587. for_each_net_continue_reverse(net)
  1588. call_netdevice_unregister_net_notifiers(nb, net);
  1589. raw_notifier_chain_unregister(&netdev_chain, nb);
  1590. goto unlock;
  1591. }
  1592. EXPORT_SYMBOL(register_netdevice_notifier);
  1593. /**
  1594. * unregister_netdevice_notifier - unregister a network notifier block
  1595. * @nb: notifier
  1596. *
  1597. * Unregister a notifier previously registered by
  1598. * register_netdevice_notifier(). The notifier is unlinked into the
  1599. * kernel structures and may then be reused. A negative errno code
  1600. * is returned on a failure.
  1601. *
  1602. * After unregistering unregister and down device events are synthesized
  1603. * for all devices on the device list to the removed notifier to remove
  1604. * the need for special case cleanup code.
  1605. */
  1606. int unregister_netdevice_notifier(struct notifier_block *nb)
  1607. {
  1608. struct net *net;
  1609. int err;
  1610. /* Close race with setup_net() and cleanup_net() */
  1611. down_write(&pernet_ops_rwsem);
  1612. rtnl_lock();
  1613. err = raw_notifier_chain_unregister(&netdev_chain, nb);
  1614. if (err)
  1615. goto unlock;
  1616. for_each_net(net)
  1617. call_netdevice_unregister_net_notifiers(nb, net);
  1618. unlock:
  1619. rtnl_unlock();
  1620. up_write(&pernet_ops_rwsem);
  1621. return err;
  1622. }
  1623. EXPORT_SYMBOL(unregister_netdevice_notifier);
  1624. static int __register_netdevice_notifier_net(struct net *net,
  1625. struct notifier_block *nb,
  1626. bool ignore_call_fail)
  1627. {
  1628. int err;
  1629. err = raw_notifier_chain_register(&net->netdev_chain, nb);
  1630. if (err)
  1631. return err;
  1632. if (dev_boot_phase)
  1633. return 0;
  1634. err = call_netdevice_register_net_notifiers(nb, net);
  1635. if (err && !ignore_call_fail)
  1636. goto chain_unregister;
  1637. return 0;
  1638. chain_unregister:
  1639. raw_notifier_chain_unregister(&net->netdev_chain, nb);
  1640. return err;
  1641. }
  1642. static int __unregister_netdevice_notifier_net(struct net *net,
  1643. struct notifier_block *nb)
  1644. {
  1645. int err;
  1646. err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
  1647. if (err)
  1648. return err;
  1649. call_netdevice_unregister_net_notifiers(nb, net);
  1650. return 0;
  1651. }
  1652. /**
  1653. * register_netdevice_notifier_net - register a per-netns network notifier block
  1654. * @net: network namespace
  1655. * @nb: notifier
  1656. *
  1657. * Register a notifier to be called when network device events occur.
  1658. * The notifier passed is linked into the kernel structures and must
  1659. * not be reused until it has been unregistered. A negative errno code
  1660. * is returned on a failure.
  1661. *
  1662. * When registered all registration and up events are replayed
  1663. * to the new notifier to allow device to have a race free
  1664. * view of the network device list.
  1665. */
  1666. int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
  1667. {
  1668. int err;
  1669. rtnl_lock();
  1670. err = __register_netdevice_notifier_net(net, nb, false);
  1671. rtnl_unlock();
  1672. return err;
  1673. }
  1674. EXPORT_SYMBOL(register_netdevice_notifier_net);
  1675. /**
  1676. * unregister_netdevice_notifier_net - unregister a per-netns
  1677. * network notifier block
  1678. * @net: network namespace
  1679. * @nb: notifier
  1680. *
  1681. * Unregister a notifier previously registered by
  1682. * register_netdevice_notifier(). The notifier is unlinked into the
  1683. * kernel structures and may then be reused. A negative errno code
  1684. * is returned on a failure.
  1685. *
  1686. * After unregistering unregister and down device events are synthesized
  1687. * for all devices on the device list to the removed notifier to remove
  1688. * the need for special case cleanup code.
  1689. */
  1690. int unregister_netdevice_notifier_net(struct net *net,
  1691. struct notifier_block *nb)
  1692. {
  1693. int err;
  1694. rtnl_lock();
  1695. err = __unregister_netdevice_notifier_net(net, nb);
  1696. rtnl_unlock();
  1697. return err;
  1698. }
  1699. EXPORT_SYMBOL(unregister_netdevice_notifier_net);
  1700. int register_netdevice_notifier_dev_net(struct net_device *dev,
  1701. struct notifier_block *nb,
  1702. struct netdev_net_notifier *nn)
  1703. {
  1704. int err;
  1705. rtnl_lock();
  1706. err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
  1707. if (!err) {
  1708. nn->nb = nb;
  1709. list_add(&nn->list, &dev->net_notifier_list);
  1710. }
  1711. rtnl_unlock();
  1712. return err;
  1713. }
  1714. EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
  1715. int unregister_netdevice_notifier_dev_net(struct net_device *dev,
  1716. struct notifier_block *nb,
  1717. struct netdev_net_notifier *nn)
  1718. {
  1719. int err;
  1720. rtnl_lock();
  1721. list_del(&nn->list);
  1722. err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
  1723. rtnl_unlock();
  1724. return err;
  1725. }
  1726. EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
  1727. static void move_netdevice_notifiers_dev_net(struct net_device *dev,
  1728. struct net *net)
  1729. {
  1730. struct netdev_net_notifier *nn;
  1731. list_for_each_entry(nn, &dev->net_notifier_list, list) {
  1732. __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
  1733. __register_netdevice_notifier_net(net, nn->nb, true);
  1734. }
  1735. }
  1736. /**
  1737. * call_netdevice_notifiers_info - call all network notifier blocks
  1738. * @val: value passed unmodified to notifier function
  1739. * @info: notifier information data
  1740. *
  1741. * Call all network notifier blocks. Parameters and return value
  1742. * are as for raw_notifier_call_chain().
  1743. */
  1744. static int call_netdevice_notifiers_info(unsigned long val,
  1745. struct netdev_notifier_info *info)
  1746. {
  1747. struct net *net = dev_net(info->dev);
  1748. int ret;
  1749. ASSERT_RTNL();
  1750. /* Run per-netns notifier block chain first, then run the global one.
  1751. * Hopefully, one day, the global one is going to be removed after
  1752. * all notifier block registrators get converted to be per-netns.
  1753. */
  1754. ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
  1755. if (ret & NOTIFY_STOP_MASK)
  1756. return ret;
  1757. return raw_notifier_call_chain(&netdev_chain, val, info);
  1758. }
  1759. static int call_netdevice_notifiers_extack(unsigned long val,
  1760. struct net_device *dev,
  1761. struct netlink_ext_ack *extack)
  1762. {
  1763. struct netdev_notifier_info info = {
  1764. .dev = dev,
  1765. .extack = extack,
  1766. };
  1767. return call_netdevice_notifiers_info(val, &info);
  1768. }
  1769. /**
  1770. * call_netdevice_notifiers - call all network notifier blocks
  1771. * @val: value passed unmodified to notifier function
  1772. * @dev: net_device pointer passed unmodified to notifier function
  1773. *
  1774. * Call all network notifier blocks. Parameters and return value
  1775. * are as for raw_notifier_call_chain().
  1776. */
  1777. int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
  1778. {
  1779. return call_netdevice_notifiers_extack(val, dev, NULL);
  1780. }
  1781. EXPORT_SYMBOL(call_netdevice_notifiers);
  1782. /**
  1783. * call_netdevice_notifiers_mtu - call all network notifier blocks
  1784. * @val: value passed unmodified to notifier function
  1785. * @dev: net_device pointer passed unmodified to notifier function
  1786. * @arg: additional u32 argument passed to the notifier function
  1787. *
  1788. * Call all network notifier blocks. Parameters and return value
  1789. * are as for raw_notifier_call_chain().
  1790. */
  1791. static int call_netdevice_notifiers_mtu(unsigned long val,
  1792. struct net_device *dev, u32 arg)
  1793. {
  1794. struct netdev_notifier_info_ext info = {
  1795. .info.dev = dev,
  1796. .ext.mtu = arg,
  1797. };
  1798. BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
  1799. return call_netdevice_notifiers_info(val, &info.info);
  1800. }
  1801. #ifdef CONFIG_NET_INGRESS
  1802. static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
  1803. void net_inc_ingress_queue(void)
  1804. {
  1805. static_branch_inc(&ingress_needed_key);
  1806. }
  1807. EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
  1808. void net_dec_ingress_queue(void)
  1809. {
  1810. static_branch_dec(&ingress_needed_key);
  1811. }
  1812. EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
  1813. #endif
  1814. #ifdef CONFIG_NET_EGRESS
  1815. static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
  1816. void net_inc_egress_queue(void)
  1817. {
  1818. static_branch_inc(&egress_needed_key);
  1819. }
  1820. EXPORT_SYMBOL_GPL(net_inc_egress_queue);
  1821. void net_dec_egress_queue(void)
  1822. {
  1823. static_branch_dec(&egress_needed_key);
  1824. }
  1825. EXPORT_SYMBOL_GPL(net_dec_egress_queue);
  1826. #endif
  1827. static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
  1828. #ifdef CONFIG_JUMP_LABEL
  1829. static atomic_t netstamp_needed_deferred;
  1830. static atomic_t netstamp_wanted;
  1831. static void netstamp_clear(struct work_struct *work)
  1832. {
  1833. int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
  1834. int wanted;
  1835. wanted = atomic_add_return(deferred, &netstamp_wanted);
  1836. if (wanted > 0)
  1837. static_branch_enable(&netstamp_needed_key);
  1838. else
  1839. static_branch_disable(&netstamp_needed_key);
  1840. }
  1841. static DECLARE_WORK(netstamp_work, netstamp_clear);
  1842. #endif
  1843. void net_enable_timestamp(void)
  1844. {
  1845. #ifdef CONFIG_JUMP_LABEL
  1846. int wanted;
  1847. while (1) {
  1848. wanted = atomic_read(&netstamp_wanted);
  1849. if (wanted <= 0)
  1850. break;
  1851. if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
  1852. return;
  1853. }
  1854. atomic_inc(&netstamp_needed_deferred);
  1855. schedule_work(&netstamp_work);
  1856. #else
  1857. static_branch_inc(&netstamp_needed_key);
  1858. #endif
  1859. }
  1860. EXPORT_SYMBOL(net_enable_timestamp);
  1861. void net_disable_timestamp(void)
  1862. {
  1863. #ifdef CONFIG_JUMP_LABEL
  1864. int wanted;
  1865. while (1) {
  1866. wanted = atomic_read(&netstamp_wanted);
  1867. if (wanted <= 1)
  1868. break;
  1869. if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
  1870. return;
  1871. }
  1872. atomic_dec(&netstamp_needed_deferred);
  1873. schedule_work(&netstamp_work);
  1874. #else
  1875. static_branch_dec(&netstamp_needed_key);
  1876. #endif
  1877. }
  1878. EXPORT_SYMBOL(net_disable_timestamp);
  1879. static inline void net_timestamp_set(struct sk_buff *skb)
  1880. {
  1881. skb->tstamp = 0;
  1882. if (static_branch_unlikely(&netstamp_needed_key))
  1883. __net_timestamp(skb);
  1884. }
  1885. #define net_timestamp_check(COND, SKB) \
  1886. if (static_branch_unlikely(&netstamp_needed_key)) { \
  1887. if ((COND) && !(SKB)->tstamp) \
  1888. __net_timestamp(SKB); \
  1889. } \
  1890. bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
  1891. {
  1892. unsigned int len;
  1893. if (!(dev->flags & IFF_UP))
  1894. return false;
  1895. len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
  1896. if (skb->len <= len)
  1897. return true;
  1898. /* if TSO is enabled, we don't care about the length as the packet
  1899. * could be forwarded without being segmented before
  1900. */
  1901. if (skb_is_gso(skb))
  1902. return true;
  1903. return false;
  1904. }
  1905. EXPORT_SYMBOL_GPL(is_skb_forwardable);
  1906. int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
  1907. {
  1908. int ret = ____dev_forward_skb(dev, skb);
  1909. if (likely(!ret)) {
  1910. skb->protocol = eth_type_trans(skb, dev);
  1911. skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
  1912. }
  1913. return ret;
  1914. }
  1915. EXPORT_SYMBOL_GPL(__dev_forward_skb);
  1916. /**
  1917. * dev_forward_skb - loopback an skb to another netif
  1918. *
  1919. * @dev: destination network device
  1920. * @skb: buffer to forward
  1921. *
  1922. * return values:
  1923. * NET_RX_SUCCESS (no congestion)
  1924. * NET_RX_DROP (packet was dropped, but freed)
  1925. *
  1926. * dev_forward_skb can be used for injecting an skb from the
  1927. * start_xmit function of one device into the receive queue
  1928. * of another device.
  1929. *
  1930. * The receiving device may be in another namespace, so
  1931. * we have to clear all information in the skb that could
  1932. * impact namespace isolation.
  1933. */
  1934. int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
  1935. {
  1936. return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
  1937. }
  1938. EXPORT_SYMBOL_GPL(dev_forward_skb);
  1939. static inline int deliver_skb(struct sk_buff *skb,
  1940. struct packet_type *pt_prev,
  1941. struct net_device *orig_dev)
  1942. {
  1943. if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
  1944. return -ENOMEM;
  1945. refcount_inc(&skb->users);
  1946. return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
  1947. }
  1948. static inline void deliver_ptype_list_skb(struct sk_buff *skb,
  1949. struct packet_type **pt,
  1950. struct net_device *orig_dev,
  1951. __be16 type,
  1952. struct list_head *ptype_list)
  1953. {
  1954. struct packet_type *ptype, *pt_prev = *pt;
  1955. list_for_each_entry_rcu(ptype, ptype_list, list) {
  1956. if (ptype->type != type)
  1957. continue;
  1958. if (pt_prev)
  1959. deliver_skb(skb, pt_prev, orig_dev);
  1960. pt_prev = ptype;
  1961. }
  1962. *pt = pt_prev;
  1963. }
  1964. static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
  1965. {
  1966. if (!ptype->af_packet_priv || !skb->sk)
  1967. return false;
  1968. if (ptype->id_match)
  1969. return ptype->id_match(ptype, skb->sk);
  1970. else if ((struct sock *)ptype->af_packet_priv == skb->sk)
  1971. return true;
  1972. return false;
  1973. }
  1974. /**
  1975. * dev_nit_active - return true if any network interface taps are in use
  1976. *
  1977. * @dev: network device to check for the presence of taps
  1978. */
  1979. bool dev_nit_active(struct net_device *dev)
  1980. {
  1981. return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
  1982. }
  1983. EXPORT_SYMBOL_GPL(dev_nit_active);
  1984. /*
  1985. * Support routine. Sends outgoing frames to any network
  1986. * taps currently in use.
  1987. */
  1988. void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
  1989. {
  1990. struct packet_type *ptype;
  1991. struct sk_buff *skb2 = NULL;
  1992. struct packet_type *pt_prev = NULL;
  1993. struct list_head *ptype_list = &ptype_all;
  1994. rcu_read_lock();
  1995. again:
  1996. list_for_each_entry_rcu(ptype, ptype_list, list) {
  1997. if (ptype->ignore_outgoing)
  1998. continue;
  1999. /* Never send packets back to the socket
  2000. * they originated from - MvS (miquels@drinkel.ow.org)
  2001. */
  2002. if (skb_loop_sk(ptype, skb))
  2003. continue;
  2004. if (pt_prev) {
  2005. deliver_skb(skb2, pt_prev, skb->dev);
  2006. pt_prev = ptype;
  2007. continue;
  2008. }
  2009. /* need to clone skb, done only once */
  2010. skb2 = skb_clone(skb, GFP_ATOMIC);
  2011. if (!skb2)
  2012. goto out_unlock;
  2013. net_timestamp_set(skb2);
  2014. /* skb->nh should be correctly
  2015. * set by sender, so that the second statement is
  2016. * just protection against buggy protocols.
  2017. */
  2018. skb_reset_mac_header(skb2);
  2019. if (skb_network_header(skb2) < skb2->data ||
  2020. skb_network_header(skb2) > skb_tail_pointer(skb2)) {
  2021. net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
  2022. ntohs(skb2->protocol),
  2023. dev->name);
  2024. skb_reset_network_header(skb2);
  2025. }
  2026. skb2->transport_header = skb2->network_header;
  2027. skb2->pkt_type = PACKET_OUTGOING;
  2028. pt_prev = ptype;
  2029. }
  2030. if (ptype_list == &ptype_all) {
  2031. ptype_list = &dev->ptype_all;
  2032. goto again;
  2033. }
  2034. out_unlock:
  2035. if (pt_prev) {
  2036. if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
  2037. pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
  2038. else
  2039. kfree_skb(skb2);
  2040. }
  2041. rcu_read_unlock();
  2042. }
  2043. EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
  2044. /**
  2045. * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
  2046. * @dev: Network device
  2047. * @txq: number of queues available
  2048. *
  2049. * If real_num_tx_queues is changed the tc mappings may no longer be
  2050. * valid. To resolve this verify the tc mapping remains valid and if
  2051. * not NULL the mapping. With no priorities mapping to this
  2052. * offset/count pair it will no longer be used. In the worst case TC0
  2053. * is invalid nothing can be done so disable priority mappings. If is
  2054. * expected that drivers will fix this mapping if they can before
  2055. * calling netif_set_real_num_tx_queues.
  2056. */
  2057. static void netif_setup_tc(struct net_device *dev, unsigned int txq)
  2058. {
  2059. int i;
  2060. struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
  2061. /* If TC0 is invalidated disable TC mapping */
  2062. if (tc->offset + tc->count > txq) {
  2063. pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
  2064. dev->num_tc = 0;
  2065. return;
  2066. }
  2067. /* Invalidated prio to tc mappings set to TC0 */
  2068. for (i = 1; i < TC_BITMASK + 1; i++) {
  2069. int q = netdev_get_prio_tc_map(dev, i);
  2070. tc = &dev->tc_to_txq[q];
  2071. if (tc->offset + tc->count > txq) {
  2072. pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
  2073. i, q);
  2074. netdev_set_prio_tc_map(dev, i, 0);
  2075. }
  2076. }
  2077. }
  2078. int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
  2079. {
  2080. if (dev->num_tc) {
  2081. struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
  2082. int i;
  2083. /* walk through the TCs and see if it falls into any of them */
  2084. for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
  2085. if ((txq - tc->offset) < tc->count)
  2086. return i;
  2087. }
  2088. /* didn't find it, just return -1 to indicate no match */
  2089. return -1;
  2090. }
  2091. return 0;
  2092. }
  2093. EXPORT_SYMBOL(netdev_txq_to_tc);
  2094. #ifdef CONFIG_XPS
  2095. struct static_key xps_needed __read_mostly;
  2096. EXPORT_SYMBOL(xps_needed);
  2097. struct static_key xps_rxqs_needed __read_mostly;
  2098. EXPORT_SYMBOL(xps_rxqs_needed);
  2099. static DEFINE_MUTEX(xps_map_mutex);
  2100. #define xmap_dereference(P) \
  2101. rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
  2102. static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
  2103. int tci, u16 index)
  2104. {
  2105. struct xps_map *map = NULL;
  2106. int pos;
  2107. if (dev_maps)
  2108. map = xmap_dereference(dev_maps->attr_map[tci]);
  2109. if (!map)
  2110. return false;
  2111. for (pos = map->len; pos--;) {
  2112. if (map->queues[pos] != index)
  2113. continue;
  2114. if (map->len > 1) {
  2115. map->queues[pos] = map->queues[--map->len];
  2116. break;
  2117. }
  2118. RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
  2119. kfree_rcu(map, rcu);
  2120. return false;
  2121. }
  2122. return true;
  2123. }
  2124. static bool remove_xps_queue_cpu(struct net_device *dev,
  2125. struct xps_dev_maps *dev_maps,
  2126. int cpu, u16 offset, u16 count)
  2127. {
  2128. int num_tc = dev->num_tc ? : 1;
  2129. bool active = false;
  2130. int tci;
  2131. for (tci = cpu * num_tc; num_tc--; tci++) {
  2132. int i, j;
  2133. for (i = count, j = offset; i--; j++) {
  2134. if (!remove_xps_queue(dev_maps, tci, j))
  2135. break;
  2136. }
  2137. active |= i < 0;
  2138. }
  2139. return active;
  2140. }
  2141. static void reset_xps_maps(struct net_device *dev,
  2142. struct xps_dev_maps *dev_maps,
  2143. bool is_rxqs_map)
  2144. {
  2145. if (is_rxqs_map) {
  2146. static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
  2147. RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
  2148. } else {
  2149. RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
  2150. }
  2151. static_key_slow_dec_cpuslocked(&xps_needed);
  2152. kfree_rcu(dev_maps, rcu);
  2153. }
  2154. static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
  2155. struct xps_dev_maps *dev_maps, unsigned int nr_ids,
  2156. u16 offset, u16 count, bool is_rxqs_map)
  2157. {
  2158. bool active = false;
  2159. int i, j;
  2160. for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
  2161. j < nr_ids;)
  2162. active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
  2163. count);
  2164. if (!active)
  2165. reset_xps_maps(dev, dev_maps, is_rxqs_map);
  2166. if (!is_rxqs_map) {
  2167. for (i = offset + (count - 1); count--; i--) {
  2168. netdev_queue_numa_node_write(
  2169. netdev_get_tx_queue(dev, i),
  2170. NUMA_NO_NODE);
  2171. }
  2172. }
  2173. }
  2174. static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
  2175. u16 count)
  2176. {
  2177. const unsigned long *possible_mask = NULL;
  2178. struct xps_dev_maps *dev_maps;
  2179. unsigned int nr_ids;
  2180. if (!static_key_false(&xps_needed))
  2181. return;
  2182. cpus_read_lock();
  2183. mutex_lock(&xps_map_mutex);
  2184. if (static_key_false(&xps_rxqs_needed)) {
  2185. dev_maps = xmap_dereference(dev->xps_rxqs_map);
  2186. if (dev_maps) {
  2187. nr_ids = dev->num_rx_queues;
  2188. clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
  2189. offset, count, true);
  2190. }
  2191. }
  2192. dev_maps = xmap_dereference(dev->xps_cpus_map);
  2193. if (!dev_maps)
  2194. goto out_no_maps;
  2195. if (num_possible_cpus() > 1)
  2196. possible_mask = cpumask_bits(cpu_possible_mask);
  2197. nr_ids = nr_cpu_ids;
  2198. clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
  2199. false);
  2200. out_no_maps:
  2201. mutex_unlock(&xps_map_mutex);
  2202. cpus_read_unlock();
  2203. }
  2204. static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
  2205. {
  2206. netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
  2207. }
  2208. static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
  2209. u16 index, bool is_rxqs_map)
  2210. {
  2211. struct xps_map *new_map;
  2212. int alloc_len = XPS_MIN_MAP_ALLOC;
  2213. int i, pos;
  2214. for (pos = 0; map && pos < map->len; pos++) {
  2215. if (map->queues[pos] != index)
  2216. continue;
  2217. return map;
  2218. }
  2219. /* Need to add tx-queue to this CPU's/rx-queue's existing map */
  2220. if (map) {
  2221. if (pos < map->alloc_len)
  2222. return map;
  2223. alloc_len = map->alloc_len * 2;
  2224. }
  2225. /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
  2226. * map
  2227. */
  2228. if (is_rxqs_map)
  2229. new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
  2230. else
  2231. new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
  2232. cpu_to_node(attr_index));
  2233. if (!new_map)
  2234. return NULL;
  2235. for (i = 0; i < pos; i++)
  2236. new_map->queues[i] = map->queues[i];
  2237. new_map->alloc_len = alloc_len;
  2238. new_map->len = pos;
  2239. return new_map;
  2240. }
  2241. /* Must be called under cpus_read_lock */
  2242. int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
  2243. u16 index, bool is_rxqs_map)
  2244. {
  2245. const unsigned long *online_mask = NULL, *possible_mask = NULL;
  2246. struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
  2247. int i, j, tci, numa_node_id = -2;
  2248. int maps_sz, num_tc = 1, tc = 0;
  2249. struct xps_map *map, *new_map;
  2250. bool active = false;
  2251. unsigned int nr_ids;
  2252. if (dev->num_tc) {
  2253. /* Do not allow XPS on subordinate device directly */
  2254. num_tc = dev->num_tc;
  2255. if (num_tc < 0)
  2256. return -EINVAL;
  2257. /* If queue belongs to subordinate dev use its map */
  2258. dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
  2259. tc = netdev_txq_to_tc(dev, index);
  2260. if (tc < 0)
  2261. return -EINVAL;
  2262. }
  2263. mutex_lock(&xps_map_mutex);
  2264. if (is_rxqs_map) {
  2265. maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
  2266. dev_maps = xmap_dereference(dev->xps_rxqs_map);
  2267. nr_ids = dev->num_rx_queues;
  2268. } else {
  2269. maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
  2270. if (num_possible_cpus() > 1) {
  2271. online_mask = cpumask_bits(cpu_online_mask);
  2272. possible_mask = cpumask_bits(cpu_possible_mask);
  2273. }
  2274. dev_maps = xmap_dereference(dev->xps_cpus_map);
  2275. nr_ids = nr_cpu_ids;
  2276. }
  2277. if (maps_sz < L1_CACHE_BYTES)
  2278. maps_sz = L1_CACHE_BYTES;
  2279. /* allocate memory for queue storage */
  2280. for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
  2281. j < nr_ids;) {
  2282. if (!new_dev_maps)
  2283. new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
  2284. if (!new_dev_maps) {
  2285. mutex_unlock(&xps_map_mutex);
  2286. return -ENOMEM;
  2287. }
  2288. tci = j * num_tc + tc;
  2289. map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
  2290. NULL;
  2291. map = expand_xps_map(map, j, index, is_rxqs_map);
  2292. if (!map)
  2293. goto error;
  2294. RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
  2295. }
  2296. if (!new_dev_maps)
  2297. goto out_no_new_maps;
  2298. if (!dev_maps) {
  2299. /* Increment static keys at most once per type */
  2300. static_key_slow_inc_cpuslocked(&xps_needed);
  2301. if (is_rxqs_map)
  2302. static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
  2303. }
  2304. for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
  2305. j < nr_ids;) {
  2306. /* copy maps belonging to foreign traffic classes */
  2307. for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
  2308. /* fill in the new device map from the old device map */
  2309. map = xmap_dereference(dev_maps->attr_map[tci]);
  2310. RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
  2311. }
  2312. /* We need to explicitly update tci as prevous loop
  2313. * could break out early if dev_maps is NULL.
  2314. */
  2315. tci = j * num_tc + tc;
  2316. if (netif_attr_test_mask(j, mask, nr_ids) &&
  2317. netif_attr_test_online(j, online_mask, nr_ids)) {
  2318. /* add tx-queue to CPU/rx-queue maps */
  2319. int pos = 0;
  2320. map = xmap_dereference(new_dev_maps->attr_map[tci]);
  2321. while ((pos < map->len) && (map->queues[pos] != index))
  2322. pos++;
  2323. if (pos == map->len)
  2324. map->queues[map->len++] = index;
  2325. #ifdef CONFIG_NUMA
  2326. if (!is_rxqs_map) {
  2327. if (numa_node_id == -2)
  2328. numa_node_id = cpu_to_node(j);
  2329. else if (numa_node_id != cpu_to_node(j))
  2330. numa_node_id = -1;
  2331. }
  2332. #endif
  2333. } else if (dev_maps) {
  2334. /* fill in the new device map from the old device map */
  2335. map = xmap_dereference(dev_maps->attr_map[tci]);
  2336. RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
  2337. }
  2338. /* copy maps belonging to foreign traffic classes */
  2339. for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
  2340. /* fill in the new device map from the old device map */
  2341. map = xmap_dereference(dev_maps->attr_map[tci]);
  2342. RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
  2343. }
  2344. }
  2345. if (is_rxqs_map)
  2346. rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
  2347. else
  2348. rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
  2349. /* Cleanup old maps */
  2350. if (!dev_maps)
  2351. goto out_no_old_maps;
  2352. for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
  2353. j < nr_ids;) {
  2354. for (i = num_tc, tci = j * num_tc; i--; tci++) {
  2355. new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
  2356. map = xmap_dereference(dev_maps->attr_map[tci]);
  2357. if (map && map != new_map)
  2358. kfree_rcu(map, rcu);
  2359. }
  2360. }
  2361. kfree_rcu(dev_maps, rcu);
  2362. out_no_old_maps:
  2363. dev_maps = new_dev_maps;
  2364. active = true;
  2365. out_no_new_maps:
  2366. if (!is_rxqs_map) {
  2367. /* update Tx queue numa node */
  2368. netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
  2369. (numa_node_id >= 0) ?
  2370. numa_node_id : NUMA_NO_NODE);
  2371. }
  2372. if (!dev_maps)
  2373. goto out_no_maps;
  2374. /* removes tx-queue from unused CPUs/rx-queues */
  2375. for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
  2376. j < nr_ids;) {
  2377. for (i = tc, tci = j * num_tc; i--; tci++)
  2378. active |= remove_xps_queue(dev_maps, tci, index);
  2379. if (!netif_attr_test_mask(j, mask, nr_ids) ||
  2380. !netif_attr_test_online(j, online_mask, nr_ids))
  2381. active |= remove_xps_queue(dev_maps, tci, index);
  2382. for (i = num_tc - tc, tci++; --i; tci++)
  2383. active |= remove_xps_queue(dev_maps, tci, index);
  2384. }
  2385. /* free map if not active */
  2386. if (!active)
  2387. reset_xps_maps(dev, dev_maps, is_rxqs_map);
  2388. out_no_maps:
  2389. mutex_unlock(&xps_map_mutex);
  2390. return 0;
  2391. error:
  2392. /* remove any maps that we added */
  2393. for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
  2394. j < nr_ids;) {
  2395. for (i = num_tc, tci = j * num_tc; i--; tci++) {
  2396. new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
  2397. map = dev_maps ?
  2398. xmap_dereference(dev_maps->attr_map[tci]) :
  2399. NULL;
  2400. if (new_map && new_map != map)
  2401. kfree(new_map);
  2402. }
  2403. }
  2404. mutex_unlock(&xps_map_mutex);
  2405. kfree(new_dev_maps);
  2406. return -ENOMEM;
  2407. }
  2408. EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
  2409. int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
  2410. u16 index)
  2411. {
  2412. int ret;
  2413. cpus_read_lock();
  2414. ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
  2415. cpus_read_unlock();
  2416. return ret;
  2417. }
  2418. EXPORT_SYMBOL(netif_set_xps_queue);
  2419. #endif
  2420. static void netdev_unbind_all_sb_channels(struct net_device *dev)
  2421. {
  2422. struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
  2423. /* Unbind any subordinate channels */
  2424. while (txq-- != &dev->_tx[0]) {
  2425. if (txq->sb_dev)
  2426. netdev_unbind_sb_channel(dev, txq->sb_dev);
  2427. }
  2428. }
  2429. void netdev_reset_tc(struct net_device *dev)
  2430. {
  2431. #ifdef CONFIG_XPS
  2432. netif_reset_xps_queues_gt(dev, 0);
  2433. #endif
  2434. netdev_unbind_all_sb_channels(dev);
  2435. /* Reset TC configuration of device */
  2436. dev->num_tc = 0;
  2437. memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
  2438. memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
  2439. }
  2440. EXPORT_SYMBOL(netdev_reset_tc);
  2441. int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
  2442. {
  2443. if (tc >= dev->num_tc)
  2444. return -EINVAL;
  2445. #ifdef CONFIG_XPS
  2446. netif_reset_xps_queues(dev, offset, count);
  2447. #endif
  2448. dev->tc_to_txq[tc].count = count;
  2449. dev->tc_to_txq[tc].offset = offset;
  2450. return 0;
  2451. }
  2452. EXPORT_SYMBOL(netdev_set_tc_queue);
  2453. int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
  2454. {
  2455. if (num_tc > TC_MAX_QUEUE)
  2456. return -EINVAL;
  2457. #ifdef CONFIG_XPS
  2458. netif_reset_xps_queues_gt(dev, 0);
  2459. #endif
  2460. netdev_unbind_all_sb_channels(dev);
  2461. dev->num_tc = num_tc;
  2462. return 0;
  2463. }
  2464. EXPORT_SYMBOL(netdev_set_num_tc);
  2465. void netdev_unbind_sb_channel(struct net_device *dev,
  2466. struct net_device *sb_dev)
  2467. {
  2468. struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
  2469. #ifdef CONFIG_XPS
  2470. netif_reset_xps_queues_gt(sb_dev, 0);
  2471. #endif
  2472. memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
  2473. memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
  2474. while (txq-- != &dev->_tx[0]) {
  2475. if (txq->sb_dev == sb_dev)
  2476. txq->sb_dev = NULL;
  2477. }
  2478. }
  2479. EXPORT_SYMBOL(netdev_unbind_sb_channel);
  2480. int netdev_bind_sb_channel_queue(struct net_device *dev,
  2481. struct net_device *sb_dev,
  2482. u8 tc, u16 count, u16 offset)
  2483. {
  2484. /* Make certain the sb_dev and dev are already configured */
  2485. if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
  2486. return -EINVAL;
  2487. /* We cannot hand out queues we don't have */
  2488. if ((offset + count) > dev->real_num_tx_queues)
  2489. return -EINVAL;
  2490. /* Record the mapping */
  2491. sb_dev->tc_to_txq[tc].count = count;
  2492. sb_dev->tc_to_txq[tc].offset = offset;
  2493. /* Provide a way for Tx queue to find the tc_to_txq map or
  2494. * XPS map for itself.
  2495. */
  2496. while (count--)
  2497. netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
  2498. return 0;
  2499. }
  2500. EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
  2501. int netdev_set_sb_channel(struct net_device *dev, u16 channel)
  2502. {
  2503. /* Do not use a multiqueue device to represent a subordinate channel */
  2504. if (netif_is_multiqueue(dev))
  2505. return -ENODEV;
  2506. /* We allow channels 1 - 32767 to be used for subordinate channels.
  2507. * Channel 0 is meant to be "native" mode and used only to represent
  2508. * the main root device. We allow writing 0 to reset the device back
  2509. * to normal mode after being used as a subordinate channel.
  2510. */
  2511. if (channel > S16_MAX)
  2512. return -EINVAL;
  2513. dev->num_tc = -channel;
  2514. return 0;
  2515. }
  2516. EXPORT_SYMBOL(netdev_set_sb_channel);
  2517. /*
  2518. * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
  2519. * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
  2520. */
  2521. int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
  2522. {
  2523. bool disabling;
  2524. int rc;
  2525. disabling = txq < dev->real_num_tx_queues;
  2526. if (txq < 1 || txq > dev->num_tx_queues)
  2527. return -EINVAL;
  2528. if (dev->reg_state == NETREG_REGISTERED ||
  2529. dev->reg_state == NETREG_UNREGISTERING) {
  2530. ASSERT_RTNL();
  2531. rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
  2532. txq);
  2533. if (rc)
  2534. return rc;
  2535. if (dev->num_tc)
  2536. netif_setup_tc(dev, txq);
  2537. dev->real_num_tx_queues = txq;
  2538. if (disabling) {
  2539. synchronize_net();
  2540. qdisc_reset_all_tx_gt(dev, txq);
  2541. #ifdef CONFIG_XPS
  2542. netif_reset_xps_queues_gt(dev, txq);
  2543. #endif
  2544. }
  2545. } else {
  2546. dev->real_num_tx_queues = txq;
  2547. }
  2548. return 0;
  2549. }
  2550. EXPORT_SYMBOL(netif_set_real_num_tx_queues);
  2551. #ifdef CONFIG_SYSFS
  2552. /**
  2553. * netif_set_real_num_rx_queues - set actual number of RX queues used
  2554. * @dev: Network device
  2555. * @rxq: Actual number of RX queues
  2556. *
  2557. * This must be called either with the rtnl_lock held or before
  2558. * registration of the net device. Returns 0 on success, or a
  2559. * negative error code. If called before registration, it always
  2560. * succeeds.
  2561. */
  2562. int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
  2563. {
  2564. int rc;
  2565. if (rxq < 1 || rxq > dev->num_rx_queues)
  2566. return -EINVAL;
  2567. if (dev->reg_state == NETREG_REGISTERED) {
  2568. ASSERT_RTNL();
  2569. rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
  2570. rxq);
  2571. if (rc)
  2572. return rc;
  2573. }
  2574. dev->real_num_rx_queues = rxq;
  2575. return 0;
  2576. }
  2577. EXPORT_SYMBOL(netif_set_real_num_rx_queues);
  2578. #endif
  2579. /**
  2580. * netif_get_num_default_rss_queues - default number of RSS queues
  2581. *
  2582. * This routine should set an upper limit on the number of RSS queues
  2583. * used by default by multiqueue devices.
  2584. */
  2585. int netif_get_num_default_rss_queues(void)
  2586. {
  2587. return is_kdump_kernel() ?
  2588. 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
  2589. }
  2590. EXPORT_SYMBOL(netif_get_num_default_rss_queues);
  2591. static void __netif_reschedule(struct Qdisc *q)
  2592. {
  2593. struct softnet_data *sd;
  2594. unsigned long flags;
  2595. local_irq_save(flags);
  2596. sd = this_cpu_ptr(&softnet_data);
  2597. q->next_sched = NULL;
  2598. *sd->output_queue_tailp = q;
  2599. sd->output_queue_tailp = &q->next_sched;
  2600. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  2601. local_irq_restore(flags);
  2602. }
  2603. void __netif_schedule(struct Qdisc *q)
  2604. {
  2605. if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
  2606. __netif_reschedule(q);
  2607. }
  2608. EXPORT_SYMBOL(__netif_schedule);
  2609. struct dev_kfree_skb_cb {
  2610. enum skb_free_reason reason;
  2611. };
  2612. static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
  2613. {
  2614. return (struct dev_kfree_skb_cb *)skb->cb;
  2615. }
  2616. void netif_schedule_queue(struct netdev_queue *txq)
  2617. {
  2618. rcu_read_lock();
  2619. if (!netif_xmit_stopped(txq)) {
  2620. struct Qdisc *q = rcu_dereference(txq->qdisc);
  2621. __netif_schedule(q);
  2622. }
  2623. rcu_read_unlock();
  2624. }
  2625. EXPORT_SYMBOL(netif_schedule_queue);
  2626. void netif_tx_wake_queue(struct netdev_queue *dev_queue)
  2627. {
  2628. if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
  2629. struct Qdisc *q;
  2630. rcu_read_lock();
  2631. q = rcu_dereference(dev_queue->qdisc);
  2632. __netif_schedule(q);
  2633. rcu_read_unlock();
  2634. }
  2635. }
  2636. EXPORT_SYMBOL(netif_tx_wake_queue);
  2637. void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
  2638. {
  2639. unsigned long flags;
  2640. if (unlikely(!skb))
  2641. return;
  2642. if (likely(refcount_read(&skb->users) == 1)) {
  2643. smp_rmb();
  2644. refcount_set(&skb->users, 0);
  2645. } else if (likely(!refcount_dec_and_test(&skb->users))) {
  2646. return;
  2647. }
  2648. get_kfree_skb_cb(skb)->reason = reason;
  2649. local_irq_save(flags);
  2650. skb->next = __this_cpu_read(softnet_data.completion_queue);
  2651. __this_cpu_write(softnet_data.completion_queue, skb);
  2652. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  2653. local_irq_restore(flags);
  2654. }
  2655. EXPORT_SYMBOL(__dev_kfree_skb_irq);
  2656. void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
  2657. {
  2658. if (in_irq() || irqs_disabled())
  2659. __dev_kfree_skb_irq(skb, reason);
  2660. else
  2661. dev_kfree_skb(skb);
  2662. }
  2663. EXPORT_SYMBOL(__dev_kfree_skb_any);
  2664. /**
  2665. * netif_device_detach - mark device as removed
  2666. * @dev: network device
  2667. *
  2668. * Mark device as removed from system and therefore no longer available.
  2669. */
  2670. void netif_device_detach(struct net_device *dev)
  2671. {
  2672. if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
  2673. netif_running(dev)) {
  2674. netif_tx_stop_all_queues(dev);
  2675. }
  2676. }
  2677. EXPORT_SYMBOL(netif_device_detach);
  2678. /**
  2679. * netif_device_attach - mark device as attached
  2680. * @dev: network device
  2681. *
  2682. * Mark device as attached from system and restart if needed.
  2683. */
  2684. void netif_device_attach(struct net_device *dev)
  2685. {
  2686. if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
  2687. netif_running(dev)) {
  2688. netif_tx_wake_all_queues(dev);
  2689. __netdev_watchdog_up(dev);
  2690. }
  2691. }
  2692. EXPORT_SYMBOL(netif_device_attach);
  2693. /*
  2694. * Returns a Tx hash based on the given packet descriptor a Tx queues' number
  2695. * to be used as a distribution range.
  2696. */
  2697. static u16 skb_tx_hash(const struct net_device *dev,
  2698. const struct net_device *sb_dev,
  2699. struct sk_buff *skb)
  2700. {
  2701. u32 hash;
  2702. u16 qoffset = 0;
  2703. u16 qcount = dev->real_num_tx_queues;
  2704. if (dev->num_tc) {
  2705. u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
  2706. qoffset = sb_dev->tc_to_txq[tc].offset;
  2707. qcount = sb_dev->tc_to_txq[tc].count;
  2708. if (unlikely(!qcount)) {
  2709. net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
  2710. sb_dev->name, qoffset, tc);
  2711. qoffset = 0;
  2712. qcount = dev->real_num_tx_queues;
  2713. }
  2714. }
  2715. if (skb_rx_queue_recorded(skb)) {
  2716. hash = skb_get_rx_queue(skb);
  2717. if (hash >= qoffset)
  2718. hash -= qoffset;
  2719. while (unlikely(hash >= qcount))
  2720. hash -= qcount;
  2721. return hash + qoffset;
  2722. }
  2723. return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
  2724. }
  2725. static void skb_warn_bad_offload(const struct sk_buff *skb)
  2726. {
  2727. static const netdev_features_t null_features;
  2728. struct net_device *dev = skb->dev;
  2729. const char *name = "";
  2730. if (!net_ratelimit())
  2731. return;
  2732. if (dev) {
  2733. if (dev->dev.parent)
  2734. name = dev_driver_string(dev->dev.parent);
  2735. else
  2736. name = netdev_name(dev);
  2737. }
  2738. skb_dump(KERN_WARNING, skb, false);
  2739. WARN(1, "%s: caps=(%pNF, %pNF)\n",
  2740. name, dev ? &dev->features : &null_features,
  2741. skb->sk ? &skb->sk->sk_route_caps : &null_features);
  2742. }
  2743. /*
  2744. * Invalidate hardware checksum when packet is to be mangled, and
  2745. * complete checksum manually on outgoing path.
  2746. */
  2747. int skb_checksum_help(struct sk_buff *skb)
  2748. {
  2749. __wsum csum;
  2750. int ret = 0, offset;
  2751. if (skb->ip_summed == CHECKSUM_COMPLETE)
  2752. goto out_set_summed;
  2753. if (unlikely(skb_shinfo(skb)->gso_size)) {
  2754. skb_warn_bad_offload(skb);
  2755. return -EINVAL;
  2756. }
  2757. /* Before computing a checksum, we should make sure no frag could
  2758. * be modified by an external entity : checksum could be wrong.
  2759. */
  2760. if (skb_has_shared_frag(skb)) {
  2761. ret = __skb_linearize(skb);
  2762. if (ret)
  2763. goto out;
  2764. }
  2765. offset = skb_checksum_start_offset(skb);
  2766. BUG_ON(offset >= skb_headlen(skb));
  2767. csum = skb_checksum(skb, offset, skb->len - offset, 0);
  2768. offset += skb->csum_offset;
  2769. BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
  2770. ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
  2771. if (ret)
  2772. goto out;
  2773. *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
  2774. out_set_summed:
  2775. skb->ip_summed = CHECKSUM_NONE;
  2776. out:
  2777. return ret;
  2778. }
  2779. EXPORT_SYMBOL(skb_checksum_help);
  2780. int skb_crc32c_csum_help(struct sk_buff *skb)
  2781. {
  2782. __le32 crc32c_csum;
  2783. int ret = 0, offset, start;
  2784. if (skb->ip_summed != CHECKSUM_PARTIAL)
  2785. goto out;
  2786. if (unlikely(skb_is_gso(skb)))
  2787. goto out;
  2788. /* Before computing a checksum, we should make sure no frag could
  2789. * be modified by an external entity : checksum could be wrong.
  2790. */
  2791. if (unlikely(skb_has_shared_frag(skb))) {
  2792. ret = __skb_linearize(skb);
  2793. if (ret)
  2794. goto out;
  2795. }
  2796. start = skb_checksum_start_offset(skb);
  2797. offset = start + offsetof(struct sctphdr, checksum);
  2798. if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
  2799. ret = -EINVAL;
  2800. goto out;
  2801. }
  2802. ret = skb_ensure_writable(skb, offset + sizeof(__le32));
  2803. if (ret)
  2804. goto out;
  2805. crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
  2806. skb->len - start, ~(__u32)0,
  2807. crc32c_csum_stub));
  2808. *(__le32 *)(skb->data + offset) = crc32c_csum;
  2809. skb->ip_summed = CHECKSUM_NONE;
  2810. skb->csum_not_inet = 0;
  2811. out:
  2812. return ret;
  2813. }
  2814. __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
  2815. {
  2816. __be16 type = skb->protocol;
  2817. /* Tunnel gso handlers can set protocol to ethernet. */
  2818. if (type == htons(ETH_P_TEB)) {
  2819. struct ethhdr *eth;
  2820. if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
  2821. return 0;
  2822. eth = (struct ethhdr *)skb->data;
  2823. type = eth->h_proto;
  2824. }
  2825. return __vlan_get_protocol(skb, type, depth);
  2826. }
  2827. /**
  2828. * skb_mac_gso_segment - mac layer segmentation handler.
  2829. * @skb: buffer to segment
  2830. * @features: features for the output path (see dev->features)
  2831. */
  2832. struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
  2833. netdev_features_t features)
  2834. {
  2835. struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
  2836. struct packet_offload *ptype;
  2837. int vlan_depth = skb->mac_len;
  2838. __be16 type = skb_network_protocol(skb, &vlan_depth);
  2839. if (unlikely(!type))
  2840. return ERR_PTR(-EINVAL);
  2841. __skb_pull(skb, vlan_depth);
  2842. rcu_read_lock();
  2843. list_for_each_entry_rcu(ptype, &offload_base, list) {
  2844. if (ptype->type == type && ptype->callbacks.gso_segment) {
  2845. segs = ptype->callbacks.gso_segment(skb, features);
  2846. break;
  2847. }
  2848. }
  2849. rcu_read_unlock();
  2850. __skb_push(skb, skb->data - skb_mac_header(skb));
  2851. return segs;
  2852. }
  2853. EXPORT_SYMBOL(skb_mac_gso_segment);
  2854. /* openvswitch calls this on rx path, so we need a different check.
  2855. */
  2856. static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
  2857. {
  2858. if (tx_path)
  2859. return skb->ip_summed != CHECKSUM_PARTIAL &&
  2860. skb->ip_summed != CHECKSUM_UNNECESSARY;
  2861. return skb->ip_summed == CHECKSUM_NONE;
  2862. }
  2863. /**
  2864. * __skb_gso_segment - Perform segmentation on skb.
  2865. * @skb: buffer to segment
  2866. * @features: features for the output path (see dev->features)
  2867. * @tx_path: whether it is called in TX path
  2868. *
  2869. * This function segments the given skb and returns a list of segments.
  2870. *
  2871. * It may return NULL if the skb requires no segmentation. This is
  2872. * only possible when GSO is used for verifying header integrity.
  2873. *
  2874. * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
  2875. */
  2876. struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
  2877. netdev_features_t features, bool tx_path)
  2878. {
  2879. struct sk_buff *segs;
  2880. if (unlikely(skb_needs_check(skb, tx_path))) {
  2881. int err;
  2882. /* We're going to init ->check field in TCP or UDP header */
  2883. err = skb_cow_head(skb, 0);
  2884. if (err < 0)
  2885. return ERR_PTR(err);
  2886. }
  2887. /* Only report GSO partial support if it will enable us to
  2888. * support segmentation on this frame without needing additional
  2889. * work.
  2890. */
  2891. if (features & NETIF_F_GSO_PARTIAL) {
  2892. netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
  2893. struct net_device *dev = skb->dev;
  2894. partial_features |= dev->features & dev->gso_partial_features;
  2895. if (!skb_gso_ok(skb, features | partial_features))
  2896. features &= ~NETIF_F_GSO_PARTIAL;
  2897. }
  2898. BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
  2899. sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
  2900. SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
  2901. SKB_GSO_CB(skb)->encap_level = 0;
  2902. skb_reset_mac_header(skb);
  2903. skb_reset_mac_len(skb);
  2904. segs = skb_mac_gso_segment(skb, features);
  2905. if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
  2906. skb_warn_bad_offload(skb);
  2907. return segs;
  2908. }
  2909. EXPORT_SYMBOL(__skb_gso_segment);
  2910. /* Take action when hardware reception checksum errors are detected. */
  2911. #ifdef CONFIG_BUG
  2912. void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
  2913. {
  2914. if (net_ratelimit()) {
  2915. pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
  2916. skb_dump(KERN_ERR, skb, true);
  2917. dump_stack();
  2918. }
  2919. }
  2920. EXPORT_SYMBOL(netdev_rx_csum_fault);
  2921. #endif
  2922. /* XXX: check that highmem exists at all on the given machine. */
  2923. static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
  2924. {
  2925. #ifdef CONFIG_HIGHMEM
  2926. int i;
  2927. if (!(dev->features & NETIF_F_HIGHDMA)) {
  2928. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2929. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2930. if (PageHighMem(skb_frag_page(frag)))
  2931. return 1;
  2932. }
  2933. }
  2934. #endif
  2935. return 0;
  2936. }
  2937. /* If MPLS offload request, verify we are testing hardware MPLS features
  2938. * instead of standard features for the netdev.
  2939. */
  2940. #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
  2941. static netdev_features_t net_mpls_features(struct sk_buff *skb,
  2942. netdev_features_t features,
  2943. __be16 type)
  2944. {
  2945. if (eth_p_mpls(type))
  2946. features &= skb->dev->mpls_features;
  2947. return features;
  2948. }
  2949. #else
  2950. static netdev_features_t net_mpls_features(struct sk_buff *skb,
  2951. netdev_features_t features,
  2952. __be16 type)
  2953. {
  2954. return features;
  2955. }
  2956. #endif
  2957. static netdev_features_t harmonize_features(struct sk_buff *skb,
  2958. netdev_features_t features)
  2959. {
  2960. __be16 type;
  2961. type = skb_network_protocol(skb, NULL);
  2962. features = net_mpls_features(skb, features, type);
  2963. if (skb->ip_summed != CHECKSUM_NONE &&
  2964. !can_checksum_protocol(features, type)) {
  2965. features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
  2966. }
  2967. if (illegal_highdma(skb->dev, skb))
  2968. features &= ~NETIF_F_SG;
  2969. return features;
  2970. }
  2971. netdev_features_t passthru_features_check(struct sk_buff *skb,
  2972. struct net_device *dev,
  2973. netdev_features_t features)
  2974. {
  2975. return features;
  2976. }
  2977. EXPORT_SYMBOL(passthru_features_check);
  2978. static netdev_features_t dflt_features_check(struct sk_buff *skb,
  2979. struct net_device *dev,
  2980. netdev_features_t features)
  2981. {
  2982. return vlan_features_check(skb, features);
  2983. }
  2984. static netdev_features_t gso_features_check(const struct sk_buff *skb,
  2985. struct net_device *dev,
  2986. netdev_features_t features)
  2987. {
  2988. u16 gso_segs = skb_shinfo(skb)->gso_segs;
  2989. if (gso_segs > dev->gso_max_segs)
  2990. return features & ~NETIF_F_GSO_MASK;
  2991. /* Support for GSO partial features requires software
  2992. * intervention before we can actually process the packets
  2993. * so we need to strip support for any partial features now
  2994. * and we can pull them back in after we have partially
  2995. * segmented the frame.
  2996. */
  2997. if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
  2998. features &= ~dev->gso_partial_features;
  2999. /* Make sure to clear the IPv4 ID mangling feature if the
  3000. * IPv4 header has the potential to be fragmented.
  3001. */
  3002. if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
  3003. struct iphdr *iph = skb->encapsulation ?
  3004. inner_ip_hdr(skb) : ip_hdr(skb);
  3005. if (!(iph->frag_off & htons(IP_DF)))
  3006. features &= ~NETIF_F_TSO_MANGLEID;
  3007. }
  3008. return features;
  3009. }
  3010. netdev_features_t netif_skb_features(struct sk_buff *skb)
  3011. {
  3012. struct net_device *dev = skb->dev;
  3013. netdev_features_t features = dev->features;
  3014. if (skb_is_gso(skb))
  3015. features = gso_features_check(skb, dev, features);
  3016. /* If encapsulation offload request, verify we are testing
  3017. * hardware encapsulation features instead of standard
  3018. * features for the netdev
  3019. */
  3020. if (skb->encapsulation)
  3021. features &= dev->hw_enc_features;
  3022. if (skb_vlan_tagged(skb))
  3023. features = netdev_intersect_features(features,
  3024. dev->vlan_features |
  3025. NETIF_F_HW_VLAN_CTAG_TX |
  3026. NETIF_F_HW_VLAN_STAG_TX);
  3027. if (dev->netdev_ops->ndo_features_check)
  3028. features &= dev->netdev_ops->ndo_features_check(skb, dev,
  3029. features);
  3030. else
  3031. features &= dflt_features_check(skb, dev, features);
  3032. return harmonize_features(skb, features);
  3033. }
  3034. EXPORT_SYMBOL(netif_skb_features);
  3035. static int xmit_one(struct sk_buff *skb, struct net_device *dev,
  3036. struct netdev_queue *txq, bool more)
  3037. {
  3038. unsigned int len;
  3039. int rc;
  3040. if (dev_nit_active(dev))
  3041. dev_queue_xmit_nit(skb, dev);
  3042. len = skb->len;
  3043. PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
  3044. trace_net_dev_start_xmit(skb, dev);
  3045. rc = netdev_start_xmit(skb, dev, txq, more);
  3046. trace_net_dev_xmit(skb, rc, dev, len);
  3047. return rc;
  3048. }
  3049. struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
  3050. struct netdev_queue *txq, int *ret)
  3051. {
  3052. struct sk_buff *skb = first;
  3053. int rc = NETDEV_TX_OK;
  3054. while (skb) {
  3055. struct sk_buff *next = skb->next;
  3056. skb_mark_not_on_list(skb);
  3057. rc = xmit_one(skb, dev, txq, next != NULL);
  3058. if (unlikely(!dev_xmit_complete(rc))) {
  3059. skb->next = next;
  3060. goto out;
  3061. }
  3062. skb = next;
  3063. if (netif_tx_queue_stopped(txq) && skb) {
  3064. rc = NETDEV_TX_BUSY;
  3065. break;
  3066. }
  3067. }
  3068. out:
  3069. *ret = rc;
  3070. return skb;
  3071. }
  3072. static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
  3073. netdev_features_t features)
  3074. {
  3075. if (skb_vlan_tag_present(skb) &&
  3076. !vlan_hw_offload_capable(features, skb->vlan_proto))
  3077. skb = __vlan_hwaccel_push_inside(skb);
  3078. return skb;
  3079. }
  3080. int skb_csum_hwoffload_help(struct sk_buff *skb,
  3081. const netdev_features_t features)
  3082. {
  3083. if (unlikely(skb->csum_not_inet))
  3084. return !!(features & NETIF_F_SCTP_CRC) ? 0 :
  3085. skb_crc32c_csum_help(skb);
  3086. return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
  3087. }
  3088. EXPORT_SYMBOL(skb_csum_hwoffload_help);
  3089. static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
  3090. {
  3091. netdev_features_t features;
  3092. features = netif_skb_features(skb);
  3093. skb = validate_xmit_vlan(skb, features);
  3094. if (unlikely(!skb))
  3095. goto out_null;
  3096. skb = sk_validate_xmit_skb(skb, dev);
  3097. if (unlikely(!skb))
  3098. goto out_null;
  3099. if (netif_needs_gso(skb, features)) {
  3100. struct sk_buff *segs;
  3101. segs = skb_gso_segment(skb, features);
  3102. if (IS_ERR(segs)) {
  3103. goto out_kfree_skb;
  3104. } else if (segs) {
  3105. consume_skb(skb);
  3106. skb = segs;
  3107. }
  3108. } else {
  3109. if (skb_needs_linearize(skb, features) &&
  3110. __skb_linearize(skb))
  3111. goto out_kfree_skb;
  3112. /* If packet is not checksummed and device does not
  3113. * support checksumming for this protocol, complete
  3114. * checksumming here.
  3115. */
  3116. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  3117. if (skb->encapsulation)
  3118. skb_set_inner_transport_header(skb,
  3119. skb_checksum_start_offset(skb));
  3120. else
  3121. skb_set_transport_header(skb,
  3122. skb_checksum_start_offset(skb));
  3123. if (skb_csum_hwoffload_help(skb, features))
  3124. goto out_kfree_skb;
  3125. }
  3126. }
  3127. skb = validate_xmit_xfrm(skb, features, again);
  3128. return skb;
  3129. out_kfree_skb:
  3130. kfree_skb(skb);
  3131. out_null:
  3132. atomic_long_inc(&dev->tx_dropped);
  3133. return NULL;
  3134. }
  3135. struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
  3136. {
  3137. struct sk_buff *next, *head = NULL, *tail;
  3138. for (; skb != NULL; skb = next) {
  3139. next = skb->next;
  3140. skb_mark_not_on_list(skb);
  3141. /* in case skb wont be segmented, point to itself */
  3142. skb->prev = skb;
  3143. skb = validate_xmit_skb(skb, dev, again);
  3144. if (!skb)
  3145. continue;
  3146. if (!head)
  3147. head = skb;
  3148. else
  3149. tail->next = skb;
  3150. /* If skb was segmented, skb->prev points to
  3151. * the last segment. If not, it still contains skb.
  3152. */
  3153. tail = skb->prev;
  3154. }
  3155. return head;
  3156. }
  3157. EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
  3158. static void qdisc_pkt_len_init(struct sk_buff *skb)
  3159. {
  3160. const struct skb_shared_info *shinfo = skb_shinfo(skb);
  3161. qdisc_skb_cb(skb)->pkt_len = skb->len;
  3162. /* To get more precise estimation of bytes sent on wire,
  3163. * we add to pkt_len the headers size of all segments
  3164. */
  3165. if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
  3166. unsigned int hdr_len;
  3167. u16 gso_segs = shinfo->gso_segs;
  3168. /* mac layer + network layer */
  3169. hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
  3170. /* + transport layer */
  3171. if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
  3172. const struct tcphdr *th;
  3173. struct tcphdr _tcphdr;
  3174. th = skb_header_pointer(skb, skb_transport_offset(skb),
  3175. sizeof(_tcphdr), &_tcphdr);
  3176. if (likely(th))
  3177. hdr_len += __tcp_hdrlen(th);
  3178. } else {
  3179. struct udphdr _udphdr;
  3180. if (skb_header_pointer(skb, skb_transport_offset(skb),
  3181. sizeof(_udphdr), &_udphdr))
  3182. hdr_len += sizeof(struct udphdr);
  3183. }
  3184. if (shinfo->gso_type & SKB_GSO_DODGY)
  3185. gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
  3186. shinfo->gso_size);
  3187. qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
  3188. }
  3189. }
  3190. static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
  3191. struct net_device *dev,
  3192. struct netdev_queue *txq)
  3193. {
  3194. spinlock_t *root_lock = qdisc_lock(q);
  3195. struct sk_buff *to_free = NULL;
  3196. bool contended;
  3197. int rc;
  3198. qdisc_calculate_pkt_len(skb, q);
  3199. if (q->flags & TCQ_F_NOLOCK) {
  3200. rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
  3201. if (likely(!netif_xmit_frozen_or_stopped(txq)))
  3202. qdisc_run(q);
  3203. if (unlikely(to_free))
  3204. kfree_skb_list(to_free);
  3205. return rc;
  3206. }
  3207. /*
  3208. * Heuristic to force contended enqueues to serialize on a
  3209. * separate lock before trying to get qdisc main lock.
  3210. * This permits qdisc->running owner to get the lock more
  3211. * often and dequeue packets faster.
  3212. */
  3213. contended = qdisc_is_running(q);
  3214. if (unlikely(contended))
  3215. spin_lock(&q->busylock);
  3216. spin_lock(root_lock);
  3217. if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
  3218. __qdisc_drop(skb, &to_free);
  3219. rc = NET_XMIT_DROP;
  3220. } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
  3221. qdisc_run_begin(q)) {
  3222. /*
  3223. * This is a work-conserving queue; there are no old skbs
  3224. * waiting to be sent out; and the qdisc is not running -
  3225. * xmit the skb directly.
  3226. */
  3227. qdisc_bstats_update(q, skb);
  3228. if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
  3229. if (unlikely(contended)) {
  3230. spin_unlock(&q->busylock);
  3231. contended = false;
  3232. }
  3233. __qdisc_run(q);
  3234. }
  3235. qdisc_run_end(q);
  3236. rc = NET_XMIT_SUCCESS;
  3237. } else {
  3238. rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
  3239. if (qdisc_run_begin(q)) {
  3240. if (unlikely(contended)) {
  3241. spin_unlock(&q->busylock);
  3242. contended = false;
  3243. }
  3244. __qdisc_run(q);
  3245. qdisc_run_end(q);
  3246. }
  3247. }
  3248. spin_unlock(root_lock);
  3249. if (unlikely(to_free))
  3250. kfree_skb_list(to_free);
  3251. if (unlikely(contended))
  3252. spin_unlock(&q->busylock);
  3253. return rc;
  3254. }
  3255. #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
  3256. static void skb_update_prio(struct sk_buff *skb)
  3257. {
  3258. const struct netprio_map *map;
  3259. const struct sock *sk;
  3260. unsigned int prioidx;
  3261. if (skb->priority)
  3262. return;
  3263. map = rcu_dereference_bh(skb->dev->priomap);
  3264. if (!map)
  3265. return;
  3266. sk = skb_to_full_sk(skb);
  3267. if (!sk)
  3268. return;
  3269. prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
  3270. if (prioidx < map->priomap_len)
  3271. skb->priority = map->priomap[prioidx];
  3272. }
  3273. #else
  3274. #define skb_update_prio(skb)
  3275. #endif
  3276. /**
  3277. * dev_loopback_xmit - loop back @skb
  3278. * @net: network namespace this loopback is happening in
  3279. * @sk: sk needed to be a netfilter okfn
  3280. * @skb: buffer to transmit
  3281. */
  3282. int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
  3283. {
  3284. skb_reset_mac_header(skb);
  3285. __skb_pull(skb, skb_network_offset(skb));
  3286. skb->pkt_type = PACKET_LOOPBACK;
  3287. if (skb->ip_summed == CHECKSUM_NONE)
  3288. skb->ip_summed = CHECKSUM_UNNECESSARY;
  3289. WARN_ON(!skb_dst(skb));
  3290. skb_dst_force(skb);
  3291. netif_rx_ni(skb);
  3292. return 0;
  3293. }
  3294. EXPORT_SYMBOL(dev_loopback_xmit);
  3295. #ifdef CONFIG_NET_EGRESS
  3296. static struct sk_buff *
  3297. sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
  3298. {
  3299. struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
  3300. struct tcf_result cl_res;
  3301. if (!miniq)
  3302. return skb;
  3303. /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
  3304. qdisc_skb_cb(skb)->mru = 0;
  3305. mini_qdisc_bstats_cpu_update(miniq, skb);
  3306. switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
  3307. case TC_ACT_OK:
  3308. case TC_ACT_RECLASSIFY:
  3309. skb->tc_index = TC_H_MIN(cl_res.classid);
  3310. break;
  3311. case TC_ACT_SHOT:
  3312. mini_qdisc_qstats_cpu_drop(miniq);
  3313. *ret = NET_XMIT_DROP;
  3314. kfree_skb(skb);
  3315. return NULL;
  3316. case TC_ACT_STOLEN:
  3317. case TC_ACT_QUEUED:
  3318. case TC_ACT_TRAP:
  3319. *ret = NET_XMIT_SUCCESS;
  3320. consume_skb(skb);
  3321. return NULL;
  3322. case TC_ACT_REDIRECT:
  3323. /* No need to push/pop skb's mac_header here on egress! */
  3324. skb_do_redirect(skb);
  3325. *ret = NET_XMIT_SUCCESS;
  3326. return NULL;
  3327. default:
  3328. break;
  3329. }
  3330. return skb;
  3331. }
  3332. #endif /* CONFIG_NET_EGRESS */
  3333. #ifdef CONFIG_XPS
  3334. static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
  3335. struct xps_dev_maps *dev_maps, unsigned int tci)
  3336. {
  3337. struct xps_map *map;
  3338. int queue_index = -1;
  3339. if (dev->num_tc) {
  3340. tci *= dev->num_tc;
  3341. tci += netdev_get_prio_tc_map(dev, skb->priority);
  3342. }
  3343. map = rcu_dereference(dev_maps->attr_map[tci]);
  3344. if (map) {
  3345. if (map->len == 1)
  3346. queue_index = map->queues[0];
  3347. else
  3348. queue_index = map->queues[reciprocal_scale(
  3349. skb_get_hash(skb), map->len)];
  3350. if (unlikely(queue_index >= dev->real_num_tx_queues))
  3351. queue_index = -1;
  3352. }
  3353. return queue_index;
  3354. }
  3355. #endif
  3356. static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
  3357. struct sk_buff *skb)
  3358. {
  3359. #ifdef CONFIG_XPS
  3360. struct xps_dev_maps *dev_maps;
  3361. struct sock *sk = skb->sk;
  3362. int queue_index = -1;
  3363. if (!static_key_false(&xps_needed))
  3364. return -1;
  3365. rcu_read_lock();
  3366. if (!static_key_false(&xps_rxqs_needed))
  3367. goto get_cpus_map;
  3368. dev_maps = rcu_dereference(sb_dev->xps_rxqs_map);
  3369. if (dev_maps) {
  3370. int tci = sk_rx_queue_get(sk);
  3371. if (tci >= 0 && tci < dev->num_rx_queues)
  3372. queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
  3373. tci);
  3374. }
  3375. get_cpus_map:
  3376. if (queue_index < 0) {
  3377. dev_maps = rcu_dereference(sb_dev->xps_cpus_map);
  3378. if (dev_maps) {
  3379. unsigned int tci = skb->sender_cpu - 1;
  3380. queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
  3381. tci);
  3382. }
  3383. }
  3384. rcu_read_unlock();
  3385. return queue_index;
  3386. #else
  3387. return -1;
  3388. #endif
  3389. }
  3390. u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
  3391. struct net_device *sb_dev)
  3392. {
  3393. return 0;
  3394. }
  3395. EXPORT_SYMBOL(dev_pick_tx_zero);
  3396. u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
  3397. struct net_device *sb_dev)
  3398. {
  3399. return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
  3400. }
  3401. EXPORT_SYMBOL(dev_pick_tx_cpu_id);
  3402. u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
  3403. struct net_device *sb_dev)
  3404. {
  3405. struct sock *sk = skb->sk;
  3406. int queue_index = sk_tx_queue_get(sk);
  3407. sb_dev = sb_dev ? : dev;
  3408. if (queue_index < 0 || skb->ooo_okay ||
  3409. queue_index >= dev->real_num_tx_queues) {
  3410. int new_index = get_xps_queue(dev, sb_dev, skb);
  3411. if (new_index < 0)
  3412. new_index = skb_tx_hash(dev, sb_dev, skb);
  3413. if (queue_index != new_index && sk &&
  3414. sk_fullsock(sk) &&
  3415. rcu_access_pointer(sk->sk_dst_cache))
  3416. sk_tx_queue_set(sk, new_index);
  3417. queue_index = new_index;
  3418. }
  3419. return queue_index;
  3420. }
  3421. EXPORT_SYMBOL(netdev_pick_tx);
  3422. struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
  3423. struct sk_buff *skb,
  3424. struct net_device *sb_dev)
  3425. {
  3426. int queue_index = 0;
  3427. #ifdef CONFIG_XPS
  3428. u32 sender_cpu = skb->sender_cpu - 1;
  3429. if (sender_cpu >= (u32)NR_CPUS)
  3430. skb->sender_cpu = raw_smp_processor_id() + 1;
  3431. #endif
  3432. if (dev->real_num_tx_queues != 1) {
  3433. const struct net_device_ops *ops = dev->netdev_ops;
  3434. if (ops->ndo_select_queue)
  3435. queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
  3436. else
  3437. queue_index = netdev_pick_tx(dev, skb, sb_dev);
  3438. queue_index = netdev_cap_txqueue(dev, queue_index);
  3439. }
  3440. skb_set_queue_mapping(skb, queue_index);
  3441. return netdev_get_tx_queue(dev, queue_index);
  3442. }
  3443. /**
  3444. * __dev_queue_xmit - transmit a buffer
  3445. * @skb: buffer to transmit
  3446. * @sb_dev: suboordinate device used for L2 forwarding offload
  3447. *
  3448. * Queue a buffer for transmission to a network device. The caller must
  3449. * have set the device and priority and built the buffer before calling
  3450. * this function. The function can be called from an interrupt.
  3451. *
  3452. * A negative errno code is returned on a failure. A success does not
  3453. * guarantee the frame will be transmitted as it may be dropped due
  3454. * to congestion or traffic shaping.
  3455. *
  3456. * -----------------------------------------------------------------------------------
  3457. * I notice this method can also return errors from the queue disciplines,
  3458. * including NET_XMIT_DROP, which is a positive value. So, errors can also
  3459. * be positive.
  3460. *
  3461. * Regardless of the return value, the skb is consumed, so it is currently
  3462. * difficult to retry a send to this method. (You can bump the ref count
  3463. * before sending to hold a reference for retry if you are careful.)
  3464. *
  3465. * When calling this method, interrupts MUST be enabled. This is because
  3466. * the BH enable code must have IRQs enabled so that it will not deadlock.
  3467. * --BLG
  3468. */
  3469. static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
  3470. {
  3471. struct net_device *dev = skb->dev;
  3472. struct netdev_queue *txq;
  3473. struct Qdisc *q;
  3474. int rc = -ENOMEM;
  3475. bool again = false;
  3476. skb_reset_mac_header(skb);
  3477. if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
  3478. __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
  3479. /* Disable soft irqs for various locks below. Also
  3480. * stops preemption for RCU.
  3481. */
  3482. rcu_read_lock_bh();
  3483. skb_update_prio(skb);
  3484. qdisc_pkt_len_init(skb);
  3485. #ifdef CONFIG_NET_CLS_ACT
  3486. skb->tc_at_ingress = 0;
  3487. # ifdef CONFIG_NET_EGRESS
  3488. if (static_branch_unlikely(&egress_needed_key)) {
  3489. skb = sch_handle_egress(skb, &rc, dev);
  3490. if (!skb)
  3491. goto out;
  3492. }
  3493. # endif
  3494. #endif
  3495. /* If device/qdisc don't need skb->dst, release it right now while
  3496. * its hot in this cpu cache.
  3497. */
  3498. if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
  3499. skb_dst_drop(skb);
  3500. else
  3501. skb_dst_force(skb);
  3502. txq = netdev_core_pick_tx(dev, skb, sb_dev);
  3503. q = rcu_dereference_bh(txq->qdisc);
  3504. trace_net_dev_queue(skb);
  3505. if (q->enqueue) {
  3506. rc = __dev_xmit_skb(skb, q, dev, txq);
  3507. goto out;
  3508. }
  3509. /* The device has no queue. Common case for software devices:
  3510. * loopback, all the sorts of tunnels...
  3511. * Really, it is unlikely that netif_tx_lock protection is necessary
  3512. * here. (f.e. loopback and IP tunnels are clean ignoring statistics
  3513. * counters.)
  3514. * However, it is possible, that they rely on protection
  3515. * made by us here.
  3516. * Check this and shot the lock. It is not prone from deadlocks.
  3517. *Either shot noqueue qdisc, it is even simpler 8)
  3518. */
  3519. if (dev->flags & IFF_UP) {
  3520. int cpu = smp_processor_id(); /* ok because BHs are off */
  3521. /* Other cpus might concurrently change txq->xmit_lock_owner
  3522. * to -1 or to their cpu id, but not to our id.
  3523. */
  3524. if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
  3525. if (dev_xmit_recursion())
  3526. goto recursion_alert;
  3527. skb = validate_xmit_skb(skb, dev, &again);
  3528. if (!skb)
  3529. goto out;
  3530. PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
  3531. HARD_TX_LOCK(dev, txq, cpu);
  3532. if (!netif_xmit_stopped(txq)) {
  3533. dev_xmit_recursion_inc();
  3534. skb = dev_hard_start_xmit(skb, dev, txq, &rc);
  3535. dev_xmit_recursion_dec();
  3536. if (dev_xmit_complete(rc)) {
  3537. HARD_TX_UNLOCK(dev, txq);
  3538. goto out;
  3539. }
  3540. }
  3541. HARD_TX_UNLOCK(dev, txq);
  3542. net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
  3543. dev->name);
  3544. } else {
  3545. /* Recursion is detected! It is possible,
  3546. * unfortunately
  3547. */
  3548. recursion_alert:
  3549. net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
  3550. dev->name);
  3551. }
  3552. }
  3553. rc = -ENETDOWN;
  3554. rcu_read_unlock_bh();
  3555. atomic_long_inc(&dev->tx_dropped);
  3556. kfree_skb_list(skb);
  3557. return rc;
  3558. out:
  3559. rcu_read_unlock_bh();
  3560. return rc;
  3561. }
  3562. int dev_queue_xmit(struct sk_buff *skb)
  3563. {
  3564. return __dev_queue_xmit(skb, NULL);
  3565. }
  3566. EXPORT_SYMBOL(dev_queue_xmit);
  3567. int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
  3568. {
  3569. return __dev_queue_xmit(skb, sb_dev);
  3570. }
  3571. EXPORT_SYMBOL(dev_queue_xmit_accel);
  3572. int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
  3573. {
  3574. struct net_device *dev = skb->dev;
  3575. struct sk_buff *orig_skb = skb;
  3576. struct netdev_queue *txq;
  3577. int ret = NETDEV_TX_BUSY;
  3578. bool again = false;
  3579. if (unlikely(!netif_running(dev) ||
  3580. !netif_carrier_ok(dev)))
  3581. goto drop;
  3582. skb = validate_xmit_skb_list(skb, dev, &again);
  3583. if (skb != orig_skb)
  3584. goto drop;
  3585. skb_set_queue_mapping(skb, queue_id);
  3586. txq = skb_get_tx_queue(dev, skb);
  3587. PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
  3588. local_bh_disable();
  3589. dev_xmit_recursion_inc();
  3590. HARD_TX_LOCK(dev, txq, smp_processor_id());
  3591. if (!netif_xmit_frozen_or_drv_stopped(txq))
  3592. ret = netdev_start_xmit(skb, dev, txq, false);
  3593. HARD_TX_UNLOCK(dev, txq);
  3594. dev_xmit_recursion_dec();
  3595. local_bh_enable();
  3596. return ret;
  3597. drop:
  3598. atomic_long_inc(&dev->tx_dropped);
  3599. kfree_skb_list(skb);
  3600. return NET_XMIT_DROP;
  3601. }
  3602. EXPORT_SYMBOL(__dev_direct_xmit);
  3603. /*************************************************************************
  3604. * Receiver routines
  3605. *************************************************************************/
  3606. int netdev_max_backlog __read_mostly = 1000;
  3607. EXPORT_SYMBOL(netdev_max_backlog);
  3608. int netdev_tstamp_prequeue __read_mostly = 1;
  3609. int netdev_budget __read_mostly = 300;
  3610. /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
  3611. unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
  3612. int weight_p __read_mostly = 64; /* old backlog weight */
  3613. int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
  3614. int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
  3615. int dev_rx_weight __read_mostly = 64;
  3616. int dev_tx_weight __read_mostly = 64;
  3617. /* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
  3618. int gro_normal_batch __read_mostly = 8;
  3619. /* Called with irq disabled */
  3620. static inline void ____napi_schedule(struct softnet_data *sd,
  3621. struct napi_struct *napi)
  3622. {
  3623. list_add_tail(&napi->poll_list, &sd->poll_list);
  3624. __raise_softirq_irqoff(NET_RX_SOFTIRQ);
  3625. }
  3626. #ifdef CONFIG_RPS
  3627. /* One global table that all flow-based protocols share. */
  3628. struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
  3629. EXPORT_SYMBOL(rps_sock_flow_table);
  3630. u32 rps_cpu_mask __read_mostly;
  3631. EXPORT_SYMBOL(rps_cpu_mask);
  3632. struct static_key_false rps_needed __read_mostly;
  3633. EXPORT_SYMBOL(rps_needed);
  3634. struct static_key_false rfs_needed __read_mostly;
  3635. EXPORT_SYMBOL(rfs_needed);
  3636. static struct rps_dev_flow *
  3637. set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
  3638. struct rps_dev_flow *rflow, u16 next_cpu)
  3639. {
  3640. if (next_cpu < nr_cpu_ids) {
  3641. #ifdef CONFIG_RFS_ACCEL
  3642. struct netdev_rx_queue *rxqueue;
  3643. struct rps_dev_flow_table *flow_table;
  3644. struct rps_dev_flow *old_rflow;
  3645. u32 flow_id;
  3646. u16 rxq_index;
  3647. int rc;
  3648. /* Should we steer this flow to a different hardware queue? */
  3649. if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
  3650. !(dev->features & NETIF_F_NTUPLE))
  3651. goto out;
  3652. rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
  3653. if (rxq_index == skb_get_rx_queue(skb))
  3654. goto out;
  3655. rxqueue = dev->_rx + rxq_index;
  3656. flow_table = rcu_dereference(rxqueue->rps_flow_table);
  3657. if (!flow_table)
  3658. goto out;
  3659. flow_id = skb_get_hash(skb) & flow_table->mask;
  3660. rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
  3661. rxq_index, flow_id);
  3662. if (rc < 0)
  3663. goto out;
  3664. old_rflow = rflow;
  3665. rflow = &flow_table->flows[flow_id];
  3666. rflow->filter = rc;
  3667. if (old_rflow->filter == rflow->filter)
  3668. old_rflow->filter = RPS_NO_FILTER;
  3669. out:
  3670. #endif
  3671. rflow->last_qtail =
  3672. per_cpu(softnet_data, next_cpu).input_queue_head;
  3673. }
  3674. rflow->cpu = next_cpu;
  3675. return rflow;
  3676. }
  3677. /*
  3678. * get_rps_cpu is called from netif_receive_skb and returns the target
  3679. * CPU from the RPS map of the receiving queue for a given skb.
  3680. * rcu_read_lock must be held on entry.
  3681. */
  3682. static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
  3683. struct rps_dev_flow **rflowp)
  3684. {
  3685. const struct rps_sock_flow_table *sock_flow_table;
  3686. struct netdev_rx_queue *rxqueue = dev->_rx;
  3687. struct rps_dev_flow_table *flow_table;
  3688. struct rps_map *map;
  3689. int cpu = -1;
  3690. u32 tcpu;
  3691. u32 hash;
  3692. if (skb_rx_queue_recorded(skb)) {
  3693. u16 index = skb_get_rx_queue(skb);
  3694. if (unlikely(index >= dev->real_num_rx_queues)) {
  3695. WARN_ONCE(dev->real_num_rx_queues > 1,
  3696. "%s received packet on queue %u, but number "
  3697. "of RX queues is %u\n",
  3698. dev->name, index, dev->real_num_rx_queues);
  3699. goto done;
  3700. }
  3701. rxqueue += index;
  3702. }
  3703. /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
  3704. flow_table = rcu_dereference(rxqueue->rps_flow_table);
  3705. map = rcu_dereference(rxqueue->rps_map);
  3706. if (!flow_table && !map)
  3707. goto done;
  3708. skb_reset_network_header(skb);
  3709. hash = skb_get_hash(skb);
  3710. if (!hash)
  3711. goto done;
  3712. sock_flow_table = rcu_dereference(rps_sock_flow_table);
  3713. if (flow_table && sock_flow_table) {
  3714. struct rps_dev_flow *rflow;
  3715. u32 next_cpu;
  3716. u32 ident;
  3717. /* First check into global flow table if there is a match */
  3718. ident = sock_flow_table->ents[hash & sock_flow_table->mask];
  3719. if ((ident ^ hash) & ~rps_cpu_mask)
  3720. goto try_rps;
  3721. next_cpu = ident & rps_cpu_mask;
  3722. /* OK, now we know there is a match,
  3723. * we can look at the local (per receive queue) flow table
  3724. */
  3725. rflow = &flow_table->flows[hash & flow_table->mask];
  3726. tcpu = rflow->cpu;
  3727. /*
  3728. * If the desired CPU (where last recvmsg was done) is
  3729. * different from current CPU (one in the rx-queue flow
  3730. * table entry), switch if one of the following holds:
  3731. * - Current CPU is unset (>= nr_cpu_ids).
  3732. * - Current CPU is offline.
  3733. * - The current CPU's queue tail has advanced beyond the
  3734. * last packet that was enqueued using this table entry.
  3735. * This guarantees that all previous packets for the flow
  3736. * have been dequeued, thus preserving in order delivery.
  3737. */
  3738. if (unlikely(tcpu != next_cpu) &&
  3739. (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
  3740. ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
  3741. rflow->last_qtail)) >= 0)) {
  3742. tcpu = next_cpu;
  3743. rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
  3744. }
  3745. if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
  3746. *rflowp = rflow;
  3747. cpu = tcpu;
  3748. goto done;
  3749. }
  3750. }
  3751. try_rps:
  3752. if (map) {
  3753. tcpu = map->cpus[reciprocal_scale(hash, map->len)];
  3754. if (cpu_online(tcpu)) {
  3755. cpu = tcpu;
  3756. goto done;
  3757. }
  3758. }
  3759. done:
  3760. return cpu;
  3761. }
  3762. #ifdef CONFIG_RFS_ACCEL
  3763. /**
  3764. * rps_may_expire_flow - check whether an RFS hardware filter may be removed
  3765. * @dev: Device on which the filter was set
  3766. * @rxq_index: RX queue index
  3767. * @flow_id: Flow ID passed to ndo_rx_flow_steer()
  3768. * @filter_id: Filter ID returned by ndo_rx_flow_steer()
  3769. *
  3770. * Drivers that implement ndo_rx_flow_steer() should periodically call
  3771. * this function for each installed filter and remove the filters for
  3772. * which it returns %true.
  3773. */
  3774. bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
  3775. u32 flow_id, u16 filter_id)
  3776. {
  3777. struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
  3778. struct rps_dev_flow_table *flow_table;
  3779. struct rps_dev_flow *rflow;
  3780. bool expire = true;
  3781. unsigned int cpu;
  3782. rcu_read_lock();
  3783. flow_table = rcu_dereference(rxqueue->rps_flow_table);
  3784. if (flow_table && flow_id <= flow_table->mask) {
  3785. rflow = &flow_table->flows[flow_id];
  3786. cpu = READ_ONCE(rflow->cpu);
  3787. if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
  3788. ((int)(per_cpu(softnet_data, cpu).input_queue_head -
  3789. rflow->last_qtail) <
  3790. (int)(10 * flow_table->mask)))
  3791. expire = false;
  3792. }
  3793. rcu_read_unlock();
  3794. return expire;
  3795. }
  3796. EXPORT_SYMBOL(rps_may_expire_flow);
  3797. #endif /* CONFIG_RFS_ACCEL */
  3798. /* Called from hardirq (IPI) context */
  3799. static void rps_trigger_softirq(void *data)
  3800. {
  3801. struct softnet_data *sd = data;
  3802. ____napi_schedule(sd, &sd->backlog);
  3803. sd->received_rps++;
  3804. }
  3805. #endif /* CONFIG_RPS */
  3806. /*
  3807. * Check if this softnet_data structure is another cpu one
  3808. * If yes, queue it to our IPI list and return 1
  3809. * If no, return 0
  3810. */
  3811. static int rps_ipi_queued(struct softnet_data *sd)
  3812. {
  3813. #ifdef CONFIG_RPS
  3814. struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
  3815. if (sd != mysd) {
  3816. sd->rps_ipi_next = mysd->rps_ipi_list;
  3817. mysd->rps_ipi_list = sd;
  3818. __raise_softirq_irqoff(NET_RX_SOFTIRQ);
  3819. return 1;
  3820. }
  3821. #endif /* CONFIG_RPS */
  3822. return 0;
  3823. }
  3824. #ifdef CONFIG_NET_FLOW_LIMIT
  3825. int netdev_flow_limit_table_len __read_mostly = (1 << 12);
  3826. #endif
  3827. static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
  3828. {
  3829. #ifdef CONFIG_NET_FLOW_LIMIT
  3830. struct sd_flow_limit *fl;
  3831. struct softnet_data *sd;
  3832. unsigned int old_flow, new_flow;
  3833. if (qlen < (netdev_max_backlog >> 1))
  3834. return false;
  3835. sd = this_cpu_ptr(&softnet_data);
  3836. rcu_read_lock();
  3837. fl = rcu_dereference(sd->flow_limit);
  3838. if (fl) {
  3839. new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
  3840. old_flow = fl->history[fl->history_head];
  3841. fl->history[fl->history_head] = new_flow;
  3842. fl->history_head++;
  3843. fl->history_head &= FLOW_LIMIT_HISTORY - 1;
  3844. if (likely(fl->buckets[old_flow]))
  3845. fl->buckets[old_flow]--;
  3846. if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
  3847. fl->count++;
  3848. rcu_read_unlock();
  3849. return true;
  3850. }
  3851. }
  3852. rcu_read_unlock();
  3853. #endif
  3854. return false;
  3855. }
  3856. /*
  3857. * enqueue_to_backlog is called to queue an skb to a per CPU backlog
  3858. * queue (may be a remote CPU queue).
  3859. */
  3860. static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
  3861. unsigned int *qtail)
  3862. {
  3863. struct softnet_data *sd;
  3864. unsigned long flags;
  3865. unsigned int qlen;
  3866. sd = &per_cpu(softnet_data, cpu);
  3867. local_irq_save(flags);
  3868. rps_lock(sd);
  3869. if (!netif_running(skb->dev))
  3870. goto drop;
  3871. qlen = skb_queue_len(&sd->input_pkt_queue);
  3872. if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
  3873. if (qlen) {
  3874. enqueue:
  3875. __skb_queue_tail(&sd->input_pkt_queue, skb);
  3876. input_queue_tail_incr_save(sd, qtail);
  3877. rps_unlock(sd);
  3878. local_irq_restore(flags);
  3879. return NET_RX_SUCCESS;
  3880. }
  3881. /* Schedule NAPI for backlog device
  3882. * We can use non atomic operation since we own the queue lock
  3883. */
  3884. if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
  3885. if (!rps_ipi_queued(sd))
  3886. ____napi_schedule(sd, &sd->backlog);
  3887. }
  3888. goto enqueue;
  3889. }
  3890. drop:
  3891. sd->dropped++;
  3892. rps_unlock(sd);
  3893. local_irq_restore(flags);
  3894. atomic_long_inc(&skb->dev->rx_dropped);
  3895. kfree_skb(skb);
  3896. return NET_RX_DROP;
  3897. }
  3898. static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
  3899. {
  3900. struct net_device *dev = skb->dev;
  3901. struct netdev_rx_queue *rxqueue;
  3902. rxqueue = dev->_rx;
  3903. if (skb_rx_queue_recorded(skb)) {
  3904. u16 index = skb_get_rx_queue(skb);
  3905. if (unlikely(index >= dev->real_num_rx_queues)) {
  3906. WARN_ONCE(dev->real_num_rx_queues > 1,
  3907. "%s received packet on queue %u, but number "
  3908. "of RX queues is %u\n",
  3909. dev->name, index, dev->real_num_rx_queues);
  3910. return rxqueue; /* Return first rxqueue */
  3911. }
  3912. rxqueue += index;
  3913. }
  3914. return rxqueue;
  3915. }
  3916. static u32 netif_receive_generic_xdp(struct sk_buff *skb,
  3917. struct xdp_buff *xdp,
  3918. struct bpf_prog *xdp_prog)
  3919. {
  3920. struct netdev_rx_queue *rxqueue;
  3921. void *orig_data, *orig_data_end;
  3922. u32 metalen, act = XDP_DROP;
  3923. __be16 orig_eth_type;
  3924. struct ethhdr *eth;
  3925. bool orig_bcast;
  3926. int hlen, off;
  3927. u32 mac_len;
  3928. /* Reinjected packets coming from act_mirred or similar should
  3929. * not get XDP generic processing.
  3930. */
  3931. if (skb_is_redirected(skb))
  3932. return XDP_PASS;
  3933. /* XDP packets must be linear and must have sufficient headroom
  3934. * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
  3935. * native XDP provides, thus we need to do it here as well.
  3936. */
  3937. if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
  3938. skb_headroom(skb) < XDP_PACKET_HEADROOM) {
  3939. int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
  3940. int troom = skb->tail + skb->data_len - skb->end;
  3941. /* In case we have to go down the path and also linearize,
  3942. * then lets do the pskb_expand_head() work just once here.
  3943. */
  3944. if (pskb_expand_head(skb,
  3945. hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
  3946. troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
  3947. goto do_drop;
  3948. if (skb_linearize(skb))
  3949. goto do_drop;
  3950. }
  3951. /* The XDP program wants to see the packet starting at the MAC
  3952. * header.
  3953. */
  3954. mac_len = skb->data - skb_mac_header(skb);
  3955. hlen = skb_headlen(skb) + mac_len;
  3956. xdp->data = skb->data - mac_len;
  3957. xdp->data_meta = xdp->data;
  3958. xdp->data_end = xdp->data + hlen;
  3959. xdp->data_hard_start = skb->data - skb_headroom(skb);
  3960. /* SKB "head" area always have tailroom for skb_shared_info */
  3961. xdp->frame_sz = (void *)skb_end_pointer(skb) - xdp->data_hard_start;
  3962. xdp->frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  3963. orig_data_end = xdp->data_end;
  3964. orig_data = xdp->data;
  3965. eth = (struct ethhdr *)xdp->data;
  3966. orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
  3967. orig_eth_type = eth->h_proto;
  3968. rxqueue = netif_get_rxqueue(skb);
  3969. xdp->rxq = &rxqueue->xdp_rxq;
  3970. act = bpf_prog_run_xdp(xdp_prog, xdp);
  3971. /* check if bpf_xdp_adjust_head was used */
  3972. off = xdp->data - orig_data;
  3973. if (off) {
  3974. if (off > 0)
  3975. __skb_pull(skb, off);
  3976. else if (off < 0)
  3977. __skb_push(skb, -off);
  3978. skb->mac_header += off;
  3979. skb_reset_network_header(skb);
  3980. }
  3981. /* check if bpf_xdp_adjust_tail was used */
  3982. off = xdp->data_end - orig_data_end;
  3983. if (off != 0) {
  3984. skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
  3985. skb->len += off; /* positive on grow, negative on shrink */
  3986. }
  3987. /* check if XDP changed eth hdr such SKB needs update */
  3988. eth = (struct ethhdr *)xdp->data;
  3989. if ((orig_eth_type != eth->h_proto) ||
  3990. (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
  3991. __skb_push(skb, ETH_HLEN);
  3992. skb->protocol = eth_type_trans(skb, skb->dev);
  3993. }
  3994. switch (act) {
  3995. case XDP_REDIRECT:
  3996. case XDP_TX:
  3997. __skb_push(skb, mac_len);
  3998. break;
  3999. case XDP_PASS:
  4000. metalen = xdp->data - xdp->data_meta;
  4001. if (metalen)
  4002. skb_metadata_set(skb, metalen);
  4003. break;
  4004. default:
  4005. bpf_warn_invalid_xdp_action(act);
  4006. fallthrough;
  4007. case XDP_ABORTED:
  4008. trace_xdp_exception(skb->dev, xdp_prog, act);
  4009. fallthrough;
  4010. case XDP_DROP:
  4011. do_drop:
  4012. kfree_skb(skb);
  4013. break;
  4014. }
  4015. return act;
  4016. }
  4017. /* When doing generic XDP we have to bypass the qdisc layer and the
  4018. * network taps in order to match in-driver-XDP behavior.
  4019. */
  4020. void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
  4021. {
  4022. struct net_device *dev = skb->dev;
  4023. struct netdev_queue *txq;
  4024. bool free_skb = true;
  4025. int cpu, rc;
  4026. txq = netdev_core_pick_tx(dev, skb, NULL);
  4027. cpu = smp_processor_id();
  4028. HARD_TX_LOCK(dev, txq, cpu);
  4029. if (!netif_xmit_stopped(txq)) {
  4030. rc = netdev_start_xmit(skb, dev, txq, 0);
  4031. if (dev_xmit_complete(rc))
  4032. free_skb = false;
  4033. }
  4034. HARD_TX_UNLOCK(dev, txq);
  4035. if (free_skb) {
  4036. trace_xdp_exception(dev, xdp_prog, XDP_TX);
  4037. kfree_skb(skb);
  4038. }
  4039. }
  4040. static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
  4041. int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
  4042. {
  4043. if (xdp_prog) {
  4044. struct xdp_buff xdp;
  4045. u32 act;
  4046. int err;
  4047. act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
  4048. if (act != XDP_PASS) {
  4049. switch (act) {
  4050. case XDP_REDIRECT:
  4051. err = xdp_do_generic_redirect(skb->dev, skb,
  4052. &xdp, xdp_prog);
  4053. if (err)
  4054. goto out_redir;
  4055. break;
  4056. case XDP_TX:
  4057. generic_xdp_tx(skb, xdp_prog);
  4058. break;
  4059. }
  4060. return XDP_DROP;
  4061. }
  4062. }
  4063. return XDP_PASS;
  4064. out_redir:
  4065. kfree_skb(skb);
  4066. return XDP_DROP;
  4067. }
  4068. EXPORT_SYMBOL_GPL(do_xdp_generic);
  4069. static int netif_rx_internal(struct sk_buff *skb)
  4070. {
  4071. int ret;
  4072. net_timestamp_check(netdev_tstamp_prequeue, skb);
  4073. trace_netif_rx(skb);
  4074. #ifdef CONFIG_RPS
  4075. if (static_branch_unlikely(&rps_needed)) {
  4076. struct rps_dev_flow voidflow, *rflow = &voidflow;
  4077. int cpu;
  4078. preempt_disable();
  4079. rcu_read_lock();
  4080. cpu = get_rps_cpu(skb->dev, skb, &rflow);
  4081. if (cpu < 0)
  4082. cpu = smp_processor_id();
  4083. ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
  4084. rcu_read_unlock();
  4085. preempt_enable();
  4086. } else
  4087. #endif
  4088. {
  4089. unsigned int qtail;
  4090. ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
  4091. put_cpu();
  4092. }
  4093. return ret;
  4094. }
  4095. /**
  4096. * netif_rx - post buffer to the network code
  4097. * @skb: buffer to post
  4098. *
  4099. * This function receives a packet from a device driver and queues it for
  4100. * the upper (protocol) levels to process. It always succeeds. The buffer
  4101. * may be dropped during processing for congestion control or by the
  4102. * protocol layers.
  4103. *
  4104. * return values:
  4105. * NET_RX_SUCCESS (no congestion)
  4106. * NET_RX_DROP (packet was dropped)
  4107. *
  4108. */
  4109. int netif_rx(struct sk_buff *skb)
  4110. {
  4111. int ret;
  4112. trace_netif_rx_entry(skb);
  4113. ret = netif_rx_internal(skb);
  4114. trace_netif_rx_exit(ret);
  4115. return ret;
  4116. }
  4117. EXPORT_SYMBOL(netif_rx);
  4118. int netif_rx_ni(struct sk_buff *skb)
  4119. {
  4120. int err;
  4121. trace_netif_rx_ni_entry(skb);
  4122. preempt_disable();
  4123. err = netif_rx_internal(skb);
  4124. if (local_softirq_pending())
  4125. do_softirq();
  4126. preempt_enable();
  4127. trace_netif_rx_ni_exit(err);
  4128. return err;
  4129. }
  4130. EXPORT_SYMBOL(netif_rx_ni);
  4131. int netif_rx_any_context(struct sk_buff *skb)
  4132. {
  4133. /*
  4134. * If invoked from contexts which do not invoke bottom half
  4135. * processing either at return from interrupt or when softrqs are
  4136. * reenabled, use netif_rx_ni() which invokes bottomhalf processing
  4137. * directly.
  4138. */
  4139. if (in_interrupt())
  4140. return netif_rx(skb);
  4141. else
  4142. return netif_rx_ni(skb);
  4143. }
  4144. EXPORT_SYMBOL(netif_rx_any_context);
  4145. static __latent_entropy void net_tx_action(struct softirq_action *h)
  4146. {
  4147. struct softnet_data *sd = this_cpu_ptr(&softnet_data);
  4148. if (sd->completion_queue) {
  4149. struct sk_buff *clist;
  4150. local_irq_disable();
  4151. clist = sd->completion_queue;
  4152. sd->completion_queue = NULL;
  4153. local_irq_enable();
  4154. while (clist) {
  4155. struct sk_buff *skb = clist;
  4156. clist = clist->next;
  4157. WARN_ON(refcount_read(&skb->users));
  4158. if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
  4159. trace_consume_skb(skb);
  4160. else
  4161. trace_kfree_skb(skb, net_tx_action);
  4162. if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
  4163. __kfree_skb(skb);
  4164. else
  4165. __kfree_skb_defer(skb);
  4166. }
  4167. __kfree_skb_flush();
  4168. }
  4169. if (sd->output_queue) {
  4170. struct Qdisc *head;
  4171. local_irq_disable();
  4172. head = sd->output_queue;
  4173. sd->output_queue = NULL;
  4174. sd->output_queue_tailp = &sd->output_queue;
  4175. local_irq_enable();
  4176. rcu_read_lock();
  4177. while (head) {
  4178. struct Qdisc *q = head;
  4179. spinlock_t *root_lock = NULL;
  4180. head = head->next_sched;
  4181. /* We need to make sure head->next_sched is read
  4182. * before clearing __QDISC_STATE_SCHED
  4183. */
  4184. smp_mb__before_atomic();
  4185. if (!(q->flags & TCQ_F_NOLOCK)) {
  4186. root_lock = qdisc_lock(q);
  4187. spin_lock(root_lock);
  4188. } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
  4189. &q->state))) {
  4190. /* There is a synchronize_net() between
  4191. * STATE_DEACTIVATED flag being set and
  4192. * qdisc_reset()/some_qdisc_is_busy() in
  4193. * dev_deactivate(), so we can safely bail out
  4194. * early here to avoid data race between
  4195. * qdisc_deactivate() and some_qdisc_is_busy()
  4196. * for lockless qdisc.
  4197. */
  4198. clear_bit(__QDISC_STATE_SCHED, &q->state);
  4199. continue;
  4200. }
  4201. clear_bit(__QDISC_STATE_SCHED, &q->state);
  4202. qdisc_run(q);
  4203. if (root_lock)
  4204. spin_unlock(root_lock);
  4205. }
  4206. rcu_read_unlock();
  4207. }
  4208. xfrm_dev_backlog(sd);
  4209. }
  4210. #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
  4211. /* This hook is defined here for ATM LANE */
  4212. int (*br_fdb_test_addr_hook)(struct net_device *dev,
  4213. unsigned char *addr) __read_mostly;
  4214. EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
  4215. #endif
  4216. static inline struct sk_buff *
  4217. sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
  4218. struct net_device *orig_dev, bool *another)
  4219. {
  4220. #ifdef CONFIG_NET_CLS_ACT
  4221. struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
  4222. struct tcf_result cl_res;
  4223. /* If there's at least one ingress present somewhere (so
  4224. * we get here via enabled static key), remaining devices
  4225. * that are not configured with an ingress qdisc will bail
  4226. * out here.
  4227. */
  4228. if (!miniq)
  4229. return skb;
  4230. if (*pt_prev) {
  4231. *ret = deliver_skb(skb, *pt_prev, orig_dev);
  4232. *pt_prev = NULL;
  4233. }
  4234. qdisc_skb_cb(skb)->pkt_len = skb->len;
  4235. qdisc_skb_cb(skb)->mru = 0;
  4236. skb->tc_at_ingress = 1;
  4237. mini_qdisc_bstats_cpu_update(miniq, skb);
  4238. switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list,
  4239. &cl_res, false)) {
  4240. case TC_ACT_OK:
  4241. case TC_ACT_RECLASSIFY:
  4242. skb->tc_index = TC_H_MIN(cl_res.classid);
  4243. break;
  4244. case TC_ACT_SHOT:
  4245. mini_qdisc_qstats_cpu_drop(miniq);
  4246. kfree_skb(skb);
  4247. return NULL;
  4248. case TC_ACT_STOLEN:
  4249. case TC_ACT_QUEUED:
  4250. case TC_ACT_TRAP:
  4251. consume_skb(skb);
  4252. return NULL;
  4253. case TC_ACT_REDIRECT:
  4254. /* skb_mac_header check was done by cls/act_bpf, so
  4255. * we can safely push the L2 header back before
  4256. * redirecting to another netdev
  4257. */
  4258. __skb_push(skb, skb->mac_len);
  4259. if (skb_do_redirect(skb) == -EAGAIN) {
  4260. __skb_pull(skb, skb->mac_len);
  4261. *another = true;
  4262. break;
  4263. }
  4264. return NULL;
  4265. case TC_ACT_CONSUMED:
  4266. return NULL;
  4267. default:
  4268. break;
  4269. }
  4270. #endif /* CONFIG_NET_CLS_ACT */
  4271. return skb;
  4272. }
  4273. /**
  4274. * netdev_is_rx_handler_busy - check if receive handler is registered
  4275. * @dev: device to check
  4276. *
  4277. * Check if a receive handler is already registered for a given device.
  4278. * Return true if there one.
  4279. *
  4280. * The caller must hold the rtnl_mutex.
  4281. */
  4282. bool netdev_is_rx_handler_busy(struct net_device *dev)
  4283. {
  4284. ASSERT_RTNL();
  4285. return dev && rtnl_dereference(dev->rx_handler);
  4286. }
  4287. EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
  4288. /**
  4289. * netdev_rx_handler_register - register receive handler
  4290. * @dev: device to register a handler for
  4291. * @rx_handler: receive handler to register
  4292. * @rx_handler_data: data pointer that is used by rx handler
  4293. *
  4294. * Register a receive handler for a device. This handler will then be
  4295. * called from __netif_receive_skb. A negative errno code is returned
  4296. * on a failure.
  4297. *
  4298. * The caller must hold the rtnl_mutex.
  4299. *
  4300. * For a general description of rx_handler, see enum rx_handler_result.
  4301. */
  4302. int netdev_rx_handler_register(struct net_device *dev,
  4303. rx_handler_func_t *rx_handler,
  4304. void *rx_handler_data)
  4305. {
  4306. if (netdev_is_rx_handler_busy(dev))
  4307. return -EBUSY;
  4308. if (dev->priv_flags & IFF_NO_RX_HANDLER)
  4309. return -EINVAL;
  4310. /* Note: rx_handler_data must be set before rx_handler */
  4311. rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
  4312. rcu_assign_pointer(dev->rx_handler, rx_handler);
  4313. return 0;
  4314. }
  4315. EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
  4316. /**
  4317. * netdev_rx_handler_unregister - unregister receive handler
  4318. * @dev: device to unregister a handler from
  4319. *
  4320. * Unregister a receive handler from a device.
  4321. *
  4322. * The caller must hold the rtnl_mutex.
  4323. */
  4324. void netdev_rx_handler_unregister(struct net_device *dev)
  4325. {
  4326. ASSERT_RTNL();
  4327. RCU_INIT_POINTER(dev->rx_handler, NULL);
  4328. /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
  4329. * section has a guarantee to see a non NULL rx_handler_data
  4330. * as well.
  4331. */
  4332. synchronize_net();
  4333. RCU_INIT_POINTER(dev->rx_handler_data, NULL);
  4334. }
  4335. EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
  4336. /*
  4337. * Limit the use of PFMEMALLOC reserves to those protocols that implement
  4338. * the special handling of PFMEMALLOC skbs.
  4339. */
  4340. static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
  4341. {
  4342. switch (skb->protocol) {
  4343. case htons(ETH_P_ARP):
  4344. case htons(ETH_P_IP):
  4345. case htons(ETH_P_IPV6):
  4346. case htons(ETH_P_8021Q):
  4347. case htons(ETH_P_8021AD):
  4348. return true;
  4349. default:
  4350. return false;
  4351. }
  4352. }
  4353. static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
  4354. int *ret, struct net_device *orig_dev)
  4355. {
  4356. if (nf_hook_ingress_active(skb)) {
  4357. int ingress_retval;
  4358. if (*pt_prev) {
  4359. *ret = deliver_skb(skb, *pt_prev, orig_dev);
  4360. *pt_prev = NULL;
  4361. }
  4362. rcu_read_lock();
  4363. ingress_retval = nf_hook_ingress(skb);
  4364. rcu_read_unlock();
  4365. return ingress_retval;
  4366. }
  4367. return 0;
  4368. }
  4369. static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
  4370. struct packet_type **ppt_prev)
  4371. {
  4372. struct packet_type *ptype, *pt_prev;
  4373. rx_handler_func_t *rx_handler;
  4374. struct sk_buff *skb = *pskb;
  4375. struct net_device *orig_dev;
  4376. bool deliver_exact = false;
  4377. int ret = NET_RX_DROP;
  4378. __be16 type;
  4379. net_timestamp_check(!netdev_tstamp_prequeue, skb);
  4380. trace_netif_receive_skb(skb);
  4381. orig_dev = skb->dev;
  4382. skb_reset_network_header(skb);
  4383. if (!skb_transport_header_was_set(skb))
  4384. skb_reset_transport_header(skb);
  4385. skb_reset_mac_len(skb);
  4386. pt_prev = NULL;
  4387. another_round:
  4388. skb->skb_iif = skb->dev->ifindex;
  4389. __this_cpu_inc(softnet_data.processed);
  4390. if (static_branch_unlikely(&generic_xdp_needed_key)) {
  4391. int ret2;
  4392. preempt_disable();
  4393. ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
  4394. preempt_enable();
  4395. if (ret2 != XDP_PASS) {
  4396. ret = NET_RX_DROP;
  4397. goto out;
  4398. }
  4399. skb_reset_mac_len(skb);
  4400. }
  4401. if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
  4402. skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
  4403. skb = skb_vlan_untag(skb);
  4404. if (unlikely(!skb))
  4405. goto out;
  4406. }
  4407. if (skb_skip_tc_classify(skb))
  4408. goto skip_classify;
  4409. if (pfmemalloc)
  4410. goto skip_taps;
  4411. list_for_each_entry_rcu(ptype, &ptype_all, list) {
  4412. if (pt_prev)
  4413. ret = deliver_skb(skb, pt_prev, orig_dev);
  4414. pt_prev = ptype;
  4415. }
  4416. list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
  4417. if (pt_prev)
  4418. ret = deliver_skb(skb, pt_prev, orig_dev);
  4419. pt_prev = ptype;
  4420. }
  4421. skip_taps:
  4422. #ifdef CONFIG_NET_INGRESS
  4423. if (static_branch_unlikely(&ingress_needed_key)) {
  4424. bool another = false;
  4425. skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
  4426. &another);
  4427. if (another)
  4428. goto another_round;
  4429. if (!skb)
  4430. goto out;
  4431. if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
  4432. goto out;
  4433. }
  4434. #endif
  4435. skb_reset_redirect(skb);
  4436. skip_classify:
  4437. if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
  4438. goto drop;
  4439. if (skb_vlan_tag_present(skb)) {
  4440. if (pt_prev) {
  4441. ret = deliver_skb(skb, pt_prev, orig_dev);
  4442. pt_prev = NULL;
  4443. }
  4444. if (vlan_do_receive(&skb))
  4445. goto another_round;
  4446. else if (unlikely(!skb))
  4447. goto out;
  4448. }
  4449. rx_handler = rcu_dereference(skb->dev->rx_handler);
  4450. if (rx_handler) {
  4451. if (pt_prev) {
  4452. ret = deliver_skb(skb, pt_prev, orig_dev);
  4453. pt_prev = NULL;
  4454. }
  4455. switch (rx_handler(&skb)) {
  4456. case RX_HANDLER_CONSUMED:
  4457. ret = NET_RX_SUCCESS;
  4458. goto out;
  4459. case RX_HANDLER_ANOTHER:
  4460. goto another_round;
  4461. case RX_HANDLER_EXACT:
  4462. deliver_exact = true;
  4463. case RX_HANDLER_PASS:
  4464. break;
  4465. default:
  4466. BUG();
  4467. }
  4468. }
  4469. if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
  4470. check_vlan_id:
  4471. if (skb_vlan_tag_get_id(skb)) {
  4472. /* Vlan id is non 0 and vlan_do_receive() above couldn't
  4473. * find vlan device.
  4474. */
  4475. skb->pkt_type = PACKET_OTHERHOST;
  4476. } else if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
  4477. skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
  4478. /* Outer header is 802.1P with vlan 0, inner header is
  4479. * 802.1Q or 802.1AD and vlan_do_receive() above could
  4480. * not find vlan dev for vlan id 0.
  4481. */
  4482. __vlan_hwaccel_clear_tag(skb);
  4483. skb = skb_vlan_untag(skb);
  4484. if (unlikely(!skb))
  4485. goto out;
  4486. if (vlan_do_receive(&skb))
  4487. /* After stripping off 802.1P header with vlan 0
  4488. * vlan dev is found for inner header.
  4489. */
  4490. goto another_round;
  4491. else if (unlikely(!skb))
  4492. goto out;
  4493. else
  4494. /* We have stripped outer 802.1P vlan 0 header.
  4495. * But could not find vlan dev.
  4496. * check again for vlan id to set OTHERHOST.
  4497. */
  4498. goto check_vlan_id;
  4499. }
  4500. /* Note: we might in the future use prio bits
  4501. * and set skb->priority like in vlan_do_receive()
  4502. * For the time being, just ignore Priority Code Point
  4503. */
  4504. __vlan_hwaccel_clear_tag(skb);
  4505. }
  4506. type = skb->protocol;
  4507. /* deliver only exact match when indicated */
  4508. if (likely(!deliver_exact)) {
  4509. deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
  4510. &ptype_base[ntohs(type) &
  4511. PTYPE_HASH_MASK]);
  4512. }
  4513. deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
  4514. &orig_dev->ptype_specific);
  4515. if (unlikely(skb->dev != orig_dev)) {
  4516. deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
  4517. &skb->dev->ptype_specific);
  4518. }
  4519. if (pt_prev) {
  4520. if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
  4521. goto drop;
  4522. *ppt_prev = pt_prev;
  4523. } else {
  4524. drop:
  4525. if (!deliver_exact)
  4526. atomic_long_inc(&skb->dev->rx_dropped);
  4527. else
  4528. atomic_long_inc(&skb->dev->rx_nohandler);
  4529. kfree_skb(skb);
  4530. /* Jamal, now you will not able to escape explaining
  4531. * me how you were going to use this. :-)
  4532. */
  4533. ret = NET_RX_DROP;
  4534. }
  4535. out:
  4536. /* The invariant here is that if *ppt_prev is not NULL
  4537. * then skb should also be non-NULL.
  4538. *
  4539. * Apparently *ppt_prev assignment above holds this invariant due to
  4540. * skb dereferencing near it.
  4541. */
  4542. *pskb = skb;
  4543. return ret;
  4544. }
  4545. static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
  4546. {
  4547. struct net_device *orig_dev = skb->dev;
  4548. struct packet_type *pt_prev = NULL;
  4549. int ret;
  4550. ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
  4551. if (pt_prev)
  4552. ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
  4553. skb->dev, pt_prev, orig_dev);
  4554. return ret;
  4555. }
  4556. /**
  4557. * netif_receive_skb_core - special purpose version of netif_receive_skb
  4558. * @skb: buffer to process
  4559. *
  4560. * More direct receive version of netif_receive_skb(). It should
  4561. * only be used by callers that have a need to skip RPS and Generic XDP.
  4562. * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
  4563. *
  4564. * This function may only be called from softirq context and interrupts
  4565. * should be enabled.
  4566. *
  4567. * Return values (usually ignored):
  4568. * NET_RX_SUCCESS: no congestion
  4569. * NET_RX_DROP: packet was dropped
  4570. */
  4571. int netif_receive_skb_core(struct sk_buff *skb)
  4572. {
  4573. int ret;
  4574. rcu_read_lock();
  4575. ret = __netif_receive_skb_one_core(skb, false);
  4576. rcu_read_unlock();
  4577. return ret;
  4578. }
  4579. EXPORT_SYMBOL(netif_receive_skb_core);
  4580. static inline void __netif_receive_skb_list_ptype(struct list_head *head,
  4581. struct packet_type *pt_prev,
  4582. struct net_device *orig_dev)
  4583. {
  4584. struct sk_buff *skb, *next;
  4585. if (!pt_prev)
  4586. return;
  4587. if (list_empty(head))
  4588. return;
  4589. if (pt_prev->list_func != NULL)
  4590. INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
  4591. ip_list_rcv, head, pt_prev, orig_dev);
  4592. else
  4593. list_for_each_entry_safe(skb, next, head, list) {
  4594. skb_list_del_init(skb);
  4595. pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
  4596. }
  4597. }
  4598. static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
  4599. {
  4600. /* Fast-path assumptions:
  4601. * - There is no RX handler.
  4602. * - Only one packet_type matches.
  4603. * If either of these fails, we will end up doing some per-packet
  4604. * processing in-line, then handling the 'last ptype' for the whole
  4605. * sublist. This can't cause out-of-order delivery to any single ptype,
  4606. * because the 'last ptype' must be constant across the sublist, and all
  4607. * other ptypes are handled per-packet.
  4608. */
  4609. /* Current (common) ptype of sublist */
  4610. struct packet_type *pt_curr = NULL;
  4611. /* Current (common) orig_dev of sublist */
  4612. struct net_device *od_curr = NULL;
  4613. struct list_head sublist;
  4614. struct sk_buff *skb, *next;
  4615. INIT_LIST_HEAD(&sublist);
  4616. list_for_each_entry_safe(skb, next, head, list) {
  4617. struct net_device *orig_dev = skb->dev;
  4618. struct packet_type *pt_prev = NULL;
  4619. skb_list_del_init(skb);
  4620. __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
  4621. if (!pt_prev)
  4622. continue;
  4623. if (pt_curr != pt_prev || od_curr != orig_dev) {
  4624. /* dispatch old sublist */
  4625. __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
  4626. /* start new sublist */
  4627. INIT_LIST_HEAD(&sublist);
  4628. pt_curr = pt_prev;
  4629. od_curr = orig_dev;
  4630. }
  4631. list_add_tail(&skb->list, &sublist);
  4632. }
  4633. /* dispatch final sublist */
  4634. __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
  4635. }
  4636. static int __netif_receive_skb(struct sk_buff *skb)
  4637. {
  4638. int ret;
  4639. if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
  4640. unsigned int noreclaim_flag;
  4641. /*
  4642. * PFMEMALLOC skbs are special, they should
  4643. * - be delivered to SOCK_MEMALLOC sockets only
  4644. * - stay away from userspace
  4645. * - have bounded memory usage
  4646. *
  4647. * Use PF_MEMALLOC as this saves us from propagating the allocation
  4648. * context down to all allocation sites.
  4649. */
  4650. noreclaim_flag = memalloc_noreclaim_save();
  4651. ret = __netif_receive_skb_one_core(skb, true);
  4652. memalloc_noreclaim_restore(noreclaim_flag);
  4653. } else
  4654. ret = __netif_receive_skb_one_core(skb, false);
  4655. return ret;
  4656. }
  4657. static void __netif_receive_skb_list(struct list_head *head)
  4658. {
  4659. unsigned long noreclaim_flag = 0;
  4660. struct sk_buff *skb, *next;
  4661. bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
  4662. list_for_each_entry_safe(skb, next, head, list) {
  4663. if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
  4664. struct list_head sublist;
  4665. /* Handle the previous sublist */
  4666. list_cut_before(&sublist, head, &skb->list);
  4667. if (!list_empty(&sublist))
  4668. __netif_receive_skb_list_core(&sublist, pfmemalloc);
  4669. pfmemalloc = !pfmemalloc;
  4670. /* See comments in __netif_receive_skb */
  4671. if (pfmemalloc)
  4672. noreclaim_flag = memalloc_noreclaim_save();
  4673. else
  4674. memalloc_noreclaim_restore(noreclaim_flag);
  4675. }
  4676. }
  4677. /* Handle the remaining sublist */
  4678. if (!list_empty(head))
  4679. __netif_receive_skb_list_core(head, pfmemalloc);
  4680. /* Restore pflags */
  4681. if (pfmemalloc)
  4682. memalloc_noreclaim_restore(noreclaim_flag);
  4683. }
  4684. static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
  4685. {
  4686. struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
  4687. struct bpf_prog *new = xdp->prog;
  4688. int ret = 0;
  4689. if (new) {
  4690. u32 i;
  4691. mutex_lock(&new->aux->used_maps_mutex);
  4692. /* generic XDP does not work with DEVMAPs that can
  4693. * have a bpf_prog installed on an entry
  4694. */
  4695. for (i = 0; i < new->aux->used_map_cnt; i++) {
  4696. if (dev_map_can_have_prog(new->aux->used_maps[i]) ||
  4697. cpu_map_prog_allowed(new->aux->used_maps[i])) {
  4698. mutex_unlock(&new->aux->used_maps_mutex);
  4699. return -EINVAL;
  4700. }
  4701. }
  4702. mutex_unlock(&new->aux->used_maps_mutex);
  4703. }
  4704. switch (xdp->command) {
  4705. case XDP_SETUP_PROG:
  4706. rcu_assign_pointer(dev->xdp_prog, new);
  4707. if (old)
  4708. bpf_prog_put(old);
  4709. if (old && !new) {
  4710. static_branch_dec(&generic_xdp_needed_key);
  4711. } else if (new && !old) {
  4712. static_branch_inc(&generic_xdp_needed_key);
  4713. dev_disable_lro(dev);
  4714. dev_disable_gro_hw(dev);
  4715. }
  4716. break;
  4717. default:
  4718. ret = -EINVAL;
  4719. break;
  4720. }
  4721. return ret;
  4722. }
  4723. static int netif_receive_skb_internal(struct sk_buff *skb)
  4724. {
  4725. int ret;
  4726. net_timestamp_check(netdev_tstamp_prequeue, skb);
  4727. if (skb_defer_rx_timestamp(skb))
  4728. return NET_RX_SUCCESS;
  4729. rcu_read_lock();
  4730. #ifdef CONFIG_RPS
  4731. if (static_branch_unlikely(&rps_needed)) {
  4732. struct rps_dev_flow voidflow, *rflow = &voidflow;
  4733. int cpu = get_rps_cpu(skb->dev, skb, &rflow);
  4734. if (cpu >= 0) {
  4735. ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
  4736. rcu_read_unlock();
  4737. return ret;
  4738. }
  4739. }
  4740. #endif
  4741. ret = __netif_receive_skb(skb);
  4742. rcu_read_unlock();
  4743. return ret;
  4744. }
  4745. static void netif_receive_skb_list_internal(struct list_head *head)
  4746. {
  4747. struct sk_buff *skb, *next;
  4748. struct list_head sublist;
  4749. INIT_LIST_HEAD(&sublist);
  4750. list_for_each_entry_safe(skb, next, head, list) {
  4751. net_timestamp_check(netdev_tstamp_prequeue, skb);
  4752. skb_list_del_init(skb);
  4753. if (!skb_defer_rx_timestamp(skb))
  4754. list_add_tail(&skb->list, &sublist);
  4755. }
  4756. list_splice_init(&sublist, head);
  4757. rcu_read_lock();
  4758. #ifdef CONFIG_RPS
  4759. if (static_branch_unlikely(&rps_needed)) {
  4760. list_for_each_entry_safe(skb, next, head, list) {
  4761. struct rps_dev_flow voidflow, *rflow = &voidflow;
  4762. int cpu = get_rps_cpu(skb->dev, skb, &rflow);
  4763. if (cpu >= 0) {
  4764. /* Will be handled, remove from list */
  4765. skb_list_del_init(skb);
  4766. enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
  4767. }
  4768. }
  4769. }
  4770. #endif
  4771. __netif_receive_skb_list(head);
  4772. rcu_read_unlock();
  4773. }
  4774. /**
  4775. * netif_receive_skb - process receive buffer from network
  4776. * @skb: buffer to process
  4777. *
  4778. * netif_receive_skb() is the main receive data processing function.
  4779. * It always succeeds. The buffer may be dropped during processing
  4780. * for congestion control or by the protocol layers.
  4781. *
  4782. * This function may only be called from softirq context and interrupts
  4783. * should be enabled.
  4784. *
  4785. * Return values (usually ignored):
  4786. * NET_RX_SUCCESS: no congestion
  4787. * NET_RX_DROP: packet was dropped
  4788. */
  4789. int netif_receive_skb(struct sk_buff *skb)
  4790. {
  4791. int ret;
  4792. trace_netif_receive_skb_entry(skb);
  4793. ret = netif_receive_skb_internal(skb);
  4794. trace_netif_receive_skb_exit(ret);
  4795. return ret;
  4796. }
  4797. EXPORT_SYMBOL(netif_receive_skb);
  4798. /**
  4799. * netif_receive_skb_list - process many receive buffers from network
  4800. * @head: list of skbs to process.
  4801. *
  4802. * Since return value of netif_receive_skb() is normally ignored, and
  4803. * wouldn't be meaningful for a list, this function returns void.
  4804. *
  4805. * This function may only be called from softirq context and interrupts
  4806. * should be enabled.
  4807. */
  4808. void netif_receive_skb_list(struct list_head *head)
  4809. {
  4810. struct sk_buff *skb;
  4811. if (list_empty(head))
  4812. return;
  4813. if (trace_netif_receive_skb_list_entry_enabled()) {
  4814. list_for_each_entry(skb, head, list)
  4815. trace_netif_receive_skb_list_entry(skb);
  4816. }
  4817. netif_receive_skb_list_internal(head);
  4818. trace_netif_receive_skb_list_exit(0);
  4819. }
  4820. EXPORT_SYMBOL(netif_receive_skb_list);
  4821. static DEFINE_PER_CPU(struct work_struct, flush_works);
  4822. /* Network device is going away, flush any packets still pending */
  4823. static void flush_backlog(struct work_struct *work)
  4824. {
  4825. struct sk_buff *skb, *tmp;
  4826. struct softnet_data *sd;
  4827. local_bh_disable();
  4828. sd = this_cpu_ptr(&softnet_data);
  4829. local_irq_disable();
  4830. rps_lock(sd);
  4831. skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
  4832. if (skb->dev->reg_state == NETREG_UNREGISTERING) {
  4833. __skb_unlink(skb, &sd->input_pkt_queue);
  4834. dev_kfree_skb_irq(skb);
  4835. input_queue_head_incr(sd);
  4836. }
  4837. }
  4838. rps_unlock(sd);
  4839. local_irq_enable();
  4840. skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
  4841. if (skb->dev->reg_state == NETREG_UNREGISTERING) {
  4842. __skb_unlink(skb, &sd->process_queue);
  4843. kfree_skb(skb);
  4844. input_queue_head_incr(sd);
  4845. }
  4846. }
  4847. local_bh_enable();
  4848. }
  4849. static bool flush_required(int cpu)
  4850. {
  4851. #if IS_ENABLED(CONFIG_RPS)
  4852. struct softnet_data *sd = &per_cpu(softnet_data, cpu);
  4853. bool do_flush;
  4854. local_irq_disable();
  4855. rps_lock(sd);
  4856. /* as insertion into process_queue happens with the rps lock held,
  4857. * process_queue access may race only with dequeue
  4858. */
  4859. do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
  4860. !skb_queue_empty_lockless(&sd->process_queue);
  4861. rps_unlock(sd);
  4862. local_irq_enable();
  4863. return do_flush;
  4864. #endif
  4865. /* without RPS we can't safely check input_pkt_queue: during a
  4866. * concurrent remote skb_queue_splice() we can detect as empty both
  4867. * input_pkt_queue and process_queue even if the latter could end-up
  4868. * containing a lot of packets.
  4869. */
  4870. return true;
  4871. }
  4872. static void flush_all_backlogs(void)
  4873. {
  4874. static cpumask_t flush_cpus;
  4875. unsigned int cpu;
  4876. /* since we are under rtnl lock protection we can use static data
  4877. * for the cpumask and avoid allocating on stack the possibly
  4878. * large mask
  4879. */
  4880. ASSERT_RTNL();
  4881. get_online_cpus();
  4882. cpumask_clear(&flush_cpus);
  4883. for_each_online_cpu(cpu) {
  4884. if (flush_required(cpu)) {
  4885. queue_work_on(cpu, system_highpri_wq,
  4886. per_cpu_ptr(&flush_works, cpu));
  4887. cpumask_set_cpu(cpu, &flush_cpus);
  4888. }
  4889. }
  4890. /* we can have in flight packet[s] on the cpus we are not flushing,
  4891. * synchronize_net() in rollback_registered_many() will take care of
  4892. * them
  4893. */
  4894. for_each_cpu(cpu, &flush_cpus)
  4895. flush_work(per_cpu_ptr(&flush_works, cpu));
  4896. put_online_cpus();
  4897. }
  4898. /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
  4899. static void gro_normal_list(struct napi_struct *napi)
  4900. {
  4901. if (!napi->rx_count)
  4902. return;
  4903. netif_receive_skb_list_internal(&napi->rx_list);
  4904. INIT_LIST_HEAD(&napi->rx_list);
  4905. napi->rx_count = 0;
  4906. }
  4907. /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
  4908. * pass the whole batch up to the stack.
  4909. */
  4910. static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
  4911. {
  4912. list_add_tail(&skb->list, &napi->rx_list);
  4913. napi->rx_count += segs;
  4914. if (napi->rx_count >= gro_normal_batch)
  4915. gro_normal_list(napi);
  4916. }
  4917. INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
  4918. INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
  4919. static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
  4920. {
  4921. struct packet_offload *ptype;
  4922. __be16 type = skb->protocol;
  4923. struct list_head *head = &offload_base;
  4924. int err = -ENOENT;
  4925. BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
  4926. if (NAPI_GRO_CB(skb)->count == 1) {
  4927. skb_shinfo(skb)->gso_size = 0;
  4928. goto out;
  4929. }
  4930. rcu_read_lock();
  4931. list_for_each_entry_rcu(ptype, head, list) {
  4932. if (ptype->type != type || !ptype->callbacks.gro_complete)
  4933. continue;
  4934. err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
  4935. ipv6_gro_complete, inet_gro_complete,
  4936. skb, 0);
  4937. break;
  4938. }
  4939. rcu_read_unlock();
  4940. if (err) {
  4941. WARN_ON(&ptype->list == head);
  4942. kfree_skb(skb);
  4943. return NET_RX_SUCCESS;
  4944. }
  4945. out:
  4946. gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
  4947. return NET_RX_SUCCESS;
  4948. }
  4949. static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
  4950. bool flush_old)
  4951. {
  4952. struct list_head *head = &napi->gro_hash[index].list;
  4953. struct sk_buff *skb, *p;
  4954. list_for_each_entry_safe_reverse(skb, p, head, list) {
  4955. if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
  4956. return;
  4957. skb_list_del_init(skb);
  4958. napi_gro_complete(napi, skb);
  4959. napi->gro_hash[index].count--;
  4960. }
  4961. if (!napi->gro_hash[index].count)
  4962. __clear_bit(index, &napi->gro_bitmask);
  4963. }
  4964. /* napi->gro_hash[].list contains packets ordered by age.
  4965. * youngest packets at the head of it.
  4966. * Complete skbs in reverse order to reduce latencies.
  4967. */
  4968. void napi_gro_flush(struct napi_struct *napi, bool flush_old)
  4969. {
  4970. unsigned long bitmask = napi->gro_bitmask;
  4971. unsigned int i, base = ~0U;
  4972. while ((i = ffs(bitmask)) != 0) {
  4973. bitmask >>= i;
  4974. base += i;
  4975. __napi_gro_flush_chain(napi, base, flush_old);
  4976. }
  4977. }
  4978. EXPORT_SYMBOL(napi_gro_flush);
  4979. static struct list_head *gro_list_prepare(struct napi_struct *napi,
  4980. struct sk_buff *skb)
  4981. {
  4982. unsigned int maclen = skb->dev->hard_header_len;
  4983. u32 hash = skb_get_hash_raw(skb);
  4984. struct list_head *head;
  4985. struct sk_buff *p;
  4986. head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;
  4987. list_for_each_entry(p, head, list) {
  4988. unsigned long diffs;
  4989. NAPI_GRO_CB(p)->flush = 0;
  4990. if (hash != skb_get_hash_raw(p)) {
  4991. NAPI_GRO_CB(p)->same_flow = 0;
  4992. continue;
  4993. }
  4994. diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
  4995. diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
  4996. if (skb_vlan_tag_present(p))
  4997. diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
  4998. diffs |= skb_metadata_dst_cmp(p, skb);
  4999. diffs |= skb_metadata_differs(p, skb);
  5000. if (maclen == ETH_HLEN)
  5001. diffs |= compare_ether_header(skb_mac_header(p),
  5002. skb_mac_header(skb));
  5003. else if (!diffs)
  5004. diffs = memcmp(skb_mac_header(p),
  5005. skb_mac_header(skb),
  5006. maclen);
  5007. diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
  5008. #if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
  5009. if (!diffs) {
  5010. struct tc_skb_ext *skb_ext = skb_ext_find(skb, TC_SKB_EXT);
  5011. struct tc_skb_ext *p_ext = skb_ext_find(p, TC_SKB_EXT);
  5012. diffs |= (!!p_ext) ^ (!!skb_ext);
  5013. if (!diffs && unlikely(skb_ext))
  5014. diffs |= p_ext->chain ^ skb_ext->chain;
  5015. }
  5016. #endif
  5017. NAPI_GRO_CB(p)->same_flow = !diffs;
  5018. }
  5019. return head;
  5020. }
  5021. static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
  5022. {
  5023. const struct skb_shared_info *pinfo = skb_shinfo(skb);
  5024. const skb_frag_t *frag0 = &pinfo->frags[0];
  5025. NAPI_GRO_CB(skb)->data_offset = 0;
  5026. NAPI_GRO_CB(skb)->frag0 = NULL;
  5027. NAPI_GRO_CB(skb)->frag0_len = 0;
  5028. if (!skb_headlen(skb) && pinfo->nr_frags &&
  5029. !PageHighMem(skb_frag_page(frag0)) &&
  5030. (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
  5031. NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
  5032. NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
  5033. skb_frag_size(frag0),
  5034. skb->end - skb->tail);
  5035. }
  5036. }
  5037. static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
  5038. {
  5039. struct skb_shared_info *pinfo = skb_shinfo(skb);
  5040. BUG_ON(skb->end - skb->tail < grow);
  5041. memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
  5042. skb->data_len -= grow;
  5043. skb->tail += grow;
  5044. skb_frag_off_add(&pinfo->frags[0], grow);
  5045. skb_frag_size_sub(&pinfo->frags[0], grow);
  5046. if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
  5047. skb_frag_unref(skb, 0);
  5048. memmove(pinfo->frags, pinfo->frags + 1,
  5049. --pinfo->nr_frags * sizeof(pinfo->frags[0]));
  5050. }
  5051. }
  5052. static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
  5053. {
  5054. struct sk_buff *oldest;
  5055. oldest = list_last_entry(head, struct sk_buff, list);
  5056. /* We are called with head length >= MAX_GRO_SKBS, so this is
  5057. * impossible.
  5058. */
  5059. if (WARN_ON_ONCE(!oldest))
  5060. return;
  5061. /* Do not adjust napi->gro_hash[].count, caller is adding a new
  5062. * SKB to the chain.
  5063. */
  5064. skb_list_del_init(oldest);
  5065. napi_gro_complete(napi, oldest);
  5066. }
  5067. INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
  5068. struct sk_buff *));
  5069. INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
  5070. struct sk_buff *));
  5071. static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
  5072. {
  5073. u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
  5074. struct list_head *head = &offload_base;
  5075. struct packet_offload *ptype;
  5076. __be16 type = skb->protocol;
  5077. struct list_head *gro_head;
  5078. struct sk_buff *pp = NULL;
  5079. enum gro_result ret;
  5080. int same_flow;
  5081. int grow;
  5082. if (netif_elide_gro(skb->dev))
  5083. goto normal;
  5084. gro_head = gro_list_prepare(napi, skb);
  5085. rcu_read_lock();
  5086. list_for_each_entry_rcu(ptype, head, list) {
  5087. if (ptype->type != type || !ptype->callbacks.gro_receive)
  5088. continue;
  5089. skb_set_network_header(skb, skb_gro_offset(skb));
  5090. skb_reset_mac_len(skb);
  5091. NAPI_GRO_CB(skb)->same_flow = 0;
  5092. NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
  5093. NAPI_GRO_CB(skb)->free = 0;
  5094. NAPI_GRO_CB(skb)->encap_mark = 0;
  5095. NAPI_GRO_CB(skb)->recursion_counter = 0;
  5096. NAPI_GRO_CB(skb)->is_fou = 0;
  5097. NAPI_GRO_CB(skb)->is_atomic = 1;
  5098. NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
  5099. /* Setup for GRO checksum validation */
  5100. switch (skb->ip_summed) {
  5101. case CHECKSUM_COMPLETE:
  5102. NAPI_GRO_CB(skb)->csum = skb->csum;
  5103. NAPI_GRO_CB(skb)->csum_valid = 1;
  5104. NAPI_GRO_CB(skb)->csum_cnt = 0;
  5105. break;
  5106. case CHECKSUM_UNNECESSARY:
  5107. NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
  5108. NAPI_GRO_CB(skb)->csum_valid = 0;
  5109. break;
  5110. default:
  5111. NAPI_GRO_CB(skb)->csum_cnt = 0;
  5112. NAPI_GRO_CB(skb)->csum_valid = 0;
  5113. }
  5114. pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
  5115. ipv6_gro_receive, inet_gro_receive,
  5116. gro_head, skb);
  5117. break;
  5118. }
  5119. rcu_read_unlock();
  5120. if (&ptype->list == head)
  5121. goto normal;
  5122. if (PTR_ERR(pp) == -EINPROGRESS) {
  5123. ret = GRO_CONSUMED;
  5124. goto ok;
  5125. }
  5126. same_flow = NAPI_GRO_CB(skb)->same_flow;
  5127. ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
  5128. if (pp) {
  5129. skb_list_del_init(pp);
  5130. napi_gro_complete(napi, pp);
  5131. napi->gro_hash[hash].count--;
  5132. }
  5133. if (same_flow)
  5134. goto ok;
  5135. if (NAPI_GRO_CB(skb)->flush)
  5136. goto normal;
  5137. if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
  5138. gro_flush_oldest(napi, gro_head);
  5139. } else {
  5140. napi->gro_hash[hash].count++;
  5141. }
  5142. NAPI_GRO_CB(skb)->count = 1;
  5143. NAPI_GRO_CB(skb)->age = jiffies;
  5144. NAPI_GRO_CB(skb)->last = skb;
  5145. skb_shinfo(skb)->gso_size = skb_gro_len(skb);
  5146. list_add(&skb->list, gro_head);
  5147. ret = GRO_HELD;
  5148. pull:
  5149. grow = skb_gro_offset(skb) - skb_headlen(skb);
  5150. if (grow > 0)
  5151. gro_pull_from_frag0(skb, grow);
  5152. ok:
  5153. if (napi->gro_hash[hash].count) {
  5154. if (!test_bit(hash, &napi->gro_bitmask))
  5155. __set_bit(hash, &napi->gro_bitmask);
  5156. } else if (test_bit(hash, &napi->gro_bitmask)) {
  5157. __clear_bit(hash, &napi->gro_bitmask);
  5158. }
  5159. return ret;
  5160. normal:
  5161. ret = GRO_NORMAL;
  5162. goto pull;
  5163. }
  5164. struct packet_offload *gro_find_receive_by_type(__be16 type)
  5165. {
  5166. struct list_head *offload_head = &offload_base;
  5167. struct packet_offload *ptype;
  5168. list_for_each_entry_rcu(ptype, offload_head, list) {
  5169. if (ptype->type != type || !ptype->callbacks.gro_receive)
  5170. continue;
  5171. return ptype;
  5172. }
  5173. return NULL;
  5174. }
  5175. EXPORT_SYMBOL(gro_find_receive_by_type);
  5176. struct packet_offload *gro_find_complete_by_type(__be16 type)
  5177. {
  5178. struct list_head *offload_head = &offload_base;
  5179. struct packet_offload *ptype;
  5180. list_for_each_entry_rcu(ptype, offload_head, list) {
  5181. if (ptype->type != type || !ptype->callbacks.gro_complete)
  5182. continue;
  5183. return ptype;
  5184. }
  5185. return NULL;
  5186. }
  5187. EXPORT_SYMBOL(gro_find_complete_by_type);
  5188. static void napi_skb_free_stolen_head(struct sk_buff *skb)
  5189. {
  5190. skb_dst_drop(skb);
  5191. skb_ext_put(skb);
  5192. kmem_cache_free(skbuff_head_cache, skb);
  5193. }
  5194. static gro_result_t napi_skb_finish(struct napi_struct *napi,
  5195. struct sk_buff *skb,
  5196. gro_result_t ret)
  5197. {
  5198. switch (ret) {
  5199. case GRO_NORMAL:
  5200. gro_normal_one(napi, skb, 1);
  5201. break;
  5202. case GRO_DROP:
  5203. kfree_skb(skb);
  5204. break;
  5205. case GRO_MERGED_FREE:
  5206. if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
  5207. napi_skb_free_stolen_head(skb);
  5208. else
  5209. __kfree_skb(skb);
  5210. break;
  5211. case GRO_HELD:
  5212. case GRO_MERGED:
  5213. case GRO_CONSUMED:
  5214. break;
  5215. }
  5216. return ret;
  5217. }
  5218. gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
  5219. {
  5220. gro_result_t ret;
  5221. skb_mark_napi_id(skb, napi);
  5222. trace_napi_gro_receive_entry(skb);
  5223. skb_gro_reset_offset(skb, 0);
  5224. ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
  5225. trace_napi_gro_receive_exit(ret);
  5226. return ret;
  5227. }
  5228. EXPORT_SYMBOL(napi_gro_receive);
  5229. static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
  5230. {
  5231. if (unlikely(skb->pfmemalloc)) {
  5232. consume_skb(skb);
  5233. return;
  5234. }
  5235. __skb_pull(skb, skb_headlen(skb));
  5236. /* restore the reserve we had after netdev_alloc_skb_ip_align() */
  5237. skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
  5238. __vlan_hwaccel_clear_tag(skb);
  5239. skb->dev = napi->dev;
  5240. skb->skb_iif = 0;
  5241. /* eth_type_trans() assumes pkt_type is PACKET_HOST */
  5242. skb->pkt_type = PACKET_HOST;
  5243. skb->encapsulation = 0;
  5244. skb_shinfo(skb)->gso_type = 0;
  5245. skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
  5246. skb_ext_reset(skb);
  5247. nf_reset_ct(skb);
  5248. napi->skb = skb;
  5249. }
  5250. struct sk_buff *napi_get_frags(struct napi_struct *napi)
  5251. {
  5252. struct sk_buff *skb = napi->skb;
  5253. if (!skb) {
  5254. skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
  5255. if (skb) {
  5256. napi->skb = skb;
  5257. skb_mark_napi_id(skb, napi);
  5258. }
  5259. }
  5260. return skb;
  5261. }
  5262. EXPORT_SYMBOL(napi_get_frags);
  5263. static gro_result_t napi_frags_finish(struct napi_struct *napi,
  5264. struct sk_buff *skb,
  5265. gro_result_t ret)
  5266. {
  5267. switch (ret) {
  5268. case GRO_NORMAL:
  5269. case GRO_HELD:
  5270. __skb_push(skb, ETH_HLEN);
  5271. skb->protocol = eth_type_trans(skb, skb->dev);
  5272. if (ret == GRO_NORMAL)
  5273. gro_normal_one(napi, skb, 1);
  5274. break;
  5275. case GRO_DROP:
  5276. napi_reuse_skb(napi, skb);
  5277. break;
  5278. case GRO_MERGED_FREE:
  5279. if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
  5280. napi_skb_free_stolen_head(skb);
  5281. else
  5282. napi_reuse_skb(napi, skb);
  5283. break;
  5284. case GRO_MERGED:
  5285. case GRO_CONSUMED:
  5286. break;
  5287. }
  5288. return ret;
  5289. }
  5290. /* Upper GRO stack assumes network header starts at gro_offset=0
  5291. * Drivers could call both napi_gro_frags() and napi_gro_receive()
  5292. * We copy ethernet header into skb->data to have a common layout.
  5293. */
  5294. static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
  5295. {
  5296. struct sk_buff *skb = napi->skb;
  5297. const struct ethhdr *eth;
  5298. unsigned int hlen = sizeof(*eth);
  5299. napi->skb = NULL;
  5300. skb_reset_mac_header(skb);
  5301. skb_gro_reset_offset(skb, hlen);
  5302. if (unlikely(skb_gro_header_hard(skb, hlen))) {
  5303. eth = skb_gro_header_slow(skb, hlen, 0);
  5304. if (unlikely(!eth)) {
  5305. net_warn_ratelimited("%s: dropping impossible skb from %s\n",
  5306. __func__, napi->dev->name);
  5307. napi_reuse_skb(napi, skb);
  5308. return NULL;
  5309. }
  5310. } else {
  5311. eth = (const struct ethhdr *)skb->data;
  5312. gro_pull_from_frag0(skb, hlen);
  5313. NAPI_GRO_CB(skb)->frag0 += hlen;
  5314. NAPI_GRO_CB(skb)->frag0_len -= hlen;
  5315. }
  5316. __skb_pull(skb, hlen);
  5317. /*
  5318. * This works because the only protocols we care about don't require
  5319. * special handling.
  5320. * We'll fix it up properly in napi_frags_finish()
  5321. */
  5322. skb->protocol = eth->h_proto;
  5323. return skb;
  5324. }
  5325. gro_result_t napi_gro_frags(struct napi_struct *napi)
  5326. {
  5327. gro_result_t ret;
  5328. struct sk_buff *skb = napi_frags_skb(napi);
  5329. if (!skb)
  5330. return GRO_DROP;
  5331. trace_napi_gro_frags_entry(skb);
  5332. ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
  5333. trace_napi_gro_frags_exit(ret);
  5334. return ret;
  5335. }
  5336. EXPORT_SYMBOL(napi_gro_frags);
  5337. /* Compute the checksum from gro_offset and return the folded value
  5338. * after adding in any pseudo checksum.
  5339. */
  5340. __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
  5341. {
  5342. __wsum wsum;
  5343. __sum16 sum;
  5344. wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
  5345. /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
  5346. sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
  5347. /* See comments in __skb_checksum_complete(). */
  5348. if (likely(!sum)) {
  5349. if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
  5350. !skb->csum_complete_sw)
  5351. netdev_rx_csum_fault(skb->dev, skb);
  5352. }
  5353. NAPI_GRO_CB(skb)->csum = wsum;
  5354. NAPI_GRO_CB(skb)->csum_valid = 1;
  5355. return sum;
  5356. }
  5357. EXPORT_SYMBOL(__skb_gro_checksum_complete);
  5358. static void net_rps_send_ipi(struct softnet_data *remsd)
  5359. {
  5360. #ifdef CONFIG_RPS
  5361. while (remsd) {
  5362. struct softnet_data *next = remsd->rps_ipi_next;
  5363. if (cpu_online(remsd->cpu))
  5364. smp_call_function_single_async(remsd->cpu, &remsd->csd);
  5365. remsd = next;
  5366. }
  5367. #endif
  5368. }
  5369. /*
  5370. * net_rps_action_and_irq_enable sends any pending IPI's for rps.
  5371. * Note: called with local irq disabled, but exits with local irq enabled.
  5372. */
  5373. static void net_rps_action_and_irq_enable(struct softnet_data *sd)
  5374. {
  5375. #ifdef CONFIG_RPS
  5376. struct softnet_data *remsd = sd->rps_ipi_list;
  5377. if (remsd) {
  5378. sd->rps_ipi_list = NULL;
  5379. local_irq_enable();
  5380. /* Send pending IPI's to kick RPS processing on remote cpus. */
  5381. net_rps_send_ipi(remsd);
  5382. } else
  5383. #endif
  5384. local_irq_enable();
  5385. }
  5386. static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
  5387. {
  5388. #ifdef CONFIG_RPS
  5389. return sd->rps_ipi_list != NULL;
  5390. #else
  5391. return false;
  5392. #endif
  5393. }
  5394. static int process_backlog(struct napi_struct *napi, int quota)
  5395. {
  5396. struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
  5397. bool again = true;
  5398. int work = 0;
  5399. /* Check if we have pending ipi, its better to send them now,
  5400. * not waiting net_rx_action() end.
  5401. */
  5402. if (sd_has_rps_ipi_waiting(sd)) {
  5403. local_irq_disable();
  5404. net_rps_action_and_irq_enable(sd);
  5405. }
  5406. napi->weight = dev_rx_weight;
  5407. while (again) {
  5408. struct sk_buff *skb;
  5409. while ((skb = __skb_dequeue(&sd->process_queue))) {
  5410. rcu_read_lock();
  5411. __netif_receive_skb(skb);
  5412. rcu_read_unlock();
  5413. input_queue_head_incr(sd);
  5414. if (++work >= quota)
  5415. return work;
  5416. }
  5417. local_irq_disable();
  5418. rps_lock(sd);
  5419. if (skb_queue_empty(&sd->input_pkt_queue)) {
  5420. /*
  5421. * Inline a custom version of __napi_complete().
  5422. * only current cpu owns and manipulates this napi,
  5423. * and NAPI_STATE_SCHED is the only possible flag set
  5424. * on backlog.
  5425. * We can use a plain write instead of clear_bit(),
  5426. * and we dont need an smp_mb() memory barrier.
  5427. */
  5428. napi->state = 0;
  5429. again = false;
  5430. } else {
  5431. skb_queue_splice_tail_init(&sd->input_pkt_queue,
  5432. &sd->process_queue);
  5433. }
  5434. rps_unlock(sd);
  5435. local_irq_enable();
  5436. }
  5437. return work;
  5438. }
  5439. /**
  5440. * __napi_schedule - schedule for receive
  5441. * @n: entry to schedule
  5442. *
  5443. * The entry's receive function will be scheduled to run.
  5444. * Consider using __napi_schedule_irqoff() if hard irqs are masked.
  5445. */
  5446. void __napi_schedule(struct napi_struct *n)
  5447. {
  5448. unsigned long flags;
  5449. local_irq_save(flags);
  5450. ____napi_schedule(this_cpu_ptr(&softnet_data), n);
  5451. local_irq_restore(flags);
  5452. }
  5453. EXPORT_SYMBOL(__napi_schedule);
  5454. /**
  5455. * napi_schedule_prep - check if napi can be scheduled
  5456. * @n: napi context
  5457. *
  5458. * Test if NAPI routine is already running, and if not mark
  5459. * it as running. This is used as a condition variable to
  5460. * insure only one NAPI poll instance runs. We also make
  5461. * sure there is no pending NAPI disable.
  5462. */
  5463. bool napi_schedule_prep(struct napi_struct *n)
  5464. {
  5465. unsigned long val, new;
  5466. do {
  5467. val = READ_ONCE(n->state);
  5468. if (unlikely(val & NAPIF_STATE_DISABLE))
  5469. return false;
  5470. new = val | NAPIF_STATE_SCHED;
  5471. /* Sets STATE_MISSED bit if STATE_SCHED was already set
  5472. * This was suggested by Alexander Duyck, as compiler
  5473. * emits better code than :
  5474. * if (val & NAPIF_STATE_SCHED)
  5475. * new |= NAPIF_STATE_MISSED;
  5476. */
  5477. new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
  5478. NAPIF_STATE_MISSED;
  5479. } while (cmpxchg(&n->state, val, new) != val);
  5480. return !(val & NAPIF_STATE_SCHED);
  5481. }
  5482. EXPORT_SYMBOL(napi_schedule_prep);
  5483. /**
  5484. * __napi_schedule_irqoff - schedule for receive
  5485. * @n: entry to schedule
  5486. *
  5487. * Variant of __napi_schedule() assuming hard irqs are masked.
  5488. *
  5489. * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
  5490. * because the interrupt disabled assumption might not be true
  5491. * due to force-threaded interrupts and spinlock substitution.
  5492. */
  5493. void __napi_schedule_irqoff(struct napi_struct *n)
  5494. {
  5495. if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  5496. ____napi_schedule(this_cpu_ptr(&softnet_data), n);
  5497. else
  5498. __napi_schedule(n);
  5499. }
  5500. EXPORT_SYMBOL(__napi_schedule_irqoff);
  5501. bool napi_complete_done(struct napi_struct *n, int work_done)
  5502. {
  5503. unsigned long flags, val, new, timeout = 0;
  5504. bool ret = true;
  5505. /*
  5506. * 1) Don't let napi dequeue from the cpu poll list
  5507. * just in case its running on a different cpu.
  5508. * 2) If we are busy polling, do nothing here, we have
  5509. * the guarantee we will be called later.
  5510. */
  5511. if (unlikely(n->state & (NAPIF_STATE_NPSVC |
  5512. NAPIF_STATE_IN_BUSY_POLL)))
  5513. return false;
  5514. if (work_done) {
  5515. if (n->gro_bitmask)
  5516. timeout = READ_ONCE(n->dev->gro_flush_timeout);
  5517. n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
  5518. }
  5519. if (n->defer_hard_irqs_count > 0) {
  5520. n->defer_hard_irqs_count--;
  5521. timeout = READ_ONCE(n->dev->gro_flush_timeout);
  5522. if (timeout)
  5523. ret = false;
  5524. }
  5525. if (n->gro_bitmask) {
  5526. /* When the NAPI instance uses a timeout and keeps postponing
  5527. * it, we need to bound somehow the time packets are kept in
  5528. * the GRO layer
  5529. */
  5530. napi_gro_flush(n, !!timeout);
  5531. }
  5532. gro_normal_list(n);
  5533. if (unlikely(!list_empty(&n->poll_list))) {
  5534. /* If n->poll_list is not empty, we need to mask irqs */
  5535. local_irq_save(flags);
  5536. list_del_init(&n->poll_list);
  5537. local_irq_restore(flags);
  5538. }
  5539. do {
  5540. val = READ_ONCE(n->state);
  5541. WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
  5542. new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
  5543. /* If STATE_MISSED was set, leave STATE_SCHED set,
  5544. * because we will call napi->poll() one more time.
  5545. * This C code was suggested by Alexander Duyck to help gcc.
  5546. */
  5547. new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
  5548. NAPIF_STATE_SCHED;
  5549. } while (cmpxchg(&n->state, val, new) != val);
  5550. if (unlikely(val & NAPIF_STATE_MISSED)) {
  5551. __napi_schedule(n);
  5552. return false;
  5553. }
  5554. if (timeout)
  5555. hrtimer_start(&n->timer, ns_to_ktime(timeout),
  5556. HRTIMER_MODE_REL_PINNED);
  5557. return ret;
  5558. }
  5559. EXPORT_SYMBOL(napi_complete_done);
  5560. /* must be called under rcu_read_lock(), as we dont take a reference */
  5561. static struct napi_struct *napi_by_id(unsigned int napi_id)
  5562. {
  5563. unsigned int hash = napi_id % HASH_SIZE(napi_hash);
  5564. struct napi_struct *napi;
  5565. hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
  5566. if (napi->napi_id == napi_id)
  5567. return napi;
  5568. return NULL;
  5569. }
  5570. #if defined(CONFIG_NET_RX_BUSY_POLL)
  5571. #define BUSY_POLL_BUDGET 8
  5572. static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
  5573. {
  5574. int rc;
  5575. /* Busy polling means there is a high chance device driver hard irq
  5576. * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
  5577. * set in napi_schedule_prep().
  5578. * Since we are about to call napi->poll() once more, we can safely
  5579. * clear NAPI_STATE_MISSED.
  5580. *
  5581. * Note: x86 could use a single "lock and ..." instruction
  5582. * to perform these two clear_bit()
  5583. */
  5584. clear_bit(NAPI_STATE_MISSED, &napi->state);
  5585. clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
  5586. local_bh_disable();
  5587. /* All we really want here is to re-enable device interrupts.
  5588. * Ideally, a new ndo_busy_poll_stop() could avoid another round.
  5589. */
  5590. rc = napi->poll(napi, BUSY_POLL_BUDGET);
  5591. /* We can't gro_normal_list() here, because napi->poll() might have
  5592. * rearmed the napi (napi_complete_done()) in which case it could
  5593. * already be running on another CPU.
  5594. */
  5595. trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
  5596. netpoll_poll_unlock(have_poll_lock);
  5597. if (rc == BUSY_POLL_BUDGET) {
  5598. /* As the whole budget was spent, we still own the napi so can
  5599. * safely handle the rx_list.
  5600. */
  5601. gro_normal_list(napi);
  5602. __napi_schedule(napi);
  5603. }
  5604. local_bh_enable();
  5605. }
  5606. void napi_busy_loop(unsigned int napi_id,
  5607. bool (*loop_end)(void *, unsigned long),
  5608. void *loop_end_arg)
  5609. {
  5610. unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
  5611. int (*napi_poll)(struct napi_struct *napi, int budget);
  5612. void *have_poll_lock = NULL;
  5613. struct napi_struct *napi;
  5614. restart:
  5615. napi_poll = NULL;
  5616. rcu_read_lock();
  5617. napi = napi_by_id(napi_id);
  5618. if (!napi)
  5619. goto out;
  5620. preempt_disable();
  5621. for (;;) {
  5622. int work = 0;
  5623. local_bh_disable();
  5624. if (!napi_poll) {
  5625. unsigned long val = READ_ONCE(napi->state);
  5626. /* If multiple threads are competing for this napi,
  5627. * we avoid dirtying napi->state as much as we can.
  5628. */
  5629. if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
  5630. NAPIF_STATE_IN_BUSY_POLL))
  5631. goto count;
  5632. if (cmpxchg(&napi->state, val,
  5633. val | NAPIF_STATE_IN_BUSY_POLL |
  5634. NAPIF_STATE_SCHED) != val)
  5635. goto count;
  5636. have_poll_lock = netpoll_poll_lock(napi);
  5637. napi_poll = napi->poll;
  5638. }
  5639. work = napi_poll(napi, BUSY_POLL_BUDGET);
  5640. trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
  5641. gro_normal_list(napi);
  5642. count:
  5643. if (work > 0)
  5644. __NET_ADD_STATS(dev_net(napi->dev),
  5645. LINUX_MIB_BUSYPOLLRXPACKETS, work);
  5646. local_bh_enable();
  5647. if (!loop_end || loop_end(loop_end_arg, start_time))
  5648. break;
  5649. if (unlikely(need_resched())) {
  5650. if (napi_poll)
  5651. busy_poll_stop(napi, have_poll_lock);
  5652. preempt_enable();
  5653. rcu_read_unlock();
  5654. cond_resched();
  5655. if (loop_end(loop_end_arg, start_time))
  5656. return;
  5657. goto restart;
  5658. }
  5659. cpu_relax();
  5660. }
  5661. if (napi_poll)
  5662. busy_poll_stop(napi, have_poll_lock);
  5663. preempt_enable();
  5664. out:
  5665. rcu_read_unlock();
  5666. }
  5667. EXPORT_SYMBOL(napi_busy_loop);
  5668. #endif /* CONFIG_NET_RX_BUSY_POLL */
  5669. static void napi_hash_add(struct napi_struct *napi)
  5670. {
  5671. if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
  5672. return;
  5673. spin_lock(&napi_hash_lock);
  5674. /* 0..NR_CPUS range is reserved for sender_cpu use */
  5675. do {
  5676. if (unlikely(++napi_gen_id < MIN_NAPI_ID))
  5677. napi_gen_id = MIN_NAPI_ID;
  5678. } while (napi_by_id(napi_gen_id));
  5679. napi->napi_id = napi_gen_id;
  5680. hlist_add_head_rcu(&napi->napi_hash_node,
  5681. &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
  5682. spin_unlock(&napi_hash_lock);
  5683. }
  5684. /* Warning : caller is responsible to make sure rcu grace period
  5685. * is respected before freeing memory containing @napi
  5686. */
  5687. static void napi_hash_del(struct napi_struct *napi)
  5688. {
  5689. spin_lock(&napi_hash_lock);
  5690. hlist_del_init_rcu(&napi->napi_hash_node);
  5691. spin_unlock(&napi_hash_lock);
  5692. }
  5693. static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
  5694. {
  5695. struct napi_struct *napi;
  5696. napi = container_of(timer, struct napi_struct, timer);
  5697. /* Note : we use a relaxed variant of napi_schedule_prep() not setting
  5698. * NAPI_STATE_MISSED, since we do not react to a device IRQ.
  5699. */
  5700. if (!napi_disable_pending(napi) &&
  5701. !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
  5702. __napi_schedule_irqoff(napi);
  5703. return HRTIMER_NORESTART;
  5704. }
  5705. static void init_gro_hash(struct napi_struct *napi)
  5706. {
  5707. int i;
  5708. for (i = 0; i < GRO_HASH_BUCKETS; i++) {
  5709. INIT_LIST_HEAD(&napi->gro_hash[i].list);
  5710. napi->gro_hash[i].count = 0;
  5711. }
  5712. napi->gro_bitmask = 0;
  5713. }
  5714. void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
  5715. int (*poll)(struct napi_struct *, int), int weight)
  5716. {
  5717. if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
  5718. return;
  5719. INIT_LIST_HEAD(&napi->poll_list);
  5720. INIT_HLIST_NODE(&napi->napi_hash_node);
  5721. hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
  5722. napi->timer.function = napi_watchdog;
  5723. init_gro_hash(napi);
  5724. napi->skb = NULL;
  5725. INIT_LIST_HEAD(&napi->rx_list);
  5726. napi->rx_count = 0;
  5727. napi->poll = poll;
  5728. if (weight > NAPI_POLL_WEIGHT)
  5729. netdev_err_once(dev, "%s() called with weight %d\n", __func__,
  5730. weight);
  5731. napi->weight = weight;
  5732. napi->dev = dev;
  5733. #ifdef CONFIG_NETPOLL
  5734. napi->poll_owner = -1;
  5735. #endif
  5736. set_bit(NAPI_STATE_SCHED, &napi->state);
  5737. set_bit(NAPI_STATE_NPSVC, &napi->state);
  5738. list_add_rcu(&napi->dev_list, &dev->napi_list);
  5739. napi_hash_add(napi);
  5740. }
  5741. EXPORT_SYMBOL(netif_napi_add);
  5742. void napi_disable(struct napi_struct *n)
  5743. {
  5744. might_sleep();
  5745. set_bit(NAPI_STATE_DISABLE, &n->state);
  5746. while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
  5747. msleep(1);
  5748. while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
  5749. msleep(1);
  5750. hrtimer_cancel(&n->timer);
  5751. clear_bit(NAPI_STATE_DISABLE, &n->state);
  5752. }
  5753. EXPORT_SYMBOL(napi_disable);
  5754. static void flush_gro_hash(struct napi_struct *napi)
  5755. {
  5756. int i;
  5757. for (i = 0; i < GRO_HASH_BUCKETS; i++) {
  5758. struct sk_buff *skb, *n;
  5759. list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
  5760. kfree_skb(skb);
  5761. napi->gro_hash[i].count = 0;
  5762. }
  5763. }
  5764. /* Must be called in process context */
  5765. void __netif_napi_del(struct napi_struct *napi)
  5766. {
  5767. if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
  5768. return;
  5769. napi_hash_del(napi);
  5770. list_del_rcu(&napi->dev_list);
  5771. napi_free_frags(napi);
  5772. flush_gro_hash(napi);
  5773. napi->gro_bitmask = 0;
  5774. }
  5775. EXPORT_SYMBOL(__netif_napi_del);
  5776. static int napi_poll(struct napi_struct *n, struct list_head *repoll)
  5777. {
  5778. void *have;
  5779. int work, weight;
  5780. list_del_init(&n->poll_list);
  5781. have = netpoll_poll_lock(n);
  5782. weight = n->weight;
  5783. /* This NAPI_STATE_SCHED test is for avoiding a race
  5784. * with netpoll's poll_napi(). Only the entity which
  5785. * obtains the lock and sees NAPI_STATE_SCHED set will
  5786. * actually make the ->poll() call. Therefore we avoid
  5787. * accidentally calling ->poll() when NAPI is not scheduled.
  5788. */
  5789. work = 0;
  5790. if (test_bit(NAPI_STATE_SCHED, &n->state)) {
  5791. work = n->poll(n, weight);
  5792. trace_napi_poll(n, work, weight);
  5793. }
  5794. if (unlikely(work > weight))
  5795. pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
  5796. n->poll, work, weight);
  5797. if (likely(work < weight))
  5798. goto out_unlock;
  5799. /* Drivers must not modify the NAPI state if they
  5800. * consume the entire weight. In such cases this code
  5801. * still "owns" the NAPI instance and therefore can
  5802. * move the instance around on the list at-will.
  5803. */
  5804. if (unlikely(napi_disable_pending(n))) {
  5805. napi_complete(n);
  5806. goto out_unlock;
  5807. }
  5808. if (n->gro_bitmask) {
  5809. /* flush too old packets
  5810. * If HZ < 1000, flush all packets.
  5811. */
  5812. napi_gro_flush(n, HZ >= 1000);
  5813. }
  5814. gro_normal_list(n);
  5815. /* Some drivers may have called napi_schedule
  5816. * prior to exhausting their budget.
  5817. */
  5818. if (unlikely(!list_empty(&n->poll_list))) {
  5819. pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
  5820. n->dev ? n->dev->name : "backlog");
  5821. goto out_unlock;
  5822. }
  5823. list_add_tail(&n->poll_list, repoll);
  5824. out_unlock:
  5825. netpoll_poll_unlock(have);
  5826. return work;
  5827. }
  5828. static __latent_entropy void net_rx_action(struct softirq_action *h)
  5829. {
  5830. struct softnet_data *sd = this_cpu_ptr(&softnet_data);
  5831. unsigned long time_limit = jiffies +
  5832. usecs_to_jiffies(netdev_budget_usecs);
  5833. int budget = netdev_budget;
  5834. LIST_HEAD(list);
  5835. LIST_HEAD(repoll);
  5836. local_irq_disable();
  5837. list_splice_init(&sd->poll_list, &list);
  5838. local_irq_enable();
  5839. for (;;) {
  5840. struct napi_struct *n;
  5841. if (list_empty(&list)) {
  5842. if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
  5843. goto out;
  5844. break;
  5845. }
  5846. n = list_first_entry(&list, struct napi_struct, poll_list);
  5847. budget -= napi_poll(n, &repoll);
  5848. /* If softirq window is exhausted then punt.
  5849. * Allow this to run for 2 jiffies since which will allow
  5850. * an average latency of 1.5/HZ.
  5851. */
  5852. if (unlikely(budget <= 0 ||
  5853. time_after_eq(jiffies, time_limit))) {
  5854. sd->time_squeeze++;
  5855. break;
  5856. }
  5857. }
  5858. local_irq_disable();
  5859. list_splice_tail_init(&sd->poll_list, &list);
  5860. list_splice_tail(&repoll, &list);
  5861. list_splice(&list, &sd->poll_list);
  5862. if (!list_empty(&sd->poll_list))
  5863. __raise_softirq_irqoff(NET_RX_SOFTIRQ);
  5864. net_rps_action_and_irq_enable(sd);
  5865. out:
  5866. __kfree_skb_flush();
  5867. }
  5868. struct netdev_adjacent {
  5869. struct net_device *dev;
  5870. /* upper master flag, there can only be one master device per list */
  5871. bool master;
  5872. /* lookup ignore flag */
  5873. bool ignore;
  5874. /* counter for the number of times this device was added to us */
  5875. u16 ref_nr;
  5876. /* private field for the users */
  5877. void *private;
  5878. struct list_head list;
  5879. struct rcu_head rcu;
  5880. };
  5881. static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
  5882. struct list_head *adj_list)
  5883. {
  5884. struct netdev_adjacent *adj;
  5885. list_for_each_entry(adj, adj_list, list) {
  5886. if (adj->dev == adj_dev)
  5887. return adj;
  5888. }
  5889. return NULL;
  5890. }
  5891. static int ____netdev_has_upper_dev(struct net_device *upper_dev,
  5892. struct netdev_nested_priv *priv)
  5893. {
  5894. struct net_device *dev = (struct net_device *)priv->data;
  5895. return upper_dev == dev;
  5896. }
  5897. /**
  5898. * netdev_has_upper_dev - Check if device is linked to an upper device
  5899. * @dev: device
  5900. * @upper_dev: upper device to check
  5901. *
  5902. * Find out if a device is linked to specified upper device and return true
  5903. * in case it is. Note that this checks only immediate upper device,
  5904. * not through a complete stack of devices. The caller must hold the RTNL lock.
  5905. */
  5906. bool netdev_has_upper_dev(struct net_device *dev,
  5907. struct net_device *upper_dev)
  5908. {
  5909. struct netdev_nested_priv priv = {
  5910. .data = (void *)upper_dev,
  5911. };
  5912. ASSERT_RTNL();
  5913. return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
  5914. &priv);
  5915. }
  5916. EXPORT_SYMBOL(netdev_has_upper_dev);
  5917. /**
  5918. * netdev_has_upper_dev_all - Check if device is linked to an upper device
  5919. * @dev: device
  5920. * @upper_dev: upper device to check
  5921. *
  5922. * Find out if a device is linked to specified upper device and return true
  5923. * in case it is. Note that this checks the entire upper device chain.
  5924. * The caller must hold rcu lock.
  5925. */
  5926. bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
  5927. struct net_device *upper_dev)
  5928. {
  5929. struct netdev_nested_priv priv = {
  5930. .data = (void *)upper_dev,
  5931. };
  5932. return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
  5933. &priv);
  5934. }
  5935. EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
  5936. /**
  5937. * netdev_has_any_upper_dev - Check if device is linked to some device
  5938. * @dev: device
  5939. *
  5940. * Find out if a device is linked to an upper device and return true in case
  5941. * it is. The caller must hold the RTNL lock.
  5942. */
  5943. bool netdev_has_any_upper_dev(struct net_device *dev)
  5944. {
  5945. ASSERT_RTNL();
  5946. return !list_empty(&dev->adj_list.upper);
  5947. }
  5948. EXPORT_SYMBOL(netdev_has_any_upper_dev);
  5949. /**
  5950. * netdev_master_upper_dev_get - Get master upper device
  5951. * @dev: device
  5952. *
  5953. * Find a master upper device and return pointer to it or NULL in case
  5954. * it's not there. The caller must hold the RTNL lock.
  5955. */
  5956. struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
  5957. {
  5958. struct netdev_adjacent *upper;
  5959. ASSERT_RTNL();
  5960. if (list_empty(&dev->adj_list.upper))
  5961. return NULL;
  5962. upper = list_first_entry(&dev->adj_list.upper,
  5963. struct netdev_adjacent, list);
  5964. if (likely(upper->master))
  5965. return upper->dev;
  5966. return NULL;
  5967. }
  5968. EXPORT_SYMBOL(netdev_master_upper_dev_get);
  5969. static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
  5970. {
  5971. struct netdev_adjacent *upper;
  5972. ASSERT_RTNL();
  5973. if (list_empty(&dev->adj_list.upper))
  5974. return NULL;
  5975. upper = list_first_entry(&dev->adj_list.upper,
  5976. struct netdev_adjacent, list);
  5977. if (likely(upper->master) && !upper->ignore)
  5978. return upper->dev;
  5979. return NULL;
  5980. }
  5981. /**
  5982. * netdev_has_any_lower_dev - Check if device is linked to some device
  5983. * @dev: device
  5984. *
  5985. * Find out if a device is linked to a lower device and return true in case
  5986. * it is. The caller must hold the RTNL lock.
  5987. */
  5988. static bool netdev_has_any_lower_dev(struct net_device *dev)
  5989. {
  5990. ASSERT_RTNL();
  5991. return !list_empty(&dev->adj_list.lower);
  5992. }
  5993. void *netdev_adjacent_get_private(struct list_head *adj_list)
  5994. {
  5995. struct netdev_adjacent *adj;
  5996. adj = list_entry(adj_list, struct netdev_adjacent, list);
  5997. return adj->private;
  5998. }
  5999. EXPORT_SYMBOL(netdev_adjacent_get_private);
  6000. /**
  6001. * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
  6002. * @dev: device
  6003. * @iter: list_head ** of the current position
  6004. *
  6005. * Gets the next device from the dev's upper list, starting from iter
  6006. * position. The caller must hold RCU read lock.
  6007. */
  6008. struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
  6009. struct list_head **iter)
  6010. {
  6011. struct netdev_adjacent *upper;
  6012. WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
  6013. upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
  6014. if (&upper->list == &dev->adj_list.upper)
  6015. return NULL;
  6016. *iter = &upper->list;
  6017. return upper->dev;
  6018. }
  6019. EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
  6020. static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
  6021. struct list_head **iter,
  6022. bool *ignore)
  6023. {
  6024. struct netdev_adjacent *upper;
  6025. upper = list_entry((*iter)->next, struct netdev_adjacent, list);
  6026. if (&upper->list == &dev->adj_list.upper)
  6027. return NULL;
  6028. *iter = &upper->list;
  6029. *ignore = upper->ignore;
  6030. return upper->dev;
  6031. }
  6032. static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
  6033. struct list_head **iter)
  6034. {
  6035. struct netdev_adjacent *upper;
  6036. WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
  6037. upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
  6038. if (&upper->list == &dev->adj_list.upper)
  6039. return NULL;
  6040. *iter = &upper->list;
  6041. return upper->dev;
  6042. }
  6043. static int __netdev_walk_all_upper_dev(struct net_device *dev,
  6044. int (*fn)(struct net_device *dev,
  6045. struct netdev_nested_priv *priv),
  6046. struct netdev_nested_priv *priv)
  6047. {
  6048. struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
  6049. struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
  6050. int ret, cur = 0;
  6051. bool ignore;
  6052. now = dev;
  6053. iter = &dev->adj_list.upper;
  6054. while (1) {
  6055. if (now != dev) {
  6056. ret = fn(now, priv);
  6057. if (ret)
  6058. return ret;
  6059. }
  6060. next = NULL;
  6061. while (1) {
  6062. udev = __netdev_next_upper_dev(now, &iter, &ignore);
  6063. if (!udev)
  6064. break;
  6065. if (ignore)
  6066. continue;
  6067. next = udev;
  6068. niter = &udev->adj_list.upper;
  6069. dev_stack[cur] = now;
  6070. iter_stack[cur++] = iter;
  6071. break;
  6072. }
  6073. if (!next) {
  6074. if (!cur)
  6075. return 0;
  6076. next = dev_stack[--cur];
  6077. niter = iter_stack[cur];
  6078. }
  6079. now = next;
  6080. iter = niter;
  6081. }
  6082. return 0;
  6083. }
  6084. int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
  6085. int (*fn)(struct net_device *dev,
  6086. struct netdev_nested_priv *priv),
  6087. struct netdev_nested_priv *priv)
  6088. {
  6089. struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
  6090. struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
  6091. int ret, cur = 0;
  6092. now = dev;
  6093. iter = &dev->adj_list.upper;
  6094. while (1) {
  6095. if (now != dev) {
  6096. ret = fn(now, priv);
  6097. if (ret)
  6098. return ret;
  6099. }
  6100. next = NULL;
  6101. while (1) {
  6102. udev = netdev_next_upper_dev_rcu(now, &iter);
  6103. if (!udev)
  6104. break;
  6105. next = udev;
  6106. niter = &udev->adj_list.upper;
  6107. dev_stack[cur] = now;
  6108. iter_stack[cur++] = iter;
  6109. break;
  6110. }
  6111. if (!next) {
  6112. if (!cur)
  6113. return 0;
  6114. next = dev_stack[--cur];
  6115. niter = iter_stack[cur];
  6116. }
  6117. now = next;
  6118. iter = niter;
  6119. }
  6120. return 0;
  6121. }
  6122. EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
  6123. static bool __netdev_has_upper_dev(struct net_device *dev,
  6124. struct net_device *upper_dev)
  6125. {
  6126. struct netdev_nested_priv priv = {
  6127. .flags = 0,
  6128. .data = (void *)upper_dev,
  6129. };
  6130. ASSERT_RTNL();
  6131. return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
  6132. &priv);
  6133. }
  6134. /**
  6135. * netdev_lower_get_next_private - Get the next ->private from the
  6136. * lower neighbour list
  6137. * @dev: device
  6138. * @iter: list_head ** of the current position
  6139. *
  6140. * Gets the next netdev_adjacent->private from the dev's lower neighbour
  6141. * list, starting from iter position. The caller must hold either hold the
  6142. * RTNL lock or its own locking that guarantees that the neighbour lower
  6143. * list will remain unchanged.
  6144. */
  6145. void *netdev_lower_get_next_private(struct net_device *dev,
  6146. struct list_head **iter)
  6147. {
  6148. struct netdev_adjacent *lower;
  6149. lower = list_entry(*iter, struct netdev_adjacent, list);
  6150. if (&lower->list == &dev->adj_list.lower)
  6151. return NULL;
  6152. *iter = lower->list.next;
  6153. return lower->private;
  6154. }
  6155. EXPORT_SYMBOL(netdev_lower_get_next_private);
  6156. /**
  6157. * netdev_lower_get_next_private_rcu - Get the next ->private from the
  6158. * lower neighbour list, RCU
  6159. * variant
  6160. * @dev: device
  6161. * @iter: list_head ** of the current position
  6162. *
  6163. * Gets the next netdev_adjacent->private from the dev's lower neighbour
  6164. * list, starting from iter position. The caller must hold RCU read lock.
  6165. */
  6166. void *netdev_lower_get_next_private_rcu(struct net_device *dev,
  6167. struct list_head **iter)
  6168. {
  6169. struct netdev_adjacent *lower;
  6170. WARN_ON_ONCE(!rcu_read_lock_held());
  6171. lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
  6172. if (&lower->list == &dev->adj_list.lower)
  6173. return NULL;
  6174. *iter = &lower->list;
  6175. return lower->private;
  6176. }
  6177. EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
  6178. /**
  6179. * netdev_lower_get_next - Get the next device from the lower neighbour
  6180. * list
  6181. * @dev: device
  6182. * @iter: list_head ** of the current position
  6183. *
  6184. * Gets the next netdev_adjacent from the dev's lower neighbour
  6185. * list, starting from iter position. The caller must hold RTNL lock or
  6186. * its own locking that guarantees that the neighbour lower
  6187. * list will remain unchanged.
  6188. */
  6189. void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
  6190. {
  6191. struct netdev_adjacent *lower;
  6192. lower = list_entry(*iter, struct netdev_adjacent, list);
  6193. if (&lower->list == &dev->adj_list.lower)
  6194. return NULL;
  6195. *iter = lower->list.next;
  6196. return lower->dev;
  6197. }
  6198. EXPORT_SYMBOL(netdev_lower_get_next);
  6199. static struct net_device *netdev_next_lower_dev(struct net_device *dev,
  6200. struct list_head **iter)
  6201. {
  6202. struct netdev_adjacent *lower;
  6203. lower = list_entry((*iter)->next, struct netdev_adjacent, list);
  6204. if (&lower->list == &dev->adj_list.lower)
  6205. return NULL;
  6206. *iter = &lower->list;
  6207. return lower->dev;
  6208. }
  6209. static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
  6210. struct list_head **iter,
  6211. bool *ignore)
  6212. {
  6213. struct netdev_adjacent *lower;
  6214. lower = list_entry((*iter)->next, struct netdev_adjacent, list);
  6215. if (&lower->list == &dev->adj_list.lower)
  6216. return NULL;
  6217. *iter = &lower->list;
  6218. *ignore = lower->ignore;
  6219. return lower->dev;
  6220. }
  6221. int netdev_walk_all_lower_dev(struct net_device *dev,
  6222. int (*fn)(struct net_device *dev,
  6223. struct netdev_nested_priv *priv),
  6224. struct netdev_nested_priv *priv)
  6225. {
  6226. struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
  6227. struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
  6228. int ret, cur = 0;
  6229. now = dev;
  6230. iter = &dev->adj_list.lower;
  6231. while (1) {
  6232. if (now != dev) {
  6233. ret = fn(now, priv);
  6234. if (ret)
  6235. return ret;
  6236. }
  6237. next = NULL;
  6238. while (1) {
  6239. ldev = netdev_next_lower_dev(now, &iter);
  6240. if (!ldev)
  6241. break;
  6242. next = ldev;
  6243. niter = &ldev->adj_list.lower;
  6244. dev_stack[cur] = now;
  6245. iter_stack[cur++] = iter;
  6246. break;
  6247. }
  6248. if (!next) {
  6249. if (!cur)
  6250. return 0;
  6251. next = dev_stack[--cur];
  6252. niter = iter_stack[cur];
  6253. }
  6254. now = next;
  6255. iter = niter;
  6256. }
  6257. return 0;
  6258. }
  6259. EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
  6260. static int __netdev_walk_all_lower_dev(struct net_device *dev,
  6261. int (*fn)(struct net_device *dev,
  6262. struct netdev_nested_priv *priv),
  6263. struct netdev_nested_priv *priv)
  6264. {
  6265. struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
  6266. struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
  6267. int ret, cur = 0;
  6268. bool ignore;
  6269. now = dev;
  6270. iter = &dev->adj_list.lower;
  6271. while (1) {
  6272. if (now != dev) {
  6273. ret = fn(now, priv);
  6274. if (ret)
  6275. return ret;
  6276. }
  6277. next = NULL;
  6278. while (1) {
  6279. ldev = __netdev_next_lower_dev(now, &iter, &ignore);
  6280. if (!ldev)
  6281. break;
  6282. if (ignore)
  6283. continue;
  6284. next = ldev;
  6285. niter = &ldev->adj_list.lower;
  6286. dev_stack[cur] = now;
  6287. iter_stack[cur++] = iter;
  6288. break;
  6289. }
  6290. if (!next) {
  6291. if (!cur)
  6292. return 0;
  6293. next = dev_stack[--cur];
  6294. niter = iter_stack[cur];
  6295. }
  6296. now = next;
  6297. iter = niter;
  6298. }
  6299. return 0;
  6300. }
  6301. struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
  6302. struct list_head **iter)
  6303. {
  6304. struct netdev_adjacent *lower;
  6305. lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
  6306. if (&lower->list == &dev->adj_list.lower)
  6307. return NULL;
  6308. *iter = &lower->list;
  6309. return lower->dev;
  6310. }
  6311. EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
  6312. static u8 __netdev_upper_depth(struct net_device *dev)
  6313. {
  6314. struct net_device *udev;
  6315. struct list_head *iter;
  6316. u8 max_depth = 0;
  6317. bool ignore;
  6318. for (iter = &dev->adj_list.upper,
  6319. udev = __netdev_next_upper_dev(dev, &iter, &ignore);
  6320. udev;
  6321. udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
  6322. if (ignore)
  6323. continue;
  6324. if (max_depth < udev->upper_level)
  6325. max_depth = udev->upper_level;
  6326. }
  6327. return max_depth;
  6328. }
  6329. static u8 __netdev_lower_depth(struct net_device *dev)
  6330. {
  6331. struct net_device *ldev;
  6332. struct list_head *iter;
  6333. u8 max_depth = 0;
  6334. bool ignore;
  6335. for (iter = &dev->adj_list.lower,
  6336. ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
  6337. ldev;
  6338. ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
  6339. if (ignore)
  6340. continue;
  6341. if (max_depth < ldev->lower_level)
  6342. max_depth = ldev->lower_level;
  6343. }
  6344. return max_depth;
  6345. }
  6346. static int __netdev_update_upper_level(struct net_device *dev,
  6347. struct netdev_nested_priv *__unused)
  6348. {
  6349. dev->upper_level = __netdev_upper_depth(dev) + 1;
  6350. return 0;
  6351. }
  6352. static int __netdev_update_lower_level(struct net_device *dev,
  6353. struct netdev_nested_priv *priv)
  6354. {
  6355. dev->lower_level = __netdev_lower_depth(dev) + 1;
  6356. #ifdef CONFIG_LOCKDEP
  6357. if (!priv)
  6358. return 0;
  6359. if (priv->flags & NESTED_SYNC_IMM)
  6360. dev->nested_level = dev->lower_level - 1;
  6361. if (priv->flags & NESTED_SYNC_TODO)
  6362. net_unlink_todo(dev);
  6363. #endif
  6364. return 0;
  6365. }
  6366. int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
  6367. int (*fn)(struct net_device *dev,
  6368. struct netdev_nested_priv *priv),
  6369. struct netdev_nested_priv *priv)
  6370. {
  6371. struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
  6372. struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
  6373. int ret, cur = 0;
  6374. now = dev;
  6375. iter = &dev->adj_list.lower;
  6376. while (1) {
  6377. if (now != dev) {
  6378. ret = fn(now, priv);
  6379. if (ret)
  6380. return ret;
  6381. }
  6382. next = NULL;
  6383. while (1) {
  6384. ldev = netdev_next_lower_dev_rcu(now, &iter);
  6385. if (!ldev)
  6386. break;
  6387. next = ldev;
  6388. niter = &ldev->adj_list.lower;
  6389. dev_stack[cur] = now;
  6390. iter_stack[cur++] = iter;
  6391. break;
  6392. }
  6393. if (!next) {
  6394. if (!cur)
  6395. return 0;
  6396. next = dev_stack[--cur];
  6397. niter = iter_stack[cur];
  6398. }
  6399. now = next;
  6400. iter = niter;
  6401. }
  6402. return 0;
  6403. }
  6404. EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
  6405. /**
  6406. * netdev_lower_get_first_private_rcu - Get the first ->private from the
  6407. * lower neighbour list, RCU
  6408. * variant
  6409. * @dev: device
  6410. *
  6411. * Gets the first netdev_adjacent->private from the dev's lower neighbour
  6412. * list. The caller must hold RCU read lock.
  6413. */
  6414. void *netdev_lower_get_first_private_rcu(struct net_device *dev)
  6415. {
  6416. struct netdev_adjacent *lower;
  6417. lower = list_first_or_null_rcu(&dev->adj_list.lower,
  6418. struct netdev_adjacent, list);
  6419. if (lower)
  6420. return lower->private;
  6421. return NULL;
  6422. }
  6423. EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
  6424. /**
  6425. * netdev_master_upper_dev_get_rcu - Get master upper device
  6426. * @dev: device
  6427. *
  6428. * Find a master upper device and return pointer to it or NULL in case
  6429. * it's not there. The caller must hold the RCU read lock.
  6430. */
  6431. struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
  6432. {
  6433. struct netdev_adjacent *upper;
  6434. upper = list_first_or_null_rcu(&dev->adj_list.upper,
  6435. struct netdev_adjacent, list);
  6436. if (upper && likely(upper->master))
  6437. return upper->dev;
  6438. return NULL;
  6439. }
  6440. EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
  6441. static int netdev_adjacent_sysfs_add(struct net_device *dev,
  6442. struct net_device *adj_dev,
  6443. struct list_head *dev_list)
  6444. {
  6445. char linkname[IFNAMSIZ+7];
  6446. sprintf(linkname, dev_list == &dev->adj_list.upper ?
  6447. "upper_%s" : "lower_%s", adj_dev->name);
  6448. return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
  6449. linkname);
  6450. }
  6451. static void netdev_adjacent_sysfs_del(struct net_device *dev,
  6452. char *name,
  6453. struct list_head *dev_list)
  6454. {
  6455. char linkname[IFNAMSIZ+7];
  6456. sprintf(linkname, dev_list == &dev->adj_list.upper ?
  6457. "upper_%s" : "lower_%s", name);
  6458. sysfs_remove_link(&(dev->dev.kobj), linkname);
  6459. }
  6460. static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
  6461. struct net_device *adj_dev,
  6462. struct list_head *dev_list)
  6463. {
  6464. return (dev_list == &dev->adj_list.upper ||
  6465. dev_list == &dev->adj_list.lower) &&
  6466. net_eq(dev_net(dev), dev_net(adj_dev));
  6467. }
  6468. static int __netdev_adjacent_dev_insert(struct net_device *dev,
  6469. struct net_device *adj_dev,
  6470. struct list_head *dev_list,
  6471. void *private, bool master)
  6472. {
  6473. struct netdev_adjacent *adj;
  6474. int ret;
  6475. adj = __netdev_find_adj(adj_dev, dev_list);
  6476. if (adj) {
  6477. adj->ref_nr += 1;
  6478. pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
  6479. dev->name, adj_dev->name, adj->ref_nr);
  6480. return 0;
  6481. }
  6482. adj = kmalloc(sizeof(*adj), GFP_KERNEL);
  6483. if (!adj)
  6484. return -ENOMEM;
  6485. adj->dev = adj_dev;
  6486. adj->master = master;
  6487. adj->ref_nr = 1;
  6488. adj->private = private;
  6489. adj->ignore = false;
  6490. dev_hold(adj_dev);
  6491. pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
  6492. dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
  6493. if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
  6494. ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
  6495. if (ret)
  6496. goto free_adj;
  6497. }
  6498. /* Ensure that master link is always the first item in list. */
  6499. if (master) {
  6500. ret = sysfs_create_link(&(dev->dev.kobj),
  6501. &(adj_dev->dev.kobj), "master");
  6502. if (ret)
  6503. goto remove_symlinks;
  6504. list_add_rcu(&adj->list, dev_list);
  6505. } else {
  6506. list_add_tail_rcu(&adj->list, dev_list);
  6507. }
  6508. return 0;
  6509. remove_symlinks:
  6510. if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
  6511. netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
  6512. free_adj:
  6513. kfree(adj);
  6514. dev_put(adj_dev);
  6515. return ret;
  6516. }
  6517. static void __netdev_adjacent_dev_remove(struct net_device *dev,
  6518. struct net_device *adj_dev,
  6519. u16 ref_nr,
  6520. struct list_head *dev_list)
  6521. {
  6522. struct netdev_adjacent *adj;
  6523. pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
  6524. dev->name, adj_dev->name, ref_nr);
  6525. adj = __netdev_find_adj(adj_dev, dev_list);
  6526. if (!adj) {
  6527. pr_err("Adjacency does not exist for device %s from %s\n",
  6528. dev->name, adj_dev->name);
  6529. WARN_ON(1);
  6530. return;
  6531. }
  6532. if (adj->ref_nr > ref_nr) {
  6533. pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
  6534. dev->name, adj_dev->name, ref_nr,
  6535. adj->ref_nr - ref_nr);
  6536. adj->ref_nr -= ref_nr;
  6537. return;
  6538. }
  6539. if (adj->master)
  6540. sysfs_remove_link(&(dev->dev.kobj), "master");
  6541. if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
  6542. netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
  6543. list_del_rcu(&adj->list);
  6544. pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
  6545. adj_dev->name, dev->name, adj_dev->name);
  6546. dev_put(adj_dev);
  6547. kfree_rcu(adj, rcu);
  6548. }
  6549. static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
  6550. struct net_device *upper_dev,
  6551. struct list_head *up_list,
  6552. struct list_head *down_list,
  6553. void *private, bool master)
  6554. {
  6555. int ret;
  6556. ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
  6557. private, master);
  6558. if (ret)
  6559. return ret;
  6560. ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
  6561. private, false);
  6562. if (ret) {
  6563. __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
  6564. return ret;
  6565. }
  6566. return 0;
  6567. }
  6568. static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
  6569. struct net_device *upper_dev,
  6570. u16 ref_nr,
  6571. struct list_head *up_list,
  6572. struct list_head *down_list)
  6573. {
  6574. __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
  6575. __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
  6576. }
  6577. static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
  6578. struct net_device *upper_dev,
  6579. void *private, bool master)
  6580. {
  6581. return __netdev_adjacent_dev_link_lists(dev, upper_dev,
  6582. &dev->adj_list.upper,
  6583. &upper_dev->adj_list.lower,
  6584. private, master);
  6585. }
  6586. static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
  6587. struct net_device *upper_dev)
  6588. {
  6589. __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
  6590. &dev->adj_list.upper,
  6591. &upper_dev->adj_list.lower);
  6592. }
  6593. static int __netdev_upper_dev_link(struct net_device *dev,
  6594. struct net_device *upper_dev, bool master,
  6595. void *upper_priv, void *upper_info,
  6596. struct netdev_nested_priv *priv,
  6597. struct netlink_ext_ack *extack)
  6598. {
  6599. struct netdev_notifier_changeupper_info changeupper_info = {
  6600. .info = {
  6601. .dev = dev,
  6602. .extack = extack,
  6603. },
  6604. .upper_dev = upper_dev,
  6605. .master = master,
  6606. .linking = true,
  6607. .upper_info = upper_info,
  6608. };
  6609. struct net_device *master_dev;
  6610. int ret = 0;
  6611. ASSERT_RTNL();
  6612. if (dev == upper_dev)
  6613. return -EBUSY;
  6614. /* To prevent loops, check if dev is not upper device to upper_dev. */
  6615. if (__netdev_has_upper_dev(upper_dev, dev))
  6616. return -EBUSY;
  6617. if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
  6618. return -EMLINK;
  6619. if (!master) {
  6620. if (__netdev_has_upper_dev(dev, upper_dev))
  6621. return -EEXIST;
  6622. } else {
  6623. master_dev = __netdev_master_upper_dev_get(dev);
  6624. if (master_dev)
  6625. return master_dev == upper_dev ? -EEXIST : -EBUSY;
  6626. }
  6627. ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
  6628. &changeupper_info.info);
  6629. ret = notifier_to_errno(ret);
  6630. if (ret)
  6631. return ret;
  6632. ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
  6633. master);
  6634. if (ret)
  6635. return ret;
  6636. ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
  6637. &changeupper_info.info);
  6638. ret = notifier_to_errno(ret);
  6639. if (ret)
  6640. goto rollback;
  6641. __netdev_update_upper_level(dev, NULL);
  6642. __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
  6643. __netdev_update_lower_level(upper_dev, priv);
  6644. __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
  6645. priv);
  6646. return 0;
  6647. rollback:
  6648. __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
  6649. return ret;
  6650. }
  6651. /**
  6652. * netdev_upper_dev_link - Add a link to the upper device
  6653. * @dev: device
  6654. * @upper_dev: new upper device
  6655. * @extack: netlink extended ack
  6656. *
  6657. * Adds a link to device which is upper to this one. The caller must hold
  6658. * the RTNL lock. On a failure a negative errno code is returned.
  6659. * On success the reference counts are adjusted and the function
  6660. * returns zero.
  6661. */
  6662. int netdev_upper_dev_link(struct net_device *dev,
  6663. struct net_device *upper_dev,
  6664. struct netlink_ext_ack *extack)
  6665. {
  6666. struct netdev_nested_priv priv = {
  6667. .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
  6668. .data = NULL,
  6669. };
  6670. return __netdev_upper_dev_link(dev, upper_dev, false,
  6671. NULL, NULL, &priv, extack);
  6672. }
  6673. EXPORT_SYMBOL(netdev_upper_dev_link);
  6674. /**
  6675. * netdev_master_upper_dev_link - Add a master link to the upper device
  6676. * @dev: device
  6677. * @upper_dev: new upper device
  6678. * @upper_priv: upper device private
  6679. * @upper_info: upper info to be passed down via notifier
  6680. * @extack: netlink extended ack
  6681. *
  6682. * Adds a link to device which is upper to this one. In this case, only
  6683. * one master upper device can be linked, although other non-master devices
  6684. * might be linked as well. The caller must hold the RTNL lock.
  6685. * On a failure a negative errno code is returned. On success the reference
  6686. * counts are adjusted and the function returns zero.
  6687. */
  6688. int netdev_master_upper_dev_link(struct net_device *dev,
  6689. struct net_device *upper_dev,
  6690. void *upper_priv, void *upper_info,
  6691. struct netlink_ext_ack *extack)
  6692. {
  6693. struct netdev_nested_priv priv = {
  6694. .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
  6695. .data = NULL,
  6696. };
  6697. return __netdev_upper_dev_link(dev, upper_dev, true,
  6698. upper_priv, upper_info, &priv, extack);
  6699. }
  6700. EXPORT_SYMBOL(netdev_master_upper_dev_link);
  6701. static void __netdev_upper_dev_unlink(struct net_device *dev,
  6702. struct net_device *upper_dev,
  6703. struct netdev_nested_priv *priv)
  6704. {
  6705. struct netdev_notifier_changeupper_info changeupper_info = {
  6706. .info = {
  6707. .dev = dev,
  6708. },
  6709. .upper_dev = upper_dev,
  6710. .linking = false,
  6711. };
  6712. ASSERT_RTNL();
  6713. changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
  6714. call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
  6715. &changeupper_info.info);
  6716. __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
  6717. call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
  6718. &changeupper_info.info);
  6719. __netdev_update_upper_level(dev, NULL);
  6720. __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
  6721. __netdev_update_lower_level(upper_dev, priv);
  6722. __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
  6723. priv);
  6724. }
  6725. /**
  6726. * netdev_upper_dev_unlink - Removes a link to upper device
  6727. * @dev: device
  6728. * @upper_dev: new upper device
  6729. *
  6730. * Removes a link to device which is upper to this one. The caller must hold
  6731. * the RTNL lock.
  6732. */
  6733. void netdev_upper_dev_unlink(struct net_device *dev,
  6734. struct net_device *upper_dev)
  6735. {
  6736. struct netdev_nested_priv priv = {
  6737. .flags = NESTED_SYNC_TODO,
  6738. .data = NULL,
  6739. };
  6740. __netdev_upper_dev_unlink(dev, upper_dev, &priv);
  6741. }
  6742. EXPORT_SYMBOL(netdev_upper_dev_unlink);
  6743. static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
  6744. struct net_device *lower_dev,
  6745. bool val)
  6746. {
  6747. struct netdev_adjacent *adj;
  6748. adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
  6749. if (adj)
  6750. adj->ignore = val;
  6751. adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
  6752. if (adj)
  6753. adj->ignore = val;
  6754. }
  6755. static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
  6756. struct net_device *lower_dev)
  6757. {
  6758. __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
  6759. }
  6760. static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
  6761. struct net_device *lower_dev)
  6762. {
  6763. __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
  6764. }
  6765. int netdev_adjacent_change_prepare(struct net_device *old_dev,
  6766. struct net_device *new_dev,
  6767. struct net_device *dev,
  6768. struct netlink_ext_ack *extack)
  6769. {
  6770. struct netdev_nested_priv priv = {
  6771. .flags = 0,
  6772. .data = NULL,
  6773. };
  6774. int err;
  6775. if (!new_dev)
  6776. return 0;
  6777. if (old_dev && new_dev != old_dev)
  6778. netdev_adjacent_dev_disable(dev, old_dev);
  6779. err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
  6780. extack);
  6781. if (err) {
  6782. if (old_dev && new_dev != old_dev)
  6783. netdev_adjacent_dev_enable(dev, old_dev);
  6784. return err;
  6785. }
  6786. return 0;
  6787. }
  6788. EXPORT_SYMBOL(netdev_adjacent_change_prepare);
  6789. void netdev_adjacent_change_commit(struct net_device *old_dev,
  6790. struct net_device *new_dev,
  6791. struct net_device *dev)
  6792. {
  6793. struct netdev_nested_priv priv = {
  6794. .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
  6795. .data = NULL,
  6796. };
  6797. if (!new_dev || !old_dev)
  6798. return;
  6799. if (new_dev == old_dev)
  6800. return;
  6801. netdev_adjacent_dev_enable(dev, old_dev);
  6802. __netdev_upper_dev_unlink(old_dev, dev, &priv);
  6803. }
  6804. EXPORT_SYMBOL(netdev_adjacent_change_commit);
  6805. void netdev_adjacent_change_abort(struct net_device *old_dev,
  6806. struct net_device *new_dev,
  6807. struct net_device *dev)
  6808. {
  6809. struct netdev_nested_priv priv = {
  6810. .flags = 0,
  6811. .data = NULL,
  6812. };
  6813. if (!new_dev)
  6814. return;
  6815. if (old_dev && new_dev != old_dev)
  6816. netdev_adjacent_dev_enable(dev, old_dev);
  6817. __netdev_upper_dev_unlink(new_dev, dev, &priv);
  6818. }
  6819. EXPORT_SYMBOL(netdev_adjacent_change_abort);
  6820. /**
  6821. * netdev_bonding_info_change - Dispatch event about slave change
  6822. * @dev: device
  6823. * @bonding_info: info to dispatch
  6824. *
  6825. * Send NETDEV_BONDING_INFO to netdev notifiers with info.
  6826. * The caller must hold the RTNL lock.
  6827. */
  6828. void netdev_bonding_info_change(struct net_device *dev,
  6829. struct netdev_bonding_info *bonding_info)
  6830. {
  6831. struct netdev_notifier_bonding_info info = {
  6832. .info.dev = dev,
  6833. };
  6834. memcpy(&info.bonding_info, bonding_info,
  6835. sizeof(struct netdev_bonding_info));
  6836. call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
  6837. &info.info);
  6838. }
  6839. EXPORT_SYMBOL(netdev_bonding_info_change);
  6840. /**
  6841. * netdev_get_xmit_slave - Get the xmit slave of master device
  6842. * @dev: device
  6843. * @skb: The packet
  6844. * @all_slaves: assume all the slaves are active
  6845. *
  6846. * The reference counters are not incremented so the caller must be
  6847. * careful with locks. The caller must hold RCU lock.
  6848. * %NULL is returned if no slave is found.
  6849. */
  6850. struct net_device *netdev_get_xmit_slave(struct net_device *dev,
  6851. struct sk_buff *skb,
  6852. bool all_slaves)
  6853. {
  6854. const struct net_device_ops *ops = dev->netdev_ops;
  6855. if (!ops->ndo_get_xmit_slave)
  6856. return NULL;
  6857. return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
  6858. }
  6859. EXPORT_SYMBOL(netdev_get_xmit_slave);
  6860. static void netdev_adjacent_add_links(struct net_device *dev)
  6861. {
  6862. struct netdev_adjacent *iter;
  6863. struct net *net = dev_net(dev);
  6864. list_for_each_entry(iter, &dev->adj_list.upper, list) {
  6865. if (!net_eq(net, dev_net(iter->dev)))
  6866. continue;
  6867. netdev_adjacent_sysfs_add(iter->dev, dev,
  6868. &iter->dev->adj_list.lower);
  6869. netdev_adjacent_sysfs_add(dev, iter->dev,
  6870. &dev->adj_list.upper);
  6871. }
  6872. list_for_each_entry(iter, &dev->adj_list.lower, list) {
  6873. if (!net_eq(net, dev_net(iter->dev)))
  6874. continue;
  6875. netdev_adjacent_sysfs_add(iter->dev, dev,
  6876. &iter->dev->adj_list.upper);
  6877. netdev_adjacent_sysfs_add(dev, iter->dev,
  6878. &dev->adj_list.lower);
  6879. }
  6880. }
  6881. static void netdev_adjacent_del_links(struct net_device *dev)
  6882. {
  6883. struct netdev_adjacent *iter;
  6884. struct net *net = dev_net(dev);
  6885. list_for_each_entry(iter, &dev->adj_list.upper, list) {
  6886. if (!net_eq(net, dev_net(iter->dev)))
  6887. continue;
  6888. netdev_adjacent_sysfs_del(iter->dev, dev->name,
  6889. &iter->dev->adj_list.lower);
  6890. netdev_adjacent_sysfs_del(dev, iter->dev->name,
  6891. &dev->adj_list.upper);
  6892. }
  6893. list_for_each_entry(iter, &dev->adj_list.lower, list) {
  6894. if (!net_eq(net, dev_net(iter->dev)))
  6895. continue;
  6896. netdev_adjacent_sysfs_del(iter->dev, dev->name,
  6897. &iter->dev->adj_list.upper);
  6898. netdev_adjacent_sysfs_del(dev, iter->dev->name,
  6899. &dev->adj_list.lower);
  6900. }
  6901. }
  6902. void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
  6903. {
  6904. struct netdev_adjacent *iter;
  6905. struct net *net = dev_net(dev);
  6906. list_for_each_entry(iter, &dev->adj_list.upper, list) {
  6907. if (!net_eq(net, dev_net(iter->dev)))
  6908. continue;
  6909. netdev_adjacent_sysfs_del(iter->dev, oldname,
  6910. &iter->dev->adj_list.lower);
  6911. netdev_adjacent_sysfs_add(iter->dev, dev,
  6912. &iter->dev->adj_list.lower);
  6913. }
  6914. list_for_each_entry(iter, &dev->adj_list.lower, list) {
  6915. if (!net_eq(net, dev_net(iter->dev)))
  6916. continue;
  6917. netdev_adjacent_sysfs_del(iter->dev, oldname,
  6918. &iter->dev->adj_list.upper);
  6919. netdev_adjacent_sysfs_add(iter->dev, dev,
  6920. &iter->dev->adj_list.upper);
  6921. }
  6922. }
  6923. void *netdev_lower_dev_get_private(struct net_device *dev,
  6924. struct net_device *lower_dev)
  6925. {
  6926. struct netdev_adjacent *lower;
  6927. if (!lower_dev)
  6928. return NULL;
  6929. lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
  6930. if (!lower)
  6931. return NULL;
  6932. return lower->private;
  6933. }
  6934. EXPORT_SYMBOL(netdev_lower_dev_get_private);
  6935. /**
  6936. * netdev_lower_change - Dispatch event about lower device state change
  6937. * @lower_dev: device
  6938. * @lower_state_info: state to dispatch
  6939. *
  6940. * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
  6941. * The caller must hold the RTNL lock.
  6942. */
  6943. void netdev_lower_state_changed(struct net_device *lower_dev,
  6944. void *lower_state_info)
  6945. {
  6946. struct netdev_notifier_changelowerstate_info changelowerstate_info = {
  6947. .info.dev = lower_dev,
  6948. };
  6949. ASSERT_RTNL();
  6950. changelowerstate_info.lower_state_info = lower_state_info;
  6951. call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
  6952. &changelowerstate_info.info);
  6953. }
  6954. EXPORT_SYMBOL(netdev_lower_state_changed);
  6955. static void dev_change_rx_flags(struct net_device *dev, int flags)
  6956. {
  6957. const struct net_device_ops *ops = dev->netdev_ops;
  6958. if (ops->ndo_change_rx_flags)
  6959. ops->ndo_change_rx_flags(dev, flags);
  6960. }
  6961. static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
  6962. {
  6963. unsigned int old_flags = dev->flags;
  6964. kuid_t uid;
  6965. kgid_t gid;
  6966. ASSERT_RTNL();
  6967. dev->flags |= IFF_PROMISC;
  6968. dev->promiscuity += inc;
  6969. if (dev->promiscuity == 0) {
  6970. /*
  6971. * Avoid overflow.
  6972. * If inc causes overflow, untouch promisc and return error.
  6973. */
  6974. if (inc < 0)
  6975. dev->flags &= ~IFF_PROMISC;
  6976. else {
  6977. dev->promiscuity -= inc;
  6978. pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
  6979. dev->name);
  6980. return -EOVERFLOW;
  6981. }
  6982. }
  6983. if (dev->flags != old_flags) {
  6984. pr_info("device %s %s promiscuous mode\n",
  6985. dev->name,
  6986. dev->flags & IFF_PROMISC ? "entered" : "left");
  6987. if (audit_enabled) {
  6988. current_uid_gid(&uid, &gid);
  6989. audit_log(audit_context(), GFP_ATOMIC,
  6990. AUDIT_ANOM_PROMISCUOUS,
  6991. "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
  6992. dev->name, (dev->flags & IFF_PROMISC),
  6993. (old_flags & IFF_PROMISC),
  6994. from_kuid(&init_user_ns, audit_get_loginuid(current)),
  6995. from_kuid(&init_user_ns, uid),
  6996. from_kgid(&init_user_ns, gid),
  6997. audit_get_sessionid(current));
  6998. }
  6999. dev_change_rx_flags(dev, IFF_PROMISC);
  7000. }
  7001. if (notify)
  7002. __dev_notify_flags(dev, old_flags, IFF_PROMISC);
  7003. return 0;
  7004. }
  7005. /**
  7006. * dev_set_promiscuity - update promiscuity count on a device
  7007. * @dev: device
  7008. * @inc: modifier
  7009. *
  7010. * Add or remove promiscuity from a device. While the count in the device
  7011. * remains above zero the interface remains promiscuous. Once it hits zero
  7012. * the device reverts back to normal filtering operation. A negative inc
  7013. * value is used to drop promiscuity on the device.
  7014. * Return 0 if successful or a negative errno code on error.
  7015. */
  7016. int dev_set_promiscuity(struct net_device *dev, int inc)
  7017. {
  7018. unsigned int old_flags = dev->flags;
  7019. int err;
  7020. err = __dev_set_promiscuity(dev, inc, true);
  7021. if (err < 0)
  7022. return err;
  7023. if (dev->flags != old_flags)
  7024. dev_set_rx_mode(dev);
  7025. return err;
  7026. }
  7027. EXPORT_SYMBOL(dev_set_promiscuity);
  7028. static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
  7029. {
  7030. unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
  7031. ASSERT_RTNL();
  7032. dev->flags |= IFF_ALLMULTI;
  7033. dev->allmulti += inc;
  7034. if (dev->allmulti == 0) {
  7035. /*
  7036. * Avoid overflow.
  7037. * If inc causes overflow, untouch allmulti and return error.
  7038. */
  7039. if (inc < 0)
  7040. dev->flags &= ~IFF_ALLMULTI;
  7041. else {
  7042. dev->allmulti -= inc;
  7043. pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
  7044. dev->name);
  7045. return -EOVERFLOW;
  7046. }
  7047. }
  7048. if (dev->flags ^ old_flags) {
  7049. dev_change_rx_flags(dev, IFF_ALLMULTI);
  7050. dev_set_rx_mode(dev);
  7051. if (notify)
  7052. __dev_notify_flags(dev, old_flags,
  7053. dev->gflags ^ old_gflags);
  7054. }
  7055. return 0;
  7056. }
  7057. /**
  7058. * dev_set_allmulti - update allmulti count on a device
  7059. * @dev: device
  7060. * @inc: modifier
  7061. *
  7062. * Add or remove reception of all multicast frames to a device. While the
  7063. * count in the device remains above zero the interface remains listening
  7064. * to all interfaces. Once it hits zero the device reverts back to normal
  7065. * filtering operation. A negative @inc value is used to drop the counter
  7066. * when releasing a resource needing all multicasts.
  7067. * Return 0 if successful or a negative errno code on error.
  7068. */
  7069. int dev_set_allmulti(struct net_device *dev, int inc)
  7070. {
  7071. return __dev_set_allmulti(dev, inc, true);
  7072. }
  7073. EXPORT_SYMBOL(dev_set_allmulti);
  7074. /*
  7075. * Upload unicast and multicast address lists to device and
  7076. * configure RX filtering. When the device doesn't support unicast
  7077. * filtering it is put in promiscuous mode while unicast addresses
  7078. * are present.
  7079. */
  7080. void __dev_set_rx_mode(struct net_device *dev)
  7081. {
  7082. const struct net_device_ops *ops = dev->netdev_ops;
  7083. /* dev_open will call this function so the list will stay sane. */
  7084. if (!(dev->flags&IFF_UP))
  7085. return;
  7086. if (!netif_device_present(dev))
  7087. return;
  7088. if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
  7089. /* Unicast addresses changes may only happen under the rtnl,
  7090. * therefore calling __dev_set_promiscuity here is safe.
  7091. */
  7092. if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
  7093. __dev_set_promiscuity(dev, 1, false);
  7094. dev->uc_promisc = true;
  7095. } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
  7096. __dev_set_promiscuity(dev, -1, false);
  7097. dev->uc_promisc = false;
  7098. }
  7099. }
  7100. if (ops->ndo_set_rx_mode)
  7101. ops->ndo_set_rx_mode(dev);
  7102. }
  7103. void dev_set_rx_mode(struct net_device *dev)
  7104. {
  7105. netif_addr_lock_bh(dev);
  7106. __dev_set_rx_mode(dev);
  7107. netif_addr_unlock_bh(dev);
  7108. }
  7109. /**
  7110. * dev_get_flags - get flags reported to userspace
  7111. * @dev: device
  7112. *
  7113. * Get the combination of flag bits exported through APIs to userspace.
  7114. */
  7115. unsigned int dev_get_flags(const struct net_device *dev)
  7116. {
  7117. unsigned int flags;
  7118. flags = (dev->flags & ~(IFF_PROMISC |
  7119. IFF_ALLMULTI |
  7120. IFF_RUNNING |
  7121. IFF_LOWER_UP |
  7122. IFF_DORMANT)) |
  7123. (dev->gflags & (IFF_PROMISC |
  7124. IFF_ALLMULTI));
  7125. if (netif_running(dev)) {
  7126. if (netif_oper_up(dev))
  7127. flags |= IFF_RUNNING;
  7128. if (netif_carrier_ok(dev))
  7129. flags |= IFF_LOWER_UP;
  7130. if (netif_dormant(dev))
  7131. flags |= IFF_DORMANT;
  7132. }
  7133. return flags;
  7134. }
  7135. EXPORT_SYMBOL(dev_get_flags);
  7136. int __dev_change_flags(struct net_device *dev, unsigned int flags,
  7137. struct netlink_ext_ack *extack)
  7138. {
  7139. unsigned int old_flags = dev->flags;
  7140. int ret;
  7141. ASSERT_RTNL();
  7142. /*
  7143. * Set the flags on our device.
  7144. */
  7145. dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
  7146. IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
  7147. IFF_AUTOMEDIA)) |
  7148. (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
  7149. IFF_ALLMULTI));
  7150. /*
  7151. * Load in the correct multicast list now the flags have changed.
  7152. */
  7153. if ((old_flags ^ flags) & IFF_MULTICAST)
  7154. dev_change_rx_flags(dev, IFF_MULTICAST);
  7155. dev_set_rx_mode(dev);
  7156. /*
  7157. * Have we downed the interface. We handle IFF_UP ourselves
  7158. * according to user attempts to set it, rather than blindly
  7159. * setting it.
  7160. */
  7161. ret = 0;
  7162. if ((old_flags ^ flags) & IFF_UP) {
  7163. if (old_flags & IFF_UP)
  7164. __dev_close(dev);
  7165. else
  7166. ret = __dev_open(dev, extack);
  7167. }
  7168. if ((flags ^ dev->gflags) & IFF_PROMISC) {
  7169. int inc = (flags & IFF_PROMISC) ? 1 : -1;
  7170. unsigned int old_flags = dev->flags;
  7171. dev->gflags ^= IFF_PROMISC;
  7172. if (__dev_set_promiscuity(dev, inc, false) >= 0)
  7173. if (dev->flags != old_flags)
  7174. dev_set_rx_mode(dev);
  7175. }
  7176. /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
  7177. * is important. Some (broken) drivers set IFF_PROMISC, when
  7178. * IFF_ALLMULTI is requested not asking us and not reporting.
  7179. */
  7180. if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
  7181. int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
  7182. dev->gflags ^= IFF_ALLMULTI;
  7183. __dev_set_allmulti(dev, inc, false);
  7184. }
  7185. return ret;
  7186. }
  7187. void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
  7188. unsigned int gchanges)
  7189. {
  7190. unsigned int changes = dev->flags ^ old_flags;
  7191. if (gchanges)
  7192. rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
  7193. if (changes & IFF_UP) {
  7194. if (dev->flags & IFF_UP)
  7195. call_netdevice_notifiers(NETDEV_UP, dev);
  7196. else
  7197. call_netdevice_notifiers(NETDEV_DOWN, dev);
  7198. }
  7199. if (dev->flags & IFF_UP &&
  7200. (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
  7201. struct netdev_notifier_change_info change_info = {
  7202. .info = {
  7203. .dev = dev,
  7204. },
  7205. .flags_changed = changes,
  7206. };
  7207. call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
  7208. }
  7209. }
  7210. /**
  7211. * dev_change_flags - change device settings
  7212. * @dev: device
  7213. * @flags: device state flags
  7214. * @extack: netlink extended ack
  7215. *
  7216. * Change settings on device based state flags. The flags are
  7217. * in the userspace exported format.
  7218. */
  7219. int dev_change_flags(struct net_device *dev, unsigned int flags,
  7220. struct netlink_ext_ack *extack)
  7221. {
  7222. int ret;
  7223. unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
  7224. ret = __dev_change_flags(dev, flags, extack);
  7225. if (ret < 0)
  7226. return ret;
  7227. changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
  7228. __dev_notify_flags(dev, old_flags, changes);
  7229. return ret;
  7230. }
  7231. EXPORT_SYMBOL(dev_change_flags);
  7232. int __dev_set_mtu(struct net_device *dev, int new_mtu)
  7233. {
  7234. const struct net_device_ops *ops = dev->netdev_ops;
  7235. if (ops->ndo_change_mtu)
  7236. return ops->ndo_change_mtu(dev, new_mtu);
  7237. /* Pairs with all the lockless reads of dev->mtu in the stack */
  7238. WRITE_ONCE(dev->mtu, new_mtu);
  7239. return 0;
  7240. }
  7241. EXPORT_SYMBOL(__dev_set_mtu);
  7242. int dev_validate_mtu(struct net_device *dev, int new_mtu,
  7243. struct netlink_ext_ack *extack)
  7244. {
  7245. /* MTU must be positive, and in range */
  7246. if (new_mtu < 0 || new_mtu < dev->min_mtu) {
  7247. NL_SET_ERR_MSG(extack, "mtu less than device minimum");
  7248. return -EINVAL;
  7249. }
  7250. if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
  7251. NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
  7252. return -EINVAL;
  7253. }
  7254. return 0;
  7255. }
  7256. /**
  7257. * dev_set_mtu_ext - Change maximum transfer unit
  7258. * @dev: device
  7259. * @new_mtu: new transfer unit
  7260. * @extack: netlink extended ack
  7261. *
  7262. * Change the maximum transfer size of the network device.
  7263. */
  7264. int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
  7265. struct netlink_ext_ack *extack)
  7266. {
  7267. int err, orig_mtu;
  7268. if (new_mtu == dev->mtu)
  7269. return 0;
  7270. err = dev_validate_mtu(dev, new_mtu, extack);
  7271. if (err)
  7272. return err;
  7273. if (!netif_device_present(dev))
  7274. return -ENODEV;
  7275. err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
  7276. err = notifier_to_errno(err);
  7277. if (err)
  7278. return err;
  7279. orig_mtu = dev->mtu;
  7280. err = __dev_set_mtu(dev, new_mtu);
  7281. if (!err) {
  7282. err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
  7283. orig_mtu);
  7284. err = notifier_to_errno(err);
  7285. if (err) {
  7286. /* setting mtu back and notifying everyone again,
  7287. * so that they have a chance to revert changes.
  7288. */
  7289. __dev_set_mtu(dev, orig_mtu);
  7290. call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
  7291. new_mtu);
  7292. }
  7293. }
  7294. return err;
  7295. }
  7296. int dev_set_mtu(struct net_device *dev, int new_mtu)
  7297. {
  7298. struct netlink_ext_ack extack;
  7299. int err;
  7300. memset(&extack, 0, sizeof(extack));
  7301. err = dev_set_mtu_ext(dev, new_mtu, &extack);
  7302. if (err && extack._msg)
  7303. net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
  7304. return err;
  7305. }
  7306. EXPORT_SYMBOL(dev_set_mtu);
  7307. /**
  7308. * dev_change_tx_queue_len - Change TX queue length of a netdevice
  7309. * @dev: device
  7310. * @new_len: new tx queue length
  7311. */
  7312. int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
  7313. {
  7314. unsigned int orig_len = dev->tx_queue_len;
  7315. int res;
  7316. if (new_len != (unsigned int)new_len)
  7317. return -ERANGE;
  7318. if (new_len != orig_len) {
  7319. dev->tx_queue_len = new_len;
  7320. res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
  7321. res = notifier_to_errno(res);
  7322. if (res)
  7323. goto err_rollback;
  7324. res = dev_qdisc_change_tx_queue_len(dev);
  7325. if (res)
  7326. goto err_rollback;
  7327. }
  7328. return 0;
  7329. err_rollback:
  7330. netdev_err(dev, "refused to change device tx_queue_len\n");
  7331. dev->tx_queue_len = orig_len;
  7332. return res;
  7333. }
  7334. /**
  7335. * dev_set_group - Change group this device belongs to
  7336. * @dev: device
  7337. * @new_group: group this device should belong to
  7338. */
  7339. void dev_set_group(struct net_device *dev, int new_group)
  7340. {
  7341. dev->group = new_group;
  7342. }
  7343. EXPORT_SYMBOL(dev_set_group);
  7344. /**
  7345. * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
  7346. * @dev: device
  7347. * @addr: new address
  7348. * @extack: netlink extended ack
  7349. */
  7350. int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
  7351. struct netlink_ext_ack *extack)
  7352. {
  7353. struct netdev_notifier_pre_changeaddr_info info = {
  7354. .info.dev = dev,
  7355. .info.extack = extack,
  7356. .dev_addr = addr,
  7357. };
  7358. int rc;
  7359. rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
  7360. return notifier_to_errno(rc);
  7361. }
  7362. EXPORT_SYMBOL(dev_pre_changeaddr_notify);
  7363. /**
  7364. * dev_set_mac_address - Change Media Access Control Address
  7365. * @dev: device
  7366. * @sa: new address
  7367. * @extack: netlink extended ack
  7368. *
  7369. * Change the hardware (MAC) address of the device
  7370. */
  7371. int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
  7372. struct netlink_ext_ack *extack)
  7373. {
  7374. const struct net_device_ops *ops = dev->netdev_ops;
  7375. int err;
  7376. if (!ops->ndo_set_mac_address)
  7377. return -EOPNOTSUPP;
  7378. if (sa->sa_family != dev->type)
  7379. return -EINVAL;
  7380. if (!netif_device_present(dev))
  7381. return -ENODEV;
  7382. err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
  7383. if (err)
  7384. return err;
  7385. err = ops->ndo_set_mac_address(dev, sa);
  7386. if (err)
  7387. return err;
  7388. dev->addr_assign_type = NET_ADDR_SET;
  7389. call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
  7390. add_device_randomness(dev->dev_addr, dev->addr_len);
  7391. return 0;
  7392. }
  7393. EXPORT_SYMBOL(dev_set_mac_address);
  7394. static DECLARE_RWSEM(dev_addr_sem);
  7395. int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
  7396. struct netlink_ext_ack *extack)
  7397. {
  7398. int ret;
  7399. down_write(&dev_addr_sem);
  7400. ret = dev_set_mac_address(dev, sa, extack);
  7401. up_write(&dev_addr_sem);
  7402. return ret;
  7403. }
  7404. EXPORT_SYMBOL(dev_set_mac_address_user);
  7405. int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
  7406. {
  7407. size_t size = sizeof(sa->sa_data);
  7408. struct net_device *dev;
  7409. int ret = 0;
  7410. down_read(&dev_addr_sem);
  7411. rcu_read_lock();
  7412. dev = dev_get_by_name_rcu(net, dev_name);
  7413. if (!dev) {
  7414. ret = -ENODEV;
  7415. goto unlock;
  7416. }
  7417. if (!dev->addr_len)
  7418. memset(sa->sa_data, 0, size);
  7419. else
  7420. memcpy(sa->sa_data, dev->dev_addr,
  7421. min_t(size_t, size, dev->addr_len));
  7422. sa->sa_family = dev->type;
  7423. unlock:
  7424. rcu_read_unlock();
  7425. up_read(&dev_addr_sem);
  7426. return ret;
  7427. }
  7428. EXPORT_SYMBOL(dev_get_mac_address);
  7429. /**
  7430. * dev_change_carrier - Change device carrier
  7431. * @dev: device
  7432. * @new_carrier: new value
  7433. *
  7434. * Change device carrier
  7435. */
  7436. int dev_change_carrier(struct net_device *dev, bool new_carrier)
  7437. {
  7438. const struct net_device_ops *ops = dev->netdev_ops;
  7439. if (!ops->ndo_change_carrier)
  7440. return -EOPNOTSUPP;
  7441. if (!netif_device_present(dev))
  7442. return -ENODEV;
  7443. return ops->ndo_change_carrier(dev, new_carrier);
  7444. }
  7445. EXPORT_SYMBOL(dev_change_carrier);
  7446. /**
  7447. * dev_get_phys_port_id - Get device physical port ID
  7448. * @dev: device
  7449. * @ppid: port ID
  7450. *
  7451. * Get device physical port ID
  7452. */
  7453. int dev_get_phys_port_id(struct net_device *dev,
  7454. struct netdev_phys_item_id *ppid)
  7455. {
  7456. const struct net_device_ops *ops = dev->netdev_ops;
  7457. if (!ops->ndo_get_phys_port_id)
  7458. return -EOPNOTSUPP;
  7459. return ops->ndo_get_phys_port_id(dev, ppid);
  7460. }
  7461. EXPORT_SYMBOL(dev_get_phys_port_id);
  7462. /**
  7463. * dev_get_phys_port_name - Get device physical port name
  7464. * @dev: device
  7465. * @name: port name
  7466. * @len: limit of bytes to copy to name
  7467. *
  7468. * Get device physical port name
  7469. */
  7470. int dev_get_phys_port_name(struct net_device *dev,
  7471. char *name, size_t len)
  7472. {
  7473. const struct net_device_ops *ops = dev->netdev_ops;
  7474. int err;
  7475. if (ops->ndo_get_phys_port_name) {
  7476. err = ops->ndo_get_phys_port_name(dev, name, len);
  7477. if (err != -EOPNOTSUPP)
  7478. return err;
  7479. }
  7480. return devlink_compat_phys_port_name_get(dev, name, len);
  7481. }
  7482. EXPORT_SYMBOL(dev_get_phys_port_name);
  7483. /**
  7484. * dev_get_port_parent_id - Get the device's port parent identifier
  7485. * @dev: network device
  7486. * @ppid: pointer to a storage for the port's parent identifier
  7487. * @recurse: allow/disallow recursion to lower devices
  7488. *
  7489. * Get the devices's port parent identifier
  7490. */
  7491. int dev_get_port_parent_id(struct net_device *dev,
  7492. struct netdev_phys_item_id *ppid,
  7493. bool recurse)
  7494. {
  7495. const struct net_device_ops *ops = dev->netdev_ops;
  7496. struct netdev_phys_item_id first = { };
  7497. struct net_device *lower_dev;
  7498. struct list_head *iter;
  7499. int err;
  7500. if (ops->ndo_get_port_parent_id) {
  7501. err = ops->ndo_get_port_parent_id(dev, ppid);
  7502. if (err != -EOPNOTSUPP)
  7503. return err;
  7504. }
  7505. err = devlink_compat_switch_id_get(dev, ppid);
  7506. if (!err || err != -EOPNOTSUPP)
  7507. return err;
  7508. if (!recurse)
  7509. return -EOPNOTSUPP;
  7510. netdev_for_each_lower_dev(dev, lower_dev, iter) {
  7511. err = dev_get_port_parent_id(lower_dev, ppid, recurse);
  7512. if (err)
  7513. break;
  7514. if (!first.id_len)
  7515. first = *ppid;
  7516. else if (memcmp(&first, ppid, sizeof(*ppid)))
  7517. return -EOPNOTSUPP;
  7518. }
  7519. return err;
  7520. }
  7521. EXPORT_SYMBOL(dev_get_port_parent_id);
  7522. /**
  7523. * netdev_port_same_parent_id - Indicate if two network devices have
  7524. * the same port parent identifier
  7525. * @a: first network device
  7526. * @b: second network device
  7527. */
  7528. bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
  7529. {
  7530. struct netdev_phys_item_id a_id = { };
  7531. struct netdev_phys_item_id b_id = { };
  7532. if (dev_get_port_parent_id(a, &a_id, true) ||
  7533. dev_get_port_parent_id(b, &b_id, true))
  7534. return false;
  7535. return netdev_phys_item_id_same(&a_id, &b_id);
  7536. }
  7537. EXPORT_SYMBOL(netdev_port_same_parent_id);
  7538. /**
  7539. * dev_change_proto_down - update protocol port state information
  7540. * @dev: device
  7541. * @proto_down: new value
  7542. *
  7543. * This info can be used by switch drivers to set the phys state of the
  7544. * port.
  7545. */
  7546. int dev_change_proto_down(struct net_device *dev, bool proto_down)
  7547. {
  7548. const struct net_device_ops *ops = dev->netdev_ops;
  7549. if (!ops->ndo_change_proto_down)
  7550. return -EOPNOTSUPP;
  7551. if (!netif_device_present(dev))
  7552. return -ENODEV;
  7553. return ops->ndo_change_proto_down(dev, proto_down);
  7554. }
  7555. EXPORT_SYMBOL(dev_change_proto_down);
  7556. /**
  7557. * dev_change_proto_down_generic - generic implementation for
  7558. * ndo_change_proto_down that sets carrier according to
  7559. * proto_down.
  7560. *
  7561. * @dev: device
  7562. * @proto_down: new value
  7563. */
  7564. int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
  7565. {
  7566. if (proto_down)
  7567. netif_carrier_off(dev);
  7568. else
  7569. netif_carrier_on(dev);
  7570. dev->proto_down = proto_down;
  7571. return 0;
  7572. }
  7573. EXPORT_SYMBOL(dev_change_proto_down_generic);
  7574. /**
  7575. * dev_change_proto_down_reason - proto down reason
  7576. *
  7577. * @dev: device
  7578. * @mask: proto down mask
  7579. * @value: proto down value
  7580. */
  7581. void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
  7582. u32 value)
  7583. {
  7584. int b;
  7585. if (!mask) {
  7586. dev->proto_down_reason = value;
  7587. } else {
  7588. for_each_set_bit(b, &mask, 32) {
  7589. if (value & (1 << b))
  7590. dev->proto_down_reason |= BIT(b);
  7591. else
  7592. dev->proto_down_reason &= ~BIT(b);
  7593. }
  7594. }
  7595. }
  7596. EXPORT_SYMBOL(dev_change_proto_down_reason);
  7597. struct bpf_xdp_link {
  7598. struct bpf_link link;
  7599. struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
  7600. int flags;
  7601. };
  7602. static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
  7603. {
  7604. if (flags & XDP_FLAGS_HW_MODE)
  7605. return XDP_MODE_HW;
  7606. if (flags & XDP_FLAGS_DRV_MODE)
  7607. return XDP_MODE_DRV;
  7608. if (flags & XDP_FLAGS_SKB_MODE)
  7609. return XDP_MODE_SKB;
  7610. return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
  7611. }
  7612. static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
  7613. {
  7614. switch (mode) {
  7615. case XDP_MODE_SKB:
  7616. return generic_xdp_install;
  7617. case XDP_MODE_DRV:
  7618. case XDP_MODE_HW:
  7619. return dev->netdev_ops->ndo_bpf;
  7620. default:
  7621. return NULL;
  7622. };
  7623. }
  7624. static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
  7625. enum bpf_xdp_mode mode)
  7626. {
  7627. return dev->xdp_state[mode].link;
  7628. }
  7629. static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
  7630. enum bpf_xdp_mode mode)
  7631. {
  7632. struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
  7633. if (link)
  7634. return link->link.prog;
  7635. return dev->xdp_state[mode].prog;
  7636. }
  7637. static u8 dev_xdp_prog_count(struct net_device *dev)
  7638. {
  7639. u8 count = 0;
  7640. int i;
  7641. for (i = 0; i < __MAX_XDP_MODE; i++)
  7642. if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
  7643. count++;
  7644. return count;
  7645. }
  7646. u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
  7647. {
  7648. struct bpf_prog *prog = dev_xdp_prog(dev, mode);
  7649. return prog ? prog->aux->id : 0;
  7650. }
  7651. static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
  7652. struct bpf_xdp_link *link)
  7653. {
  7654. dev->xdp_state[mode].link = link;
  7655. dev->xdp_state[mode].prog = NULL;
  7656. }
  7657. static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
  7658. struct bpf_prog *prog)
  7659. {
  7660. dev->xdp_state[mode].link = NULL;
  7661. dev->xdp_state[mode].prog = prog;
  7662. }
  7663. static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
  7664. bpf_op_t bpf_op, struct netlink_ext_ack *extack,
  7665. u32 flags, struct bpf_prog *prog)
  7666. {
  7667. struct netdev_bpf xdp;
  7668. int err;
  7669. memset(&xdp, 0, sizeof(xdp));
  7670. xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
  7671. xdp.extack = extack;
  7672. xdp.flags = flags;
  7673. xdp.prog = prog;
  7674. /* Drivers assume refcnt is already incremented (i.e, prog pointer is
  7675. * "moved" into driver), so they don't increment it on their own, but
  7676. * they do decrement refcnt when program is detached or replaced.
  7677. * Given net_device also owns link/prog, we need to bump refcnt here
  7678. * to prevent drivers from underflowing it.
  7679. */
  7680. if (prog)
  7681. bpf_prog_inc(prog);
  7682. err = bpf_op(dev, &xdp);
  7683. if (err) {
  7684. if (prog)
  7685. bpf_prog_put(prog);
  7686. return err;
  7687. }
  7688. if (mode != XDP_MODE_HW)
  7689. bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
  7690. return 0;
  7691. }
  7692. static void dev_xdp_uninstall(struct net_device *dev)
  7693. {
  7694. struct bpf_xdp_link *link;
  7695. struct bpf_prog *prog;
  7696. enum bpf_xdp_mode mode;
  7697. bpf_op_t bpf_op;
  7698. ASSERT_RTNL();
  7699. for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
  7700. prog = dev_xdp_prog(dev, mode);
  7701. if (!prog)
  7702. continue;
  7703. bpf_op = dev_xdp_bpf_op(dev, mode);
  7704. if (!bpf_op)
  7705. continue;
  7706. WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
  7707. /* auto-detach link from net device */
  7708. link = dev_xdp_link(dev, mode);
  7709. if (link)
  7710. link->dev = NULL;
  7711. else
  7712. bpf_prog_put(prog);
  7713. dev_xdp_set_link(dev, mode, NULL);
  7714. }
  7715. }
  7716. static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
  7717. struct bpf_xdp_link *link, struct bpf_prog *new_prog,
  7718. struct bpf_prog *old_prog, u32 flags)
  7719. {
  7720. unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
  7721. struct bpf_prog *cur_prog;
  7722. enum bpf_xdp_mode mode;
  7723. bpf_op_t bpf_op;
  7724. int err;
  7725. ASSERT_RTNL();
  7726. /* either link or prog attachment, never both */
  7727. if (link && (new_prog || old_prog))
  7728. return -EINVAL;
  7729. /* link supports only XDP mode flags */
  7730. if (link && (flags & ~XDP_FLAGS_MODES)) {
  7731. NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
  7732. return -EINVAL;
  7733. }
  7734. /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
  7735. if (num_modes > 1) {
  7736. NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
  7737. return -EINVAL;
  7738. }
  7739. /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
  7740. if (!num_modes && dev_xdp_prog_count(dev) > 1) {
  7741. NL_SET_ERR_MSG(extack,
  7742. "More than one program loaded, unset mode is ambiguous");
  7743. return -EINVAL;
  7744. }
  7745. /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
  7746. if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
  7747. NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
  7748. return -EINVAL;
  7749. }
  7750. mode = dev_xdp_mode(dev, flags);
  7751. /* can't replace attached link */
  7752. if (dev_xdp_link(dev, mode)) {
  7753. NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
  7754. return -EBUSY;
  7755. }
  7756. cur_prog = dev_xdp_prog(dev, mode);
  7757. /* can't replace attached prog with link */
  7758. if (link && cur_prog) {
  7759. NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
  7760. return -EBUSY;
  7761. }
  7762. if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
  7763. NL_SET_ERR_MSG(extack, "Active program does not match expected");
  7764. return -EEXIST;
  7765. }
  7766. /* put effective new program into new_prog */
  7767. if (link)
  7768. new_prog = link->link.prog;
  7769. if (new_prog) {
  7770. bool offload = mode == XDP_MODE_HW;
  7771. enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
  7772. ? XDP_MODE_DRV : XDP_MODE_SKB;
  7773. if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
  7774. NL_SET_ERR_MSG(extack, "XDP program already attached");
  7775. return -EBUSY;
  7776. }
  7777. if (!offload && dev_xdp_prog(dev, other_mode)) {
  7778. NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
  7779. return -EEXIST;
  7780. }
  7781. if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) {
  7782. NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported");
  7783. return -EINVAL;
  7784. }
  7785. if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
  7786. NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
  7787. return -EINVAL;
  7788. }
  7789. if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
  7790. NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
  7791. return -EINVAL;
  7792. }
  7793. }
  7794. /* don't call drivers if the effective program didn't change */
  7795. if (new_prog != cur_prog) {
  7796. bpf_op = dev_xdp_bpf_op(dev, mode);
  7797. if (!bpf_op) {
  7798. NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
  7799. return -EOPNOTSUPP;
  7800. }
  7801. err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
  7802. if (err)
  7803. return err;
  7804. }
  7805. if (link)
  7806. dev_xdp_set_link(dev, mode, link);
  7807. else
  7808. dev_xdp_set_prog(dev, mode, new_prog);
  7809. if (cur_prog)
  7810. bpf_prog_put(cur_prog);
  7811. return 0;
  7812. }
  7813. static int dev_xdp_attach_link(struct net_device *dev,
  7814. struct netlink_ext_ack *extack,
  7815. struct bpf_xdp_link *link)
  7816. {
  7817. return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
  7818. }
  7819. static int dev_xdp_detach_link(struct net_device *dev,
  7820. struct netlink_ext_ack *extack,
  7821. struct bpf_xdp_link *link)
  7822. {
  7823. enum bpf_xdp_mode mode;
  7824. bpf_op_t bpf_op;
  7825. ASSERT_RTNL();
  7826. mode = dev_xdp_mode(dev, link->flags);
  7827. if (dev_xdp_link(dev, mode) != link)
  7828. return -EINVAL;
  7829. bpf_op = dev_xdp_bpf_op(dev, mode);
  7830. WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
  7831. dev_xdp_set_link(dev, mode, NULL);
  7832. return 0;
  7833. }
  7834. static void bpf_xdp_link_release(struct bpf_link *link)
  7835. {
  7836. struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
  7837. rtnl_lock();
  7838. /* if racing with net_device's tear down, xdp_link->dev might be
  7839. * already NULL, in which case link was already auto-detached
  7840. */
  7841. if (xdp_link->dev) {
  7842. WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
  7843. xdp_link->dev = NULL;
  7844. }
  7845. rtnl_unlock();
  7846. }
  7847. static int bpf_xdp_link_detach(struct bpf_link *link)
  7848. {
  7849. bpf_xdp_link_release(link);
  7850. return 0;
  7851. }
  7852. static void bpf_xdp_link_dealloc(struct bpf_link *link)
  7853. {
  7854. struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
  7855. kfree(xdp_link);
  7856. }
  7857. static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
  7858. struct seq_file *seq)
  7859. {
  7860. struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
  7861. u32 ifindex = 0;
  7862. rtnl_lock();
  7863. if (xdp_link->dev)
  7864. ifindex = xdp_link->dev->ifindex;
  7865. rtnl_unlock();
  7866. seq_printf(seq, "ifindex:\t%u\n", ifindex);
  7867. }
  7868. static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
  7869. struct bpf_link_info *info)
  7870. {
  7871. struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
  7872. u32 ifindex = 0;
  7873. rtnl_lock();
  7874. if (xdp_link->dev)
  7875. ifindex = xdp_link->dev->ifindex;
  7876. rtnl_unlock();
  7877. info->xdp.ifindex = ifindex;
  7878. return 0;
  7879. }
  7880. static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
  7881. struct bpf_prog *old_prog)
  7882. {
  7883. struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
  7884. enum bpf_xdp_mode mode;
  7885. bpf_op_t bpf_op;
  7886. int err = 0;
  7887. rtnl_lock();
  7888. /* link might have been auto-released already, so fail */
  7889. if (!xdp_link->dev) {
  7890. err = -ENOLINK;
  7891. goto out_unlock;
  7892. }
  7893. if (old_prog && link->prog != old_prog) {
  7894. err = -EPERM;
  7895. goto out_unlock;
  7896. }
  7897. old_prog = link->prog;
  7898. if (old_prog->type != new_prog->type ||
  7899. old_prog->expected_attach_type != new_prog->expected_attach_type) {
  7900. err = -EINVAL;
  7901. goto out_unlock;
  7902. }
  7903. if (old_prog == new_prog) {
  7904. /* no-op, don't disturb drivers */
  7905. bpf_prog_put(new_prog);
  7906. goto out_unlock;
  7907. }
  7908. mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
  7909. bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
  7910. err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
  7911. xdp_link->flags, new_prog);
  7912. if (err)
  7913. goto out_unlock;
  7914. old_prog = xchg(&link->prog, new_prog);
  7915. bpf_prog_put(old_prog);
  7916. out_unlock:
  7917. rtnl_unlock();
  7918. return err;
  7919. }
  7920. static const struct bpf_link_ops bpf_xdp_link_lops = {
  7921. .release = bpf_xdp_link_release,
  7922. .dealloc = bpf_xdp_link_dealloc,
  7923. .detach = bpf_xdp_link_detach,
  7924. .show_fdinfo = bpf_xdp_link_show_fdinfo,
  7925. .fill_link_info = bpf_xdp_link_fill_link_info,
  7926. .update_prog = bpf_xdp_link_update,
  7927. };
  7928. int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
  7929. {
  7930. struct net *net = current->nsproxy->net_ns;
  7931. struct bpf_link_primer link_primer;
  7932. struct bpf_xdp_link *link;
  7933. struct net_device *dev;
  7934. int err, fd;
  7935. rtnl_lock();
  7936. dev = dev_get_by_index(net, attr->link_create.target_ifindex);
  7937. if (!dev) {
  7938. rtnl_unlock();
  7939. return -EINVAL;
  7940. }
  7941. link = kzalloc(sizeof(*link), GFP_USER);
  7942. if (!link) {
  7943. err = -ENOMEM;
  7944. goto unlock;
  7945. }
  7946. bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
  7947. link->dev = dev;
  7948. link->flags = attr->link_create.flags;
  7949. err = bpf_link_prime(&link->link, &link_primer);
  7950. if (err) {
  7951. kfree(link);
  7952. goto unlock;
  7953. }
  7954. err = dev_xdp_attach_link(dev, NULL, link);
  7955. rtnl_unlock();
  7956. if (err) {
  7957. link->dev = NULL;
  7958. bpf_link_cleanup(&link_primer);
  7959. goto out_put_dev;
  7960. }
  7961. fd = bpf_link_settle(&link_primer);
  7962. /* link itself doesn't hold dev's refcnt to not complicate shutdown */
  7963. dev_put(dev);
  7964. return fd;
  7965. unlock:
  7966. rtnl_unlock();
  7967. out_put_dev:
  7968. dev_put(dev);
  7969. return err;
  7970. }
  7971. /**
  7972. * dev_change_xdp_fd - set or clear a bpf program for a device rx path
  7973. * @dev: device
  7974. * @extack: netlink extended ack
  7975. * @fd: new program fd or negative value to clear
  7976. * @expected_fd: old program fd that userspace expects to replace or clear
  7977. * @flags: xdp-related flags
  7978. *
  7979. * Set or clear a bpf program for a device
  7980. */
  7981. int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
  7982. int fd, int expected_fd, u32 flags)
  7983. {
  7984. enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
  7985. struct bpf_prog *new_prog = NULL, *old_prog = NULL;
  7986. int err;
  7987. ASSERT_RTNL();
  7988. if (fd >= 0) {
  7989. new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
  7990. mode != XDP_MODE_SKB);
  7991. if (IS_ERR(new_prog))
  7992. return PTR_ERR(new_prog);
  7993. }
  7994. if (expected_fd >= 0) {
  7995. old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
  7996. mode != XDP_MODE_SKB);
  7997. if (IS_ERR(old_prog)) {
  7998. err = PTR_ERR(old_prog);
  7999. old_prog = NULL;
  8000. goto err_out;
  8001. }
  8002. }
  8003. err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
  8004. err_out:
  8005. if (err && new_prog)
  8006. bpf_prog_put(new_prog);
  8007. if (old_prog)
  8008. bpf_prog_put(old_prog);
  8009. return err;
  8010. }
  8011. /**
  8012. * dev_new_index - allocate an ifindex
  8013. * @net: the applicable net namespace
  8014. *
  8015. * Returns a suitable unique value for a new device interface
  8016. * number. The caller must hold the rtnl semaphore or the
  8017. * dev_base_lock to be sure it remains unique.
  8018. */
  8019. static int dev_new_index(struct net *net)
  8020. {
  8021. int ifindex = net->ifindex;
  8022. for (;;) {
  8023. if (++ifindex <= 0)
  8024. ifindex = 1;
  8025. if (!__dev_get_by_index(net, ifindex))
  8026. return net->ifindex = ifindex;
  8027. }
  8028. }
  8029. /* Delayed registration/unregisteration */
  8030. static LIST_HEAD(net_todo_list);
  8031. DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
  8032. static void net_set_todo(struct net_device *dev)
  8033. {
  8034. list_add_tail(&dev->todo_list, &net_todo_list);
  8035. dev_net(dev)->dev_unreg_count++;
  8036. }
  8037. static void rollback_registered_many(struct list_head *head)
  8038. {
  8039. struct net_device *dev, *tmp;
  8040. LIST_HEAD(close_head);
  8041. BUG_ON(dev_boot_phase);
  8042. ASSERT_RTNL();
  8043. list_for_each_entry_safe(dev, tmp, head, unreg_list) {
  8044. /* Some devices call without registering
  8045. * for initialization unwind. Remove those
  8046. * devices and proceed with the remaining.
  8047. */
  8048. if (dev->reg_state == NETREG_UNINITIALIZED) {
  8049. pr_debug("unregister_netdevice: device %s/%p never was registered\n",
  8050. dev->name, dev);
  8051. WARN_ON(1);
  8052. list_del(&dev->unreg_list);
  8053. continue;
  8054. }
  8055. dev->dismantle = true;
  8056. BUG_ON(dev->reg_state != NETREG_REGISTERED);
  8057. }
  8058. /* If device is running, close it first. */
  8059. list_for_each_entry(dev, head, unreg_list)
  8060. list_add_tail(&dev->close_list, &close_head);
  8061. dev_close_many(&close_head, true);
  8062. list_for_each_entry(dev, head, unreg_list) {
  8063. /* And unlink it from device chain. */
  8064. unlist_netdevice(dev);
  8065. dev->reg_state = NETREG_UNREGISTERING;
  8066. }
  8067. flush_all_backlogs();
  8068. synchronize_net();
  8069. list_for_each_entry(dev, head, unreg_list) {
  8070. struct sk_buff *skb = NULL;
  8071. /* Shutdown queueing discipline. */
  8072. dev_shutdown(dev);
  8073. dev_xdp_uninstall(dev);
  8074. /* Notify protocols, that we are about to destroy
  8075. * this device. They should clean all the things.
  8076. */
  8077. call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
  8078. if (!dev->rtnl_link_ops ||
  8079. dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
  8080. skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
  8081. GFP_KERNEL, NULL, 0);
  8082. /*
  8083. * Flush the unicast and multicast chains
  8084. */
  8085. dev_uc_flush(dev);
  8086. dev_mc_flush(dev);
  8087. netdev_name_node_alt_flush(dev);
  8088. netdev_name_node_free(dev->name_node);
  8089. if (dev->netdev_ops->ndo_uninit)
  8090. dev->netdev_ops->ndo_uninit(dev);
  8091. if (skb)
  8092. rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
  8093. /* Notifier chain MUST detach us all upper devices. */
  8094. WARN_ON(netdev_has_any_upper_dev(dev));
  8095. WARN_ON(netdev_has_any_lower_dev(dev));
  8096. /* Remove entries from kobject tree */
  8097. netdev_unregister_kobject(dev);
  8098. #ifdef CONFIG_XPS
  8099. /* Remove XPS queueing entries */
  8100. netif_reset_xps_queues_gt(dev, 0);
  8101. #endif
  8102. }
  8103. synchronize_net();
  8104. list_for_each_entry(dev, head, unreg_list)
  8105. dev_put(dev);
  8106. }
  8107. static void rollback_registered(struct net_device *dev)
  8108. {
  8109. LIST_HEAD(single);
  8110. list_add(&dev->unreg_list, &single);
  8111. rollback_registered_many(&single);
  8112. list_del(&single);
  8113. }
  8114. static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
  8115. struct net_device *upper, netdev_features_t features)
  8116. {
  8117. netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
  8118. netdev_features_t feature;
  8119. int feature_bit;
  8120. for_each_netdev_feature(upper_disables, feature_bit) {
  8121. feature = __NETIF_F_BIT(feature_bit);
  8122. if (!(upper->wanted_features & feature)
  8123. && (features & feature)) {
  8124. netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
  8125. &feature, upper->name);
  8126. features &= ~feature;
  8127. }
  8128. }
  8129. return features;
  8130. }
  8131. static void netdev_sync_lower_features(struct net_device *upper,
  8132. struct net_device *lower, netdev_features_t features)
  8133. {
  8134. netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
  8135. netdev_features_t feature;
  8136. int feature_bit;
  8137. for_each_netdev_feature(upper_disables, feature_bit) {
  8138. feature = __NETIF_F_BIT(feature_bit);
  8139. if (!(features & feature) && (lower->features & feature)) {
  8140. netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
  8141. &feature, lower->name);
  8142. lower->wanted_features &= ~feature;
  8143. __netdev_update_features(lower);
  8144. if (unlikely(lower->features & feature))
  8145. netdev_WARN(upper, "failed to disable %pNF on %s!\n",
  8146. &feature, lower->name);
  8147. else
  8148. netdev_features_change(lower);
  8149. }
  8150. }
  8151. }
  8152. static netdev_features_t netdev_fix_features(struct net_device *dev,
  8153. netdev_features_t features)
  8154. {
  8155. /* Fix illegal checksum combinations */
  8156. if ((features & NETIF_F_HW_CSUM) &&
  8157. (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
  8158. netdev_warn(dev, "mixed HW and IP checksum settings.\n");
  8159. features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
  8160. }
  8161. /* TSO requires that SG is present as well. */
  8162. if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
  8163. netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
  8164. features &= ~NETIF_F_ALL_TSO;
  8165. }
  8166. if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
  8167. !(features & NETIF_F_IP_CSUM)) {
  8168. netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
  8169. features &= ~NETIF_F_TSO;
  8170. features &= ~NETIF_F_TSO_ECN;
  8171. }
  8172. if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
  8173. !(features & NETIF_F_IPV6_CSUM)) {
  8174. netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
  8175. features &= ~NETIF_F_TSO6;
  8176. }
  8177. /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
  8178. if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
  8179. features &= ~NETIF_F_TSO_MANGLEID;
  8180. /* TSO ECN requires that TSO is present as well. */
  8181. if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
  8182. features &= ~NETIF_F_TSO_ECN;
  8183. /* Software GSO depends on SG. */
  8184. if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
  8185. netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
  8186. features &= ~NETIF_F_GSO;
  8187. }
  8188. /* GSO partial features require GSO partial be set */
  8189. if ((features & dev->gso_partial_features) &&
  8190. !(features & NETIF_F_GSO_PARTIAL)) {
  8191. netdev_dbg(dev,
  8192. "Dropping partially supported GSO features since no GSO partial.\n");
  8193. features &= ~dev->gso_partial_features;
  8194. }
  8195. if (!(features & NETIF_F_RXCSUM)) {
  8196. /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
  8197. * successfully merged by hardware must also have the
  8198. * checksum verified by hardware. If the user does not
  8199. * want to enable RXCSUM, logically, we should disable GRO_HW.
  8200. */
  8201. if (features & NETIF_F_GRO_HW) {
  8202. netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
  8203. features &= ~NETIF_F_GRO_HW;
  8204. }
  8205. }
  8206. /* LRO/HW-GRO features cannot be combined with RX-FCS */
  8207. if (features & NETIF_F_RXFCS) {
  8208. if (features & NETIF_F_LRO) {
  8209. netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
  8210. features &= ~NETIF_F_LRO;
  8211. }
  8212. if (features & NETIF_F_GRO_HW) {
  8213. netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
  8214. features &= ~NETIF_F_GRO_HW;
  8215. }
  8216. }
  8217. if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
  8218. netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
  8219. features &= ~NETIF_F_HW_TLS_RX;
  8220. }
  8221. return features;
  8222. }
  8223. int __netdev_update_features(struct net_device *dev)
  8224. {
  8225. struct net_device *upper, *lower;
  8226. netdev_features_t features;
  8227. struct list_head *iter;
  8228. int err = -1;
  8229. ASSERT_RTNL();
  8230. features = netdev_get_wanted_features(dev);
  8231. if (dev->netdev_ops->ndo_fix_features)
  8232. features = dev->netdev_ops->ndo_fix_features(dev, features);
  8233. /* driver might be less strict about feature dependencies */
  8234. features = netdev_fix_features(dev, features);
  8235. /* some features can't be enabled if they're off on an upper device */
  8236. netdev_for_each_upper_dev_rcu(dev, upper, iter)
  8237. features = netdev_sync_upper_features(dev, upper, features);
  8238. if (dev->features == features)
  8239. goto sync_lower;
  8240. netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
  8241. &dev->features, &features);
  8242. if (dev->netdev_ops->ndo_set_features)
  8243. err = dev->netdev_ops->ndo_set_features(dev, features);
  8244. else
  8245. err = 0;
  8246. if (unlikely(err < 0)) {
  8247. netdev_err(dev,
  8248. "set_features() failed (%d); wanted %pNF, left %pNF\n",
  8249. err, &features, &dev->features);
  8250. /* return non-0 since some features might have changed and
  8251. * it's better to fire a spurious notification than miss it
  8252. */
  8253. return -1;
  8254. }
  8255. sync_lower:
  8256. /* some features must be disabled on lower devices when disabled
  8257. * on an upper device (think: bonding master or bridge)
  8258. */
  8259. netdev_for_each_lower_dev(dev, lower, iter)
  8260. netdev_sync_lower_features(dev, lower, features);
  8261. if (!err) {
  8262. netdev_features_t diff = features ^ dev->features;
  8263. if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
  8264. /* udp_tunnel_{get,drop}_rx_info both need
  8265. * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
  8266. * device, or they won't do anything.
  8267. * Thus we need to update dev->features
  8268. * *before* calling udp_tunnel_get_rx_info,
  8269. * but *after* calling udp_tunnel_drop_rx_info.
  8270. */
  8271. if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
  8272. dev->features = features;
  8273. udp_tunnel_get_rx_info(dev);
  8274. } else {
  8275. udp_tunnel_drop_rx_info(dev);
  8276. }
  8277. }
  8278. if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
  8279. if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
  8280. dev->features = features;
  8281. err |= vlan_get_rx_ctag_filter_info(dev);
  8282. } else {
  8283. vlan_drop_rx_ctag_filter_info(dev);
  8284. }
  8285. }
  8286. if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
  8287. if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
  8288. dev->features = features;
  8289. err |= vlan_get_rx_stag_filter_info(dev);
  8290. } else {
  8291. vlan_drop_rx_stag_filter_info(dev);
  8292. }
  8293. }
  8294. dev->features = features;
  8295. }
  8296. return err < 0 ? 0 : 1;
  8297. }
  8298. /**
  8299. * netdev_update_features - recalculate device features
  8300. * @dev: the device to check
  8301. *
  8302. * Recalculate dev->features set and send notifications if it
  8303. * has changed. Should be called after driver or hardware dependent
  8304. * conditions might have changed that influence the features.
  8305. */
  8306. void netdev_update_features(struct net_device *dev)
  8307. {
  8308. if (__netdev_update_features(dev))
  8309. netdev_features_change(dev);
  8310. }
  8311. EXPORT_SYMBOL(netdev_update_features);
  8312. /**
  8313. * netdev_change_features - recalculate device features
  8314. * @dev: the device to check
  8315. *
  8316. * Recalculate dev->features set and send notifications even
  8317. * if they have not changed. Should be called instead of
  8318. * netdev_update_features() if also dev->vlan_features might
  8319. * have changed to allow the changes to be propagated to stacked
  8320. * VLAN devices.
  8321. */
  8322. void netdev_change_features(struct net_device *dev)
  8323. {
  8324. __netdev_update_features(dev);
  8325. netdev_features_change(dev);
  8326. }
  8327. EXPORT_SYMBOL(netdev_change_features);
  8328. /**
  8329. * netif_stacked_transfer_operstate - transfer operstate
  8330. * @rootdev: the root or lower level device to transfer state from
  8331. * @dev: the device to transfer operstate to
  8332. *
  8333. * Transfer operational state from root to device. This is normally
  8334. * called when a stacking relationship exists between the root
  8335. * device and the device(a leaf device).
  8336. */
  8337. void netif_stacked_transfer_operstate(const struct net_device *rootdev,
  8338. struct net_device *dev)
  8339. {
  8340. if (rootdev->operstate == IF_OPER_DORMANT)
  8341. netif_dormant_on(dev);
  8342. else
  8343. netif_dormant_off(dev);
  8344. if (rootdev->operstate == IF_OPER_TESTING)
  8345. netif_testing_on(dev);
  8346. else
  8347. netif_testing_off(dev);
  8348. if (netif_carrier_ok(rootdev))
  8349. netif_carrier_on(dev);
  8350. else
  8351. netif_carrier_off(dev);
  8352. }
  8353. EXPORT_SYMBOL(netif_stacked_transfer_operstate);
  8354. static int netif_alloc_rx_queues(struct net_device *dev)
  8355. {
  8356. unsigned int i, count = dev->num_rx_queues;
  8357. struct netdev_rx_queue *rx;
  8358. size_t sz = count * sizeof(*rx);
  8359. int err = 0;
  8360. BUG_ON(count < 1);
  8361. rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
  8362. if (!rx)
  8363. return -ENOMEM;
  8364. dev->_rx = rx;
  8365. for (i = 0; i < count; i++) {
  8366. rx[i].dev = dev;
  8367. /* XDP RX-queue setup */
  8368. err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i);
  8369. if (err < 0)
  8370. goto err_rxq_info;
  8371. }
  8372. return 0;
  8373. err_rxq_info:
  8374. /* Rollback successful reg's and free other resources */
  8375. while (i--)
  8376. xdp_rxq_info_unreg(&rx[i].xdp_rxq);
  8377. kvfree(dev->_rx);
  8378. dev->_rx = NULL;
  8379. return err;
  8380. }
  8381. static void netif_free_rx_queues(struct net_device *dev)
  8382. {
  8383. unsigned int i, count = dev->num_rx_queues;
  8384. /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
  8385. if (!dev->_rx)
  8386. return;
  8387. for (i = 0; i < count; i++)
  8388. xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
  8389. kvfree(dev->_rx);
  8390. }
  8391. static void netdev_init_one_queue(struct net_device *dev,
  8392. struct netdev_queue *queue, void *_unused)
  8393. {
  8394. /* Initialize queue lock */
  8395. spin_lock_init(&queue->_xmit_lock);
  8396. netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
  8397. queue->xmit_lock_owner = -1;
  8398. netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
  8399. queue->dev = dev;
  8400. #ifdef CONFIG_BQL
  8401. dql_init(&queue->dql, HZ);
  8402. #endif
  8403. }
  8404. static void netif_free_tx_queues(struct net_device *dev)
  8405. {
  8406. kvfree(dev->_tx);
  8407. }
  8408. static int netif_alloc_netdev_queues(struct net_device *dev)
  8409. {
  8410. unsigned int count = dev->num_tx_queues;
  8411. struct netdev_queue *tx;
  8412. size_t sz = count * sizeof(*tx);
  8413. if (count < 1 || count > 0xffff)
  8414. return -EINVAL;
  8415. tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
  8416. if (!tx)
  8417. return -ENOMEM;
  8418. dev->_tx = tx;
  8419. netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
  8420. spin_lock_init(&dev->tx_global_lock);
  8421. return 0;
  8422. }
  8423. void netif_tx_stop_all_queues(struct net_device *dev)
  8424. {
  8425. unsigned int i;
  8426. for (i = 0; i < dev->num_tx_queues; i++) {
  8427. struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
  8428. netif_tx_stop_queue(txq);
  8429. }
  8430. }
  8431. EXPORT_SYMBOL(netif_tx_stop_all_queues);
  8432. /**
  8433. * register_netdevice - register a network device
  8434. * @dev: device to register
  8435. *
  8436. * Take a completed network device structure and add it to the kernel
  8437. * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
  8438. * chain. 0 is returned on success. A negative errno code is returned
  8439. * on a failure to set up the device, or if the name is a duplicate.
  8440. *
  8441. * Callers must hold the rtnl semaphore. You may want
  8442. * register_netdev() instead of this.
  8443. *
  8444. * BUGS:
  8445. * The locking appears insufficient to guarantee two parallel registers
  8446. * will not get the same name.
  8447. */
  8448. int register_netdevice(struct net_device *dev)
  8449. {
  8450. int ret;
  8451. struct net *net = dev_net(dev);
  8452. BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
  8453. NETDEV_FEATURE_COUNT);
  8454. BUG_ON(dev_boot_phase);
  8455. ASSERT_RTNL();
  8456. might_sleep();
  8457. /* When net_device's are persistent, this will be fatal. */
  8458. BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
  8459. BUG_ON(!net);
  8460. ret = ethtool_check_ops(dev->ethtool_ops);
  8461. if (ret)
  8462. return ret;
  8463. spin_lock_init(&dev->addr_list_lock);
  8464. netdev_set_addr_lockdep_class(dev);
  8465. ret = dev_get_valid_name(net, dev, dev->name);
  8466. if (ret < 0)
  8467. goto out;
  8468. ret = -ENOMEM;
  8469. dev->name_node = netdev_name_node_head_alloc(dev);
  8470. if (!dev->name_node)
  8471. goto out;
  8472. /* Init, if this function is available */
  8473. if (dev->netdev_ops->ndo_init) {
  8474. ret = dev->netdev_ops->ndo_init(dev);
  8475. if (ret) {
  8476. if (ret > 0)
  8477. ret = -EIO;
  8478. goto err_free_name;
  8479. }
  8480. }
  8481. if (((dev->hw_features | dev->features) &
  8482. NETIF_F_HW_VLAN_CTAG_FILTER) &&
  8483. (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
  8484. !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
  8485. netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
  8486. ret = -EINVAL;
  8487. goto err_uninit;
  8488. }
  8489. ret = -EBUSY;
  8490. if (!dev->ifindex)
  8491. dev->ifindex = dev_new_index(net);
  8492. else if (__dev_get_by_index(net, dev->ifindex))
  8493. goto err_uninit;
  8494. /* Transfer changeable features to wanted_features and enable
  8495. * software offloads (GSO and GRO).
  8496. */
  8497. dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
  8498. dev->features |= NETIF_F_SOFT_FEATURES;
  8499. if (dev->netdev_ops->ndo_udp_tunnel_add) {
  8500. dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
  8501. dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
  8502. }
  8503. dev->wanted_features = dev->features & dev->hw_features;
  8504. if (!(dev->flags & IFF_LOOPBACK))
  8505. dev->hw_features |= NETIF_F_NOCACHE_COPY;
  8506. /* If IPv4 TCP segmentation offload is supported we should also
  8507. * allow the device to enable segmenting the frame with the option
  8508. * of ignoring a static IP ID value. This doesn't enable the
  8509. * feature itself but allows the user to enable it later.
  8510. */
  8511. if (dev->hw_features & NETIF_F_TSO)
  8512. dev->hw_features |= NETIF_F_TSO_MANGLEID;
  8513. if (dev->vlan_features & NETIF_F_TSO)
  8514. dev->vlan_features |= NETIF_F_TSO_MANGLEID;
  8515. if (dev->mpls_features & NETIF_F_TSO)
  8516. dev->mpls_features |= NETIF_F_TSO_MANGLEID;
  8517. if (dev->hw_enc_features & NETIF_F_TSO)
  8518. dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
  8519. /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
  8520. */
  8521. dev->vlan_features |= NETIF_F_HIGHDMA;
  8522. /* Make NETIF_F_SG inheritable to tunnel devices.
  8523. */
  8524. dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
  8525. /* Make NETIF_F_SG inheritable to MPLS.
  8526. */
  8527. dev->mpls_features |= NETIF_F_SG;
  8528. ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
  8529. ret = notifier_to_errno(ret);
  8530. if (ret)
  8531. goto err_uninit;
  8532. ret = netdev_register_kobject(dev);
  8533. if (ret) {
  8534. dev->reg_state = NETREG_UNREGISTERED;
  8535. goto err_uninit;
  8536. }
  8537. dev->reg_state = NETREG_REGISTERED;
  8538. __netdev_update_features(dev);
  8539. /*
  8540. * Default initial state at registry is that the
  8541. * device is present.
  8542. */
  8543. set_bit(__LINK_STATE_PRESENT, &dev->state);
  8544. linkwatch_init_dev(dev);
  8545. dev_init_scheduler(dev);
  8546. dev_hold(dev);
  8547. list_netdevice(dev);
  8548. add_device_randomness(dev->dev_addr, dev->addr_len);
  8549. /* If the device has permanent device address, driver should
  8550. * set dev_addr and also addr_assign_type should be set to
  8551. * NET_ADDR_PERM (default value).
  8552. */
  8553. if (dev->addr_assign_type == NET_ADDR_PERM)
  8554. memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
  8555. /* Notify protocols, that a new device appeared. */
  8556. ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
  8557. ret = notifier_to_errno(ret);
  8558. if (ret) {
  8559. rollback_registered(dev);
  8560. rcu_barrier();
  8561. dev->reg_state = NETREG_UNREGISTERED;
  8562. /* We should put the kobject that hold in
  8563. * netdev_unregister_kobject(), otherwise
  8564. * the net device cannot be freed when
  8565. * driver calls free_netdev(), because the
  8566. * kobject is being hold.
  8567. */
  8568. kobject_put(&dev->dev.kobj);
  8569. }
  8570. /*
  8571. * Prevent userspace races by waiting until the network
  8572. * device is fully setup before sending notifications.
  8573. */
  8574. if (!dev->rtnl_link_ops ||
  8575. dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
  8576. rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
  8577. out:
  8578. return ret;
  8579. err_uninit:
  8580. if (dev->netdev_ops->ndo_uninit)
  8581. dev->netdev_ops->ndo_uninit(dev);
  8582. if (dev->priv_destructor)
  8583. dev->priv_destructor(dev);
  8584. err_free_name:
  8585. netdev_name_node_free(dev->name_node);
  8586. goto out;
  8587. }
  8588. EXPORT_SYMBOL(register_netdevice);
  8589. /**
  8590. * init_dummy_netdev - init a dummy network device for NAPI
  8591. * @dev: device to init
  8592. *
  8593. * This takes a network device structure and initialize the minimum
  8594. * amount of fields so it can be used to schedule NAPI polls without
  8595. * registering a full blown interface. This is to be used by drivers
  8596. * that need to tie several hardware interfaces to a single NAPI
  8597. * poll scheduler due to HW limitations.
  8598. */
  8599. int init_dummy_netdev(struct net_device *dev)
  8600. {
  8601. /* Clear everything. Note we don't initialize spinlocks
  8602. * are they aren't supposed to be taken by any of the
  8603. * NAPI code and this dummy netdev is supposed to be
  8604. * only ever used for NAPI polls
  8605. */
  8606. memset(dev, 0, sizeof(struct net_device));
  8607. /* make sure we BUG if trying to hit standard
  8608. * register/unregister code path
  8609. */
  8610. dev->reg_state = NETREG_DUMMY;
  8611. /* NAPI wants this */
  8612. INIT_LIST_HEAD(&dev->napi_list);
  8613. /* a dummy interface is started by default */
  8614. set_bit(__LINK_STATE_PRESENT, &dev->state);
  8615. set_bit(__LINK_STATE_START, &dev->state);
  8616. /* napi_busy_loop stats accounting wants this */
  8617. dev_net_set(dev, &init_net);
  8618. /* Note : We dont allocate pcpu_refcnt for dummy devices,
  8619. * because users of this 'device' dont need to change
  8620. * its refcount.
  8621. */
  8622. return 0;
  8623. }
  8624. EXPORT_SYMBOL_GPL(init_dummy_netdev);
  8625. /**
  8626. * register_netdev - register a network device
  8627. * @dev: device to register
  8628. *
  8629. * Take a completed network device structure and add it to the kernel
  8630. * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
  8631. * chain. 0 is returned on success. A negative errno code is returned
  8632. * on a failure to set up the device, or if the name is a duplicate.
  8633. *
  8634. * This is a wrapper around register_netdevice that takes the rtnl semaphore
  8635. * and expands the device name if you passed a format string to
  8636. * alloc_netdev.
  8637. */
  8638. int register_netdev(struct net_device *dev)
  8639. {
  8640. int err;
  8641. if (rtnl_lock_killable())
  8642. return -EINTR;
  8643. err = register_netdevice(dev);
  8644. rtnl_unlock();
  8645. return err;
  8646. }
  8647. EXPORT_SYMBOL(register_netdev);
  8648. int netdev_refcnt_read(const struct net_device *dev)
  8649. {
  8650. int i, refcnt = 0;
  8651. for_each_possible_cpu(i)
  8652. refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
  8653. return refcnt;
  8654. }
  8655. EXPORT_SYMBOL(netdev_refcnt_read);
  8656. #define WAIT_REFS_MIN_MSECS 1
  8657. #define WAIT_REFS_MAX_MSECS 250
  8658. /**
  8659. * netdev_wait_allrefs - wait until all references are gone.
  8660. * @dev: target net_device
  8661. *
  8662. * This is called when unregistering network devices.
  8663. *
  8664. * Any protocol or device that holds a reference should register
  8665. * for netdevice notification, and cleanup and put back the
  8666. * reference if they receive an UNREGISTER event.
  8667. * We can get stuck here if buggy protocols don't correctly
  8668. * call dev_put.
  8669. */
  8670. static void netdev_wait_allrefs(struct net_device *dev)
  8671. {
  8672. unsigned long rebroadcast_time, warning_time;
  8673. int wait = 0, refcnt;
  8674. linkwatch_forget_dev(dev);
  8675. rebroadcast_time = warning_time = jiffies;
  8676. refcnt = netdev_refcnt_read(dev);
  8677. while (refcnt != 0) {
  8678. if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
  8679. rtnl_lock();
  8680. /* Rebroadcast unregister notification */
  8681. call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
  8682. __rtnl_unlock();
  8683. rcu_barrier();
  8684. rtnl_lock();
  8685. if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
  8686. &dev->state)) {
  8687. /* We must not have linkwatch events
  8688. * pending on unregister. If this
  8689. * happens, we simply run the queue
  8690. * unscheduled, resulting in a noop
  8691. * for this device.
  8692. */
  8693. linkwatch_run_queue();
  8694. }
  8695. __rtnl_unlock();
  8696. rebroadcast_time = jiffies;
  8697. }
  8698. if (!wait) {
  8699. rcu_barrier();
  8700. wait = WAIT_REFS_MIN_MSECS;
  8701. } else {
  8702. msleep(wait);
  8703. wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
  8704. }
  8705. refcnt = netdev_refcnt_read(dev);
  8706. if (refcnt && time_after(jiffies, warning_time + 10 * HZ)) {
  8707. pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
  8708. dev->name, refcnt);
  8709. warning_time = jiffies;
  8710. }
  8711. }
  8712. }
  8713. /* The sequence is:
  8714. *
  8715. * rtnl_lock();
  8716. * ...
  8717. * register_netdevice(x1);
  8718. * register_netdevice(x2);
  8719. * ...
  8720. * unregister_netdevice(y1);
  8721. * unregister_netdevice(y2);
  8722. * ...
  8723. * rtnl_unlock();
  8724. * free_netdev(y1);
  8725. * free_netdev(y2);
  8726. *
  8727. * We are invoked by rtnl_unlock().
  8728. * This allows us to deal with problems:
  8729. * 1) We can delete sysfs objects which invoke hotplug
  8730. * without deadlocking with linkwatch via keventd.
  8731. * 2) Since we run with the RTNL semaphore not held, we can sleep
  8732. * safely in order to wait for the netdev refcnt to drop to zero.
  8733. *
  8734. * We must not return until all unregister events added during
  8735. * the interval the lock was held have been completed.
  8736. */
  8737. void netdev_run_todo(void)
  8738. {
  8739. struct list_head list;
  8740. #ifdef CONFIG_LOCKDEP
  8741. struct list_head unlink_list;
  8742. list_replace_init(&net_unlink_list, &unlink_list);
  8743. while (!list_empty(&unlink_list)) {
  8744. struct net_device *dev = list_first_entry(&unlink_list,
  8745. struct net_device,
  8746. unlink_list);
  8747. list_del_init(&dev->unlink_list);
  8748. dev->nested_level = dev->lower_level - 1;
  8749. }
  8750. #endif
  8751. /* Snapshot list, allow later requests */
  8752. list_replace_init(&net_todo_list, &list);
  8753. __rtnl_unlock();
  8754. /* Wait for rcu callbacks to finish before next phase */
  8755. if (!list_empty(&list))
  8756. rcu_barrier();
  8757. while (!list_empty(&list)) {
  8758. struct net_device *dev
  8759. = list_first_entry(&list, struct net_device, todo_list);
  8760. list_del(&dev->todo_list);
  8761. if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
  8762. pr_err("network todo '%s' but state %d\n",
  8763. dev->name, dev->reg_state);
  8764. dump_stack();
  8765. continue;
  8766. }
  8767. dev->reg_state = NETREG_UNREGISTERED;
  8768. netdev_wait_allrefs(dev);
  8769. /* paranoia */
  8770. BUG_ON(netdev_refcnt_read(dev));
  8771. BUG_ON(!list_empty(&dev->ptype_all));
  8772. BUG_ON(!list_empty(&dev->ptype_specific));
  8773. WARN_ON(rcu_access_pointer(dev->ip_ptr));
  8774. WARN_ON(rcu_access_pointer(dev->ip6_ptr));
  8775. #if IS_ENABLED(CONFIG_DECNET)
  8776. WARN_ON(dev->dn_ptr);
  8777. #endif
  8778. if (dev->priv_destructor)
  8779. dev->priv_destructor(dev);
  8780. if (dev->needs_free_netdev)
  8781. free_netdev(dev);
  8782. /* Report a network device has been unregistered */
  8783. rtnl_lock();
  8784. dev_net(dev)->dev_unreg_count--;
  8785. __rtnl_unlock();
  8786. wake_up(&netdev_unregistering_wq);
  8787. /* Free network device */
  8788. kobject_put(&dev->dev.kobj);
  8789. }
  8790. }
  8791. /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
  8792. * all the same fields in the same order as net_device_stats, with only
  8793. * the type differing, but rtnl_link_stats64 may have additional fields
  8794. * at the end for newer counters.
  8795. */
  8796. void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
  8797. const struct net_device_stats *netdev_stats)
  8798. {
  8799. #if BITS_PER_LONG == 64
  8800. BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
  8801. memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
  8802. /* zero out counters that only exist in rtnl_link_stats64 */
  8803. memset((char *)stats64 + sizeof(*netdev_stats), 0,
  8804. sizeof(*stats64) - sizeof(*netdev_stats));
  8805. #else
  8806. size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
  8807. const unsigned long *src = (const unsigned long *)netdev_stats;
  8808. u64 *dst = (u64 *)stats64;
  8809. BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
  8810. for (i = 0; i < n; i++)
  8811. dst[i] = src[i];
  8812. /* zero out counters that only exist in rtnl_link_stats64 */
  8813. memset((char *)stats64 + n * sizeof(u64), 0,
  8814. sizeof(*stats64) - n * sizeof(u64));
  8815. #endif
  8816. }
  8817. EXPORT_SYMBOL(netdev_stats_to_stats64);
  8818. /**
  8819. * dev_get_stats - get network device statistics
  8820. * @dev: device to get statistics from
  8821. * @storage: place to store stats
  8822. *
  8823. * Get network statistics from device. Return @storage.
  8824. * The device driver may provide its own method by setting
  8825. * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
  8826. * otherwise the internal statistics structure is used.
  8827. */
  8828. struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
  8829. struct rtnl_link_stats64 *storage)
  8830. {
  8831. const struct net_device_ops *ops = dev->netdev_ops;
  8832. if (ops->ndo_get_stats64) {
  8833. memset(storage, 0, sizeof(*storage));
  8834. ops->ndo_get_stats64(dev, storage);
  8835. } else if (ops->ndo_get_stats) {
  8836. netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
  8837. } else {
  8838. netdev_stats_to_stats64(storage, &dev->stats);
  8839. }
  8840. storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
  8841. storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
  8842. storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
  8843. return storage;
  8844. }
  8845. EXPORT_SYMBOL(dev_get_stats);
  8846. /**
  8847. * dev_fetch_sw_netstats - get per-cpu network device statistics
  8848. * @s: place to store stats
  8849. * @netstats: per-cpu network stats to read from
  8850. *
  8851. * Read per-cpu network statistics and populate the related fields in @s.
  8852. */
  8853. void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
  8854. const struct pcpu_sw_netstats __percpu *netstats)
  8855. {
  8856. int cpu;
  8857. for_each_possible_cpu(cpu) {
  8858. const struct pcpu_sw_netstats *stats;
  8859. struct pcpu_sw_netstats tmp;
  8860. unsigned int start;
  8861. stats = per_cpu_ptr(netstats, cpu);
  8862. do {
  8863. start = u64_stats_fetch_begin_irq(&stats->syncp);
  8864. tmp.rx_packets = stats->rx_packets;
  8865. tmp.rx_bytes = stats->rx_bytes;
  8866. tmp.tx_packets = stats->tx_packets;
  8867. tmp.tx_bytes = stats->tx_bytes;
  8868. } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
  8869. s->rx_packets += tmp.rx_packets;
  8870. s->rx_bytes += tmp.rx_bytes;
  8871. s->tx_packets += tmp.tx_packets;
  8872. s->tx_bytes += tmp.tx_bytes;
  8873. }
  8874. }
  8875. EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
  8876. struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
  8877. {
  8878. struct netdev_queue *queue = dev_ingress_queue(dev);
  8879. #ifdef CONFIG_NET_CLS_ACT
  8880. if (queue)
  8881. return queue;
  8882. queue = kzalloc(sizeof(*queue), GFP_KERNEL);
  8883. if (!queue)
  8884. return NULL;
  8885. netdev_init_one_queue(dev, queue, NULL);
  8886. RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
  8887. queue->qdisc_sleeping = &noop_qdisc;
  8888. rcu_assign_pointer(dev->ingress_queue, queue);
  8889. #endif
  8890. return queue;
  8891. }
  8892. static const struct ethtool_ops default_ethtool_ops;
  8893. void netdev_set_default_ethtool_ops(struct net_device *dev,
  8894. const struct ethtool_ops *ops)
  8895. {
  8896. if (dev->ethtool_ops == &default_ethtool_ops)
  8897. dev->ethtool_ops = ops;
  8898. }
  8899. EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
  8900. void netdev_freemem(struct net_device *dev)
  8901. {
  8902. char *addr = (char *)dev - dev->padded;
  8903. kvfree(addr);
  8904. }
  8905. /**
  8906. * alloc_netdev_mqs - allocate network device
  8907. * @sizeof_priv: size of private data to allocate space for
  8908. * @name: device name format string
  8909. * @name_assign_type: origin of device name
  8910. * @setup: callback to initialize device
  8911. * @txqs: the number of TX subqueues to allocate
  8912. * @rxqs: the number of RX subqueues to allocate
  8913. *
  8914. * Allocates a struct net_device with private data area for driver use
  8915. * and performs basic initialization. Also allocates subqueue structs
  8916. * for each queue on the device.
  8917. */
  8918. struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
  8919. unsigned char name_assign_type,
  8920. void (*setup)(struct net_device *),
  8921. unsigned int txqs, unsigned int rxqs)
  8922. {
  8923. struct net_device *dev;
  8924. unsigned int alloc_size;
  8925. struct net_device *p;
  8926. BUG_ON(strlen(name) >= sizeof(dev->name));
  8927. if (txqs < 1) {
  8928. pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
  8929. return NULL;
  8930. }
  8931. if (rxqs < 1) {
  8932. pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
  8933. return NULL;
  8934. }
  8935. alloc_size = sizeof(struct net_device);
  8936. if (sizeof_priv) {
  8937. /* ensure 32-byte alignment of private area */
  8938. alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
  8939. alloc_size += sizeof_priv;
  8940. }
  8941. /* ensure 32-byte alignment of whole construct */
  8942. alloc_size += NETDEV_ALIGN - 1;
  8943. p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
  8944. if (!p)
  8945. return NULL;
  8946. dev = PTR_ALIGN(p, NETDEV_ALIGN);
  8947. dev->padded = (char *)dev - (char *)p;
  8948. dev->pcpu_refcnt = alloc_percpu(int);
  8949. if (!dev->pcpu_refcnt)
  8950. goto free_dev;
  8951. if (dev_addr_init(dev))
  8952. goto free_pcpu;
  8953. dev_mc_init(dev);
  8954. dev_uc_init(dev);
  8955. dev_net_set(dev, &init_net);
  8956. dev->gso_max_size = GSO_MAX_SIZE;
  8957. dev->gso_max_segs = GSO_MAX_SEGS;
  8958. dev->upper_level = 1;
  8959. dev->lower_level = 1;
  8960. #ifdef CONFIG_LOCKDEP
  8961. dev->nested_level = 0;
  8962. INIT_LIST_HEAD(&dev->unlink_list);
  8963. #endif
  8964. INIT_LIST_HEAD(&dev->napi_list);
  8965. INIT_LIST_HEAD(&dev->unreg_list);
  8966. INIT_LIST_HEAD(&dev->close_list);
  8967. INIT_LIST_HEAD(&dev->link_watch_list);
  8968. INIT_LIST_HEAD(&dev->adj_list.upper);
  8969. INIT_LIST_HEAD(&dev->adj_list.lower);
  8970. INIT_LIST_HEAD(&dev->ptype_all);
  8971. INIT_LIST_HEAD(&dev->ptype_specific);
  8972. INIT_LIST_HEAD(&dev->net_notifier_list);
  8973. #ifdef CONFIG_NET_SCHED
  8974. hash_init(dev->qdisc_hash);
  8975. #endif
  8976. dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
  8977. setup(dev);
  8978. if (!dev->tx_queue_len) {
  8979. dev->priv_flags |= IFF_NO_QUEUE;
  8980. dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
  8981. }
  8982. dev->num_tx_queues = txqs;
  8983. dev->real_num_tx_queues = txqs;
  8984. if (netif_alloc_netdev_queues(dev))
  8985. goto free_all;
  8986. dev->num_rx_queues = rxqs;
  8987. dev->real_num_rx_queues = rxqs;
  8988. if (netif_alloc_rx_queues(dev))
  8989. goto free_all;
  8990. strcpy(dev->name, name);
  8991. dev->name_assign_type = name_assign_type;
  8992. dev->group = INIT_NETDEV_GROUP;
  8993. if (!dev->ethtool_ops)
  8994. dev->ethtool_ops = &default_ethtool_ops;
  8995. nf_hook_ingress_init(dev);
  8996. return dev;
  8997. free_all:
  8998. free_netdev(dev);
  8999. return NULL;
  9000. free_pcpu:
  9001. free_percpu(dev->pcpu_refcnt);
  9002. free_dev:
  9003. netdev_freemem(dev);
  9004. return NULL;
  9005. }
  9006. EXPORT_SYMBOL(alloc_netdev_mqs);
  9007. /**
  9008. * free_netdev - free network device
  9009. * @dev: device
  9010. *
  9011. * This function does the last stage of destroying an allocated device
  9012. * interface. The reference to the device object is released. If this
  9013. * is the last reference then it will be freed.Must be called in process
  9014. * context.
  9015. */
  9016. void free_netdev(struct net_device *dev)
  9017. {
  9018. struct napi_struct *p, *n;
  9019. might_sleep();
  9020. netif_free_tx_queues(dev);
  9021. netif_free_rx_queues(dev);
  9022. kfree(rcu_dereference_protected(dev->ingress_queue, 1));
  9023. /* Flush device addresses */
  9024. dev_addr_flush(dev);
  9025. list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
  9026. netif_napi_del(p);
  9027. free_percpu(dev->pcpu_refcnt);
  9028. dev->pcpu_refcnt = NULL;
  9029. free_percpu(dev->xdp_bulkq);
  9030. dev->xdp_bulkq = NULL;
  9031. /* Compatibility with error handling in drivers */
  9032. if (dev->reg_state == NETREG_UNINITIALIZED) {
  9033. netdev_freemem(dev);
  9034. return;
  9035. }
  9036. BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
  9037. dev->reg_state = NETREG_RELEASED;
  9038. /* will free via device release */
  9039. put_device(&dev->dev);
  9040. }
  9041. EXPORT_SYMBOL(free_netdev);
  9042. /**
  9043. * synchronize_net - Synchronize with packet receive processing
  9044. *
  9045. * Wait for packets currently being received to be done.
  9046. * Does not block later packets from starting.
  9047. */
  9048. void synchronize_net(void)
  9049. {
  9050. might_sleep();
  9051. if (rtnl_is_locked())
  9052. synchronize_rcu_expedited();
  9053. else
  9054. synchronize_rcu();
  9055. }
  9056. EXPORT_SYMBOL(synchronize_net);
  9057. /**
  9058. * unregister_netdevice_queue - remove device from the kernel
  9059. * @dev: device
  9060. * @head: list
  9061. *
  9062. * This function shuts down a device interface and removes it
  9063. * from the kernel tables.
  9064. * If head not NULL, device is queued to be unregistered later.
  9065. *
  9066. * Callers must hold the rtnl semaphore. You may want
  9067. * unregister_netdev() instead of this.
  9068. */
  9069. void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
  9070. {
  9071. ASSERT_RTNL();
  9072. if (head) {
  9073. list_move_tail(&dev->unreg_list, head);
  9074. } else {
  9075. rollback_registered(dev);
  9076. /* Finish processing unregister after unlock */
  9077. net_set_todo(dev);
  9078. }
  9079. }
  9080. EXPORT_SYMBOL(unregister_netdevice_queue);
  9081. /**
  9082. * unregister_netdevice_many - unregister many devices
  9083. * @head: list of devices
  9084. *
  9085. * Note: As most callers use a stack allocated list_head,
  9086. * we force a list_del() to make sure stack wont be corrupted later.
  9087. */
  9088. void unregister_netdevice_many(struct list_head *head)
  9089. {
  9090. struct net_device *dev;
  9091. if (!list_empty(head)) {
  9092. rollback_registered_many(head);
  9093. list_for_each_entry(dev, head, unreg_list)
  9094. net_set_todo(dev);
  9095. list_del(head);
  9096. }
  9097. }
  9098. EXPORT_SYMBOL(unregister_netdevice_many);
  9099. /**
  9100. * unregister_netdev - remove device from the kernel
  9101. * @dev: device
  9102. *
  9103. * This function shuts down a device interface and removes it
  9104. * from the kernel tables.
  9105. *
  9106. * This is just a wrapper for unregister_netdevice that takes
  9107. * the rtnl semaphore. In general you want to use this and not
  9108. * unregister_netdevice.
  9109. */
  9110. void unregister_netdev(struct net_device *dev)
  9111. {
  9112. rtnl_lock();
  9113. unregister_netdevice(dev);
  9114. rtnl_unlock();
  9115. }
  9116. EXPORT_SYMBOL(unregister_netdev);
  9117. /**
  9118. * dev_change_net_namespace - move device to different nethost namespace
  9119. * @dev: device
  9120. * @net: network namespace
  9121. * @pat: If not NULL name pattern to try if the current device name
  9122. * is already taken in the destination network namespace.
  9123. *
  9124. * This function shuts down a device interface and moves it
  9125. * to a new network namespace. On success 0 is returned, on
  9126. * a failure a netagive errno code is returned.
  9127. *
  9128. * Callers must hold the rtnl semaphore.
  9129. */
  9130. int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
  9131. {
  9132. struct net *net_old = dev_net(dev);
  9133. int err, new_nsid, new_ifindex;
  9134. ASSERT_RTNL();
  9135. /* Don't allow namespace local devices to be moved. */
  9136. err = -EINVAL;
  9137. if (dev->features & NETIF_F_NETNS_LOCAL)
  9138. goto out;
  9139. /* Ensure the device has been registrered */
  9140. if (dev->reg_state != NETREG_REGISTERED)
  9141. goto out;
  9142. /* Get out if there is nothing todo */
  9143. err = 0;
  9144. if (net_eq(net_old, net))
  9145. goto out;
  9146. /* Pick the destination device name, and ensure
  9147. * we can use it in the destination network namespace.
  9148. */
  9149. err = -EEXIST;
  9150. if (__dev_get_by_name(net, dev->name)) {
  9151. /* We get here if we can't use the current device name */
  9152. if (!pat)
  9153. goto out;
  9154. err = dev_get_valid_name(net, dev, pat);
  9155. if (err < 0)
  9156. goto out;
  9157. }
  9158. /*
  9159. * And now a mini version of register_netdevice unregister_netdevice.
  9160. */
  9161. /* If device is running close it first. */
  9162. dev_close(dev);
  9163. /* And unlink it from device chain */
  9164. unlist_netdevice(dev);
  9165. synchronize_net();
  9166. /* Shutdown queueing discipline. */
  9167. dev_shutdown(dev);
  9168. /* Notify protocols, that we are about to destroy
  9169. * this device. They should clean all the things.
  9170. *
  9171. * Note that dev->reg_state stays at NETREG_REGISTERED.
  9172. * This is wanted because this way 8021q and macvlan know
  9173. * the device is just moving and can keep their slaves up.
  9174. */
  9175. call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
  9176. rcu_barrier();
  9177. new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
  9178. /* If there is an ifindex conflict assign a new one */
  9179. if (__dev_get_by_index(net, dev->ifindex))
  9180. new_ifindex = dev_new_index(net);
  9181. else
  9182. new_ifindex = dev->ifindex;
  9183. rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
  9184. new_ifindex);
  9185. /*
  9186. * Flush the unicast and multicast chains
  9187. */
  9188. dev_uc_flush(dev);
  9189. dev_mc_flush(dev);
  9190. /* Send a netdev-removed uevent to the old namespace */
  9191. kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
  9192. netdev_adjacent_del_links(dev);
  9193. /* Move per-net netdevice notifiers that are following the netdevice */
  9194. move_netdevice_notifiers_dev_net(dev, net);
  9195. /* Actually switch the network namespace */
  9196. dev_net_set(dev, net);
  9197. dev->ifindex = new_ifindex;
  9198. /* Send a netdev-add uevent to the new namespace */
  9199. kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
  9200. netdev_adjacent_add_links(dev);
  9201. /* Fixup kobjects */
  9202. err = device_rename(&dev->dev, dev->name);
  9203. WARN_ON(err);
  9204. /* Adapt owner in case owning user namespace of target network
  9205. * namespace is different from the original one.
  9206. */
  9207. err = netdev_change_owner(dev, net_old, net);
  9208. WARN_ON(err);
  9209. /* Add the device back in the hashes */
  9210. list_netdevice(dev);
  9211. /* Notify protocols, that a new device appeared. */
  9212. call_netdevice_notifiers(NETDEV_REGISTER, dev);
  9213. /*
  9214. * Prevent userspace races by waiting until the network
  9215. * device is fully setup before sending notifications.
  9216. */
  9217. rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
  9218. synchronize_net();
  9219. err = 0;
  9220. out:
  9221. return err;
  9222. }
  9223. EXPORT_SYMBOL_GPL(dev_change_net_namespace);
  9224. static int dev_cpu_dead(unsigned int oldcpu)
  9225. {
  9226. struct sk_buff **list_skb;
  9227. struct sk_buff *skb;
  9228. unsigned int cpu;
  9229. struct softnet_data *sd, *oldsd, *remsd = NULL;
  9230. local_irq_disable();
  9231. cpu = smp_processor_id();
  9232. sd = &per_cpu(softnet_data, cpu);
  9233. oldsd = &per_cpu(softnet_data, oldcpu);
  9234. /* Find end of our completion_queue. */
  9235. list_skb = &sd->completion_queue;
  9236. while (*list_skb)
  9237. list_skb = &(*list_skb)->next;
  9238. /* Append completion queue from offline CPU. */
  9239. *list_skb = oldsd->completion_queue;
  9240. oldsd->completion_queue = NULL;
  9241. /* Append output queue from offline CPU. */
  9242. if (oldsd->output_queue) {
  9243. *sd->output_queue_tailp = oldsd->output_queue;
  9244. sd->output_queue_tailp = oldsd->output_queue_tailp;
  9245. oldsd->output_queue = NULL;
  9246. oldsd->output_queue_tailp = &oldsd->output_queue;
  9247. }
  9248. /* Append NAPI poll list from offline CPU, with one exception :
  9249. * process_backlog() must be called by cpu owning percpu backlog.
  9250. * We properly handle process_queue & input_pkt_queue later.
  9251. */
  9252. while (!list_empty(&oldsd->poll_list)) {
  9253. struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
  9254. struct napi_struct,
  9255. poll_list);
  9256. list_del_init(&napi->poll_list);
  9257. if (napi->poll == process_backlog)
  9258. napi->state = 0;
  9259. else
  9260. ____napi_schedule(sd, napi);
  9261. }
  9262. raise_softirq_irqoff(NET_TX_SOFTIRQ);
  9263. local_irq_enable();
  9264. #ifdef CONFIG_RPS
  9265. remsd = oldsd->rps_ipi_list;
  9266. oldsd->rps_ipi_list = NULL;
  9267. #endif
  9268. /* send out pending IPI's on offline CPU */
  9269. net_rps_send_ipi(remsd);
  9270. /* Process offline CPU's input_pkt_queue */
  9271. while ((skb = __skb_dequeue(&oldsd->process_queue))) {
  9272. netif_rx_ni(skb);
  9273. input_queue_head_incr(oldsd);
  9274. }
  9275. while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
  9276. netif_rx_ni(skb);
  9277. input_queue_head_incr(oldsd);
  9278. }
  9279. return 0;
  9280. }
  9281. /**
  9282. * netdev_increment_features - increment feature set by one
  9283. * @all: current feature set
  9284. * @one: new feature set
  9285. * @mask: mask feature set
  9286. *
  9287. * Computes a new feature set after adding a device with feature set
  9288. * @one to the master device with current feature set @all. Will not
  9289. * enable anything that is off in @mask. Returns the new feature set.
  9290. */
  9291. netdev_features_t netdev_increment_features(netdev_features_t all,
  9292. netdev_features_t one, netdev_features_t mask)
  9293. {
  9294. if (mask & NETIF_F_HW_CSUM)
  9295. mask |= NETIF_F_CSUM_MASK;
  9296. mask |= NETIF_F_VLAN_CHALLENGED;
  9297. all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
  9298. all &= one | ~NETIF_F_ALL_FOR_ALL;
  9299. /* If one device supports hw checksumming, set for all. */
  9300. if (all & NETIF_F_HW_CSUM)
  9301. all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
  9302. return all;
  9303. }
  9304. EXPORT_SYMBOL(netdev_increment_features);
  9305. static struct hlist_head * __net_init netdev_create_hash(void)
  9306. {
  9307. int i;
  9308. struct hlist_head *hash;
  9309. hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
  9310. if (hash != NULL)
  9311. for (i = 0; i < NETDEV_HASHENTRIES; i++)
  9312. INIT_HLIST_HEAD(&hash[i]);
  9313. return hash;
  9314. }
  9315. /* Initialize per network namespace state */
  9316. static int __net_init netdev_init(struct net *net)
  9317. {
  9318. BUILD_BUG_ON(GRO_HASH_BUCKETS >
  9319. 8 * sizeof_field(struct napi_struct, gro_bitmask));
  9320. if (net != &init_net)
  9321. INIT_LIST_HEAD(&net->dev_base_head);
  9322. net->dev_name_head = netdev_create_hash();
  9323. if (net->dev_name_head == NULL)
  9324. goto err_name;
  9325. net->dev_index_head = netdev_create_hash();
  9326. if (net->dev_index_head == NULL)
  9327. goto err_idx;
  9328. RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
  9329. return 0;
  9330. err_idx:
  9331. kfree(net->dev_name_head);
  9332. err_name:
  9333. return -ENOMEM;
  9334. }
  9335. /**
  9336. * netdev_drivername - network driver for the device
  9337. * @dev: network device
  9338. *
  9339. * Determine network driver for device.
  9340. */
  9341. const char *netdev_drivername(const struct net_device *dev)
  9342. {
  9343. const struct device_driver *driver;
  9344. const struct device *parent;
  9345. const char *empty = "";
  9346. parent = dev->dev.parent;
  9347. if (!parent)
  9348. return empty;
  9349. driver = parent->driver;
  9350. if (driver && driver->name)
  9351. return driver->name;
  9352. return empty;
  9353. }
  9354. static void __netdev_printk(const char *level, const struct net_device *dev,
  9355. struct va_format *vaf)
  9356. {
  9357. if (dev && dev->dev.parent) {
  9358. dev_printk_emit(level[1] - '0',
  9359. dev->dev.parent,
  9360. "%s %s %s%s: %pV",
  9361. dev_driver_string(dev->dev.parent),
  9362. dev_name(dev->dev.parent),
  9363. netdev_name(dev), netdev_reg_state(dev),
  9364. vaf);
  9365. } else if (dev) {
  9366. printk("%s%s%s: %pV",
  9367. level, netdev_name(dev), netdev_reg_state(dev), vaf);
  9368. } else {
  9369. printk("%s(NULL net_device): %pV", level, vaf);
  9370. }
  9371. }
  9372. void netdev_printk(const char *level, const struct net_device *dev,
  9373. const char *format, ...)
  9374. {
  9375. struct va_format vaf;
  9376. va_list args;
  9377. va_start(args, format);
  9378. vaf.fmt = format;
  9379. vaf.va = &args;
  9380. __netdev_printk(level, dev, &vaf);
  9381. va_end(args);
  9382. }
  9383. EXPORT_SYMBOL(netdev_printk);
  9384. #define define_netdev_printk_level(func, level) \
  9385. void func(const struct net_device *dev, const char *fmt, ...) \
  9386. { \
  9387. struct va_format vaf; \
  9388. va_list args; \
  9389. \
  9390. va_start(args, fmt); \
  9391. \
  9392. vaf.fmt = fmt; \
  9393. vaf.va = &args; \
  9394. \
  9395. __netdev_printk(level, dev, &vaf); \
  9396. \
  9397. va_end(args); \
  9398. } \
  9399. EXPORT_SYMBOL(func);
  9400. define_netdev_printk_level(netdev_emerg, KERN_EMERG);
  9401. define_netdev_printk_level(netdev_alert, KERN_ALERT);
  9402. define_netdev_printk_level(netdev_crit, KERN_CRIT);
  9403. define_netdev_printk_level(netdev_err, KERN_ERR);
  9404. define_netdev_printk_level(netdev_warn, KERN_WARNING);
  9405. define_netdev_printk_level(netdev_notice, KERN_NOTICE);
  9406. define_netdev_printk_level(netdev_info, KERN_INFO);
  9407. static void __net_exit netdev_exit(struct net *net)
  9408. {
  9409. kfree(net->dev_name_head);
  9410. kfree(net->dev_index_head);
  9411. if (net != &init_net)
  9412. WARN_ON_ONCE(!list_empty(&net->dev_base_head));
  9413. }
  9414. static struct pernet_operations __net_initdata netdev_net_ops = {
  9415. .init = netdev_init,
  9416. .exit = netdev_exit,
  9417. };
  9418. static void __net_exit default_device_exit(struct net *net)
  9419. {
  9420. struct net_device *dev, *aux;
  9421. /*
  9422. * Push all migratable network devices back to the
  9423. * initial network namespace
  9424. */
  9425. rtnl_lock();
  9426. for_each_netdev_safe(net, dev, aux) {
  9427. int err;
  9428. char fb_name[IFNAMSIZ];
  9429. /* Ignore unmoveable devices (i.e. loopback) */
  9430. if (dev->features & NETIF_F_NETNS_LOCAL)
  9431. continue;
  9432. /* Leave virtual devices for the generic cleanup */
  9433. if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
  9434. continue;
  9435. /* Push remaining network devices to init_net */
  9436. snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
  9437. if (__dev_get_by_name(&init_net, fb_name))
  9438. snprintf(fb_name, IFNAMSIZ, "dev%%d");
  9439. err = dev_change_net_namespace(dev, &init_net, fb_name);
  9440. if (err) {
  9441. pr_emerg("%s: failed to move %s to init_net: %d\n",
  9442. __func__, dev->name, err);
  9443. BUG();
  9444. }
  9445. }
  9446. rtnl_unlock();
  9447. }
  9448. static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
  9449. {
  9450. /* Return with the rtnl_lock held when there are no network
  9451. * devices unregistering in any network namespace in net_list.
  9452. */
  9453. struct net *net;
  9454. bool unregistering;
  9455. DEFINE_WAIT_FUNC(wait, woken_wake_function);
  9456. add_wait_queue(&netdev_unregistering_wq, &wait);
  9457. for (;;) {
  9458. unregistering = false;
  9459. rtnl_lock();
  9460. list_for_each_entry(net, net_list, exit_list) {
  9461. if (net->dev_unreg_count > 0) {
  9462. unregistering = true;
  9463. break;
  9464. }
  9465. }
  9466. if (!unregistering)
  9467. break;
  9468. __rtnl_unlock();
  9469. wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  9470. }
  9471. remove_wait_queue(&netdev_unregistering_wq, &wait);
  9472. }
  9473. static void __net_exit default_device_exit_batch(struct list_head *net_list)
  9474. {
  9475. /* At exit all network devices most be removed from a network
  9476. * namespace. Do this in the reverse order of registration.
  9477. * Do this across as many network namespaces as possible to
  9478. * improve batching efficiency.
  9479. */
  9480. struct net_device *dev;
  9481. struct net *net;
  9482. LIST_HEAD(dev_kill_list);
  9483. /* To prevent network device cleanup code from dereferencing
  9484. * loopback devices or network devices that have been freed
  9485. * wait here for all pending unregistrations to complete,
  9486. * before unregistring the loopback device and allowing the
  9487. * network namespace be freed.
  9488. *
  9489. * The netdev todo list containing all network devices
  9490. * unregistrations that happen in default_device_exit_batch
  9491. * will run in the rtnl_unlock() at the end of
  9492. * default_device_exit_batch.
  9493. */
  9494. rtnl_lock_unregistering(net_list);
  9495. list_for_each_entry(net, net_list, exit_list) {
  9496. for_each_netdev_reverse(net, dev) {
  9497. if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
  9498. dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
  9499. else
  9500. unregister_netdevice_queue(dev, &dev_kill_list);
  9501. }
  9502. }
  9503. unregister_netdevice_many(&dev_kill_list);
  9504. rtnl_unlock();
  9505. }
  9506. static struct pernet_operations __net_initdata default_device_ops = {
  9507. .exit = default_device_exit,
  9508. .exit_batch = default_device_exit_batch,
  9509. };
  9510. /*
  9511. * Initialize the DEV module. At boot time this walks the device list and
  9512. * unhooks any devices that fail to initialise (normally hardware not
  9513. * present) and leaves us with a valid list of present and active devices.
  9514. *
  9515. */
  9516. /*
  9517. * This is called single threaded during boot, so no need
  9518. * to take the rtnl semaphore.
  9519. */
  9520. static int __init net_dev_init(void)
  9521. {
  9522. int i, rc = -ENOMEM;
  9523. BUG_ON(!dev_boot_phase);
  9524. if (dev_proc_init())
  9525. goto out;
  9526. if (netdev_kobject_init())
  9527. goto out;
  9528. INIT_LIST_HEAD(&ptype_all);
  9529. for (i = 0; i < PTYPE_HASH_SIZE; i++)
  9530. INIT_LIST_HEAD(&ptype_base[i]);
  9531. INIT_LIST_HEAD(&offload_base);
  9532. if (register_pernet_subsys(&netdev_net_ops))
  9533. goto out;
  9534. /*
  9535. * Initialise the packet receive queues.
  9536. */
  9537. for_each_possible_cpu(i) {
  9538. struct work_struct *flush = per_cpu_ptr(&flush_works, i);
  9539. struct softnet_data *sd = &per_cpu(softnet_data, i);
  9540. INIT_WORK(flush, flush_backlog);
  9541. skb_queue_head_init(&sd->input_pkt_queue);
  9542. skb_queue_head_init(&sd->process_queue);
  9543. #ifdef CONFIG_XFRM_OFFLOAD
  9544. skb_queue_head_init(&sd->xfrm_backlog);
  9545. #endif
  9546. INIT_LIST_HEAD(&sd->poll_list);
  9547. sd->output_queue_tailp = &sd->output_queue;
  9548. #ifdef CONFIG_RPS
  9549. sd->csd.func = rps_trigger_softirq;
  9550. sd->csd.info = sd;
  9551. sd->cpu = i;
  9552. #endif
  9553. init_gro_hash(&sd->backlog);
  9554. sd->backlog.poll = process_backlog;
  9555. sd->backlog.weight = weight_p;
  9556. }
  9557. dev_boot_phase = 0;
  9558. /* The loopback device is special if any other network devices
  9559. * is present in a network namespace the loopback device must
  9560. * be present. Since we now dynamically allocate and free the
  9561. * loopback device ensure this invariant is maintained by
  9562. * keeping the loopback device as the first device on the
  9563. * list of network devices. Ensuring the loopback devices
  9564. * is the first device that appears and the last network device
  9565. * that disappears.
  9566. */
  9567. if (register_pernet_device(&loopback_net_ops))
  9568. goto out;
  9569. if (register_pernet_device(&default_device_ops))
  9570. goto out;
  9571. open_softirq(NET_TX_SOFTIRQ, net_tx_action);
  9572. open_softirq(NET_RX_SOFTIRQ, net_rx_action);
  9573. rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
  9574. NULL, dev_cpu_dead);
  9575. WARN_ON(rc < 0);
  9576. rc = 0;
  9577. out:
  9578. return rc;
  9579. }
  9580. subsys_initcall(net_dev_init);