skbuff.c 157 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Routines having to do with the 'struct sk_buff' memory handlers.
  4. *
  5. * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
  6. * Florian La Roche <rzsfl@rz.uni-sb.de>
  7. *
  8. * Fixes:
  9. * Alan Cox : Fixed the worst of the load
  10. * balancer bugs.
  11. * Dave Platt : Interrupt stacking fix.
  12. * Richard Kooijman : Timestamp fixes.
  13. * Alan Cox : Changed buffer format.
  14. * Alan Cox : destructor hook for AF_UNIX etc.
  15. * Linus Torvalds : Better skb_clone.
  16. * Alan Cox : Added skb_copy.
  17. * Alan Cox : Added all the changed routines Linus
  18. * only put in the headers
  19. * Ray VanTassle : Fixed --skb->lock in free
  20. * Alan Cox : skb_copy copy arp field
  21. * Andi Kleen : slabified it.
  22. * Robert Olsson : Removed skb_head_pool
  23. *
  24. * NOTE:
  25. * The __skb_ routines should be called with interrupts
  26. * disabled, or you better be *real* sure that the operation is atomic
  27. * with respect to whatever list is being frobbed (e.g. via lock_sock()
  28. * or via disabling bottom half handlers, etc).
  29. */
  30. /*
  31. * The functions in this file will not compile correctly with gcc 2.4.x
  32. */
  33. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34. #include <linux/module.h>
  35. #include <linux/types.h>
  36. #include <linux/kernel.h>
  37. #include <linux/mm.h>
  38. #include <linux/interrupt.h>
  39. #include <linux/in.h>
  40. #include <linux/inet.h>
  41. #include <linux/slab.h>
  42. #include <linux/tcp.h>
  43. #include <linux/udp.h>
  44. #include <linux/sctp.h>
  45. #include <linux/netdevice.h>
  46. #ifdef CONFIG_NET_CLS_ACT
  47. #include <net/pkt_sched.h>
  48. #endif
  49. #include <linux/string.h>
  50. #include <linux/skbuff.h>
  51. #include <linux/splice.h>
  52. #include <linux/cache.h>
  53. #include <linux/rtnetlink.h>
  54. #include <linux/init.h>
  55. #include <linux/scatterlist.h>
  56. #include <linux/errqueue.h>
  57. #include <linux/prefetch.h>
  58. #include <linux/if_vlan.h>
  59. #include <linux/mpls.h>
  60. #include <net/protocol.h>
  61. #include <net/dst.h>
  62. #include <net/sock.h>
  63. #include <net/checksum.h>
  64. #include <net/ip6_checksum.h>
  65. #include <net/xfrm.h>
  66. #include <net/mpls.h>
  67. #include <net/mptcp.h>
  68. #include <linux/uaccess.h>
  69. #include <trace/events/skb.h>
  70. #include <linux/highmem.h>
  71. #include <linux/capability.h>
  72. #include <linux/user_namespace.h>
  73. #include <linux/indirect_call_wrapper.h>
  74. #include <trace/hooks/net.h>
  75. #include "datagram.h"
  76. struct kmem_cache *skbuff_head_cache __ro_after_init;
  77. static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
  78. #ifdef CONFIG_SKB_EXTENSIONS
  79. static struct kmem_cache *skbuff_ext_cache __ro_after_init;
  80. #endif
  81. int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
  82. EXPORT_SYMBOL(sysctl_max_skb_frags);
  83. /**
  84. * skb_panic - private function for out-of-line support
  85. * @skb: buffer
  86. * @sz: size
  87. * @addr: address
  88. * @msg: skb_over_panic or skb_under_panic
  89. *
  90. * Out-of-line support for skb_put() and skb_push().
  91. * Called via the wrapper skb_over_panic() or skb_under_panic().
  92. * Keep out of line to prevent kernel bloat.
  93. * __builtin_return_address is not used because it is not always reliable.
  94. */
  95. static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
  96. const char msg[])
  97. {
  98. pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
  99. msg, addr, skb->len, sz, skb->head, skb->data,
  100. (unsigned long)skb->tail, (unsigned long)skb->end,
  101. skb->dev ? skb->dev->name : "<NULL>");
  102. BUG();
  103. }
  104. static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
  105. {
  106. skb_panic(skb, sz, addr, __func__);
  107. }
  108. static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
  109. {
  110. skb_panic(skb, sz, addr, __func__);
  111. }
  112. /*
  113. * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
  114. * the caller if emergency pfmemalloc reserves are being used. If it is and
  115. * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
  116. * may be used. Otherwise, the packet data may be discarded until enough
  117. * memory is free
  118. */
  119. #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
  120. __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
  121. static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
  122. unsigned long ip, bool *pfmemalloc)
  123. {
  124. void *obj;
  125. bool ret_pfmemalloc = false;
  126. /*
  127. * Try a regular allocation, when that fails and we're not entitled
  128. * to the reserves, fail.
  129. */
  130. obj = kmalloc_node_track_caller(size,
  131. flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
  132. node);
  133. if (obj || !(gfp_pfmemalloc_allowed(flags)))
  134. goto out;
  135. /* Try again but now we are using pfmemalloc reserves */
  136. ret_pfmemalloc = true;
  137. obj = kmalloc_node_track_caller(size, flags, node);
  138. out:
  139. if (pfmemalloc)
  140. *pfmemalloc = ret_pfmemalloc;
  141. return obj;
  142. }
  143. /* Allocate a new skbuff. We do this ourselves so we can fill in a few
  144. * 'private' fields and also do memory statistics to find all the
  145. * [BEEP] leaks.
  146. *
  147. */
  148. /**
  149. * __alloc_skb - allocate a network buffer
  150. * @size: size to allocate
  151. * @gfp_mask: allocation mask
  152. * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
  153. * instead of head cache and allocate a cloned (child) skb.
  154. * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
  155. * allocations in case the data is required for writeback
  156. * @node: numa node to allocate memory on
  157. *
  158. * Allocate a new &sk_buff. The returned buffer has no headroom and a
  159. * tail room of at least size bytes. The object has a reference count
  160. * of one. The return is the buffer. On a failure the return is %NULL.
  161. *
  162. * Buffers may only be allocated from interrupts using a @gfp_mask of
  163. * %GFP_ATOMIC.
  164. */
  165. struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
  166. int flags, int node)
  167. {
  168. struct kmem_cache *cache;
  169. struct skb_shared_info *shinfo;
  170. struct sk_buff *skb;
  171. u8 *data;
  172. bool pfmemalloc;
  173. cache = (flags & SKB_ALLOC_FCLONE)
  174. ? skbuff_fclone_cache : skbuff_head_cache;
  175. if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
  176. gfp_mask |= __GFP_MEMALLOC;
  177. /* Get the HEAD */
  178. skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
  179. if (!skb)
  180. goto out;
  181. prefetchw(skb);
  182. /* We do our best to align skb_shared_info on a separate cache
  183. * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
  184. * aligned memory blocks, unless SLUB/SLAB debug is enabled.
  185. * Both skb->head and skb_shared_info are cache line aligned.
  186. */
  187. size = SKB_DATA_ALIGN(size);
  188. size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  189. data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
  190. if (!data)
  191. goto nodata;
  192. /* kmalloc(size) might give us more room than requested.
  193. * Put skb_shared_info exactly at the end of allocated zone,
  194. * to allow max possible filling before reallocation.
  195. */
  196. size = SKB_WITH_OVERHEAD(ksize(data));
  197. prefetchw(data + size);
  198. /*
  199. * Only clear those fields we need to clear, not those that we will
  200. * actually initialise below. Hence, don't put any more fields after
  201. * the tail pointer in struct sk_buff!
  202. */
  203. memset(skb, 0, offsetof(struct sk_buff, tail));
  204. /* Account for allocated memory : skb + skb->head */
  205. skb->truesize = SKB_TRUESIZE(size);
  206. skb->pfmemalloc = pfmemalloc;
  207. refcount_set(&skb->users, 1);
  208. skb->head = data;
  209. skb->data = data;
  210. skb_reset_tail_pointer(skb);
  211. skb->end = skb->tail + size;
  212. skb->mac_header = (typeof(skb->mac_header))~0U;
  213. skb->transport_header = (typeof(skb->transport_header))~0U;
  214. /* make sure we initialize shinfo sequentially */
  215. shinfo = skb_shinfo(skb);
  216. memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
  217. atomic_set(&shinfo->dataref, 1);
  218. if (flags & SKB_ALLOC_FCLONE) {
  219. struct sk_buff_fclones *fclones;
  220. fclones = container_of(skb, struct sk_buff_fclones, skb1);
  221. skb->fclone = SKB_FCLONE_ORIG;
  222. refcount_set(&fclones->fclone_ref, 1);
  223. fclones->skb2.fclone = SKB_FCLONE_CLONE;
  224. }
  225. skb_set_kcov_handle(skb, kcov_common_handle());
  226. out:
  227. return skb;
  228. nodata:
  229. kmem_cache_free(cache, skb);
  230. skb = NULL;
  231. goto out;
  232. }
  233. EXPORT_SYMBOL(__alloc_skb);
  234. /* Caller must provide SKB that is memset cleared */
  235. static struct sk_buff *__build_skb_around(struct sk_buff *skb,
  236. void *data, unsigned int frag_size)
  237. {
  238. struct skb_shared_info *shinfo;
  239. unsigned int size = frag_size ? : ksize(data);
  240. size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  241. /* Assumes caller memset cleared SKB */
  242. skb->truesize = SKB_TRUESIZE(size);
  243. refcount_set(&skb->users, 1);
  244. skb->head = data;
  245. skb->data = data;
  246. skb_reset_tail_pointer(skb);
  247. skb->end = skb->tail + size;
  248. skb->mac_header = (typeof(skb->mac_header))~0U;
  249. skb->transport_header = (typeof(skb->transport_header))~0U;
  250. /* make sure we initialize shinfo sequentially */
  251. shinfo = skb_shinfo(skb);
  252. memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
  253. atomic_set(&shinfo->dataref, 1);
  254. skb_set_kcov_handle(skb, kcov_common_handle());
  255. return skb;
  256. }
  257. /**
  258. * __build_skb - build a network buffer
  259. * @data: data buffer provided by caller
  260. * @frag_size: size of data, or 0 if head was kmalloced
  261. *
  262. * Allocate a new &sk_buff. Caller provides space holding head and
  263. * skb_shared_info. @data must have been allocated by kmalloc() only if
  264. * @frag_size is 0, otherwise data should come from the page allocator
  265. * or vmalloc()
  266. * The return is the new skb buffer.
  267. * On a failure the return is %NULL, and @data is not freed.
  268. * Notes :
  269. * Before IO, driver allocates only data buffer where NIC put incoming frame
  270. * Driver should add room at head (NET_SKB_PAD) and
  271. * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
  272. * After IO, driver calls build_skb(), to allocate sk_buff and populate it
  273. * before giving packet to stack.
  274. * RX rings only contains data buffers, not full skbs.
  275. */
  276. struct sk_buff *__build_skb(void *data, unsigned int frag_size)
  277. {
  278. struct sk_buff *skb;
  279. skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
  280. if (unlikely(!skb))
  281. return NULL;
  282. memset(skb, 0, offsetof(struct sk_buff, tail));
  283. return __build_skb_around(skb, data, frag_size);
  284. }
  285. /* build_skb() is wrapper over __build_skb(), that specifically
  286. * takes care of skb->head and skb->pfmemalloc
  287. * This means that if @frag_size is not zero, then @data must be backed
  288. * by a page fragment, not kmalloc() or vmalloc()
  289. */
  290. struct sk_buff *build_skb(void *data, unsigned int frag_size)
  291. {
  292. struct sk_buff *skb = __build_skb(data, frag_size);
  293. if (skb && frag_size) {
  294. skb->head_frag = 1;
  295. if (page_is_pfmemalloc(virt_to_head_page(data)))
  296. skb->pfmemalloc = 1;
  297. }
  298. return skb;
  299. }
  300. EXPORT_SYMBOL(build_skb);
  301. /**
  302. * build_skb_around - build a network buffer around provided skb
  303. * @skb: sk_buff provide by caller, must be memset cleared
  304. * @data: data buffer provided by caller
  305. * @frag_size: size of data, or 0 if head was kmalloced
  306. */
  307. struct sk_buff *build_skb_around(struct sk_buff *skb,
  308. void *data, unsigned int frag_size)
  309. {
  310. if (unlikely(!skb))
  311. return NULL;
  312. skb = __build_skb_around(skb, data, frag_size);
  313. if (skb && frag_size) {
  314. skb->head_frag = 1;
  315. if (page_is_pfmemalloc(virt_to_head_page(data)))
  316. skb->pfmemalloc = 1;
  317. }
  318. return skb;
  319. }
  320. EXPORT_SYMBOL(build_skb_around);
  321. #define NAPI_SKB_CACHE_SIZE 64
  322. struct napi_alloc_cache {
  323. struct page_frag_cache page;
  324. unsigned int skb_count;
  325. void *skb_cache[NAPI_SKB_CACHE_SIZE];
  326. };
  327. static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
  328. static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
  329. static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
  330. {
  331. struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  332. return page_frag_alloc(&nc->page, fragsz, gfp_mask);
  333. }
  334. void *napi_alloc_frag(unsigned int fragsz)
  335. {
  336. fragsz = SKB_DATA_ALIGN(fragsz);
  337. return __napi_alloc_frag(fragsz, GFP_ATOMIC);
  338. }
  339. EXPORT_SYMBOL(napi_alloc_frag);
  340. /**
  341. * netdev_alloc_frag - allocate a page fragment
  342. * @fragsz: fragment size
  343. *
  344. * Allocates a frag from a page for receive buffer.
  345. * Uses GFP_ATOMIC allocations.
  346. */
  347. void *netdev_alloc_frag(unsigned int fragsz)
  348. {
  349. struct page_frag_cache *nc;
  350. void *data;
  351. fragsz = SKB_DATA_ALIGN(fragsz);
  352. if (in_irq() || irqs_disabled()) {
  353. nc = this_cpu_ptr(&netdev_alloc_cache);
  354. data = page_frag_alloc(nc, fragsz, GFP_ATOMIC);
  355. } else {
  356. local_bh_disable();
  357. data = __napi_alloc_frag(fragsz, GFP_ATOMIC);
  358. local_bh_enable();
  359. }
  360. return data;
  361. }
  362. EXPORT_SYMBOL(netdev_alloc_frag);
  363. /**
  364. * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
  365. * @dev: network device to receive on
  366. * @len: length to allocate
  367. * @gfp_mask: get_free_pages mask, passed to alloc_skb
  368. *
  369. * Allocate a new &sk_buff and assign it a usage count of one. The
  370. * buffer has NET_SKB_PAD headroom built in. Users should allocate
  371. * the headroom they think they need without accounting for the
  372. * built in space. The built in space is used for optimisations.
  373. *
  374. * %NULL is returned if there is no free memory.
  375. */
  376. struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
  377. gfp_t gfp_mask)
  378. {
  379. struct page_frag_cache *nc;
  380. struct sk_buff *skb;
  381. bool pfmemalloc;
  382. void *data;
  383. len += NET_SKB_PAD;
  384. /* If requested length is either too small or too big,
  385. * we use kmalloc() for skb->head allocation.
  386. */
  387. if (len <= SKB_WITH_OVERHEAD(1024) ||
  388. len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
  389. (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
  390. skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
  391. if (!skb)
  392. goto skb_fail;
  393. goto skb_success;
  394. }
  395. len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  396. len = SKB_DATA_ALIGN(len);
  397. if (sk_memalloc_socks())
  398. gfp_mask |= __GFP_MEMALLOC;
  399. if (in_irq() || irqs_disabled()) {
  400. nc = this_cpu_ptr(&netdev_alloc_cache);
  401. data = page_frag_alloc(nc, len, gfp_mask);
  402. pfmemalloc = nc->pfmemalloc;
  403. } else {
  404. local_bh_disable();
  405. nc = this_cpu_ptr(&napi_alloc_cache.page);
  406. data = page_frag_alloc(nc, len, gfp_mask);
  407. pfmemalloc = nc->pfmemalloc;
  408. local_bh_enable();
  409. }
  410. if (unlikely(!data))
  411. return NULL;
  412. skb = __build_skb(data, len);
  413. if (unlikely(!skb)) {
  414. skb_free_frag(data);
  415. return NULL;
  416. }
  417. if (pfmemalloc)
  418. skb->pfmemalloc = 1;
  419. skb->head_frag = 1;
  420. skb_success:
  421. skb_reserve(skb, NET_SKB_PAD);
  422. skb->dev = dev;
  423. skb_fail:
  424. return skb;
  425. }
  426. EXPORT_SYMBOL(__netdev_alloc_skb);
  427. /**
  428. * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
  429. * @napi: napi instance this buffer was allocated for
  430. * @len: length to allocate
  431. * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
  432. *
  433. * Allocate a new sk_buff for use in NAPI receive. This buffer will
  434. * attempt to allocate the head from a special reserved region used
  435. * only for NAPI Rx allocation. By doing this we can save several
  436. * CPU cycles by avoiding having to disable and re-enable IRQs.
  437. *
  438. * %NULL is returned if there is no free memory.
  439. */
  440. struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
  441. gfp_t gfp_mask)
  442. {
  443. struct napi_alloc_cache *nc;
  444. struct sk_buff *skb;
  445. void *data;
  446. len += NET_SKB_PAD + NET_IP_ALIGN;
  447. /* If requested length is either too small or too big,
  448. * we use kmalloc() for skb->head allocation.
  449. */
  450. if (len <= SKB_WITH_OVERHEAD(1024) ||
  451. len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
  452. (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
  453. skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
  454. if (!skb)
  455. goto skb_fail;
  456. goto skb_success;
  457. }
  458. nc = this_cpu_ptr(&napi_alloc_cache);
  459. len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  460. len = SKB_DATA_ALIGN(len);
  461. if (sk_memalloc_socks())
  462. gfp_mask |= __GFP_MEMALLOC;
  463. data = page_frag_alloc(&nc->page, len, gfp_mask);
  464. if (unlikely(!data))
  465. return NULL;
  466. skb = __build_skb(data, len);
  467. if (unlikely(!skb)) {
  468. skb_free_frag(data);
  469. return NULL;
  470. }
  471. if (nc->page.pfmemalloc)
  472. skb->pfmemalloc = 1;
  473. skb->head_frag = 1;
  474. skb_success:
  475. skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
  476. skb->dev = napi->dev;
  477. skb_fail:
  478. return skb;
  479. }
  480. EXPORT_SYMBOL(__napi_alloc_skb);
  481. void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
  482. int size, unsigned int truesize)
  483. {
  484. skb_fill_page_desc(skb, i, page, off, size);
  485. skb->len += size;
  486. skb->data_len += size;
  487. skb->truesize += truesize;
  488. }
  489. EXPORT_SYMBOL(skb_add_rx_frag);
  490. void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
  491. unsigned int truesize)
  492. {
  493. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  494. skb_frag_size_add(frag, size);
  495. skb->len += size;
  496. skb->data_len += size;
  497. skb->truesize += truesize;
  498. }
  499. EXPORT_SYMBOL(skb_coalesce_rx_frag);
  500. static void skb_drop_list(struct sk_buff **listp)
  501. {
  502. kfree_skb_list(*listp);
  503. *listp = NULL;
  504. }
  505. static inline void skb_drop_fraglist(struct sk_buff *skb)
  506. {
  507. skb_drop_list(&skb_shinfo(skb)->frag_list);
  508. }
  509. static void skb_clone_fraglist(struct sk_buff *skb)
  510. {
  511. struct sk_buff *list;
  512. skb_walk_frags(skb, list)
  513. skb_get(list);
  514. }
  515. static void skb_free_head(struct sk_buff *skb)
  516. {
  517. unsigned char *head = skb->head;
  518. if (skb->head_frag)
  519. skb_free_frag(head);
  520. else
  521. kfree(head);
  522. }
  523. static void skb_release_data(struct sk_buff *skb)
  524. {
  525. struct skb_shared_info *shinfo = skb_shinfo(skb);
  526. int i;
  527. if (skb->cloned &&
  528. atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
  529. &shinfo->dataref))
  530. return;
  531. for (i = 0; i < shinfo->nr_frags; i++)
  532. __skb_frag_unref(&shinfo->frags[i]);
  533. if (shinfo->frag_list)
  534. kfree_skb_list(shinfo->frag_list);
  535. skb_zcopy_clear(skb, true);
  536. skb_free_head(skb);
  537. }
  538. /*
  539. * Free an skbuff by memory without cleaning the state.
  540. */
  541. static void kfree_skbmem(struct sk_buff *skb)
  542. {
  543. struct sk_buff_fclones *fclones;
  544. switch (skb->fclone) {
  545. case SKB_FCLONE_UNAVAILABLE:
  546. kmem_cache_free(skbuff_head_cache, skb);
  547. return;
  548. case SKB_FCLONE_ORIG:
  549. fclones = container_of(skb, struct sk_buff_fclones, skb1);
  550. /* We usually free the clone (TX completion) before original skb
  551. * This test would have no chance to be true for the clone,
  552. * while here, branch prediction will be good.
  553. */
  554. if (refcount_read(&fclones->fclone_ref) == 1)
  555. goto fastpath;
  556. break;
  557. default: /* SKB_FCLONE_CLONE */
  558. fclones = container_of(skb, struct sk_buff_fclones, skb2);
  559. break;
  560. }
  561. if (!refcount_dec_and_test(&fclones->fclone_ref))
  562. return;
  563. fastpath:
  564. kmem_cache_free(skbuff_fclone_cache, fclones);
  565. }
  566. void skb_release_head_state(struct sk_buff *skb)
  567. {
  568. nf_reset_ct(skb);
  569. skb_dst_drop(skb);
  570. if (skb->destructor) {
  571. WARN_ON(in_irq());
  572. skb->destructor(skb);
  573. }
  574. #if IS_ENABLED(CONFIG_NF_CONNTRACK)
  575. nf_conntrack_put(skb_nfct(skb));
  576. #endif
  577. skb_ext_put(skb);
  578. }
  579. /* Free everything but the sk_buff shell. */
  580. static void skb_release_all(struct sk_buff *skb)
  581. {
  582. skb_release_head_state(skb);
  583. if (likely(skb->head))
  584. skb_release_data(skb);
  585. }
  586. /**
  587. * __kfree_skb - private function
  588. * @skb: buffer
  589. *
  590. * Free an sk_buff. Release anything attached to the buffer.
  591. * Clean the state. This is an internal helper function. Users should
  592. * always call kfree_skb
  593. */
  594. void __kfree_skb(struct sk_buff *skb)
  595. {
  596. skb_release_all(skb);
  597. kfree_skbmem(skb);
  598. }
  599. EXPORT_SYMBOL(__kfree_skb);
  600. /**
  601. * kfree_skb - free an sk_buff
  602. * @skb: buffer to free
  603. *
  604. * Drop a reference to the buffer and free it if the usage count has
  605. * hit zero.
  606. */
  607. void kfree_skb(struct sk_buff *skb)
  608. {
  609. if (!skb_unref(skb))
  610. return;
  611. trace_android_vh_kfree_skb(skb);
  612. trace_kfree_skb(skb, __builtin_return_address(0));
  613. __kfree_skb(skb);
  614. }
  615. EXPORT_SYMBOL(kfree_skb);
  616. void kfree_skb_list(struct sk_buff *segs)
  617. {
  618. while (segs) {
  619. struct sk_buff *next = segs->next;
  620. kfree_skb(segs);
  621. segs = next;
  622. }
  623. }
  624. EXPORT_SYMBOL(kfree_skb_list);
  625. /* Dump skb information and contents.
  626. *
  627. * Must only be called from net_ratelimit()-ed paths.
  628. *
  629. * Dumps whole packets if full_pkt, only headers otherwise.
  630. */
  631. void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
  632. {
  633. struct skb_shared_info *sh = skb_shinfo(skb);
  634. struct net_device *dev = skb->dev;
  635. struct sock *sk = skb->sk;
  636. struct sk_buff *list_skb;
  637. bool has_mac, has_trans;
  638. int headroom, tailroom;
  639. int i, len, seg_len;
  640. if (full_pkt)
  641. len = skb->len;
  642. else
  643. len = min_t(int, skb->len, MAX_HEADER + 128);
  644. headroom = skb_headroom(skb);
  645. tailroom = skb_tailroom(skb);
  646. has_mac = skb_mac_header_was_set(skb);
  647. has_trans = skb_transport_header_was_set(skb);
  648. printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
  649. "mac=(%d,%d) net=(%d,%d) trans=%d\n"
  650. "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
  651. "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
  652. "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
  653. level, skb->len, headroom, skb_headlen(skb), tailroom,
  654. has_mac ? skb->mac_header : -1,
  655. has_mac ? skb_mac_header_len(skb) : -1,
  656. skb->network_header,
  657. has_trans ? skb_network_header_len(skb) : -1,
  658. has_trans ? skb->transport_header : -1,
  659. sh->tx_flags, sh->nr_frags,
  660. sh->gso_size, sh->gso_type, sh->gso_segs,
  661. skb->csum, skb->ip_summed, skb->csum_complete_sw,
  662. skb->csum_valid, skb->csum_level,
  663. skb->hash, skb->sw_hash, skb->l4_hash,
  664. ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
  665. if (dev)
  666. printk("%sdev name=%s feat=%pNF\n",
  667. level, dev->name, &dev->features);
  668. if (sk)
  669. printk("%ssk family=%hu type=%u proto=%u\n",
  670. level, sk->sk_family, sk->sk_type, sk->sk_protocol);
  671. if (full_pkt && headroom)
  672. print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
  673. 16, 1, skb->head, headroom, false);
  674. seg_len = min_t(int, skb_headlen(skb), len);
  675. if (seg_len)
  676. print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
  677. 16, 1, skb->data, seg_len, false);
  678. len -= seg_len;
  679. if (full_pkt && tailroom)
  680. print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
  681. 16, 1, skb_tail_pointer(skb), tailroom, false);
  682. for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
  683. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  684. u32 p_off, p_len, copied;
  685. struct page *p;
  686. u8 *vaddr;
  687. skb_frag_foreach_page(frag, skb_frag_off(frag),
  688. skb_frag_size(frag), p, p_off, p_len,
  689. copied) {
  690. seg_len = min_t(int, p_len, len);
  691. vaddr = kmap_atomic(p);
  692. print_hex_dump(level, "skb frag: ",
  693. DUMP_PREFIX_OFFSET,
  694. 16, 1, vaddr + p_off, seg_len, false);
  695. kunmap_atomic(vaddr);
  696. len -= seg_len;
  697. if (!len)
  698. break;
  699. }
  700. }
  701. if (full_pkt && skb_has_frag_list(skb)) {
  702. printk("skb fraglist:\n");
  703. skb_walk_frags(skb, list_skb)
  704. skb_dump(level, list_skb, true);
  705. }
  706. }
  707. EXPORT_SYMBOL(skb_dump);
  708. /**
  709. * skb_tx_error - report an sk_buff xmit error
  710. * @skb: buffer that triggered an error
  711. *
  712. * Report xmit error if a device callback is tracking this skb.
  713. * skb must be freed afterwards.
  714. */
  715. void skb_tx_error(struct sk_buff *skb)
  716. {
  717. skb_zcopy_clear(skb, true);
  718. }
  719. EXPORT_SYMBOL(skb_tx_error);
  720. #ifdef CONFIG_TRACEPOINTS
  721. /**
  722. * consume_skb - free an skbuff
  723. * @skb: buffer to free
  724. *
  725. * Drop a ref to the buffer and free it if the usage count has hit zero
  726. * Functions identically to kfree_skb, but kfree_skb assumes that the frame
  727. * is being dropped after a failure and notes that
  728. */
  729. void consume_skb(struct sk_buff *skb)
  730. {
  731. if (!skb_unref(skb))
  732. return;
  733. trace_consume_skb(skb);
  734. __kfree_skb(skb);
  735. }
  736. EXPORT_SYMBOL(consume_skb);
  737. #endif
  738. /**
  739. * consume_stateless_skb - free an skbuff, assuming it is stateless
  740. * @skb: buffer to free
  741. *
  742. * Alike consume_skb(), but this variant assumes that this is the last
  743. * skb reference and all the head states have been already dropped
  744. */
  745. void __consume_stateless_skb(struct sk_buff *skb)
  746. {
  747. trace_consume_skb(skb);
  748. skb_release_data(skb);
  749. kfree_skbmem(skb);
  750. }
  751. void __kfree_skb_flush(void)
  752. {
  753. struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  754. /* flush skb_cache if containing objects */
  755. if (nc->skb_count) {
  756. kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
  757. nc->skb_cache);
  758. nc->skb_count = 0;
  759. }
  760. }
  761. static inline void _kfree_skb_defer(struct sk_buff *skb)
  762. {
  763. struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
  764. /* drop skb->head and call any destructors for packet */
  765. skb_release_all(skb);
  766. /* record skb to CPU local list */
  767. nc->skb_cache[nc->skb_count++] = skb;
  768. #ifdef CONFIG_SLUB
  769. /* SLUB writes into objects when freeing */
  770. prefetchw(skb);
  771. #endif
  772. /* flush skb_cache if it is filled */
  773. if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
  774. kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
  775. nc->skb_cache);
  776. nc->skb_count = 0;
  777. }
  778. }
  779. void __kfree_skb_defer(struct sk_buff *skb)
  780. {
  781. _kfree_skb_defer(skb);
  782. }
  783. void napi_consume_skb(struct sk_buff *skb, int budget)
  784. {
  785. /* Zero budget indicate non-NAPI context called us, like netpoll */
  786. if (unlikely(!budget)) {
  787. dev_consume_skb_any(skb);
  788. return;
  789. }
  790. if (!skb_unref(skb))
  791. return;
  792. /* if reaching here SKB is ready to free */
  793. trace_consume_skb(skb);
  794. /* if SKB is a clone, don't handle this case */
  795. if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
  796. __kfree_skb(skb);
  797. return;
  798. }
  799. _kfree_skb_defer(skb);
  800. }
  801. EXPORT_SYMBOL(napi_consume_skb);
  802. /* Make sure a field is enclosed inside headers_start/headers_end section */
  803. #define CHECK_SKB_FIELD(field) \
  804. BUILD_BUG_ON(offsetof(struct sk_buff, field) < \
  805. offsetof(struct sk_buff, headers_start)); \
  806. BUILD_BUG_ON(offsetof(struct sk_buff, field) > \
  807. offsetof(struct sk_buff, headers_end)); \
  808. static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
  809. {
  810. new->tstamp = old->tstamp;
  811. /* We do not copy old->sk */
  812. new->dev = old->dev;
  813. memcpy(new->cb, old->cb, sizeof(old->cb));
  814. skb_dst_copy(new, old);
  815. __skb_ext_copy(new, old);
  816. __nf_copy(new, old, false);
  817. /* Note : this field could be in headers_start/headers_end section
  818. * It is not yet because we do not want to have a 16 bit hole
  819. */
  820. new->queue_mapping = old->queue_mapping;
  821. memcpy(&new->headers_start, &old->headers_start,
  822. offsetof(struct sk_buff, headers_end) -
  823. offsetof(struct sk_buff, headers_start));
  824. CHECK_SKB_FIELD(protocol);
  825. CHECK_SKB_FIELD(csum);
  826. CHECK_SKB_FIELD(hash);
  827. CHECK_SKB_FIELD(priority);
  828. CHECK_SKB_FIELD(skb_iif);
  829. CHECK_SKB_FIELD(vlan_proto);
  830. CHECK_SKB_FIELD(vlan_tci);
  831. CHECK_SKB_FIELD(transport_header);
  832. CHECK_SKB_FIELD(network_header);
  833. CHECK_SKB_FIELD(mac_header);
  834. CHECK_SKB_FIELD(inner_protocol);
  835. CHECK_SKB_FIELD(inner_transport_header);
  836. CHECK_SKB_FIELD(inner_network_header);
  837. CHECK_SKB_FIELD(inner_mac_header);
  838. CHECK_SKB_FIELD(mark);
  839. #ifdef CONFIG_NETWORK_SECMARK
  840. CHECK_SKB_FIELD(secmark);
  841. #endif
  842. #ifdef CONFIG_NET_RX_BUSY_POLL
  843. CHECK_SKB_FIELD(napi_id);
  844. #endif
  845. #ifdef CONFIG_XPS
  846. CHECK_SKB_FIELD(sender_cpu);
  847. #endif
  848. #ifdef CONFIG_NET_SCHED
  849. CHECK_SKB_FIELD(tc_index);
  850. #endif
  851. }
  852. /*
  853. * You should not add any new code to this function. Add it to
  854. * __copy_skb_header above instead.
  855. */
  856. static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
  857. {
  858. #define C(x) n->x = skb->x
  859. n->next = n->prev = NULL;
  860. n->sk = NULL;
  861. __copy_skb_header(n, skb);
  862. C(len);
  863. C(data_len);
  864. C(mac_len);
  865. n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
  866. n->cloned = 1;
  867. n->nohdr = 0;
  868. n->peeked = 0;
  869. C(pfmemalloc);
  870. n->destructor = NULL;
  871. C(tail);
  872. C(end);
  873. C(head);
  874. C(head_frag);
  875. C(data);
  876. C(truesize);
  877. refcount_set(&n->users, 1);
  878. atomic_inc(&(skb_shinfo(skb)->dataref));
  879. skb->cloned = 1;
  880. return n;
  881. #undef C
  882. }
  883. /**
  884. * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
  885. * @first: first sk_buff of the msg
  886. */
  887. struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
  888. {
  889. struct sk_buff *n;
  890. n = alloc_skb(0, GFP_ATOMIC);
  891. if (!n)
  892. return NULL;
  893. n->len = first->len;
  894. n->data_len = first->len;
  895. n->truesize = first->truesize;
  896. skb_shinfo(n)->frag_list = first;
  897. __copy_skb_header(n, first);
  898. n->destructor = NULL;
  899. return n;
  900. }
  901. EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
  902. /**
  903. * skb_morph - morph one skb into another
  904. * @dst: the skb to receive the contents
  905. * @src: the skb to supply the contents
  906. *
  907. * This is identical to skb_clone except that the target skb is
  908. * supplied by the user.
  909. *
  910. * The target skb is returned upon exit.
  911. */
  912. struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
  913. {
  914. skb_release_all(dst);
  915. return __skb_clone(dst, src);
  916. }
  917. EXPORT_SYMBOL_GPL(skb_morph);
  918. int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
  919. {
  920. unsigned long max_pg, num_pg, new_pg, old_pg;
  921. struct user_struct *user;
  922. if (capable(CAP_IPC_LOCK) || !size)
  923. return 0;
  924. num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
  925. max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  926. user = mmp->user ? : current_user();
  927. do {
  928. old_pg = atomic_long_read(&user->locked_vm);
  929. new_pg = old_pg + num_pg;
  930. if (new_pg > max_pg)
  931. return -ENOBUFS;
  932. } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
  933. old_pg);
  934. if (!mmp->user) {
  935. mmp->user = get_uid(user);
  936. mmp->num_pg = num_pg;
  937. } else {
  938. mmp->num_pg += num_pg;
  939. }
  940. return 0;
  941. }
  942. EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
  943. void mm_unaccount_pinned_pages(struct mmpin *mmp)
  944. {
  945. if (mmp->user) {
  946. atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
  947. free_uid(mmp->user);
  948. }
  949. }
  950. EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
  951. struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
  952. {
  953. struct ubuf_info *uarg;
  954. struct sk_buff *skb;
  955. WARN_ON_ONCE(!in_task());
  956. skb = sock_omalloc(sk, 0, GFP_KERNEL);
  957. if (!skb)
  958. return NULL;
  959. BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
  960. uarg = (void *)skb->cb;
  961. uarg->mmp.user = NULL;
  962. if (mm_account_pinned_pages(&uarg->mmp, size)) {
  963. kfree_skb(skb);
  964. return NULL;
  965. }
  966. uarg->callback = sock_zerocopy_callback;
  967. uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
  968. uarg->len = 1;
  969. uarg->bytelen = size;
  970. uarg->zerocopy = 1;
  971. refcount_set(&uarg->refcnt, 1);
  972. sock_hold(sk);
  973. return uarg;
  974. }
  975. EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
  976. static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
  977. {
  978. return container_of((void *)uarg, struct sk_buff, cb);
  979. }
  980. struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
  981. struct ubuf_info *uarg)
  982. {
  983. if (uarg) {
  984. const u32 byte_limit = 1 << 19; /* limit to a few TSO */
  985. u32 bytelen, next;
  986. /* realloc only when socket is locked (TCP, UDP cork),
  987. * so uarg->len and sk_zckey access is serialized
  988. */
  989. if (!sock_owned_by_user(sk)) {
  990. WARN_ON_ONCE(1);
  991. return NULL;
  992. }
  993. bytelen = uarg->bytelen + size;
  994. if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
  995. /* TCP can create new skb to attach new uarg */
  996. if (sk->sk_type == SOCK_STREAM)
  997. goto new_alloc;
  998. return NULL;
  999. }
  1000. next = (u32)atomic_read(&sk->sk_zckey);
  1001. if ((u32)(uarg->id + uarg->len) == next) {
  1002. if (mm_account_pinned_pages(&uarg->mmp, size))
  1003. return NULL;
  1004. uarg->len++;
  1005. uarg->bytelen = bytelen;
  1006. atomic_set(&sk->sk_zckey, ++next);
  1007. /* no extra ref when appending to datagram (MSG_MORE) */
  1008. if (sk->sk_type == SOCK_STREAM)
  1009. sock_zerocopy_get(uarg);
  1010. return uarg;
  1011. }
  1012. }
  1013. new_alloc:
  1014. return sock_zerocopy_alloc(sk, size);
  1015. }
  1016. EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
  1017. static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
  1018. {
  1019. struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
  1020. u32 old_lo, old_hi;
  1021. u64 sum_len;
  1022. old_lo = serr->ee.ee_info;
  1023. old_hi = serr->ee.ee_data;
  1024. sum_len = old_hi - old_lo + 1ULL + len;
  1025. if (sum_len >= (1ULL << 32))
  1026. return false;
  1027. if (lo != old_hi + 1)
  1028. return false;
  1029. serr->ee.ee_data += len;
  1030. return true;
  1031. }
  1032. void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
  1033. {
  1034. struct sk_buff *tail, *skb = skb_from_uarg(uarg);
  1035. struct sock_exterr_skb *serr;
  1036. struct sock *sk = skb->sk;
  1037. struct sk_buff_head *q;
  1038. unsigned long flags;
  1039. u32 lo, hi;
  1040. u16 len;
  1041. mm_unaccount_pinned_pages(&uarg->mmp);
  1042. /* if !len, there was only 1 call, and it was aborted
  1043. * so do not queue a completion notification
  1044. */
  1045. if (!uarg->len || sock_flag(sk, SOCK_DEAD))
  1046. goto release;
  1047. len = uarg->len;
  1048. lo = uarg->id;
  1049. hi = uarg->id + len - 1;
  1050. serr = SKB_EXT_ERR(skb);
  1051. memset(serr, 0, sizeof(*serr));
  1052. serr->ee.ee_errno = 0;
  1053. serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
  1054. serr->ee.ee_data = hi;
  1055. serr->ee.ee_info = lo;
  1056. if (!success)
  1057. serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
  1058. q = &sk->sk_error_queue;
  1059. spin_lock_irqsave(&q->lock, flags);
  1060. tail = skb_peek_tail(q);
  1061. if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
  1062. !skb_zerocopy_notify_extend(tail, lo, len)) {
  1063. __skb_queue_tail(q, skb);
  1064. skb = NULL;
  1065. }
  1066. spin_unlock_irqrestore(&q->lock, flags);
  1067. sk->sk_error_report(sk);
  1068. release:
  1069. consume_skb(skb);
  1070. sock_put(sk);
  1071. }
  1072. EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
  1073. void sock_zerocopy_put(struct ubuf_info *uarg)
  1074. {
  1075. if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
  1076. if (uarg->callback)
  1077. uarg->callback(uarg, uarg->zerocopy);
  1078. else
  1079. consume_skb(skb_from_uarg(uarg));
  1080. }
  1081. }
  1082. EXPORT_SYMBOL_GPL(sock_zerocopy_put);
  1083. void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
  1084. {
  1085. if (uarg) {
  1086. struct sock *sk = skb_from_uarg(uarg)->sk;
  1087. atomic_dec(&sk->sk_zckey);
  1088. uarg->len--;
  1089. if (have_uref)
  1090. sock_zerocopy_put(uarg);
  1091. }
  1092. }
  1093. EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
  1094. int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
  1095. {
  1096. return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
  1097. }
  1098. EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
  1099. int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
  1100. struct msghdr *msg, int len,
  1101. struct ubuf_info *uarg)
  1102. {
  1103. struct ubuf_info *orig_uarg = skb_zcopy(skb);
  1104. struct iov_iter orig_iter = msg->msg_iter;
  1105. int err, orig_len = skb->len;
  1106. /* An skb can only point to one uarg. This edge case happens when
  1107. * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
  1108. */
  1109. if (orig_uarg && uarg != orig_uarg)
  1110. return -EEXIST;
  1111. err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
  1112. if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
  1113. struct sock *save_sk = skb->sk;
  1114. /* Streams do not free skb on error. Reset to prev state. */
  1115. msg->msg_iter = orig_iter;
  1116. skb->sk = sk;
  1117. ___pskb_trim(skb, orig_len);
  1118. skb->sk = save_sk;
  1119. return err;
  1120. }
  1121. skb_zcopy_set(skb, uarg, NULL);
  1122. return skb->len - orig_len;
  1123. }
  1124. EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
  1125. static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
  1126. gfp_t gfp_mask)
  1127. {
  1128. if (skb_zcopy(orig)) {
  1129. if (skb_zcopy(nskb)) {
  1130. /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
  1131. if (!gfp_mask) {
  1132. WARN_ON_ONCE(1);
  1133. return -ENOMEM;
  1134. }
  1135. if (skb_uarg(nskb) == skb_uarg(orig))
  1136. return 0;
  1137. if (skb_copy_ubufs(nskb, GFP_ATOMIC))
  1138. return -EIO;
  1139. }
  1140. skb_zcopy_set(nskb, skb_uarg(orig), NULL);
  1141. }
  1142. return 0;
  1143. }
  1144. /**
  1145. * skb_copy_ubufs - copy userspace skb frags buffers to kernel
  1146. * @skb: the skb to modify
  1147. * @gfp_mask: allocation priority
  1148. *
  1149. * This must be called on SKBTX_DEV_ZEROCOPY skb.
  1150. * It will copy all frags into kernel and drop the reference
  1151. * to userspace pages.
  1152. *
  1153. * If this function is called from an interrupt gfp_mask() must be
  1154. * %GFP_ATOMIC.
  1155. *
  1156. * Returns 0 on success or a negative error code on failure
  1157. * to allocate kernel memory to copy to.
  1158. */
  1159. int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
  1160. {
  1161. int num_frags = skb_shinfo(skb)->nr_frags;
  1162. struct page *page, *head = NULL;
  1163. int i, new_frags;
  1164. u32 d_off;
  1165. if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
  1166. return -EINVAL;
  1167. if (!num_frags)
  1168. goto release;
  1169. new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1170. for (i = 0; i < new_frags; i++) {
  1171. page = alloc_page(gfp_mask);
  1172. if (!page) {
  1173. while (head) {
  1174. struct page *next = (struct page *)page_private(head);
  1175. put_page(head);
  1176. head = next;
  1177. }
  1178. return -ENOMEM;
  1179. }
  1180. set_page_private(page, (unsigned long)head);
  1181. head = page;
  1182. }
  1183. page = head;
  1184. d_off = 0;
  1185. for (i = 0; i < num_frags; i++) {
  1186. skb_frag_t *f = &skb_shinfo(skb)->frags[i];
  1187. u32 p_off, p_len, copied;
  1188. struct page *p;
  1189. u8 *vaddr;
  1190. skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
  1191. p, p_off, p_len, copied) {
  1192. u32 copy, done = 0;
  1193. vaddr = kmap_atomic(p);
  1194. while (done < p_len) {
  1195. if (d_off == PAGE_SIZE) {
  1196. d_off = 0;
  1197. page = (struct page *)page_private(page);
  1198. }
  1199. copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
  1200. memcpy(page_address(page) + d_off,
  1201. vaddr + p_off + done, copy);
  1202. done += copy;
  1203. d_off += copy;
  1204. }
  1205. kunmap_atomic(vaddr);
  1206. }
  1207. }
  1208. /* skb frags release userspace buffers */
  1209. for (i = 0; i < num_frags; i++)
  1210. skb_frag_unref(skb, i);
  1211. /* skb frags point to kernel buffers */
  1212. for (i = 0; i < new_frags - 1; i++) {
  1213. __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
  1214. head = (struct page *)page_private(head);
  1215. }
  1216. __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
  1217. skb_shinfo(skb)->nr_frags = new_frags;
  1218. release:
  1219. skb_zcopy_clear(skb, false);
  1220. return 0;
  1221. }
  1222. EXPORT_SYMBOL_GPL(skb_copy_ubufs);
  1223. /**
  1224. * skb_clone - duplicate an sk_buff
  1225. * @skb: buffer to clone
  1226. * @gfp_mask: allocation priority
  1227. *
  1228. * Duplicate an &sk_buff. The new one is not owned by a socket. Both
  1229. * copies share the same packet data but not structure. The new
  1230. * buffer has a reference count of 1. If the allocation fails the
  1231. * function returns %NULL otherwise the new buffer is returned.
  1232. *
  1233. * If this function is called from an interrupt gfp_mask() must be
  1234. * %GFP_ATOMIC.
  1235. */
  1236. struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
  1237. {
  1238. struct sk_buff_fclones *fclones = container_of(skb,
  1239. struct sk_buff_fclones,
  1240. skb1);
  1241. struct sk_buff *n;
  1242. if (skb_orphan_frags(skb, gfp_mask))
  1243. return NULL;
  1244. if (skb->fclone == SKB_FCLONE_ORIG &&
  1245. refcount_read(&fclones->fclone_ref) == 1) {
  1246. n = &fclones->skb2;
  1247. refcount_set(&fclones->fclone_ref, 2);
  1248. } else {
  1249. if (skb_pfmemalloc(skb))
  1250. gfp_mask |= __GFP_MEMALLOC;
  1251. n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
  1252. if (!n)
  1253. return NULL;
  1254. n->fclone = SKB_FCLONE_UNAVAILABLE;
  1255. }
  1256. return __skb_clone(n, skb);
  1257. }
  1258. EXPORT_SYMBOL(skb_clone);
  1259. void skb_headers_offset_update(struct sk_buff *skb, int off)
  1260. {
  1261. /* Only adjust this if it actually is csum_start rather than csum */
  1262. if (skb->ip_summed == CHECKSUM_PARTIAL)
  1263. skb->csum_start += off;
  1264. /* {transport,network,mac}_header and tail are relative to skb->head */
  1265. skb->transport_header += off;
  1266. skb->network_header += off;
  1267. if (skb_mac_header_was_set(skb))
  1268. skb->mac_header += off;
  1269. skb->inner_transport_header += off;
  1270. skb->inner_network_header += off;
  1271. skb->inner_mac_header += off;
  1272. }
  1273. EXPORT_SYMBOL(skb_headers_offset_update);
  1274. void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
  1275. {
  1276. __copy_skb_header(new, old);
  1277. skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
  1278. skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
  1279. skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
  1280. }
  1281. EXPORT_SYMBOL(skb_copy_header);
  1282. static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
  1283. {
  1284. if (skb_pfmemalloc(skb))
  1285. return SKB_ALLOC_RX;
  1286. return 0;
  1287. }
  1288. /**
  1289. * skb_copy - create private copy of an sk_buff
  1290. * @skb: buffer to copy
  1291. * @gfp_mask: allocation priority
  1292. *
  1293. * Make a copy of both an &sk_buff and its data. This is used when the
  1294. * caller wishes to modify the data and needs a private copy of the
  1295. * data to alter. Returns %NULL on failure or the pointer to the buffer
  1296. * on success. The returned buffer has a reference count of 1.
  1297. *
  1298. * As by-product this function converts non-linear &sk_buff to linear
  1299. * one, so that &sk_buff becomes completely private and caller is allowed
  1300. * to modify all the data of returned buffer. This means that this
  1301. * function is not recommended for use in circumstances when only
  1302. * header is going to be modified. Use pskb_copy() instead.
  1303. */
  1304. struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
  1305. {
  1306. int headerlen = skb_headroom(skb);
  1307. unsigned int size = skb_end_offset(skb) + skb->data_len;
  1308. struct sk_buff *n = __alloc_skb(size, gfp_mask,
  1309. skb_alloc_rx_flag(skb), NUMA_NO_NODE);
  1310. if (!n)
  1311. return NULL;
  1312. /* Set the data pointer */
  1313. skb_reserve(n, headerlen);
  1314. /* Set the tail pointer and length */
  1315. skb_put(n, skb->len);
  1316. BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
  1317. skb_copy_header(n, skb);
  1318. return n;
  1319. }
  1320. EXPORT_SYMBOL(skb_copy);
  1321. /**
  1322. * __pskb_copy_fclone - create copy of an sk_buff with private head.
  1323. * @skb: buffer to copy
  1324. * @headroom: headroom of new skb
  1325. * @gfp_mask: allocation priority
  1326. * @fclone: if true allocate the copy of the skb from the fclone
  1327. * cache instead of the head cache; it is recommended to set this
  1328. * to true for the cases where the copy will likely be cloned
  1329. *
  1330. * Make a copy of both an &sk_buff and part of its data, located
  1331. * in header. Fragmented data remain shared. This is used when
  1332. * the caller wishes to modify only header of &sk_buff and needs
  1333. * private copy of the header to alter. Returns %NULL on failure
  1334. * or the pointer to the buffer on success.
  1335. * The returned buffer has a reference count of 1.
  1336. */
  1337. struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
  1338. gfp_t gfp_mask, bool fclone)
  1339. {
  1340. unsigned int size = skb_headlen(skb) + headroom;
  1341. int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
  1342. struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
  1343. if (!n)
  1344. goto out;
  1345. /* Set the data pointer */
  1346. skb_reserve(n, headroom);
  1347. /* Set the tail pointer and length */
  1348. skb_put(n, skb_headlen(skb));
  1349. /* Copy the bytes */
  1350. skb_copy_from_linear_data(skb, n->data, n->len);
  1351. n->truesize += skb->data_len;
  1352. n->data_len = skb->data_len;
  1353. n->len = skb->len;
  1354. if (skb_shinfo(skb)->nr_frags) {
  1355. int i;
  1356. if (skb_orphan_frags(skb, gfp_mask) ||
  1357. skb_zerocopy_clone(n, skb, gfp_mask)) {
  1358. kfree_skb(n);
  1359. n = NULL;
  1360. goto out;
  1361. }
  1362. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1363. skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
  1364. skb_frag_ref(skb, i);
  1365. }
  1366. skb_shinfo(n)->nr_frags = i;
  1367. }
  1368. if (skb_has_frag_list(skb)) {
  1369. skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
  1370. skb_clone_fraglist(n);
  1371. }
  1372. skb_copy_header(n, skb);
  1373. out:
  1374. return n;
  1375. }
  1376. EXPORT_SYMBOL(__pskb_copy_fclone);
  1377. /**
  1378. * pskb_expand_head - reallocate header of &sk_buff
  1379. * @skb: buffer to reallocate
  1380. * @nhead: room to add at head
  1381. * @ntail: room to add at tail
  1382. * @gfp_mask: allocation priority
  1383. *
  1384. * Expands (or creates identical copy, if @nhead and @ntail are zero)
  1385. * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
  1386. * reference count of 1. Returns zero in the case of success or error,
  1387. * if expansion failed. In the last case, &sk_buff is not changed.
  1388. *
  1389. * All the pointers pointing into skb header may change and must be
  1390. * reloaded after call to this function.
  1391. */
  1392. int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
  1393. gfp_t gfp_mask)
  1394. {
  1395. int i, osize = skb_end_offset(skb);
  1396. int size = osize + nhead + ntail;
  1397. long off;
  1398. u8 *data;
  1399. BUG_ON(nhead < 0);
  1400. BUG_ON(skb_shared(skb));
  1401. size = SKB_DATA_ALIGN(size);
  1402. if (skb_pfmemalloc(skb))
  1403. gfp_mask |= __GFP_MEMALLOC;
  1404. data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
  1405. gfp_mask, NUMA_NO_NODE, NULL);
  1406. if (!data)
  1407. goto nodata;
  1408. size = SKB_WITH_OVERHEAD(ksize(data));
  1409. /* Copy only real data... and, alas, header. This should be
  1410. * optimized for the cases when header is void.
  1411. */
  1412. memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
  1413. memcpy((struct skb_shared_info *)(data + size),
  1414. skb_shinfo(skb),
  1415. offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
  1416. /*
  1417. * if shinfo is shared we must drop the old head gracefully, but if it
  1418. * is not we can just drop the old head and let the existing refcount
  1419. * be since all we did is relocate the values
  1420. */
  1421. if (skb_cloned(skb)) {
  1422. if (skb_orphan_frags(skb, gfp_mask))
  1423. goto nofrags;
  1424. if (skb_zcopy(skb))
  1425. refcount_inc(&skb_uarg(skb)->refcnt);
  1426. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  1427. skb_frag_ref(skb, i);
  1428. if (skb_has_frag_list(skb))
  1429. skb_clone_fraglist(skb);
  1430. skb_release_data(skb);
  1431. } else {
  1432. skb_free_head(skb);
  1433. }
  1434. off = (data + nhead) - skb->head;
  1435. skb->head = data;
  1436. skb->head_frag = 0;
  1437. skb->data += off;
  1438. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  1439. skb->end = size;
  1440. off = nhead;
  1441. #else
  1442. skb->end = skb->head + size;
  1443. #endif
  1444. skb->tail += off;
  1445. skb_headers_offset_update(skb, nhead);
  1446. skb->cloned = 0;
  1447. skb->hdr_len = 0;
  1448. skb->nohdr = 0;
  1449. atomic_set(&skb_shinfo(skb)->dataref, 1);
  1450. skb_metadata_clear(skb);
  1451. /* It is not generally safe to change skb->truesize.
  1452. * For the moment, we really care of rx path, or
  1453. * when skb is orphaned (not attached to a socket).
  1454. */
  1455. if (!skb->sk || skb->destructor == sock_edemux)
  1456. skb->truesize += size - osize;
  1457. return 0;
  1458. nofrags:
  1459. kfree(data);
  1460. nodata:
  1461. return -ENOMEM;
  1462. }
  1463. EXPORT_SYMBOL(pskb_expand_head);
  1464. /* Make private copy of skb with writable head and some headroom */
  1465. struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
  1466. {
  1467. struct sk_buff *skb2;
  1468. int delta = headroom - skb_headroom(skb);
  1469. if (delta <= 0)
  1470. skb2 = pskb_copy(skb, GFP_ATOMIC);
  1471. else {
  1472. skb2 = skb_clone(skb, GFP_ATOMIC);
  1473. if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
  1474. GFP_ATOMIC)) {
  1475. kfree_skb(skb2);
  1476. skb2 = NULL;
  1477. }
  1478. }
  1479. return skb2;
  1480. }
  1481. EXPORT_SYMBOL(skb_realloc_headroom);
  1482. /**
  1483. * skb_copy_expand - copy and expand sk_buff
  1484. * @skb: buffer to copy
  1485. * @newheadroom: new free bytes at head
  1486. * @newtailroom: new free bytes at tail
  1487. * @gfp_mask: allocation priority
  1488. *
  1489. * Make a copy of both an &sk_buff and its data and while doing so
  1490. * allocate additional space.
  1491. *
  1492. * This is used when the caller wishes to modify the data and needs a
  1493. * private copy of the data to alter as well as more space for new fields.
  1494. * Returns %NULL on failure or the pointer to the buffer
  1495. * on success. The returned buffer has a reference count of 1.
  1496. *
  1497. * You must pass %GFP_ATOMIC as the allocation priority if this function
  1498. * is called from an interrupt.
  1499. */
  1500. struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
  1501. int newheadroom, int newtailroom,
  1502. gfp_t gfp_mask)
  1503. {
  1504. /*
  1505. * Allocate the copy buffer
  1506. */
  1507. struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
  1508. gfp_mask, skb_alloc_rx_flag(skb),
  1509. NUMA_NO_NODE);
  1510. int oldheadroom = skb_headroom(skb);
  1511. int head_copy_len, head_copy_off;
  1512. if (!n)
  1513. return NULL;
  1514. skb_reserve(n, newheadroom);
  1515. /* Set the tail pointer and length */
  1516. skb_put(n, skb->len);
  1517. head_copy_len = oldheadroom;
  1518. head_copy_off = 0;
  1519. if (newheadroom <= head_copy_len)
  1520. head_copy_len = newheadroom;
  1521. else
  1522. head_copy_off = newheadroom - head_copy_len;
  1523. /* Copy the linear header and data. */
  1524. BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
  1525. skb->len + head_copy_len));
  1526. skb_copy_header(n, skb);
  1527. skb_headers_offset_update(n, newheadroom - oldheadroom);
  1528. return n;
  1529. }
  1530. EXPORT_SYMBOL(skb_copy_expand);
  1531. /**
  1532. * __skb_pad - zero pad the tail of an skb
  1533. * @skb: buffer to pad
  1534. * @pad: space to pad
  1535. * @free_on_error: free buffer on error
  1536. *
  1537. * Ensure that a buffer is followed by a padding area that is zero
  1538. * filled. Used by network drivers which may DMA or transfer data
  1539. * beyond the buffer end onto the wire.
  1540. *
  1541. * May return error in out of memory cases. The skb is freed on error
  1542. * if @free_on_error is true.
  1543. */
  1544. int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
  1545. {
  1546. int err;
  1547. int ntail;
  1548. /* If the skbuff is non linear tailroom is always zero.. */
  1549. if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
  1550. memset(skb->data+skb->len, 0, pad);
  1551. return 0;
  1552. }
  1553. ntail = skb->data_len + pad - (skb->end - skb->tail);
  1554. if (likely(skb_cloned(skb) || ntail > 0)) {
  1555. err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
  1556. if (unlikely(err))
  1557. goto free_skb;
  1558. }
  1559. /* FIXME: The use of this function with non-linear skb's really needs
  1560. * to be audited.
  1561. */
  1562. err = skb_linearize(skb);
  1563. if (unlikely(err))
  1564. goto free_skb;
  1565. memset(skb->data + skb->len, 0, pad);
  1566. return 0;
  1567. free_skb:
  1568. if (free_on_error)
  1569. kfree_skb(skb);
  1570. return err;
  1571. }
  1572. EXPORT_SYMBOL(__skb_pad);
  1573. /**
  1574. * pskb_put - add data to the tail of a potentially fragmented buffer
  1575. * @skb: start of the buffer to use
  1576. * @tail: tail fragment of the buffer to use
  1577. * @len: amount of data to add
  1578. *
  1579. * This function extends the used data area of the potentially
  1580. * fragmented buffer. @tail must be the last fragment of @skb -- or
  1581. * @skb itself. If this would exceed the total buffer size the kernel
  1582. * will panic. A pointer to the first byte of the extra data is
  1583. * returned.
  1584. */
  1585. void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
  1586. {
  1587. if (tail != skb) {
  1588. skb->data_len += len;
  1589. skb->len += len;
  1590. }
  1591. return skb_put(tail, len);
  1592. }
  1593. EXPORT_SYMBOL_GPL(pskb_put);
  1594. /**
  1595. * skb_put - add data to a buffer
  1596. * @skb: buffer to use
  1597. * @len: amount of data to add
  1598. *
  1599. * This function extends the used data area of the buffer. If this would
  1600. * exceed the total buffer size the kernel will panic. A pointer to the
  1601. * first byte of the extra data is returned.
  1602. */
  1603. void *skb_put(struct sk_buff *skb, unsigned int len)
  1604. {
  1605. void *tmp = skb_tail_pointer(skb);
  1606. SKB_LINEAR_ASSERT(skb);
  1607. skb->tail += len;
  1608. skb->len += len;
  1609. if (unlikely(skb->tail > skb->end))
  1610. skb_over_panic(skb, len, __builtin_return_address(0));
  1611. return tmp;
  1612. }
  1613. EXPORT_SYMBOL(skb_put);
  1614. /**
  1615. * skb_push - add data to the start of a buffer
  1616. * @skb: buffer to use
  1617. * @len: amount of data to add
  1618. *
  1619. * This function extends the used data area of the buffer at the buffer
  1620. * start. If this would exceed the total buffer headroom the kernel will
  1621. * panic. A pointer to the first byte of the extra data is returned.
  1622. */
  1623. void *skb_push(struct sk_buff *skb, unsigned int len)
  1624. {
  1625. skb->data -= len;
  1626. skb->len += len;
  1627. if (unlikely(skb->data < skb->head))
  1628. skb_under_panic(skb, len, __builtin_return_address(0));
  1629. return skb->data;
  1630. }
  1631. EXPORT_SYMBOL(skb_push);
  1632. /**
  1633. * skb_pull - remove data from the start of a buffer
  1634. * @skb: buffer to use
  1635. * @len: amount of data to remove
  1636. *
  1637. * This function removes data from the start of a buffer, returning
  1638. * the memory to the headroom. A pointer to the next data in the buffer
  1639. * is returned. Once the data has been pulled future pushes will overwrite
  1640. * the old data.
  1641. */
  1642. void *skb_pull(struct sk_buff *skb, unsigned int len)
  1643. {
  1644. return skb_pull_inline(skb, len);
  1645. }
  1646. EXPORT_SYMBOL(skb_pull);
  1647. /**
  1648. * skb_trim - remove end from a buffer
  1649. * @skb: buffer to alter
  1650. * @len: new length
  1651. *
  1652. * Cut the length of a buffer down by removing data from the tail. If
  1653. * the buffer is already under the length specified it is not modified.
  1654. * The skb must be linear.
  1655. */
  1656. void skb_trim(struct sk_buff *skb, unsigned int len)
  1657. {
  1658. if (skb->len > len)
  1659. __skb_trim(skb, len);
  1660. }
  1661. EXPORT_SYMBOL(skb_trim);
  1662. /* Trims skb to length len. It can change skb pointers.
  1663. */
  1664. int ___pskb_trim(struct sk_buff *skb, unsigned int len)
  1665. {
  1666. struct sk_buff **fragp;
  1667. struct sk_buff *frag;
  1668. int offset = skb_headlen(skb);
  1669. int nfrags = skb_shinfo(skb)->nr_frags;
  1670. int i;
  1671. int err;
  1672. if (skb_cloned(skb) &&
  1673. unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
  1674. return err;
  1675. i = 0;
  1676. if (offset >= len)
  1677. goto drop_pages;
  1678. for (; i < nfrags; i++) {
  1679. int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1680. if (end < len) {
  1681. offset = end;
  1682. continue;
  1683. }
  1684. skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
  1685. drop_pages:
  1686. skb_shinfo(skb)->nr_frags = i;
  1687. for (; i < nfrags; i++)
  1688. skb_frag_unref(skb, i);
  1689. if (skb_has_frag_list(skb))
  1690. skb_drop_fraglist(skb);
  1691. goto done;
  1692. }
  1693. for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
  1694. fragp = &frag->next) {
  1695. int end = offset + frag->len;
  1696. if (skb_shared(frag)) {
  1697. struct sk_buff *nfrag;
  1698. nfrag = skb_clone(frag, GFP_ATOMIC);
  1699. if (unlikely(!nfrag))
  1700. return -ENOMEM;
  1701. nfrag->next = frag->next;
  1702. consume_skb(frag);
  1703. frag = nfrag;
  1704. *fragp = frag;
  1705. }
  1706. if (end < len) {
  1707. offset = end;
  1708. continue;
  1709. }
  1710. if (end > len &&
  1711. unlikely((err = pskb_trim(frag, len - offset))))
  1712. return err;
  1713. if (frag->next)
  1714. skb_drop_list(&frag->next);
  1715. break;
  1716. }
  1717. done:
  1718. if (len > skb_headlen(skb)) {
  1719. skb->data_len -= skb->len - len;
  1720. skb->len = len;
  1721. } else {
  1722. skb->len = len;
  1723. skb->data_len = 0;
  1724. skb_set_tail_pointer(skb, len);
  1725. }
  1726. if (!skb->sk || skb->destructor == sock_edemux)
  1727. skb_condense(skb);
  1728. return 0;
  1729. }
  1730. EXPORT_SYMBOL(___pskb_trim);
  1731. /* Note : use pskb_trim_rcsum() instead of calling this directly
  1732. */
  1733. int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
  1734. {
  1735. if (skb->ip_summed == CHECKSUM_COMPLETE) {
  1736. int delta = skb->len - len;
  1737. skb->csum = csum_block_sub(skb->csum,
  1738. skb_checksum(skb, len, delta, 0),
  1739. len);
  1740. } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
  1741. int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
  1742. int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
  1743. if (offset + sizeof(__sum16) > hdlen)
  1744. return -EINVAL;
  1745. }
  1746. return __pskb_trim(skb, len);
  1747. }
  1748. EXPORT_SYMBOL(pskb_trim_rcsum_slow);
  1749. /**
  1750. * __pskb_pull_tail - advance tail of skb header
  1751. * @skb: buffer to reallocate
  1752. * @delta: number of bytes to advance tail
  1753. *
  1754. * The function makes a sense only on a fragmented &sk_buff,
  1755. * it expands header moving its tail forward and copying necessary
  1756. * data from fragmented part.
  1757. *
  1758. * &sk_buff MUST have reference count of 1.
  1759. *
  1760. * Returns %NULL (and &sk_buff does not change) if pull failed
  1761. * or value of new tail of skb in the case of success.
  1762. *
  1763. * All the pointers pointing into skb header may change and must be
  1764. * reloaded after call to this function.
  1765. */
  1766. /* Moves tail of skb head forward, copying data from fragmented part,
  1767. * when it is necessary.
  1768. * 1. It may fail due to malloc failure.
  1769. * 2. It may change skb pointers.
  1770. *
  1771. * It is pretty complicated. Luckily, it is called only in exceptional cases.
  1772. */
  1773. void *__pskb_pull_tail(struct sk_buff *skb, int delta)
  1774. {
  1775. /* If skb has not enough free space at tail, get new one
  1776. * plus 128 bytes for future expansions. If we have enough
  1777. * room at tail, reallocate without expansion only if skb is cloned.
  1778. */
  1779. int i, k, eat = (skb->tail + delta) - skb->end;
  1780. if (eat > 0 || skb_cloned(skb)) {
  1781. if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
  1782. GFP_ATOMIC))
  1783. return NULL;
  1784. }
  1785. BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
  1786. skb_tail_pointer(skb), delta));
  1787. /* Optimization: no fragments, no reasons to preestimate
  1788. * size of pulled pages. Superb.
  1789. */
  1790. if (!skb_has_frag_list(skb))
  1791. goto pull_pages;
  1792. /* Estimate size of pulled pages. */
  1793. eat = delta;
  1794. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1795. int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1796. if (size >= eat)
  1797. goto pull_pages;
  1798. eat -= size;
  1799. }
  1800. /* If we need update frag list, we are in troubles.
  1801. * Certainly, it is possible to add an offset to skb data,
  1802. * but taking into account that pulling is expected to
  1803. * be very rare operation, it is worth to fight against
  1804. * further bloating skb head and crucify ourselves here instead.
  1805. * Pure masohism, indeed. 8)8)
  1806. */
  1807. if (eat) {
  1808. struct sk_buff *list = skb_shinfo(skb)->frag_list;
  1809. struct sk_buff *clone = NULL;
  1810. struct sk_buff *insp = NULL;
  1811. do {
  1812. if (list->len <= eat) {
  1813. /* Eaten as whole. */
  1814. eat -= list->len;
  1815. list = list->next;
  1816. insp = list;
  1817. } else {
  1818. /* Eaten partially. */
  1819. if (skb_shared(list)) {
  1820. /* Sucks! We need to fork list. :-( */
  1821. clone = skb_clone(list, GFP_ATOMIC);
  1822. if (!clone)
  1823. return NULL;
  1824. insp = list->next;
  1825. list = clone;
  1826. } else {
  1827. /* This may be pulled without
  1828. * problems. */
  1829. insp = list;
  1830. }
  1831. if (!pskb_pull(list, eat)) {
  1832. kfree_skb(clone);
  1833. return NULL;
  1834. }
  1835. break;
  1836. }
  1837. } while (eat);
  1838. /* Free pulled out fragments. */
  1839. while ((list = skb_shinfo(skb)->frag_list) != insp) {
  1840. skb_shinfo(skb)->frag_list = list->next;
  1841. consume_skb(list);
  1842. }
  1843. /* And insert new clone at head. */
  1844. if (clone) {
  1845. clone->next = list;
  1846. skb_shinfo(skb)->frag_list = clone;
  1847. }
  1848. }
  1849. /* Success! Now we may commit changes to skb data. */
  1850. pull_pages:
  1851. eat = delta;
  1852. k = 0;
  1853. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1854. int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  1855. if (size <= eat) {
  1856. skb_frag_unref(skb, i);
  1857. eat -= size;
  1858. } else {
  1859. skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
  1860. *frag = skb_shinfo(skb)->frags[i];
  1861. if (eat) {
  1862. skb_frag_off_add(frag, eat);
  1863. skb_frag_size_sub(frag, eat);
  1864. if (!i)
  1865. goto end;
  1866. eat = 0;
  1867. }
  1868. k++;
  1869. }
  1870. }
  1871. skb_shinfo(skb)->nr_frags = k;
  1872. end:
  1873. skb->tail += delta;
  1874. skb->data_len -= delta;
  1875. if (!skb->data_len)
  1876. skb_zcopy_clear(skb, false);
  1877. return skb_tail_pointer(skb);
  1878. }
  1879. EXPORT_SYMBOL(__pskb_pull_tail);
  1880. /**
  1881. * skb_copy_bits - copy bits from skb to kernel buffer
  1882. * @skb: source skb
  1883. * @offset: offset in source
  1884. * @to: destination buffer
  1885. * @len: number of bytes to copy
  1886. *
  1887. * Copy the specified number of bytes from the source skb to the
  1888. * destination buffer.
  1889. *
  1890. * CAUTION ! :
  1891. * If its prototype is ever changed,
  1892. * check arch/{*}/net/{*}.S files,
  1893. * since it is called from BPF assembly code.
  1894. */
  1895. int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
  1896. {
  1897. int start = skb_headlen(skb);
  1898. struct sk_buff *frag_iter;
  1899. int i, copy;
  1900. if (offset > (int)skb->len - len)
  1901. goto fault;
  1902. /* Copy header. */
  1903. if ((copy = start - offset) > 0) {
  1904. if (copy > len)
  1905. copy = len;
  1906. skb_copy_from_linear_data_offset(skb, offset, to, copy);
  1907. if ((len -= copy) == 0)
  1908. return 0;
  1909. offset += copy;
  1910. to += copy;
  1911. }
  1912. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  1913. int end;
  1914. skb_frag_t *f = &skb_shinfo(skb)->frags[i];
  1915. WARN_ON(start > offset + len);
  1916. end = start + skb_frag_size(f);
  1917. if ((copy = end - offset) > 0) {
  1918. u32 p_off, p_len, copied;
  1919. struct page *p;
  1920. u8 *vaddr;
  1921. if (copy > len)
  1922. copy = len;
  1923. skb_frag_foreach_page(f,
  1924. skb_frag_off(f) + offset - start,
  1925. copy, p, p_off, p_len, copied) {
  1926. vaddr = kmap_atomic(p);
  1927. memcpy(to + copied, vaddr + p_off, p_len);
  1928. kunmap_atomic(vaddr);
  1929. }
  1930. if ((len -= copy) == 0)
  1931. return 0;
  1932. offset += copy;
  1933. to += copy;
  1934. }
  1935. start = end;
  1936. }
  1937. skb_walk_frags(skb, frag_iter) {
  1938. int end;
  1939. WARN_ON(start > offset + len);
  1940. end = start + frag_iter->len;
  1941. if ((copy = end - offset) > 0) {
  1942. if (copy > len)
  1943. copy = len;
  1944. if (skb_copy_bits(frag_iter, offset - start, to, copy))
  1945. goto fault;
  1946. if ((len -= copy) == 0)
  1947. return 0;
  1948. offset += copy;
  1949. to += copy;
  1950. }
  1951. start = end;
  1952. }
  1953. if (!len)
  1954. return 0;
  1955. fault:
  1956. return -EFAULT;
  1957. }
  1958. EXPORT_SYMBOL(skb_copy_bits);
  1959. /*
  1960. * Callback from splice_to_pipe(), if we need to release some pages
  1961. * at the end of the spd in case we error'ed out in filling the pipe.
  1962. */
  1963. static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
  1964. {
  1965. put_page(spd->pages[i]);
  1966. }
  1967. static struct page *linear_to_page(struct page *page, unsigned int *len,
  1968. unsigned int *offset,
  1969. struct sock *sk)
  1970. {
  1971. struct page_frag *pfrag = sk_page_frag(sk);
  1972. if (!sk_page_frag_refill(sk, pfrag))
  1973. return NULL;
  1974. *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
  1975. memcpy(page_address(pfrag->page) + pfrag->offset,
  1976. page_address(page) + *offset, *len);
  1977. *offset = pfrag->offset;
  1978. pfrag->offset += *len;
  1979. return pfrag->page;
  1980. }
  1981. static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
  1982. struct page *page,
  1983. unsigned int offset)
  1984. {
  1985. return spd->nr_pages &&
  1986. spd->pages[spd->nr_pages - 1] == page &&
  1987. (spd->partial[spd->nr_pages - 1].offset +
  1988. spd->partial[spd->nr_pages - 1].len == offset);
  1989. }
  1990. /*
  1991. * Fill page/offset/length into spd, if it can hold more pages.
  1992. */
  1993. static bool spd_fill_page(struct splice_pipe_desc *spd,
  1994. struct pipe_inode_info *pipe, struct page *page,
  1995. unsigned int *len, unsigned int offset,
  1996. bool linear,
  1997. struct sock *sk)
  1998. {
  1999. if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
  2000. return true;
  2001. if (linear) {
  2002. page = linear_to_page(page, len, &offset, sk);
  2003. if (!page)
  2004. return true;
  2005. }
  2006. if (spd_can_coalesce(spd, page, offset)) {
  2007. spd->partial[spd->nr_pages - 1].len += *len;
  2008. return false;
  2009. }
  2010. get_page(page);
  2011. spd->pages[spd->nr_pages] = page;
  2012. spd->partial[spd->nr_pages].len = *len;
  2013. spd->partial[spd->nr_pages].offset = offset;
  2014. spd->nr_pages++;
  2015. return false;
  2016. }
  2017. static bool __splice_segment(struct page *page, unsigned int poff,
  2018. unsigned int plen, unsigned int *off,
  2019. unsigned int *len,
  2020. struct splice_pipe_desc *spd, bool linear,
  2021. struct sock *sk,
  2022. struct pipe_inode_info *pipe)
  2023. {
  2024. if (!*len)
  2025. return true;
  2026. /* skip this segment if already processed */
  2027. if (*off >= plen) {
  2028. *off -= plen;
  2029. return false;
  2030. }
  2031. /* ignore any bits we already processed */
  2032. poff += *off;
  2033. plen -= *off;
  2034. *off = 0;
  2035. do {
  2036. unsigned int flen = min(*len, plen);
  2037. if (spd_fill_page(spd, pipe, page, &flen, poff,
  2038. linear, sk))
  2039. return true;
  2040. poff += flen;
  2041. plen -= flen;
  2042. *len -= flen;
  2043. } while (*len && plen);
  2044. return false;
  2045. }
  2046. /*
  2047. * Map linear and fragment data from the skb to spd. It reports true if the
  2048. * pipe is full or if we already spliced the requested length.
  2049. */
  2050. static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
  2051. unsigned int *offset, unsigned int *len,
  2052. struct splice_pipe_desc *spd, struct sock *sk)
  2053. {
  2054. int seg;
  2055. struct sk_buff *iter;
  2056. /* map the linear part :
  2057. * If skb->head_frag is set, this 'linear' part is backed by a
  2058. * fragment, and if the head is not shared with any clones then
  2059. * we can avoid a copy since we own the head portion of this page.
  2060. */
  2061. if (__splice_segment(virt_to_page(skb->data),
  2062. (unsigned long) skb->data & (PAGE_SIZE - 1),
  2063. skb_headlen(skb),
  2064. offset, len, spd,
  2065. skb_head_is_locked(skb),
  2066. sk, pipe))
  2067. return true;
  2068. /*
  2069. * then map the fragments
  2070. */
  2071. for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
  2072. const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
  2073. if (__splice_segment(skb_frag_page(f),
  2074. skb_frag_off(f), skb_frag_size(f),
  2075. offset, len, spd, false, sk, pipe))
  2076. return true;
  2077. }
  2078. skb_walk_frags(skb, iter) {
  2079. if (*offset >= iter->len) {
  2080. *offset -= iter->len;
  2081. continue;
  2082. }
  2083. /* __skb_splice_bits() only fails if the output has no room
  2084. * left, so no point in going over the frag_list for the error
  2085. * case.
  2086. */
  2087. if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
  2088. return true;
  2089. }
  2090. return false;
  2091. }
  2092. /*
  2093. * Map data from the skb to a pipe. Should handle both the linear part,
  2094. * the fragments, and the frag list.
  2095. */
  2096. int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
  2097. struct pipe_inode_info *pipe, unsigned int tlen,
  2098. unsigned int flags)
  2099. {
  2100. struct partial_page partial[MAX_SKB_FRAGS];
  2101. struct page *pages[MAX_SKB_FRAGS];
  2102. struct splice_pipe_desc spd = {
  2103. .pages = pages,
  2104. .partial = partial,
  2105. .nr_pages_max = MAX_SKB_FRAGS,
  2106. .ops = &nosteal_pipe_buf_ops,
  2107. .spd_release = sock_spd_release,
  2108. };
  2109. int ret = 0;
  2110. __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
  2111. if (spd.nr_pages)
  2112. ret = splice_to_pipe(pipe, &spd);
  2113. return ret;
  2114. }
  2115. EXPORT_SYMBOL_GPL(skb_splice_bits);
  2116. /* Send skb data on a socket. Socket must be locked. */
  2117. int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
  2118. int len)
  2119. {
  2120. unsigned int orig_len = len;
  2121. struct sk_buff *head = skb;
  2122. unsigned short fragidx;
  2123. int slen, ret;
  2124. do_frag_list:
  2125. /* Deal with head data */
  2126. while (offset < skb_headlen(skb) && len) {
  2127. struct kvec kv;
  2128. struct msghdr msg;
  2129. slen = min_t(int, len, skb_headlen(skb) - offset);
  2130. kv.iov_base = skb->data + offset;
  2131. kv.iov_len = slen;
  2132. memset(&msg, 0, sizeof(msg));
  2133. msg.msg_flags = MSG_DONTWAIT;
  2134. ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
  2135. if (ret <= 0)
  2136. goto error;
  2137. offset += ret;
  2138. len -= ret;
  2139. }
  2140. /* All the data was skb head? */
  2141. if (!len)
  2142. goto out;
  2143. /* Make offset relative to start of frags */
  2144. offset -= skb_headlen(skb);
  2145. /* Find where we are in frag list */
  2146. for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
  2147. skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
  2148. if (offset < skb_frag_size(frag))
  2149. break;
  2150. offset -= skb_frag_size(frag);
  2151. }
  2152. for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
  2153. skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
  2154. slen = min_t(size_t, len, skb_frag_size(frag) - offset);
  2155. while (slen) {
  2156. ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
  2157. skb_frag_off(frag) + offset,
  2158. slen, MSG_DONTWAIT);
  2159. if (ret <= 0)
  2160. goto error;
  2161. len -= ret;
  2162. offset += ret;
  2163. slen -= ret;
  2164. }
  2165. offset = 0;
  2166. }
  2167. if (len) {
  2168. /* Process any frag lists */
  2169. if (skb == head) {
  2170. if (skb_has_frag_list(skb)) {
  2171. skb = skb_shinfo(skb)->frag_list;
  2172. goto do_frag_list;
  2173. }
  2174. } else if (skb->next) {
  2175. skb = skb->next;
  2176. goto do_frag_list;
  2177. }
  2178. }
  2179. out:
  2180. return orig_len - len;
  2181. error:
  2182. return orig_len == len ? ret : orig_len - len;
  2183. }
  2184. EXPORT_SYMBOL_GPL(skb_send_sock_locked);
  2185. /**
  2186. * skb_store_bits - store bits from kernel buffer to skb
  2187. * @skb: destination buffer
  2188. * @offset: offset in destination
  2189. * @from: source buffer
  2190. * @len: number of bytes to copy
  2191. *
  2192. * Copy the specified number of bytes from the source buffer to the
  2193. * destination skb. This function handles all the messy bits of
  2194. * traversing fragment lists and such.
  2195. */
  2196. int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
  2197. {
  2198. int start = skb_headlen(skb);
  2199. struct sk_buff *frag_iter;
  2200. int i, copy;
  2201. if (offset > (int)skb->len - len)
  2202. goto fault;
  2203. if ((copy = start - offset) > 0) {
  2204. if (copy > len)
  2205. copy = len;
  2206. skb_copy_to_linear_data_offset(skb, offset, from, copy);
  2207. if ((len -= copy) == 0)
  2208. return 0;
  2209. offset += copy;
  2210. from += copy;
  2211. }
  2212. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2213. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2214. int end;
  2215. WARN_ON(start > offset + len);
  2216. end = start + skb_frag_size(frag);
  2217. if ((copy = end - offset) > 0) {
  2218. u32 p_off, p_len, copied;
  2219. struct page *p;
  2220. u8 *vaddr;
  2221. if (copy > len)
  2222. copy = len;
  2223. skb_frag_foreach_page(frag,
  2224. skb_frag_off(frag) + offset - start,
  2225. copy, p, p_off, p_len, copied) {
  2226. vaddr = kmap_atomic(p);
  2227. memcpy(vaddr + p_off, from + copied, p_len);
  2228. kunmap_atomic(vaddr);
  2229. }
  2230. if ((len -= copy) == 0)
  2231. return 0;
  2232. offset += copy;
  2233. from += copy;
  2234. }
  2235. start = end;
  2236. }
  2237. skb_walk_frags(skb, frag_iter) {
  2238. int end;
  2239. WARN_ON(start > offset + len);
  2240. end = start + frag_iter->len;
  2241. if ((copy = end - offset) > 0) {
  2242. if (copy > len)
  2243. copy = len;
  2244. if (skb_store_bits(frag_iter, offset - start,
  2245. from, copy))
  2246. goto fault;
  2247. if ((len -= copy) == 0)
  2248. return 0;
  2249. offset += copy;
  2250. from += copy;
  2251. }
  2252. start = end;
  2253. }
  2254. if (!len)
  2255. return 0;
  2256. fault:
  2257. return -EFAULT;
  2258. }
  2259. EXPORT_SYMBOL(skb_store_bits);
  2260. /* Checksum skb data. */
  2261. __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
  2262. __wsum csum, const struct skb_checksum_ops *ops)
  2263. {
  2264. int start = skb_headlen(skb);
  2265. int i, copy = start - offset;
  2266. struct sk_buff *frag_iter;
  2267. int pos = 0;
  2268. /* Checksum header. */
  2269. if (copy > 0) {
  2270. if (copy > len)
  2271. copy = len;
  2272. csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
  2273. skb->data + offset, copy, csum);
  2274. if ((len -= copy) == 0)
  2275. return csum;
  2276. offset += copy;
  2277. pos = copy;
  2278. }
  2279. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2280. int end;
  2281. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2282. WARN_ON(start > offset + len);
  2283. end = start + skb_frag_size(frag);
  2284. if ((copy = end - offset) > 0) {
  2285. u32 p_off, p_len, copied;
  2286. struct page *p;
  2287. __wsum csum2;
  2288. u8 *vaddr;
  2289. if (copy > len)
  2290. copy = len;
  2291. skb_frag_foreach_page(frag,
  2292. skb_frag_off(frag) + offset - start,
  2293. copy, p, p_off, p_len, copied) {
  2294. vaddr = kmap_atomic(p);
  2295. csum2 = INDIRECT_CALL_1(ops->update,
  2296. csum_partial_ext,
  2297. vaddr + p_off, p_len, 0);
  2298. kunmap_atomic(vaddr);
  2299. csum = INDIRECT_CALL_1(ops->combine,
  2300. csum_block_add_ext, csum,
  2301. csum2, pos, p_len);
  2302. pos += p_len;
  2303. }
  2304. if (!(len -= copy))
  2305. return csum;
  2306. offset += copy;
  2307. }
  2308. start = end;
  2309. }
  2310. skb_walk_frags(skb, frag_iter) {
  2311. int end;
  2312. WARN_ON(start > offset + len);
  2313. end = start + frag_iter->len;
  2314. if ((copy = end - offset) > 0) {
  2315. __wsum csum2;
  2316. if (copy > len)
  2317. copy = len;
  2318. csum2 = __skb_checksum(frag_iter, offset - start,
  2319. copy, 0, ops);
  2320. csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
  2321. csum, csum2, pos, copy);
  2322. if ((len -= copy) == 0)
  2323. return csum;
  2324. offset += copy;
  2325. pos += copy;
  2326. }
  2327. start = end;
  2328. }
  2329. BUG_ON(len);
  2330. return csum;
  2331. }
  2332. EXPORT_SYMBOL(__skb_checksum);
  2333. __wsum skb_checksum(const struct sk_buff *skb, int offset,
  2334. int len, __wsum csum)
  2335. {
  2336. const struct skb_checksum_ops ops = {
  2337. .update = csum_partial_ext,
  2338. .combine = csum_block_add_ext,
  2339. };
  2340. return __skb_checksum(skb, offset, len, csum, &ops);
  2341. }
  2342. EXPORT_SYMBOL(skb_checksum);
  2343. /* Both of above in one bottle. */
  2344. __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
  2345. u8 *to, int len)
  2346. {
  2347. int start = skb_headlen(skb);
  2348. int i, copy = start - offset;
  2349. struct sk_buff *frag_iter;
  2350. int pos = 0;
  2351. __wsum csum = 0;
  2352. /* Copy header. */
  2353. if (copy > 0) {
  2354. if (copy > len)
  2355. copy = len;
  2356. csum = csum_partial_copy_nocheck(skb->data + offset, to,
  2357. copy);
  2358. if ((len -= copy) == 0)
  2359. return csum;
  2360. offset += copy;
  2361. to += copy;
  2362. pos = copy;
  2363. }
  2364. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  2365. int end;
  2366. WARN_ON(start > offset + len);
  2367. end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
  2368. if ((copy = end - offset) > 0) {
  2369. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  2370. u32 p_off, p_len, copied;
  2371. struct page *p;
  2372. __wsum csum2;
  2373. u8 *vaddr;
  2374. if (copy > len)
  2375. copy = len;
  2376. skb_frag_foreach_page(frag,
  2377. skb_frag_off(frag) + offset - start,
  2378. copy, p, p_off, p_len, copied) {
  2379. vaddr = kmap_atomic(p);
  2380. csum2 = csum_partial_copy_nocheck(vaddr + p_off,
  2381. to + copied,
  2382. p_len);
  2383. kunmap_atomic(vaddr);
  2384. csum = csum_block_add(csum, csum2, pos);
  2385. pos += p_len;
  2386. }
  2387. if (!(len -= copy))
  2388. return csum;
  2389. offset += copy;
  2390. to += copy;
  2391. }
  2392. start = end;
  2393. }
  2394. skb_walk_frags(skb, frag_iter) {
  2395. __wsum csum2;
  2396. int end;
  2397. WARN_ON(start > offset + len);
  2398. end = start + frag_iter->len;
  2399. if ((copy = end - offset) > 0) {
  2400. if (copy > len)
  2401. copy = len;
  2402. csum2 = skb_copy_and_csum_bits(frag_iter,
  2403. offset - start,
  2404. to, copy);
  2405. csum = csum_block_add(csum, csum2, pos);
  2406. if ((len -= copy) == 0)
  2407. return csum;
  2408. offset += copy;
  2409. to += copy;
  2410. pos += copy;
  2411. }
  2412. start = end;
  2413. }
  2414. BUG_ON(len);
  2415. return csum;
  2416. }
  2417. EXPORT_SYMBOL(skb_copy_and_csum_bits);
  2418. __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
  2419. {
  2420. __sum16 sum;
  2421. sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
  2422. /* See comments in __skb_checksum_complete(). */
  2423. if (likely(!sum)) {
  2424. if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
  2425. !skb->csum_complete_sw)
  2426. netdev_rx_csum_fault(skb->dev, skb);
  2427. }
  2428. if (!skb_shared(skb))
  2429. skb->csum_valid = !sum;
  2430. return sum;
  2431. }
  2432. EXPORT_SYMBOL(__skb_checksum_complete_head);
  2433. /* This function assumes skb->csum already holds pseudo header's checksum,
  2434. * which has been changed from the hardware checksum, for example, by
  2435. * __skb_checksum_validate_complete(). And, the original skb->csum must
  2436. * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
  2437. *
  2438. * It returns non-zero if the recomputed checksum is still invalid, otherwise
  2439. * zero. The new checksum is stored back into skb->csum unless the skb is
  2440. * shared.
  2441. */
  2442. __sum16 __skb_checksum_complete(struct sk_buff *skb)
  2443. {
  2444. __wsum csum;
  2445. __sum16 sum;
  2446. csum = skb_checksum(skb, 0, skb->len, 0);
  2447. sum = csum_fold(csum_add(skb->csum, csum));
  2448. /* This check is inverted, because we already knew the hardware
  2449. * checksum is invalid before calling this function. So, if the
  2450. * re-computed checksum is valid instead, then we have a mismatch
  2451. * between the original skb->csum and skb_checksum(). This means either
  2452. * the original hardware checksum is incorrect or we screw up skb->csum
  2453. * when moving skb->data around.
  2454. */
  2455. if (likely(!sum)) {
  2456. if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
  2457. !skb->csum_complete_sw)
  2458. netdev_rx_csum_fault(skb->dev, skb);
  2459. }
  2460. if (!skb_shared(skb)) {
  2461. /* Save full packet checksum */
  2462. skb->csum = csum;
  2463. skb->ip_summed = CHECKSUM_COMPLETE;
  2464. skb->csum_complete_sw = 1;
  2465. skb->csum_valid = !sum;
  2466. }
  2467. return sum;
  2468. }
  2469. EXPORT_SYMBOL(__skb_checksum_complete);
  2470. static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
  2471. {
  2472. net_warn_ratelimited(
  2473. "%s: attempt to compute crc32c without libcrc32c.ko\n",
  2474. __func__);
  2475. return 0;
  2476. }
  2477. static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
  2478. int offset, int len)
  2479. {
  2480. net_warn_ratelimited(
  2481. "%s: attempt to compute crc32c without libcrc32c.ko\n",
  2482. __func__);
  2483. return 0;
  2484. }
  2485. static const struct skb_checksum_ops default_crc32c_ops = {
  2486. .update = warn_crc32c_csum_update,
  2487. .combine = warn_crc32c_csum_combine,
  2488. };
  2489. const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
  2490. &default_crc32c_ops;
  2491. EXPORT_SYMBOL(crc32c_csum_stub);
  2492. /**
  2493. * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
  2494. * @from: source buffer
  2495. *
  2496. * Calculates the amount of linear headroom needed in the 'to' skb passed
  2497. * into skb_zerocopy().
  2498. */
  2499. unsigned int
  2500. skb_zerocopy_headlen(const struct sk_buff *from)
  2501. {
  2502. unsigned int hlen = 0;
  2503. if (!from->head_frag ||
  2504. skb_headlen(from) < L1_CACHE_BYTES ||
  2505. skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
  2506. hlen = skb_headlen(from);
  2507. if (!hlen)
  2508. hlen = from->len;
  2509. }
  2510. if (skb_has_frag_list(from))
  2511. hlen = from->len;
  2512. return hlen;
  2513. }
  2514. EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
  2515. /**
  2516. * skb_zerocopy - Zero copy skb to skb
  2517. * @to: destination buffer
  2518. * @from: source buffer
  2519. * @len: number of bytes to copy from source buffer
  2520. * @hlen: size of linear headroom in destination buffer
  2521. *
  2522. * Copies up to `len` bytes from `from` to `to` by creating references
  2523. * to the frags in the source buffer.
  2524. *
  2525. * The `hlen` as calculated by skb_zerocopy_headlen() specifies the
  2526. * headroom in the `to` buffer.
  2527. *
  2528. * Return value:
  2529. * 0: everything is OK
  2530. * -ENOMEM: couldn't orphan frags of @from due to lack of memory
  2531. * -EFAULT: skb_copy_bits() found some problem with skb geometry
  2532. */
  2533. int
  2534. skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
  2535. {
  2536. int i, j = 0;
  2537. int plen = 0; /* length of skb->head fragment */
  2538. int ret;
  2539. struct page *page;
  2540. unsigned int offset;
  2541. BUG_ON(!from->head_frag && !hlen);
  2542. /* dont bother with small payloads */
  2543. if (len <= skb_tailroom(to))
  2544. return skb_copy_bits(from, 0, skb_put(to, len), len);
  2545. if (hlen) {
  2546. ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
  2547. if (unlikely(ret))
  2548. return ret;
  2549. len -= hlen;
  2550. } else {
  2551. plen = min_t(int, skb_headlen(from), len);
  2552. if (plen) {
  2553. page = virt_to_head_page(from->head);
  2554. offset = from->data - (unsigned char *)page_address(page);
  2555. __skb_fill_page_desc(to, 0, page, offset, plen);
  2556. get_page(page);
  2557. j = 1;
  2558. len -= plen;
  2559. }
  2560. }
  2561. to->truesize += len + plen;
  2562. to->len += len + plen;
  2563. to->data_len += len + plen;
  2564. if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
  2565. skb_tx_error(from);
  2566. return -ENOMEM;
  2567. }
  2568. skb_zerocopy_clone(to, from, GFP_ATOMIC);
  2569. for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
  2570. int size;
  2571. if (!len)
  2572. break;
  2573. skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
  2574. size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
  2575. len);
  2576. skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
  2577. len -= size;
  2578. skb_frag_ref(to, j);
  2579. j++;
  2580. }
  2581. skb_shinfo(to)->nr_frags = j;
  2582. return 0;
  2583. }
  2584. EXPORT_SYMBOL_GPL(skb_zerocopy);
  2585. void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
  2586. {
  2587. __wsum csum;
  2588. long csstart;
  2589. if (skb->ip_summed == CHECKSUM_PARTIAL)
  2590. csstart = skb_checksum_start_offset(skb);
  2591. else
  2592. csstart = skb_headlen(skb);
  2593. BUG_ON(csstart > skb_headlen(skb));
  2594. skb_copy_from_linear_data(skb, to, csstart);
  2595. csum = 0;
  2596. if (csstart != skb->len)
  2597. csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
  2598. skb->len - csstart);
  2599. if (skb->ip_summed == CHECKSUM_PARTIAL) {
  2600. long csstuff = csstart + skb->csum_offset;
  2601. *((__sum16 *)(to + csstuff)) = csum_fold(csum);
  2602. }
  2603. }
  2604. EXPORT_SYMBOL(skb_copy_and_csum_dev);
  2605. /**
  2606. * skb_dequeue - remove from the head of the queue
  2607. * @list: list to dequeue from
  2608. *
  2609. * Remove the head of the list. The list lock is taken so the function
  2610. * may be used safely with other locking list functions. The head item is
  2611. * returned or %NULL if the list is empty.
  2612. */
  2613. struct sk_buff *skb_dequeue(struct sk_buff_head *list)
  2614. {
  2615. unsigned long flags;
  2616. struct sk_buff *result;
  2617. spin_lock_irqsave(&list->lock, flags);
  2618. result = __skb_dequeue(list);
  2619. spin_unlock_irqrestore(&list->lock, flags);
  2620. return result;
  2621. }
  2622. EXPORT_SYMBOL(skb_dequeue);
  2623. /**
  2624. * skb_dequeue_tail - remove from the tail of the queue
  2625. * @list: list to dequeue from
  2626. *
  2627. * Remove the tail of the list. The list lock is taken so the function
  2628. * may be used safely with other locking list functions. The tail item is
  2629. * returned or %NULL if the list is empty.
  2630. */
  2631. struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
  2632. {
  2633. unsigned long flags;
  2634. struct sk_buff *result;
  2635. spin_lock_irqsave(&list->lock, flags);
  2636. result = __skb_dequeue_tail(list);
  2637. spin_unlock_irqrestore(&list->lock, flags);
  2638. return result;
  2639. }
  2640. EXPORT_SYMBOL(skb_dequeue_tail);
  2641. /**
  2642. * skb_queue_purge - empty a list
  2643. * @list: list to empty
  2644. *
  2645. * Delete all buffers on an &sk_buff list. Each buffer is removed from
  2646. * the list and one reference dropped. This function takes the list
  2647. * lock and is atomic with respect to other list locking functions.
  2648. */
  2649. void skb_queue_purge(struct sk_buff_head *list)
  2650. {
  2651. struct sk_buff *skb;
  2652. while ((skb = skb_dequeue(list)) != NULL)
  2653. kfree_skb(skb);
  2654. }
  2655. EXPORT_SYMBOL(skb_queue_purge);
  2656. /**
  2657. * skb_rbtree_purge - empty a skb rbtree
  2658. * @root: root of the rbtree to empty
  2659. * Return value: the sum of truesizes of all purged skbs.
  2660. *
  2661. * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
  2662. * the list and one reference dropped. This function does not take
  2663. * any lock. Synchronization should be handled by the caller (e.g., TCP
  2664. * out-of-order queue is protected by the socket lock).
  2665. */
  2666. unsigned int skb_rbtree_purge(struct rb_root *root)
  2667. {
  2668. struct rb_node *p = rb_first(root);
  2669. unsigned int sum = 0;
  2670. while (p) {
  2671. struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
  2672. p = rb_next(p);
  2673. rb_erase(&skb->rbnode, root);
  2674. sum += skb->truesize;
  2675. kfree_skb(skb);
  2676. }
  2677. return sum;
  2678. }
  2679. /**
  2680. * skb_queue_head - queue a buffer at the list head
  2681. * @list: list to use
  2682. * @newsk: buffer to queue
  2683. *
  2684. * Queue a buffer at the start of the list. This function takes the
  2685. * list lock and can be used safely with other locking &sk_buff functions
  2686. * safely.
  2687. *
  2688. * A buffer cannot be placed on two lists at the same time.
  2689. */
  2690. void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
  2691. {
  2692. unsigned long flags;
  2693. spin_lock_irqsave(&list->lock, flags);
  2694. __skb_queue_head(list, newsk);
  2695. spin_unlock_irqrestore(&list->lock, flags);
  2696. }
  2697. EXPORT_SYMBOL(skb_queue_head);
  2698. /**
  2699. * skb_queue_tail - queue a buffer at the list tail
  2700. * @list: list to use
  2701. * @newsk: buffer to queue
  2702. *
  2703. * Queue a buffer at the tail of the list. This function takes the
  2704. * list lock and can be used safely with other locking &sk_buff functions
  2705. * safely.
  2706. *
  2707. * A buffer cannot be placed on two lists at the same time.
  2708. */
  2709. void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
  2710. {
  2711. unsigned long flags;
  2712. spin_lock_irqsave(&list->lock, flags);
  2713. __skb_queue_tail(list, newsk);
  2714. spin_unlock_irqrestore(&list->lock, flags);
  2715. }
  2716. EXPORT_SYMBOL(skb_queue_tail);
  2717. /**
  2718. * skb_unlink - remove a buffer from a list
  2719. * @skb: buffer to remove
  2720. * @list: list to use
  2721. *
  2722. * Remove a packet from a list. The list locks are taken and this
  2723. * function is atomic with respect to other list locked calls
  2724. *
  2725. * You must know what list the SKB is on.
  2726. */
  2727. void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
  2728. {
  2729. unsigned long flags;
  2730. spin_lock_irqsave(&list->lock, flags);
  2731. __skb_unlink(skb, list);
  2732. spin_unlock_irqrestore(&list->lock, flags);
  2733. }
  2734. EXPORT_SYMBOL(skb_unlink);
  2735. /**
  2736. * skb_append - append a buffer
  2737. * @old: buffer to insert after
  2738. * @newsk: buffer to insert
  2739. * @list: list to use
  2740. *
  2741. * Place a packet after a given packet in a list. The list locks are taken
  2742. * and this function is atomic with respect to other list locked calls.
  2743. * A buffer cannot be placed on two lists at the same time.
  2744. */
  2745. void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
  2746. {
  2747. unsigned long flags;
  2748. spin_lock_irqsave(&list->lock, flags);
  2749. __skb_queue_after(list, old, newsk);
  2750. spin_unlock_irqrestore(&list->lock, flags);
  2751. }
  2752. EXPORT_SYMBOL(skb_append);
  2753. static inline void skb_split_inside_header(struct sk_buff *skb,
  2754. struct sk_buff* skb1,
  2755. const u32 len, const int pos)
  2756. {
  2757. int i;
  2758. skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
  2759. pos - len);
  2760. /* And move data appendix as is. */
  2761. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  2762. skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
  2763. skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
  2764. skb_shinfo(skb)->nr_frags = 0;
  2765. skb1->data_len = skb->data_len;
  2766. skb1->len += skb1->data_len;
  2767. skb->data_len = 0;
  2768. skb->len = len;
  2769. skb_set_tail_pointer(skb, len);
  2770. }
  2771. static inline void skb_split_no_header(struct sk_buff *skb,
  2772. struct sk_buff* skb1,
  2773. const u32 len, int pos)
  2774. {
  2775. int i, k = 0;
  2776. const int nfrags = skb_shinfo(skb)->nr_frags;
  2777. skb_shinfo(skb)->nr_frags = 0;
  2778. skb1->len = skb1->data_len = skb->len - len;
  2779. skb->len = len;
  2780. skb->data_len = len - pos;
  2781. for (i = 0; i < nfrags; i++) {
  2782. int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  2783. if (pos + size > len) {
  2784. skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
  2785. if (pos < len) {
  2786. /* Split frag.
  2787. * We have two variants in this case:
  2788. * 1. Move all the frag to the second
  2789. * part, if it is possible. F.e.
  2790. * this approach is mandatory for TUX,
  2791. * where splitting is expensive.
  2792. * 2. Split is accurately. We make this.
  2793. */
  2794. skb_frag_ref(skb, i);
  2795. skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
  2796. skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
  2797. skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
  2798. skb_shinfo(skb)->nr_frags++;
  2799. }
  2800. k++;
  2801. } else
  2802. skb_shinfo(skb)->nr_frags++;
  2803. pos += size;
  2804. }
  2805. skb_shinfo(skb1)->nr_frags = k;
  2806. }
  2807. /**
  2808. * skb_split - Split fragmented skb to two parts at length len.
  2809. * @skb: the buffer to split
  2810. * @skb1: the buffer to receive the second part
  2811. * @len: new length for skb
  2812. */
  2813. void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
  2814. {
  2815. int pos = skb_headlen(skb);
  2816. skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
  2817. SKBTX_SHARED_FRAG;
  2818. skb_zerocopy_clone(skb1, skb, 0);
  2819. if (len < pos) /* Split line is inside header. */
  2820. skb_split_inside_header(skb, skb1, len, pos);
  2821. else /* Second chunk has no header, nothing to copy. */
  2822. skb_split_no_header(skb, skb1, len, pos);
  2823. }
  2824. EXPORT_SYMBOL(skb_split);
  2825. /* Shifting from/to a cloned skb is a no-go.
  2826. *
  2827. * Caller cannot keep skb_shinfo related pointers past calling here!
  2828. */
  2829. static int skb_prepare_for_shift(struct sk_buff *skb)
  2830. {
  2831. return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
  2832. }
  2833. /**
  2834. * skb_shift - Shifts paged data partially from skb to another
  2835. * @tgt: buffer into which tail data gets added
  2836. * @skb: buffer from which the paged data comes from
  2837. * @shiftlen: shift up to this many bytes
  2838. *
  2839. * Attempts to shift up to shiftlen worth of bytes, which may be less than
  2840. * the length of the skb, from skb to tgt. Returns number bytes shifted.
  2841. * It's up to caller to free skb if everything was shifted.
  2842. *
  2843. * If @tgt runs out of frags, the whole operation is aborted.
  2844. *
  2845. * Skb cannot include anything else but paged data while tgt is allowed
  2846. * to have non-paged data as well.
  2847. *
  2848. * TODO: full sized shift could be optimized but that would need
  2849. * specialized skb free'er to handle frags without up-to-date nr_frags.
  2850. */
  2851. int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
  2852. {
  2853. int from, to, merge, todo;
  2854. skb_frag_t *fragfrom, *fragto;
  2855. BUG_ON(shiftlen > skb->len);
  2856. if (skb_headlen(skb))
  2857. return 0;
  2858. if (skb_zcopy(tgt) || skb_zcopy(skb))
  2859. return 0;
  2860. todo = shiftlen;
  2861. from = 0;
  2862. to = skb_shinfo(tgt)->nr_frags;
  2863. fragfrom = &skb_shinfo(skb)->frags[from];
  2864. /* Actual merge is delayed until the point when we know we can
  2865. * commit all, so that we don't have to undo partial changes
  2866. */
  2867. if (!to ||
  2868. !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
  2869. skb_frag_off(fragfrom))) {
  2870. merge = -1;
  2871. } else {
  2872. merge = to - 1;
  2873. todo -= skb_frag_size(fragfrom);
  2874. if (todo < 0) {
  2875. if (skb_prepare_for_shift(skb) ||
  2876. skb_prepare_for_shift(tgt))
  2877. return 0;
  2878. /* All previous frag pointers might be stale! */
  2879. fragfrom = &skb_shinfo(skb)->frags[from];
  2880. fragto = &skb_shinfo(tgt)->frags[merge];
  2881. skb_frag_size_add(fragto, shiftlen);
  2882. skb_frag_size_sub(fragfrom, shiftlen);
  2883. skb_frag_off_add(fragfrom, shiftlen);
  2884. goto onlymerged;
  2885. }
  2886. from++;
  2887. }
  2888. /* Skip full, not-fitting skb to avoid expensive operations */
  2889. if ((shiftlen == skb->len) &&
  2890. (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
  2891. return 0;
  2892. if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
  2893. return 0;
  2894. while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
  2895. if (to == MAX_SKB_FRAGS)
  2896. return 0;
  2897. fragfrom = &skb_shinfo(skb)->frags[from];
  2898. fragto = &skb_shinfo(tgt)->frags[to];
  2899. if (todo >= skb_frag_size(fragfrom)) {
  2900. *fragto = *fragfrom;
  2901. todo -= skb_frag_size(fragfrom);
  2902. from++;
  2903. to++;
  2904. } else {
  2905. __skb_frag_ref(fragfrom);
  2906. skb_frag_page_copy(fragto, fragfrom);
  2907. skb_frag_off_copy(fragto, fragfrom);
  2908. skb_frag_size_set(fragto, todo);
  2909. skb_frag_off_add(fragfrom, todo);
  2910. skb_frag_size_sub(fragfrom, todo);
  2911. todo = 0;
  2912. to++;
  2913. break;
  2914. }
  2915. }
  2916. /* Ready to "commit" this state change to tgt */
  2917. skb_shinfo(tgt)->nr_frags = to;
  2918. if (merge >= 0) {
  2919. fragfrom = &skb_shinfo(skb)->frags[0];
  2920. fragto = &skb_shinfo(tgt)->frags[merge];
  2921. skb_frag_size_add(fragto, skb_frag_size(fragfrom));
  2922. __skb_frag_unref(fragfrom);
  2923. }
  2924. /* Reposition in the original skb */
  2925. to = 0;
  2926. while (from < skb_shinfo(skb)->nr_frags)
  2927. skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
  2928. skb_shinfo(skb)->nr_frags = to;
  2929. BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
  2930. onlymerged:
  2931. /* Most likely the tgt won't ever need its checksum anymore, skb on
  2932. * the other hand might need it if it needs to be resent
  2933. */
  2934. tgt->ip_summed = CHECKSUM_PARTIAL;
  2935. skb->ip_summed = CHECKSUM_PARTIAL;
  2936. /* Yak, is it really working this way? Some helper please? */
  2937. skb->len -= shiftlen;
  2938. skb->data_len -= shiftlen;
  2939. skb->truesize -= shiftlen;
  2940. tgt->len += shiftlen;
  2941. tgt->data_len += shiftlen;
  2942. tgt->truesize += shiftlen;
  2943. return shiftlen;
  2944. }
  2945. /**
  2946. * skb_prepare_seq_read - Prepare a sequential read of skb data
  2947. * @skb: the buffer to read
  2948. * @from: lower offset of data to be read
  2949. * @to: upper offset of data to be read
  2950. * @st: state variable
  2951. *
  2952. * Initializes the specified state variable. Must be called before
  2953. * invoking skb_seq_read() for the first time.
  2954. */
  2955. void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
  2956. unsigned int to, struct skb_seq_state *st)
  2957. {
  2958. st->lower_offset = from;
  2959. st->upper_offset = to;
  2960. st->root_skb = st->cur_skb = skb;
  2961. st->frag_idx = st->stepped_offset = 0;
  2962. st->frag_data = NULL;
  2963. }
  2964. EXPORT_SYMBOL(skb_prepare_seq_read);
  2965. /**
  2966. * skb_seq_read - Sequentially read skb data
  2967. * @consumed: number of bytes consumed by the caller so far
  2968. * @data: destination pointer for data to be returned
  2969. * @st: state variable
  2970. *
  2971. * Reads a block of skb data at @consumed relative to the
  2972. * lower offset specified to skb_prepare_seq_read(). Assigns
  2973. * the head of the data block to @data and returns the length
  2974. * of the block or 0 if the end of the skb data or the upper
  2975. * offset has been reached.
  2976. *
  2977. * The caller is not required to consume all of the data
  2978. * returned, i.e. @consumed is typically set to the number
  2979. * of bytes already consumed and the next call to
  2980. * skb_seq_read() will return the remaining part of the block.
  2981. *
  2982. * Note 1: The size of each block of data returned can be arbitrary,
  2983. * this limitation is the cost for zerocopy sequential
  2984. * reads of potentially non linear data.
  2985. *
  2986. * Note 2: Fragment lists within fragments are not implemented
  2987. * at the moment, state->root_skb could be replaced with
  2988. * a stack for this purpose.
  2989. */
  2990. unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
  2991. struct skb_seq_state *st)
  2992. {
  2993. unsigned int block_limit, abs_offset = consumed + st->lower_offset;
  2994. skb_frag_t *frag;
  2995. if (unlikely(abs_offset >= st->upper_offset)) {
  2996. if (st->frag_data) {
  2997. kunmap_atomic(st->frag_data);
  2998. st->frag_data = NULL;
  2999. }
  3000. return 0;
  3001. }
  3002. next_skb:
  3003. block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
  3004. if (abs_offset < block_limit && !st->frag_data) {
  3005. *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
  3006. return block_limit - abs_offset;
  3007. }
  3008. if (st->frag_idx == 0 && !st->frag_data)
  3009. st->stepped_offset += skb_headlen(st->cur_skb);
  3010. while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
  3011. frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
  3012. block_limit = skb_frag_size(frag) + st->stepped_offset;
  3013. if (abs_offset < block_limit) {
  3014. if (!st->frag_data)
  3015. st->frag_data = kmap_atomic(skb_frag_page(frag));
  3016. *data = (u8 *) st->frag_data + skb_frag_off(frag) +
  3017. (abs_offset - st->stepped_offset);
  3018. return block_limit - abs_offset;
  3019. }
  3020. if (st->frag_data) {
  3021. kunmap_atomic(st->frag_data);
  3022. st->frag_data = NULL;
  3023. }
  3024. st->frag_idx++;
  3025. st->stepped_offset += skb_frag_size(frag);
  3026. }
  3027. if (st->frag_data) {
  3028. kunmap_atomic(st->frag_data);
  3029. st->frag_data = NULL;
  3030. }
  3031. if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
  3032. st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
  3033. st->frag_idx = 0;
  3034. goto next_skb;
  3035. } else if (st->cur_skb->next) {
  3036. st->cur_skb = st->cur_skb->next;
  3037. st->frag_idx = 0;
  3038. goto next_skb;
  3039. }
  3040. return 0;
  3041. }
  3042. EXPORT_SYMBOL(skb_seq_read);
  3043. /**
  3044. * skb_abort_seq_read - Abort a sequential read of skb data
  3045. * @st: state variable
  3046. *
  3047. * Must be called if skb_seq_read() was not called until it
  3048. * returned 0.
  3049. */
  3050. void skb_abort_seq_read(struct skb_seq_state *st)
  3051. {
  3052. if (st->frag_data)
  3053. kunmap_atomic(st->frag_data);
  3054. }
  3055. EXPORT_SYMBOL(skb_abort_seq_read);
  3056. #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
  3057. static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
  3058. struct ts_config *conf,
  3059. struct ts_state *state)
  3060. {
  3061. return skb_seq_read(offset, text, TS_SKB_CB(state));
  3062. }
  3063. static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
  3064. {
  3065. skb_abort_seq_read(TS_SKB_CB(state));
  3066. }
  3067. /**
  3068. * skb_find_text - Find a text pattern in skb data
  3069. * @skb: the buffer to look in
  3070. * @from: search offset
  3071. * @to: search limit
  3072. * @config: textsearch configuration
  3073. *
  3074. * Finds a pattern in the skb data according to the specified
  3075. * textsearch configuration. Use textsearch_next() to retrieve
  3076. * subsequent occurrences of the pattern. Returns the offset
  3077. * to the first occurrence or UINT_MAX if no match was found.
  3078. */
  3079. unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
  3080. unsigned int to, struct ts_config *config)
  3081. {
  3082. struct ts_state state;
  3083. unsigned int ret;
  3084. config->get_next_block = skb_ts_get_next_block;
  3085. config->finish = skb_ts_finish;
  3086. skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
  3087. ret = textsearch_find(config, &state);
  3088. return (ret <= to - from ? ret : UINT_MAX);
  3089. }
  3090. EXPORT_SYMBOL(skb_find_text);
  3091. int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
  3092. int offset, size_t size)
  3093. {
  3094. int i = skb_shinfo(skb)->nr_frags;
  3095. if (skb_can_coalesce(skb, i, page, offset)) {
  3096. skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
  3097. } else if (i < MAX_SKB_FRAGS) {
  3098. get_page(page);
  3099. skb_fill_page_desc(skb, i, page, offset, size);
  3100. } else {
  3101. return -EMSGSIZE;
  3102. }
  3103. return 0;
  3104. }
  3105. EXPORT_SYMBOL_GPL(skb_append_pagefrags);
  3106. /**
  3107. * skb_pull_rcsum - pull skb and update receive checksum
  3108. * @skb: buffer to update
  3109. * @len: length of data pulled
  3110. *
  3111. * This function performs an skb_pull on the packet and updates
  3112. * the CHECKSUM_COMPLETE checksum. It should be used on
  3113. * receive path processing instead of skb_pull unless you know
  3114. * that the checksum difference is zero (e.g., a valid IP header)
  3115. * or you are setting ip_summed to CHECKSUM_NONE.
  3116. */
  3117. void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
  3118. {
  3119. unsigned char *data = skb->data;
  3120. BUG_ON(len > skb->len);
  3121. __skb_pull(skb, len);
  3122. skb_postpull_rcsum(skb, data, len);
  3123. return skb->data;
  3124. }
  3125. EXPORT_SYMBOL_GPL(skb_pull_rcsum);
  3126. static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
  3127. {
  3128. skb_frag_t head_frag;
  3129. struct page *page;
  3130. page = virt_to_head_page(frag_skb->head);
  3131. __skb_frag_set_page(&head_frag, page);
  3132. skb_frag_off_set(&head_frag, frag_skb->data -
  3133. (unsigned char *)page_address(page));
  3134. skb_frag_size_set(&head_frag, skb_headlen(frag_skb));
  3135. return head_frag;
  3136. }
  3137. struct sk_buff *skb_segment_list(struct sk_buff *skb,
  3138. netdev_features_t features,
  3139. unsigned int offset)
  3140. {
  3141. struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
  3142. unsigned int tnl_hlen = skb_tnl_header_len(skb);
  3143. unsigned int delta_truesize = 0;
  3144. unsigned int delta_len = 0;
  3145. struct sk_buff *tail = NULL;
  3146. struct sk_buff *nskb, *tmp;
  3147. int len_diff, err;
  3148. skb_push(skb, -skb_network_offset(skb) + offset);
  3149. skb_shinfo(skb)->frag_list = NULL;
  3150. do {
  3151. nskb = list_skb;
  3152. list_skb = list_skb->next;
  3153. err = 0;
  3154. delta_truesize += nskb->truesize;
  3155. if (skb_shared(nskb)) {
  3156. tmp = skb_clone(nskb, GFP_ATOMIC);
  3157. if (tmp) {
  3158. consume_skb(nskb);
  3159. nskb = tmp;
  3160. err = skb_unclone(nskb, GFP_ATOMIC);
  3161. } else {
  3162. err = -ENOMEM;
  3163. }
  3164. }
  3165. if (!tail)
  3166. skb->next = nskb;
  3167. else
  3168. tail->next = nskb;
  3169. if (unlikely(err)) {
  3170. nskb->next = list_skb;
  3171. goto err_linearize;
  3172. }
  3173. tail = nskb;
  3174. delta_len += nskb->len;
  3175. skb_push(nskb, -skb_network_offset(nskb) + offset);
  3176. skb_release_head_state(nskb);
  3177. len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
  3178. __copy_skb_header(nskb, skb);
  3179. skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
  3180. nskb->transport_header += len_diff;
  3181. skb_copy_from_linear_data_offset(skb, -tnl_hlen,
  3182. nskb->data - tnl_hlen,
  3183. offset + tnl_hlen);
  3184. if (skb_needs_linearize(nskb, features) &&
  3185. __skb_linearize(nskb))
  3186. goto err_linearize;
  3187. } while (list_skb);
  3188. skb->truesize = skb->truesize - delta_truesize;
  3189. skb->data_len = skb->data_len - delta_len;
  3190. skb->len = skb->len - delta_len;
  3191. skb_gso_reset(skb);
  3192. skb->prev = tail;
  3193. if (skb_needs_linearize(skb, features) &&
  3194. __skb_linearize(skb))
  3195. goto err_linearize;
  3196. skb_get(skb);
  3197. return skb;
  3198. err_linearize:
  3199. kfree_skb_list(skb->next);
  3200. skb->next = NULL;
  3201. return ERR_PTR(-ENOMEM);
  3202. }
  3203. EXPORT_SYMBOL_GPL(skb_segment_list);
  3204. int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
  3205. {
  3206. if (unlikely(p->len + skb->len >= 65536))
  3207. return -E2BIG;
  3208. if (NAPI_GRO_CB(p)->last == p)
  3209. skb_shinfo(p)->frag_list = skb;
  3210. else
  3211. NAPI_GRO_CB(p)->last->next = skb;
  3212. skb_pull(skb, skb_gro_offset(skb));
  3213. NAPI_GRO_CB(p)->last = skb;
  3214. NAPI_GRO_CB(p)->count++;
  3215. p->data_len += skb->len;
  3216. p->truesize += skb->truesize;
  3217. p->len += skb->len;
  3218. NAPI_GRO_CB(skb)->same_flow = 1;
  3219. return 0;
  3220. }
  3221. /**
  3222. * skb_segment - Perform protocol segmentation on skb.
  3223. * @head_skb: buffer to segment
  3224. * @features: features for the output path (see dev->features)
  3225. *
  3226. * This function performs segmentation on the given skb. It returns
  3227. * a pointer to the first in a list of new skbs for the segments.
  3228. * In case of error it returns ERR_PTR(err).
  3229. */
  3230. struct sk_buff *skb_segment(struct sk_buff *head_skb,
  3231. netdev_features_t features)
  3232. {
  3233. struct sk_buff *segs = NULL;
  3234. struct sk_buff *tail = NULL;
  3235. struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
  3236. skb_frag_t *frag = skb_shinfo(head_skb)->frags;
  3237. unsigned int mss = skb_shinfo(head_skb)->gso_size;
  3238. unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
  3239. struct sk_buff *frag_skb = head_skb;
  3240. unsigned int offset = doffset;
  3241. unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
  3242. unsigned int partial_segs = 0;
  3243. unsigned int headroom;
  3244. unsigned int len = head_skb->len;
  3245. __be16 proto;
  3246. bool csum, sg;
  3247. int nfrags = skb_shinfo(head_skb)->nr_frags;
  3248. int err = -ENOMEM;
  3249. int i = 0;
  3250. int pos;
  3251. if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) &&
  3252. (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) {
  3253. /* gso_size is untrusted, and we have a frag_list with a linear
  3254. * non head_frag head.
  3255. *
  3256. * (we assume checking the first list_skb member suffices;
  3257. * i.e if either of the list_skb members have non head_frag
  3258. * head, then the first one has too).
  3259. *
  3260. * If head_skb's headlen does not fit requested gso_size, it
  3261. * means that the frag_list members do NOT terminate on exact
  3262. * gso_size boundaries. Hence we cannot perform skb_frag_t page
  3263. * sharing. Therefore we must fallback to copying the frag_list
  3264. * skbs; we do so by disabling SG.
  3265. */
  3266. if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb))
  3267. features &= ~NETIF_F_SG;
  3268. }
  3269. __skb_push(head_skb, doffset);
  3270. proto = skb_network_protocol(head_skb, NULL);
  3271. if (unlikely(!proto))
  3272. return ERR_PTR(-EINVAL);
  3273. sg = !!(features & NETIF_F_SG);
  3274. csum = !!can_checksum_protocol(features, proto);
  3275. if (sg && csum && (mss != GSO_BY_FRAGS)) {
  3276. if (!(features & NETIF_F_GSO_PARTIAL)) {
  3277. struct sk_buff *iter;
  3278. unsigned int frag_len;
  3279. if (!list_skb ||
  3280. !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
  3281. goto normal;
  3282. /* If we get here then all the required
  3283. * GSO features except frag_list are supported.
  3284. * Try to split the SKB to multiple GSO SKBs
  3285. * with no frag_list.
  3286. * Currently we can do that only when the buffers don't
  3287. * have a linear part and all the buffers except
  3288. * the last are of the same length.
  3289. */
  3290. frag_len = list_skb->len;
  3291. skb_walk_frags(head_skb, iter) {
  3292. if (frag_len != iter->len && iter->next)
  3293. goto normal;
  3294. if (skb_headlen(iter) && !iter->head_frag)
  3295. goto normal;
  3296. len -= iter->len;
  3297. }
  3298. if (len != frag_len)
  3299. goto normal;
  3300. }
  3301. /* GSO partial only requires that we trim off any excess that
  3302. * doesn't fit into an MSS sized block, so take care of that
  3303. * now.
  3304. */
  3305. partial_segs = len / mss;
  3306. if (partial_segs > 1)
  3307. mss *= partial_segs;
  3308. else
  3309. partial_segs = 0;
  3310. }
  3311. normal:
  3312. headroom = skb_headroom(head_skb);
  3313. pos = skb_headlen(head_skb);
  3314. do {
  3315. struct sk_buff *nskb;
  3316. skb_frag_t *nskb_frag;
  3317. int hsize;
  3318. int size;
  3319. if (unlikely(mss == GSO_BY_FRAGS)) {
  3320. len = list_skb->len;
  3321. } else {
  3322. len = head_skb->len - offset;
  3323. if (len > mss)
  3324. len = mss;
  3325. }
  3326. hsize = skb_headlen(head_skb) - offset;
  3327. if (hsize < 0)
  3328. hsize = 0;
  3329. if (hsize > len || !sg)
  3330. hsize = len;
  3331. if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
  3332. (skb_headlen(list_skb) == len || sg)) {
  3333. BUG_ON(skb_headlen(list_skb) > len);
  3334. i = 0;
  3335. nfrags = skb_shinfo(list_skb)->nr_frags;
  3336. frag = skb_shinfo(list_skb)->frags;
  3337. frag_skb = list_skb;
  3338. pos += skb_headlen(list_skb);
  3339. while (pos < offset + len) {
  3340. BUG_ON(i >= nfrags);
  3341. size = skb_frag_size(frag);
  3342. if (pos + size > offset + len)
  3343. break;
  3344. i++;
  3345. pos += size;
  3346. frag++;
  3347. }
  3348. nskb = skb_clone(list_skb, GFP_ATOMIC);
  3349. list_skb = list_skb->next;
  3350. if (unlikely(!nskb))
  3351. goto err;
  3352. if (unlikely(pskb_trim(nskb, len))) {
  3353. kfree_skb(nskb);
  3354. goto err;
  3355. }
  3356. hsize = skb_end_offset(nskb);
  3357. if (skb_cow_head(nskb, doffset + headroom)) {
  3358. kfree_skb(nskb);
  3359. goto err;
  3360. }
  3361. nskb->truesize += skb_end_offset(nskb) - hsize;
  3362. skb_release_head_state(nskb);
  3363. __skb_push(nskb, doffset);
  3364. } else {
  3365. nskb = __alloc_skb(hsize + doffset + headroom,
  3366. GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
  3367. NUMA_NO_NODE);
  3368. if (unlikely(!nskb))
  3369. goto err;
  3370. skb_reserve(nskb, headroom);
  3371. __skb_put(nskb, doffset);
  3372. }
  3373. if (segs)
  3374. tail->next = nskb;
  3375. else
  3376. segs = nskb;
  3377. tail = nskb;
  3378. __copy_skb_header(nskb, head_skb);
  3379. skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
  3380. skb_reset_mac_len(nskb);
  3381. skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
  3382. nskb->data - tnl_hlen,
  3383. doffset + tnl_hlen);
  3384. if (nskb->len == len + doffset)
  3385. goto perform_csum_check;
  3386. if (!sg) {
  3387. if (!csum) {
  3388. if (!nskb->remcsum_offload)
  3389. nskb->ip_summed = CHECKSUM_NONE;
  3390. SKB_GSO_CB(nskb)->csum =
  3391. skb_copy_and_csum_bits(head_skb, offset,
  3392. skb_put(nskb,
  3393. len),
  3394. len);
  3395. SKB_GSO_CB(nskb)->csum_start =
  3396. skb_headroom(nskb) + doffset;
  3397. } else {
  3398. skb_copy_bits(head_skb, offset,
  3399. skb_put(nskb, len),
  3400. len);
  3401. }
  3402. continue;
  3403. }
  3404. nskb_frag = skb_shinfo(nskb)->frags;
  3405. skb_copy_from_linear_data_offset(head_skb, offset,
  3406. skb_put(nskb, hsize), hsize);
  3407. skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
  3408. SKBTX_SHARED_FRAG;
  3409. if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
  3410. skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
  3411. goto err;
  3412. while (pos < offset + len) {
  3413. if (i >= nfrags) {
  3414. i = 0;
  3415. nfrags = skb_shinfo(list_skb)->nr_frags;
  3416. frag = skb_shinfo(list_skb)->frags;
  3417. frag_skb = list_skb;
  3418. if (!skb_headlen(list_skb)) {
  3419. BUG_ON(!nfrags);
  3420. } else {
  3421. BUG_ON(!list_skb->head_frag);
  3422. /* to make room for head_frag. */
  3423. i--;
  3424. frag--;
  3425. }
  3426. if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
  3427. skb_zerocopy_clone(nskb, frag_skb,
  3428. GFP_ATOMIC))
  3429. goto err;
  3430. list_skb = list_skb->next;
  3431. }
  3432. if (unlikely(skb_shinfo(nskb)->nr_frags >=
  3433. MAX_SKB_FRAGS)) {
  3434. net_warn_ratelimited(
  3435. "skb_segment: too many frags: %u %u\n",
  3436. pos, mss);
  3437. err = -EINVAL;
  3438. goto err;
  3439. }
  3440. *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
  3441. __skb_frag_ref(nskb_frag);
  3442. size = skb_frag_size(nskb_frag);
  3443. if (pos < offset) {
  3444. skb_frag_off_add(nskb_frag, offset - pos);
  3445. skb_frag_size_sub(nskb_frag, offset - pos);
  3446. }
  3447. skb_shinfo(nskb)->nr_frags++;
  3448. if (pos + size <= offset + len) {
  3449. i++;
  3450. frag++;
  3451. pos += size;
  3452. } else {
  3453. skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
  3454. goto skip_fraglist;
  3455. }
  3456. nskb_frag++;
  3457. }
  3458. skip_fraglist:
  3459. nskb->data_len = len - hsize;
  3460. nskb->len += nskb->data_len;
  3461. nskb->truesize += nskb->data_len;
  3462. perform_csum_check:
  3463. if (!csum) {
  3464. if (skb_has_shared_frag(nskb) &&
  3465. __skb_linearize(nskb))
  3466. goto err;
  3467. if (!nskb->remcsum_offload)
  3468. nskb->ip_summed = CHECKSUM_NONE;
  3469. SKB_GSO_CB(nskb)->csum =
  3470. skb_checksum(nskb, doffset,
  3471. nskb->len - doffset, 0);
  3472. SKB_GSO_CB(nskb)->csum_start =
  3473. skb_headroom(nskb) + doffset;
  3474. }
  3475. } while ((offset += len) < head_skb->len);
  3476. /* Some callers want to get the end of the list.
  3477. * Put it in segs->prev to avoid walking the list.
  3478. * (see validate_xmit_skb_list() for example)
  3479. */
  3480. segs->prev = tail;
  3481. if (partial_segs) {
  3482. struct sk_buff *iter;
  3483. int type = skb_shinfo(head_skb)->gso_type;
  3484. unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
  3485. /* Update type to add partial and then remove dodgy if set */
  3486. type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
  3487. type &= ~SKB_GSO_DODGY;
  3488. /* Update GSO info and prepare to start updating headers on
  3489. * our way back down the stack of protocols.
  3490. */
  3491. for (iter = segs; iter; iter = iter->next) {
  3492. skb_shinfo(iter)->gso_size = gso_size;
  3493. skb_shinfo(iter)->gso_segs = partial_segs;
  3494. skb_shinfo(iter)->gso_type = type;
  3495. SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
  3496. }
  3497. if (tail->len - doffset <= gso_size)
  3498. skb_shinfo(tail)->gso_size = 0;
  3499. else if (tail != segs)
  3500. skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
  3501. }
  3502. /* Following permits correct backpressure, for protocols
  3503. * using skb_set_owner_w().
  3504. * Idea is to tranfert ownership from head_skb to last segment.
  3505. */
  3506. if (head_skb->destructor == sock_wfree) {
  3507. swap(tail->truesize, head_skb->truesize);
  3508. swap(tail->destructor, head_skb->destructor);
  3509. swap(tail->sk, head_skb->sk);
  3510. }
  3511. return segs;
  3512. err:
  3513. kfree_skb_list(segs);
  3514. return ERR_PTR(err);
  3515. }
  3516. EXPORT_SYMBOL_GPL(skb_segment);
  3517. int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
  3518. {
  3519. struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
  3520. unsigned int offset = skb_gro_offset(skb);
  3521. unsigned int headlen = skb_headlen(skb);
  3522. unsigned int len = skb_gro_len(skb);
  3523. unsigned int delta_truesize;
  3524. struct sk_buff *lp;
  3525. if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
  3526. return -E2BIG;
  3527. lp = NAPI_GRO_CB(p)->last;
  3528. pinfo = skb_shinfo(lp);
  3529. if (headlen <= offset) {
  3530. skb_frag_t *frag;
  3531. skb_frag_t *frag2;
  3532. int i = skbinfo->nr_frags;
  3533. int nr_frags = pinfo->nr_frags + i;
  3534. if (nr_frags > MAX_SKB_FRAGS)
  3535. goto merge;
  3536. offset -= headlen;
  3537. pinfo->nr_frags = nr_frags;
  3538. skbinfo->nr_frags = 0;
  3539. frag = pinfo->frags + nr_frags;
  3540. frag2 = skbinfo->frags + i;
  3541. do {
  3542. *--frag = *--frag2;
  3543. } while (--i);
  3544. skb_frag_off_add(frag, offset);
  3545. skb_frag_size_sub(frag, offset);
  3546. /* all fragments truesize : remove (head size + sk_buff) */
  3547. delta_truesize = skb->truesize -
  3548. SKB_TRUESIZE(skb_end_offset(skb));
  3549. skb->truesize -= skb->data_len;
  3550. skb->len -= skb->data_len;
  3551. skb->data_len = 0;
  3552. NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
  3553. goto done;
  3554. } else if (skb->head_frag) {
  3555. int nr_frags = pinfo->nr_frags;
  3556. skb_frag_t *frag = pinfo->frags + nr_frags;
  3557. struct page *page = virt_to_head_page(skb->head);
  3558. unsigned int first_size = headlen - offset;
  3559. unsigned int first_offset;
  3560. if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
  3561. goto merge;
  3562. first_offset = skb->data -
  3563. (unsigned char *)page_address(page) +
  3564. offset;
  3565. pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
  3566. __skb_frag_set_page(frag, page);
  3567. skb_frag_off_set(frag, first_offset);
  3568. skb_frag_size_set(frag, first_size);
  3569. memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
  3570. /* We dont need to clear skbinfo->nr_frags here */
  3571. delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
  3572. NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
  3573. goto done;
  3574. }
  3575. merge:
  3576. delta_truesize = skb->truesize;
  3577. if (offset > headlen) {
  3578. unsigned int eat = offset - headlen;
  3579. skb_frag_off_add(&skbinfo->frags[0], eat);
  3580. skb_frag_size_sub(&skbinfo->frags[0], eat);
  3581. skb->data_len -= eat;
  3582. skb->len -= eat;
  3583. offset = headlen;
  3584. }
  3585. __skb_pull(skb, offset);
  3586. if (NAPI_GRO_CB(p)->last == p)
  3587. skb_shinfo(p)->frag_list = skb;
  3588. else
  3589. NAPI_GRO_CB(p)->last->next = skb;
  3590. NAPI_GRO_CB(p)->last = skb;
  3591. __skb_header_release(skb);
  3592. lp = p;
  3593. done:
  3594. NAPI_GRO_CB(p)->count++;
  3595. p->data_len += len;
  3596. p->truesize += delta_truesize;
  3597. p->len += len;
  3598. if (lp != p) {
  3599. lp->data_len += len;
  3600. lp->truesize += delta_truesize;
  3601. lp->len += len;
  3602. }
  3603. NAPI_GRO_CB(skb)->same_flow = 1;
  3604. return 0;
  3605. }
  3606. #ifdef CONFIG_SKB_EXTENSIONS
  3607. #define SKB_EXT_ALIGN_VALUE 8
  3608. #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
  3609. static const u8 skb_ext_type_len[] = {
  3610. #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
  3611. [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
  3612. #endif
  3613. #ifdef CONFIG_XFRM
  3614. [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
  3615. #endif
  3616. #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
  3617. [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
  3618. #endif
  3619. #if IS_ENABLED(CONFIG_MPTCP)
  3620. [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
  3621. #endif
  3622. #if IS_ENABLED(CONFIG_KCOV)
  3623. [SKB_EXT_KCOV_HANDLE] = SKB_EXT_CHUNKSIZEOF(u64),
  3624. #endif
  3625. };
  3626. static __always_inline unsigned int skb_ext_total_length(void)
  3627. {
  3628. return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
  3629. #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
  3630. skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
  3631. #endif
  3632. #ifdef CONFIG_XFRM
  3633. skb_ext_type_len[SKB_EXT_SEC_PATH] +
  3634. #endif
  3635. #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
  3636. skb_ext_type_len[TC_SKB_EXT] +
  3637. #endif
  3638. #if IS_ENABLED(CONFIG_MPTCP)
  3639. skb_ext_type_len[SKB_EXT_MPTCP] +
  3640. #endif
  3641. #if IS_ENABLED(CONFIG_KCOV)
  3642. skb_ext_type_len[SKB_EXT_KCOV_HANDLE] +
  3643. #endif
  3644. 0;
  3645. }
  3646. static void skb_extensions_init(void)
  3647. {
  3648. BUILD_BUG_ON(SKB_EXT_NUM >= 8);
  3649. BUILD_BUG_ON(skb_ext_total_length() > 255);
  3650. skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
  3651. SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
  3652. 0,
  3653. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  3654. NULL);
  3655. }
  3656. #else
  3657. static void skb_extensions_init(void) {}
  3658. #endif
  3659. void __init skb_init(void)
  3660. {
  3661. skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
  3662. sizeof(struct sk_buff),
  3663. 0,
  3664. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  3665. offsetof(struct sk_buff, cb),
  3666. sizeof_field(struct sk_buff, cb),
  3667. NULL);
  3668. skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
  3669. sizeof(struct sk_buff_fclones),
  3670. 0,
  3671. SLAB_HWCACHE_ALIGN|SLAB_PANIC,
  3672. NULL);
  3673. skb_extensions_init();
  3674. }
  3675. static int
  3676. __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
  3677. unsigned int recursion_level)
  3678. {
  3679. int start = skb_headlen(skb);
  3680. int i, copy = start - offset;
  3681. struct sk_buff *frag_iter;
  3682. int elt = 0;
  3683. if (unlikely(recursion_level >= 24))
  3684. return -EMSGSIZE;
  3685. if (copy > 0) {
  3686. if (copy > len)
  3687. copy = len;
  3688. sg_set_buf(sg, skb->data + offset, copy);
  3689. elt++;
  3690. if ((len -= copy) == 0)
  3691. return elt;
  3692. offset += copy;
  3693. }
  3694. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
  3695. int end;
  3696. WARN_ON(start > offset + len);
  3697. end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
  3698. if ((copy = end - offset) > 0) {
  3699. skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
  3700. if (unlikely(elt && sg_is_last(&sg[elt - 1])))
  3701. return -EMSGSIZE;
  3702. if (copy > len)
  3703. copy = len;
  3704. sg_set_page(&sg[elt], skb_frag_page(frag), copy,
  3705. skb_frag_off(frag) + offset - start);
  3706. elt++;
  3707. if (!(len -= copy))
  3708. return elt;
  3709. offset += copy;
  3710. }
  3711. start = end;
  3712. }
  3713. skb_walk_frags(skb, frag_iter) {
  3714. int end, ret;
  3715. WARN_ON(start > offset + len);
  3716. end = start + frag_iter->len;
  3717. if ((copy = end - offset) > 0) {
  3718. if (unlikely(elt && sg_is_last(&sg[elt - 1])))
  3719. return -EMSGSIZE;
  3720. if (copy > len)
  3721. copy = len;
  3722. ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
  3723. copy, recursion_level + 1);
  3724. if (unlikely(ret < 0))
  3725. return ret;
  3726. elt += ret;
  3727. if ((len -= copy) == 0)
  3728. return elt;
  3729. offset += copy;
  3730. }
  3731. start = end;
  3732. }
  3733. BUG_ON(len);
  3734. return elt;
  3735. }
  3736. /**
  3737. * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
  3738. * @skb: Socket buffer containing the buffers to be mapped
  3739. * @sg: The scatter-gather list to map into
  3740. * @offset: The offset into the buffer's contents to start mapping
  3741. * @len: Length of buffer space to be mapped
  3742. *
  3743. * Fill the specified scatter-gather list with mappings/pointers into a
  3744. * region of the buffer space attached to a socket buffer. Returns either
  3745. * the number of scatterlist items used, or -EMSGSIZE if the contents
  3746. * could not fit.
  3747. */
  3748. int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
  3749. {
  3750. int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
  3751. if (nsg <= 0)
  3752. return nsg;
  3753. sg_mark_end(&sg[nsg - 1]);
  3754. return nsg;
  3755. }
  3756. EXPORT_SYMBOL_GPL(skb_to_sgvec);
  3757. /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
  3758. * sglist without mark the sg which contain last skb data as the end.
  3759. * So the caller can mannipulate sg list as will when padding new data after
  3760. * the first call without calling sg_unmark_end to expend sg list.
  3761. *
  3762. * Scenario to use skb_to_sgvec_nomark:
  3763. * 1. sg_init_table
  3764. * 2. skb_to_sgvec_nomark(payload1)
  3765. * 3. skb_to_sgvec_nomark(payload2)
  3766. *
  3767. * This is equivalent to:
  3768. * 1. sg_init_table
  3769. * 2. skb_to_sgvec(payload1)
  3770. * 3. sg_unmark_end
  3771. * 4. skb_to_sgvec(payload2)
  3772. *
  3773. * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
  3774. * is more preferable.
  3775. */
  3776. int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
  3777. int offset, int len)
  3778. {
  3779. return __skb_to_sgvec(skb, sg, offset, len, 0);
  3780. }
  3781. EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
  3782. /**
  3783. * skb_cow_data - Check that a socket buffer's data buffers are writable
  3784. * @skb: The socket buffer to check.
  3785. * @tailbits: Amount of trailing space to be added
  3786. * @trailer: Returned pointer to the skb where the @tailbits space begins
  3787. *
  3788. * Make sure that the data buffers attached to a socket buffer are
  3789. * writable. If they are not, private copies are made of the data buffers
  3790. * and the socket buffer is set to use these instead.
  3791. *
  3792. * If @tailbits is given, make sure that there is space to write @tailbits
  3793. * bytes of data beyond current end of socket buffer. @trailer will be
  3794. * set to point to the skb in which this space begins.
  3795. *
  3796. * The number of scatterlist elements required to completely map the
  3797. * COW'd and extended socket buffer will be returned.
  3798. */
  3799. int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
  3800. {
  3801. int copyflag;
  3802. int elt;
  3803. struct sk_buff *skb1, **skb_p;
  3804. /* If skb is cloned or its head is paged, reallocate
  3805. * head pulling out all the pages (pages are considered not writable
  3806. * at the moment even if they are anonymous).
  3807. */
  3808. if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
  3809. !__pskb_pull_tail(skb, __skb_pagelen(skb)))
  3810. return -ENOMEM;
  3811. /* Easy case. Most of packets will go this way. */
  3812. if (!skb_has_frag_list(skb)) {
  3813. /* A little of trouble, not enough of space for trailer.
  3814. * This should not happen, when stack is tuned to generate
  3815. * good frames. OK, on miss we reallocate and reserve even more
  3816. * space, 128 bytes is fair. */
  3817. if (skb_tailroom(skb) < tailbits &&
  3818. pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
  3819. return -ENOMEM;
  3820. /* Voila! */
  3821. *trailer = skb;
  3822. return 1;
  3823. }
  3824. /* Misery. We are in troubles, going to mincer fragments... */
  3825. elt = 1;
  3826. skb_p = &skb_shinfo(skb)->frag_list;
  3827. copyflag = 0;
  3828. while ((skb1 = *skb_p) != NULL) {
  3829. int ntail = 0;
  3830. /* The fragment is partially pulled by someone,
  3831. * this can happen on input. Copy it and everything
  3832. * after it. */
  3833. if (skb_shared(skb1))
  3834. copyflag = 1;
  3835. /* If the skb is the last, worry about trailer. */
  3836. if (skb1->next == NULL && tailbits) {
  3837. if (skb_shinfo(skb1)->nr_frags ||
  3838. skb_has_frag_list(skb1) ||
  3839. skb_tailroom(skb1) < tailbits)
  3840. ntail = tailbits + 128;
  3841. }
  3842. if (copyflag ||
  3843. skb_cloned(skb1) ||
  3844. ntail ||
  3845. skb_shinfo(skb1)->nr_frags ||
  3846. skb_has_frag_list(skb1)) {
  3847. struct sk_buff *skb2;
  3848. /* Fuck, we are miserable poor guys... */
  3849. if (ntail == 0)
  3850. skb2 = skb_copy(skb1, GFP_ATOMIC);
  3851. else
  3852. skb2 = skb_copy_expand(skb1,
  3853. skb_headroom(skb1),
  3854. ntail,
  3855. GFP_ATOMIC);
  3856. if (unlikely(skb2 == NULL))
  3857. return -ENOMEM;
  3858. if (skb1->sk)
  3859. skb_set_owner_w(skb2, skb1->sk);
  3860. /* Looking around. Are we still alive?
  3861. * OK, link new skb, drop old one */
  3862. skb2->next = skb1->next;
  3863. *skb_p = skb2;
  3864. kfree_skb(skb1);
  3865. skb1 = skb2;
  3866. }
  3867. elt++;
  3868. *trailer = skb1;
  3869. skb_p = &skb1->next;
  3870. }
  3871. return elt;
  3872. }
  3873. EXPORT_SYMBOL_GPL(skb_cow_data);
  3874. static void sock_rmem_free(struct sk_buff *skb)
  3875. {
  3876. struct sock *sk = skb->sk;
  3877. atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
  3878. }
  3879. static void skb_set_err_queue(struct sk_buff *skb)
  3880. {
  3881. /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
  3882. * So, it is safe to (mis)use it to mark skbs on the error queue.
  3883. */
  3884. skb->pkt_type = PACKET_OUTGOING;
  3885. BUILD_BUG_ON(PACKET_OUTGOING == 0);
  3886. }
  3887. /*
  3888. * Note: We dont mem charge error packets (no sk_forward_alloc changes)
  3889. */
  3890. int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
  3891. {
  3892. if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
  3893. (unsigned int)READ_ONCE(sk->sk_rcvbuf))
  3894. return -ENOMEM;
  3895. skb_orphan(skb);
  3896. skb->sk = sk;
  3897. skb->destructor = sock_rmem_free;
  3898. atomic_add(skb->truesize, &sk->sk_rmem_alloc);
  3899. skb_set_err_queue(skb);
  3900. /* before exiting rcu section, make sure dst is refcounted */
  3901. skb_dst_force(skb);
  3902. skb_queue_tail(&sk->sk_error_queue, skb);
  3903. if (!sock_flag(sk, SOCK_DEAD))
  3904. sk->sk_error_report(sk);
  3905. return 0;
  3906. }
  3907. EXPORT_SYMBOL(sock_queue_err_skb);
  3908. static bool is_icmp_err_skb(const struct sk_buff *skb)
  3909. {
  3910. return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
  3911. SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
  3912. }
  3913. struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
  3914. {
  3915. struct sk_buff_head *q = &sk->sk_error_queue;
  3916. struct sk_buff *skb, *skb_next = NULL;
  3917. bool icmp_next = false;
  3918. unsigned long flags;
  3919. spin_lock_irqsave(&q->lock, flags);
  3920. skb = __skb_dequeue(q);
  3921. if (skb && (skb_next = skb_peek(q))) {
  3922. icmp_next = is_icmp_err_skb(skb_next);
  3923. if (icmp_next)
  3924. sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
  3925. }
  3926. spin_unlock_irqrestore(&q->lock, flags);
  3927. if (is_icmp_err_skb(skb) && !icmp_next)
  3928. sk->sk_err = 0;
  3929. if (skb_next)
  3930. sk->sk_error_report(sk);
  3931. return skb;
  3932. }
  3933. EXPORT_SYMBOL(sock_dequeue_err_skb);
  3934. /**
  3935. * skb_clone_sk - create clone of skb, and take reference to socket
  3936. * @skb: the skb to clone
  3937. *
  3938. * This function creates a clone of a buffer that holds a reference on
  3939. * sk_refcnt. Buffers created via this function are meant to be
  3940. * returned using sock_queue_err_skb, or free via kfree_skb.
  3941. *
  3942. * When passing buffers allocated with this function to sock_queue_err_skb
  3943. * it is necessary to wrap the call with sock_hold/sock_put in order to
  3944. * prevent the socket from being released prior to being enqueued on
  3945. * the sk_error_queue.
  3946. */
  3947. struct sk_buff *skb_clone_sk(struct sk_buff *skb)
  3948. {
  3949. struct sock *sk = skb->sk;
  3950. struct sk_buff *clone;
  3951. if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
  3952. return NULL;
  3953. clone = skb_clone(skb, GFP_ATOMIC);
  3954. if (!clone) {
  3955. sock_put(sk);
  3956. return NULL;
  3957. }
  3958. clone->sk = sk;
  3959. clone->destructor = sock_efree;
  3960. return clone;
  3961. }
  3962. EXPORT_SYMBOL(skb_clone_sk);
  3963. static void __skb_complete_tx_timestamp(struct sk_buff *skb,
  3964. struct sock *sk,
  3965. int tstype,
  3966. bool opt_stats)
  3967. {
  3968. struct sock_exterr_skb *serr;
  3969. int err;
  3970. BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
  3971. serr = SKB_EXT_ERR(skb);
  3972. memset(serr, 0, sizeof(*serr));
  3973. serr->ee.ee_errno = ENOMSG;
  3974. serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
  3975. serr->ee.ee_info = tstype;
  3976. serr->opt_stats = opt_stats;
  3977. serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
  3978. if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
  3979. serr->ee.ee_data = skb_shinfo(skb)->tskey;
  3980. if (sk->sk_protocol == IPPROTO_TCP &&
  3981. sk->sk_type == SOCK_STREAM)
  3982. serr->ee.ee_data -= sk->sk_tskey;
  3983. }
  3984. err = sock_queue_err_skb(sk, skb);
  3985. if (err)
  3986. kfree_skb(skb);
  3987. }
  3988. static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
  3989. {
  3990. bool ret;
  3991. if (likely(sysctl_tstamp_allow_data || tsonly))
  3992. return true;
  3993. read_lock_bh(&sk->sk_callback_lock);
  3994. ret = sk->sk_socket && sk->sk_socket->file &&
  3995. file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
  3996. read_unlock_bh(&sk->sk_callback_lock);
  3997. return ret;
  3998. }
  3999. void skb_complete_tx_timestamp(struct sk_buff *skb,
  4000. struct skb_shared_hwtstamps *hwtstamps)
  4001. {
  4002. struct sock *sk = skb->sk;
  4003. if (!skb_may_tx_timestamp(sk, false))
  4004. goto err;
  4005. /* Take a reference to prevent skb_orphan() from freeing the socket,
  4006. * but only if the socket refcount is not zero.
  4007. */
  4008. if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
  4009. *skb_hwtstamps(skb) = *hwtstamps;
  4010. __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
  4011. sock_put(sk);
  4012. return;
  4013. }
  4014. err:
  4015. kfree_skb(skb);
  4016. }
  4017. EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
  4018. void __skb_tstamp_tx(struct sk_buff *orig_skb,
  4019. struct skb_shared_hwtstamps *hwtstamps,
  4020. struct sock *sk, int tstype)
  4021. {
  4022. struct sk_buff *skb;
  4023. bool tsonly, opt_stats = false;
  4024. if (!sk)
  4025. return;
  4026. if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
  4027. skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
  4028. return;
  4029. tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
  4030. if (!skb_may_tx_timestamp(sk, tsonly))
  4031. return;
  4032. if (tsonly) {
  4033. #ifdef CONFIG_INET
  4034. if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
  4035. sk->sk_protocol == IPPROTO_TCP &&
  4036. sk->sk_type == SOCK_STREAM) {
  4037. skb = tcp_get_timestamping_opt_stats(sk, orig_skb);
  4038. opt_stats = true;
  4039. } else
  4040. #endif
  4041. skb = alloc_skb(0, GFP_ATOMIC);
  4042. } else {
  4043. skb = skb_clone(orig_skb, GFP_ATOMIC);
  4044. }
  4045. if (!skb)
  4046. return;
  4047. if (tsonly) {
  4048. skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
  4049. SKBTX_ANY_TSTAMP;
  4050. skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
  4051. }
  4052. if (hwtstamps)
  4053. *skb_hwtstamps(skb) = *hwtstamps;
  4054. else
  4055. skb->tstamp = ktime_get_real();
  4056. __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
  4057. }
  4058. EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
  4059. void skb_tstamp_tx(struct sk_buff *orig_skb,
  4060. struct skb_shared_hwtstamps *hwtstamps)
  4061. {
  4062. return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
  4063. SCM_TSTAMP_SND);
  4064. }
  4065. EXPORT_SYMBOL_GPL(skb_tstamp_tx);
  4066. void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
  4067. {
  4068. struct sock *sk = skb->sk;
  4069. struct sock_exterr_skb *serr;
  4070. int err = 1;
  4071. skb->wifi_acked_valid = 1;
  4072. skb->wifi_acked = acked;
  4073. serr = SKB_EXT_ERR(skb);
  4074. memset(serr, 0, sizeof(*serr));
  4075. serr->ee.ee_errno = ENOMSG;
  4076. serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
  4077. /* Take a reference to prevent skb_orphan() from freeing the socket,
  4078. * but only if the socket refcount is not zero.
  4079. */
  4080. if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
  4081. err = sock_queue_err_skb(sk, skb);
  4082. sock_put(sk);
  4083. }
  4084. if (err)
  4085. kfree_skb(skb);
  4086. }
  4087. EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
  4088. /**
  4089. * skb_partial_csum_set - set up and verify partial csum values for packet
  4090. * @skb: the skb to set
  4091. * @start: the number of bytes after skb->data to start checksumming.
  4092. * @off: the offset from start to place the checksum.
  4093. *
  4094. * For untrusted partially-checksummed packets, we need to make sure the values
  4095. * for skb->csum_start and skb->csum_offset are valid so we don't oops.
  4096. *
  4097. * This function checks and sets those values and skb->ip_summed: if this
  4098. * returns false you should drop the packet.
  4099. */
  4100. bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
  4101. {
  4102. u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
  4103. u32 csum_start = skb_headroom(skb) + (u32)start;
  4104. if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
  4105. net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
  4106. start, off, skb_headroom(skb), skb_headlen(skb));
  4107. return false;
  4108. }
  4109. skb->ip_summed = CHECKSUM_PARTIAL;
  4110. skb->csum_start = csum_start;
  4111. skb->csum_offset = off;
  4112. skb_set_transport_header(skb, start);
  4113. return true;
  4114. }
  4115. EXPORT_SYMBOL_GPL(skb_partial_csum_set);
  4116. static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
  4117. unsigned int max)
  4118. {
  4119. if (skb_headlen(skb) >= len)
  4120. return 0;
  4121. /* If we need to pullup then pullup to the max, so we
  4122. * won't need to do it again.
  4123. */
  4124. if (max > skb->len)
  4125. max = skb->len;
  4126. if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
  4127. return -ENOMEM;
  4128. if (skb_headlen(skb) < len)
  4129. return -EPROTO;
  4130. return 0;
  4131. }
  4132. #define MAX_TCP_HDR_LEN (15 * 4)
  4133. static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
  4134. typeof(IPPROTO_IP) proto,
  4135. unsigned int off)
  4136. {
  4137. int err;
  4138. switch (proto) {
  4139. case IPPROTO_TCP:
  4140. err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
  4141. off + MAX_TCP_HDR_LEN);
  4142. if (!err && !skb_partial_csum_set(skb, off,
  4143. offsetof(struct tcphdr,
  4144. check)))
  4145. err = -EPROTO;
  4146. return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
  4147. case IPPROTO_UDP:
  4148. err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
  4149. off + sizeof(struct udphdr));
  4150. if (!err && !skb_partial_csum_set(skb, off,
  4151. offsetof(struct udphdr,
  4152. check)))
  4153. err = -EPROTO;
  4154. return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
  4155. }
  4156. return ERR_PTR(-EPROTO);
  4157. }
  4158. /* This value should be large enough to cover a tagged ethernet header plus
  4159. * maximally sized IP and TCP or UDP headers.
  4160. */
  4161. #define MAX_IP_HDR_LEN 128
  4162. static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
  4163. {
  4164. unsigned int off;
  4165. bool fragment;
  4166. __sum16 *csum;
  4167. int err;
  4168. fragment = false;
  4169. err = skb_maybe_pull_tail(skb,
  4170. sizeof(struct iphdr),
  4171. MAX_IP_HDR_LEN);
  4172. if (err < 0)
  4173. goto out;
  4174. if (ip_is_fragment(ip_hdr(skb)))
  4175. fragment = true;
  4176. off = ip_hdrlen(skb);
  4177. err = -EPROTO;
  4178. if (fragment)
  4179. goto out;
  4180. csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
  4181. if (IS_ERR(csum))
  4182. return PTR_ERR(csum);
  4183. if (recalculate)
  4184. *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
  4185. ip_hdr(skb)->daddr,
  4186. skb->len - off,
  4187. ip_hdr(skb)->protocol, 0);
  4188. err = 0;
  4189. out:
  4190. return err;
  4191. }
  4192. /* This value should be large enough to cover a tagged ethernet header plus
  4193. * an IPv6 header, all options, and a maximal TCP or UDP header.
  4194. */
  4195. #define MAX_IPV6_HDR_LEN 256
  4196. #define OPT_HDR(type, skb, off) \
  4197. (type *)(skb_network_header(skb) + (off))
  4198. static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
  4199. {
  4200. int err;
  4201. u8 nexthdr;
  4202. unsigned int off;
  4203. unsigned int len;
  4204. bool fragment;
  4205. bool done;
  4206. __sum16 *csum;
  4207. fragment = false;
  4208. done = false;
  4209. off = sizeof(struct ipv6hdr);
  4210. err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
  4211. if (err < 0)
  4212. goto out;
  4213. nexthdr = ipv6_hdr(skb)->nexthdr;
  4214. len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
  4215. while (off <= len && !done) {
  4216. switch (nexthdr) {
  4217. case IPPROTO_DSTOPTS:
  4218. case IPPROTO_HOPOPTS:
  4219. case IPPROTO_ROUTING: {
  4220. struct ipv6_opt_hdr *hp;
  4221. err = skb_maybe_pull_tail(skb,
  4222. off +
  4223. sizeof(struct ipv6_opt_hdr),
  4224. MAX_IPV6_HDR_LEN);
  4225. if (err < 0)
  4226. goto out;
  4227. hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
  4228. nexthdr = hp->nexthdr;
  4229. off += ipv6_optlen(hp);
  4230. break;
  4231. }
  4232. case IPPROTO_AH: {
  4233. struct ip_auth_hdr *hp;
  4234. err = skb_maybe_pull_tail(skb,
  4235. off +
  4236. sizeof(struct ip_auth_hdr),
  4237. MAX_IPV6_HDR_LEN);
  4238. if (err < 0)
  4239. goto out;
  4240. hp = OPT_HDR(struct ip_auth_hdr, skb, off);
  4241. nexthdr = hp->nexthdr;
  4242. off += ipv6_authlen(hp);
  4243. break;
  4244. }
  4245. case IPPROTO_FRAGMENT: {
  4246. struct frag_hdr *hp;
  4247. err = skb_maybe_pull_tail(skb,
  4248. off +
  4249. sizeof(struct frag_hdr),
  4250. MAX_IPV6_HDR_LEN);
  4251. if (err < 0)
  4252. goto out;
  4253. hp = OPT_HDR(struct frag_hdr, skb, off);
  4254. if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
  4255. fragment = true;
  4256. nexthdr = hp->nexthdr;
  4257. off += sizeof(struct frag_hdr);
  4258. break;
  4259. }
  4260. default:
  4261. done = true;
  4262. break;
  4263. }
  4264. }
  4265. err = -EPROTO;
  4266. if (!done || fragment)
  4267. goto out;
  4268. csum = skb_checksum_setup_ip(skb, nexthdr, off);
  4269. if (IS_ERR(csum))
  4270. return PTR_ERR(csum);
  4271. if (recalculate)
  4272. *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
  4273. &ipv6_hdr(skb)->daddr,
  4274. skb->len - off, nexthdr, 0);
  4275. err = 0;
  4276. out:
  4277. return err;
  4278. }
  4279. /**
  4280. * skb_checksum_setup - set up partial checksum offset
  4281. * @skb: the skb to set up
  4282. * @recalculate: if true the pseudo-header checksum will be recalculated
  4283. */
  4284. int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
  4285. {
  4286. int err;
  4287. switch (skb->protocol) {
  4288. case htons(ETH_P_IP):
  4289. err = skb_checksum_setup_ipv4(skb, recalculate);
  4290. break;
  4291. case htons(ETH_P_IPV6):
  4292. err = skb_checksum_setup_ipv6(skb, recalculate);
  4293. break;
  4294. default:
  4295. err = -EPROTO;
  4296. break;
  4297. }
  4298. return err;
  4299. }
  4300. EXPORT_SYMBOL(skb_checksum_setup);
  4301. /**
  4302. * skb_checksum_maybe_trim - maybe trims the given skb
  4303. * @skb: the skb to check
  4304. * @transport_len: the data length beyond the network header
  4305. *
  4306. * Checks whether the given skb has data beyond the given transport length.
  4307. * If so, returns a cloned skb trimmed to this transport length.
  4308. * Otherwise returns the provided skb. Returns NULL in error cases
  4309. * (e.g. transport_len exceeds skb length or out-of-memory).
  4310. *
  4311. * Caller needs to set the skb transport header and free any returned skb if it
  4312. * differs from the provided skb.
  4313. */
  4314. static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
  4315. unsigned int transport_len)
  4316. {
  4317. struct sk_buff *skb_chk;
  4318. unsigned int len = skb_transport_offset(skb) + transport_len;
  4319. int ret;
  4320. if (skb->len < len)
  4321. return NULL;
  4322. else if (skb->len == len)
  4323. return skb;
  4324. skb_chk = skb_clone(skb, GFP_ATOMIC);
  4325. if (!skb_chk)
  4326. return NULL;
  4327. ret = pskb_trim_rcsum(skb_chk, len);
  4328. if (ret) {
  4329. kfree_skb(skb_chk);
  4330. return NULL;
  4331. }
  4332. return skb_chk;
  4333. }
  4334. /**
  4335. * skb_checksum_trimmed - validate checksum of an skb
  4336. * @skb: the skb to check
  4337. * @transport_len: the data length beyond the network header
  4338. * @skb_chkf: checksum function to use
  4339. *
  4340. * Applies the given checksum function skb_chkf to the provided skb.
  4341. * Returns a checked and maybe trimmed skb. Returns NULL on error.
  4342. *
  4343. * If the skb has data beyond the given transport length, then a
  4344. * trimmed & cloned skb is checked and returned.
  4345. *
  4346. * Caller needs to set the skb transport header and free any returned skb if it
  4347. * differs from the provided skb.
  4348. */
  4349. struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
  4350. unsigned int transport_len,
  4351. __sum16(*skb_chkf)(struct sk_buff *skb))
  4352. {
  4353. struct sk_buff *skb_chk;
  4354. unsigned int offset = skb_transport_offset(skb);
  4355. __sum16 ret;
  4356. skb_chk = skb_checksum_maybe_trim(skb, transport_len);
  4357. if (!skb_chk)
  4358. goto err;
  4359. if (!pskb_may_pull(skb_chk, offset))
  4360. goto err;
  4361. skb_pull_rcsum(skb_chk, offset);
  4362. ret = skb_chkf(skb_chk);
  4363. skb_push_rcsum(skb_chk, offset);
  4364. if (ret)
  4365. goto err;
  4366. return skb_chk;
  4367. err:
  4368. if (skb_chk && skb_chk != skb)
  4369. kfree_skb(skb_chk);
  4370. return NULL;
  4371. }
  4372. EXPORT_SYMBOL(skb_checksum_trimmed);
  4373. void __skb_warn_lro_forwarding(const struct sk_buff *skb)
  4374. {
  4375. net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
  4376. skb->dev->name);
  4377. }
  4378. EXPORT_SYMBOL(__skb_warn_lro_forwarding);
  4379. void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
  4380. {
  4381. if (head_stolen) {
  4382. skb_release_head_state(skb);
  4383. kmem_cache_free(skbuff_head_cache, skb);
  4384. } else {
  4385. __kfree_skb(skb);
  4386. }
  4387. }
  4388. EXPORT_SYMBOL(kfree_skb_partial);
  4389. /**
  4390. * skb_try_coalesce - try to merge skb to prior one
  4391. * @to: prior buffer
  4392. * @from: buffer to add
  4393. * @fragstolen: pointer to boolean
  4394. * @delta_truesize: how much more was allocated than was requested
  4395. */
  4396. bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
  4397. bool *fragstolen, int *delta_truesize)
  4398. {
  4399. struct skb_shared_info *to_shinfo, *from_shinfo;
  4400. int i, delta, len = from->len;
  4401. *fragstolen = false;
  4402. if (skb_cloned(to))
  4403. return false;
  4404. if (len <= skb_tailroom(to)) {
  4405. if (len)
  4406. BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
  4407. *delta_truesize = 0;
  4408. return true;
  4409. }
  4410. to_shinfo = skb_shinfo(to);
  4411. from_shinfo = skb_shinfo(from);
  4412. if (to_shinfo->frag_list || from_shinfo->frag_list)
  4413. return false;
  4414. if (skb_zcopy(to) || skb_zcopy(from))
  4415. return false;
  4416. if (skb_headlen(from) != 0) {
  4417. struct page *page;
  4418. unsigned int offset;
  4419. if (to_shinfo->nr_frags +
  4420. from_shinfo->nr_frags >= MAX_SKB_FRAGS)
  4421. return false;
  4422. if (skb_head_is_locked(from))
  4423. return false;
  4424. delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
  4425. page = virt_to_head_page(from->head);
  4426. offset = from->data - (unsigned char *)page_address(page);
  4427. skb_fill_page_desc(to, to_shinfo->nr_frags,
  4428. page, offset, skb_headlen(from));
  4429. *fragstolen = true;
  4430. } else {
  4431. if (to_shinfo->nr_frags +
  4432. from_shinfo->nr_frags > MAX_SKB_FRAGS)
  4433. return false;
  4434. delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
  4435. }
  4436. WARN_ON_ONCE(delta < len);
  4437. memcpy(to_shinfo->frags + to_shinfo->nr_frags,
  4438. from_shinfo->frags,
  4439. from_shinfo->nr_frags * sizeof(skb_frag_t));
  4440. to_shinfo->nr_frags += from_shinfo->nr_frags;
  4441. if (!skb_cloned(from))
  4442. from_shinfo->nr_frags = 0;
  4443. /* if the skb is not cloned this does nothing
  4444. * since we set nr_frags to 0.
  4445. */
  4446. for (i = 0; i < from_shinfo->nr_frags; i++)
  4447. __skb_frag_ref(&from_shinfo->frags[i]);
  4448. to->truesize += delta;
  4449. to->len += len;
  4450. to->data_len += len;
  4451. *delta_truesize = delta;
  4452. return true;
  4453. }
  4454. EXPORT_SYMBOL(skb_try_coalesce);
  4455. /**
  4456. * skb_scrub_packet - scrub an skb
  4457. *
  4458. * @skb: buffer to clean
  4459. * @xnet: packet is crossing netns
  4460. *
  4461. * skb_scrub_packet can be used after encapsulating or decapsulting a packet
  4462. * into/from a tunnel. Some information have to be cleared during these
  4463. * operations.
  4464. * skb_scrub_packet can also be used to clean a skb before injecting it in
  4465. * another namespace (@xnet == true). We have to clear all information in the
  4466. * skb that could impact namespace isolation.
  4467. */
  4468. void skb_scrub_packet(struct sk_buff *skb, bool xnet)
  4469. {
  4470. skb->pkt_type = PACKET_HOST;
  4471. skb->skb_iif = 0;
  4472. skb->ignore_df = 0;
  4473. skb_dst_drop(skb);
  4474. skb_ext_reset(skb);
  4475. nf_reset_ct(skb);
  4476. nf_reset_trace(skb);
  4477. #ifdef CONFIG_NET_SWITCHDEV
  4478. skb->offload_fwd_mark = 0;
  4479. skb->offload_l3_fwd_mark = 0;
  4480. #endif
  4481. if (!xnet)
  4482. return;
  4483. ipvs_reset(skb);
  4484. skb->mark = 0;
  4485. skb->tstamp = 0;
  4486. }
  4487. EXPORT_SYMBOL_GPL(skb_scrub_packet);
  4488. /**
  4489. * skb_gso_transport_seglen - Return length of individual segments of a gso packet
  4490. *
  4491. * @skb: GSO skb
  4492. *
  4493. * skb_gso_transport_seglen is used to determine the real size of the
  4494. * individual segments, including Layer4 headers (TCP/UDP).
  4495. *
  4496. * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
  4497. */
  4498. static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
  4499. {
  4500. const struct skb_shared_info *shinfo = skb_shinfo(skb);
  4501. unsigned int thlen = 0;
  4502. if (skb->encapsulation) {
  4503. thlen = skb_inner_transport_header(skb) -
  4504. skb_transport_header(skb);
  4505. if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
  4506. thlen += inner_tcp_hdrlen(skb);
  4507. } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
  4508. thlen = tcp_hdrlen(skb);
  4509. } else if (unlikely(skb_is_gso_sctp(skb))) {
  4510. thlen = sizeof(struct sctphdr);
  4511. } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
  4512. thlen = sizeof(struct udphdr);
  4513. }
  4514. /* UFO sets gso_size to the size of the fragmentation
  4515. * payload, i.e. the size of the L4 (UDP) header is already
  4516. * accounted for.
  4517. */
  4518. return thlen + shinfo->gso_size;
  4519. }
  4520. /**
  4521. * skb_gso_network_seglen - Return length of individual segments of a gso packet
  4522. *
  4523. * @skb: GSO skb
  4524. *
  4525. * skb_gso_network_seglen is used to determine the real size of the
  4526. * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
  4527. *
  4528. * The MAC/L2 header is not accounted for.
  4529. */
  4530. static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
  4531. {
  4532. unsigned int hdr_len = skb_transport_header(skb) -
  4533. skb_network_header(skb);
  4534. return hdr_len + skb_gso_transport_seglen(skb);
  4535. }
  4536. /**
  4537. * skb_gso_mac_seglen - Return length of individual segments of a gso packet
  4538. *
  4539. * @skb: GSO skb
  4540. *
  4541. * skb_gso_mac_seglen is used to determine the real size of the
  4542. * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
  4543. * headers (TCP/UDP).
  4544. */
  4545. static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
  4546. {
  4547. unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
  4548. return hdr_len + skb_gso_transport_seglen(skb);
  4549. }
  4550. /**
  4551. * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
  4552. *
  4553. * There are a couple of instances where we have a GSO skb, and we
  4554. * want to determine what size it would be after it is segmented.
  4555. *
  4556. * We might want to check:
  4557. * - L3+L4+payload size (e.g. IP forwarding)
  4558. * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
  4559. *
  4560. * This is a helper to do that correctly considering GSO_BY_FRAGS.
  4561. *
  4562. * @skb: GSO skb
  4563. *
  4564. * @seg_len: The segmented length (from skb_gso_*_seglen). In the
  4565. * GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
  4566. *
  4567. * @max_len: The maximum permissible length.
  4568. *
  4569. * Returns true if the segmented length <= max length.
  4570. */
  4571. static inline bool skb_gso_size_check(const struct sk_buff *skb,
  4572. unsigned int seg_len,
  4573. unsigned int max_len) {
  4574. const struct skb_shared_info *shinfo = skb_shinfo(skb);
  4575. const struct sk_buff *iter;
  4576. if (shinfo->gso_size != GSO_BY_FRAGS)
  4577. return seg_len <= max_len;
  4578. /* Undo this so we can re-use header sizes */
  4579. seg_len -= GSO_BY_FRAGS;
  4580. skb_walk_frags(skb, iter) {
  4581. if (seg_len + skb_headlen(iter) > max_len)
  4582. return false;
  4583. }
  4584. return true;
  4585. }
  4586. /**
  4587. * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
  4588. *
  4589. * @skb: GSO skb
  4590. * @mtu: MTU to validate against
  4591. *
  4592. * skb_gso_validate_network_len validates if a given skb will fit a
  4593. * wanted MTU once split. It considers L3 headers, L4 headers, and the
  4594. * payload.
  4595. */
  4596. bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
  4597. {
  4598. return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
  4599. }
  4600. EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
  4601. /**
  4602. * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
  4603. *
  4604. * @skb: GSO skb
  4605. * @len: length to validate against
  4606. *
  4607. * skb_gso_validate_mac_len validates if a given skb will fit a wanted
  4608. * length once split, including L2, L3 and L4 headers and the payload.
  4609. */
  4610. bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
  4611. {
  4612. return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
  4613. }
  4614. EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);
  4615. static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
  4616. {
  4617. int mac_len, meta_len;
  4618. void *meta;
  4619. if (skb_cow(skb, skb_headroom(skb)) < 0) {
  4620. kfree_skb(skb);
  4621. return NULL;
  4622. }
  4623. mac_len = skb->data - skb_mac_header(skb);
  4624. if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
  4625. memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
  4626. mac_len - VLAN_HLEN - ETH_TLEN);
  4627. }
  4628. meta_len = skb_metadata_len(skb);
  4629. if (meta_len) {
  4630. meta = skb_metadata_end(skb) - meta_len;
  4631. memmove(meta + VLAN_HLEN, meta, meta_len);
  4632. }
  4633. skb->mac_header += VLAN_HLEN;
  4634. return skb;
  4635. }
  4636. struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
  4637. {
  4638. struct vlan_hdr *vhdr;
  4639. u16 vlan_tci;
  4640. if (unlikely(skb_vlan_tag_present(skb))) {
  4641. /* vlan_tci is already set-up so leave this for another time */
  4642. return skb;
  4643. }
  4644. skb = skb_share_check(skb, GFP_ATOMIC);
  4645. if (unlikely(!skb))
  4646. goto err_free;
  4647. /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */
  4648. if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
  4649. goto err_free;
  4650. vhdr = (struct vlan_hdr *)skb->data;
  4651. vlan_tci = ntohs(vhdr->h_vlan_TCI);
  4652. __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
  4653. skb_pull_rcsum(skb, VLAN_HLEN);
  4654. vlan_set_encap_proto(skb, vhdr);
  4655. skb = skb_reorder_vlan_header(skb);
  4656. if (unlikely(!skb))
  4657. goto err_free;
  4658. skb_reset_network_header(skb);
  4659. skb_reset_transport_header(skb);
  4660. skb_reset_mac_len(skb);
  4661. return skb;
  4662. err_free:
  4663. kfree_skb(skb);
  4664. return NULL;
  4665. }
  4666. EXPORT_SYMBOL(skb_vlan_untag);
  4667. int skb_ensure_writable(struct sk_buff *skb, int write_len)
  4668. {
  4669. if (!pskb_may_pull(skb, write_len))
  4670. return -ENOMEM;
  4671. if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
  4672. return 0;
  4673. return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
  4674. }
  4675. EXPORT_SYMBOL(skb_ensure_writable);
  4676. /* remove VLAN header from packet and update csum accordingly.
  4677. * expects a non skb_vlan_tag_present skb with a vlan tag payload
  4678. */
  4679. int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
  4680. {
  4681. struct vlan_hdr *vhdr;
  4682. int offset = skb->data - skb_mac_header(skb);
  4683. int err;
  4684. if (WARN_ONCE(offset,
  4685. "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
  4686. offset)) {
  4687. return -EINVAL;
  4688. }
  4689. err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
  4690. if (unlikely(err))
  4691. return err;
  4692. skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
  4693. vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
  4694. *vlan_tci = ntohs(vhdr->h_vlan_TCI);
  4695. memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
  4696. __skb_pull(skb, VLAN_HLEN);
  4697. vlan_set_encap_proto(skb, vhdr);
  4698. skb->mac_header += VLAN_HLEN;
  4699. if (skb_network_offset(skb) < ETH_HLEN)
  4700. skb_set_network_header(skb, ETH_HLEN);
  4701. skb_reset_mac_len(skb);
  4702. return err;
  4703. }
  4704. EXPORT_SYMBOL(__skb_vlan_pop);
  4705. /* Pop a vlan tag either from hwaccel or from payload.
  4706. * Expects skb->data at mac header.
  4707. */
  4708. int skb_vlan_pop(struct sk_buff *skb)
  4709. {
  4710. u16 vlan_tci;
  4711. __be16 vlan_proto;
  4712. int err;
  4713. if (likely(skb_vlan_tag_present(skb))) {
  4714. __vlan_hwaccel_clear_tag(skb);
  4715. } else {
  4716. if (unlikely(!eth_type_vlan(skb->protocol)))
  4717. return 0;
  4718. err = __skb_vlan_pop(skb, &vlan_tci);
  4719. if (err)
  4720. return err;
  4721. }
  4722. /* move next vlan tag to hw accel tag */
  4723. if (likely(!eth_type_vlan(skb->protocol)))
  4724. return 0;
  4725. vlan_proto = skb->protocol;
  4726. err = __skb_vlan_pop(skb, &vlan_tci);
  4727. if (unlikely(err))
  4728. return err;
  4729. __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
  4730. return 0;
  4731. }
  4732. EXPORT_SYMBOL(skb_vlan_pop);
  4733. /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
  4734. * Expects skb->data at mac header.
  4735. */
  4736. int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
  4737. {
  4738. if (skb_vlan_tag_present(skb)) {
  4739. int offset = skb->data - skb_mac_header(skb);
  4740. int err;
  4741. if (WARN_ONCE(offset,
  4742. "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
  4743. offset)) {
  4744. return -EINVAL;
  4745. }
  4746. err = __vlan_insert_tag(skb, skb->vlan_proto,
  4747. skb_vlan_tag_get(skb));
  4748. if (err)
  4749. return err;
  4750. skb->protocol = skb->vlan_proto;
  4751. skb->mac_len += VLAN_HLEN;
  4752. skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
  4753. }
  4754. __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
  4755. return 0;
  4756. }
  4757. EXPORT_SYMBOL(skb_vlan_push);
  4758. /**
  4759. * skb_eth_pop() - Drop the Ethernet header at the head of a packet
  4760. *
  4761. * @skb: Socket buffer to modify
  4762. *
  4763. * Drop the Ethernet header of @skb.
  4764. *
  4765. * Expects that skb->data points to the mac header and that no VLAN tags are
  4766. * present.
  4767. *
  4768. * Returns 0 on success, -errno otherwise.
  4769. */
  4770. int skb_eth_pop(struct sk_buff *skb)
  4771. {
  4772. if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
  4773. skb_network_offset(skb) < ETH_HLEN)
  4774. return -EPROTO;
  4775. skb_pull_rcsum(skb, ETH_HLEN);
  4776. skb_reset_mac_header(skb);
  4777. skb_reset_mac_len(skb);
  4778. return 0;
  4779. }
  4780. EXPORT_SYMBOL(skb_eth_pop);
  4781. /**
  4782. * skb_eth_push() - Add a new Ethernet header at the head of a packet
  4783. *
  4784. * @skb: Socket buffer to modify
  4785. * @dst: Destination MAC address of the new header
  4786. * @src: Source MAC address of the new header
  4787. *
  4788. * Prepend @skb with a new Ethernet header.
  4789. *
  4790. * Expects that skb->data points to the mac header, which must be empty.
  4791. *
  4792. * Returns 0 on success, -errno otherwise.
  4793. */
  4794. int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
  4795. const unsigned char *src)
  4796. {
  4797. struct ethhdr *eth;
  4798. int err;
  4799. if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
  4800. return -EPROTO;
  4801. err = skb_cow_head(skb, sizeof(*eth));
  4802. if (err < 0)
  4803. return err;
  4804. skb_push(skb, sizeof(*eth));
  4805. skb_reset_mac_header(skb);
  4806. skb_reset_mac_len(skb);
  4807. eth = eth_hdr(skb);
  4808. ether_addr_copy(eth->h_dest, dst);
  4809. ether_addr_copy(eth->h_source, src);
  4810. eth->h_proto = skb->protocol;
  4811. skb_postpush_rcsum(skb, eth, sizeof(*eth));
  4812. return 0;
  4813. }
  4814. EXPORT_SYMBOL(skb_eth_push);
  4815. /* Update the ethertype of hdr and the skb csum value if required. */
  4816. static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
  4817. __be16 ethertype)
  4818. {
  4819. if (skb->ip_summed == CHECKSUM_COMPLETE) {
  4820. __be16 diff[] = { ~hdr->h_proto, ethertype };
  4821. skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
  4822. }
  4823. hdr->h_proto = ethertype;
  4824. }
  4825. /**
  4826. * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
  4827. * the packet
  4828. *
  4829. * @skb: buffer
  4830. * @mpls_lse: MPLS label stack entry to push
  4831. * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
  4832. * @mac_len: length of the MAC header
  4833. * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
  4834. * ethernet
  4835. *
  4836. * Expects skb->data at mac header.
  4837. *
  4838. * Returns 0 on success, -errno otherwise.
  4839. */
  4840. int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
  4841. int mac_len, bool ethernet)
  4842. {
  4843. struct mpls_shim_hdr *lse;
  4844. int err;
  4845. if (unlikely(!eth_p_mpls(mpls_proto)))
  4846. return -EINVAL;
  4847. /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
  4848. if (skb->encapsulation)
  4849. return -EINVAL;
  4850. err = skb_cow_head(skb, MPLS_HLEN);
  4851. if (unlikely(err))
  4852. return err;
  4853. if (!skb->inner_protocol) {
  4854. skb_set_inner_network_header(skb, skb_network_offset(skb));
  4855. skb_set_inner_protocol(skb, skb->protocol);
  4856. }
  4857. skb_push(skb, MPLS_HLEN);
  4858. memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
  4859. mac_len);
  4860. skb_reset_mac_header(skb);
  4861. skb_set_network_header(skb, mac_len);
  4862. skb_reset_mac_len(skb);
  4863. lse = mpls_hdr(skb);
  4864. lse->label_stack_entry = mpls_lse;
  4865. skb_postpush_rcsum(skb, lse, MPLS_HLEN);
  4866. if (ethernet && mac_len >= ETH_HLEN)
  4867. skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
  4868. skb->protocol = mpls_proto;
  4869. return 0;
  4870. }
  4871. EXPORT_SYMBOL_GPL(skb_mpls_push);
  4872. /**
  4873. * skb_mpls_pop() - pop the outermost MPLS header
  4874. *
  4875. * @skb: buffer
  4876. * @next_proto: ethertype of header after popped MPLS header
  4877. * @mac_len: length of the MAC header
  4878. * @ethernet: flag to indicate if the packet is ethernet
  4879. *
  4880. * Expects skb->data at mac header.
  4881. *
  4882. * Returns 0 on success, -errno otherwise.
  4883. */
  4884. int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
  4885. bool ethernet)
  4886. {
  4887. int err;
  4888. if (unlikely(!eth_p_mpls(skb->protocol)))
  4889. return 0;
  4890. err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
  4891. if (unlikely(err))
  4892. return err;
  4893. skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
  4894. memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
  4895. mac_len);
  4896. __skb_pull(skb, MPLS_HLEN);
  4897. skb_reset_mac_header(skb);
  4898. skb_set_network_header(skb, mac_len);
  4899. if (ethernet && mac_len >= ETH_HLEN) {
  4900. struct ethhdr *hdr;
  4901. /* use mpls_hdr() to get ethertype to account for VLANs. */
  4902. hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
  4903. skb_mod_eth_type(skb, hdr, next_proto);
  4904. }
  4905. skb->protocol = next_proto;
  4906. return 0;
  4907. }
  4908. EXPORT_SYMBOL_GPL(skb_mpls_pop);
  4909. /**
  4910. * skb_mpls_update_lse() - modify outermost MPLS header and update csum
  4911. *
  4912. * @skb: buffer
  4913. * @mpls_lse: new MPLS label stack entry to update to
  4914. *
  4915. * Expects skb->data at mac header.
  4916. *
  4917. * Returns 0 on success, -errno otherwise.
  4918. */
  4919. int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
  4920. {
  4921. int err;
  4922. if (unlikely(!eth_p_mpls(skb->protocol)))
  4923. return -EINVAL;
  4924. err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
  4925. if (unlikely(err))
  4926. return err;
  4927. if (skb->ip_summed == CHECKSUM_COMPLETE) {
  4928. __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
  4929. skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
  4930. }
  4931. mpls_hdr(skb)->label_stack_entry = mpls_lse;
  4932. return 0;
  4933. }
  4934. EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
  4935. /**
  4936. * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
  4937. *
  4938. * @skb: buffer
  4939. *
  4940. * Expects skb->data at mac header.
  4941. *
  4942. * Returns 0 on success, -errno otherwise.
  4943. */
  4944. int skb_mpls_dec_ttl(struct sk_buff *skb)
  4945. {
  4946. u32 lse;
  4947. u8 ttl;
  4948. if (unlikely(!eth_p_mpls(skb->protocol)))
  4949. return -EINVAL;
  4950. if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
  4951. return -ENOMEM;
  4952. lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
  4953. ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
  4954. if (!--ttl)
  4955. return -EINVAL;
  4956. lse &= ~MPLS_LS_TTL_MASK;
  4957. lse |= ttl << MPLS_LS_TTL_SHIFT;
  4958. return skb_mpls_update_lse(skb, cpu_to_be32(lse));
  4959. }
  4960. EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
  4961. /**
  4962. * alloc_skb_with_frags - allocate skb with page frags
  4963. *
  4964. * @header_len: size of linear part
  4965. * @data_len: needed length in frags
  4966. * @max_page_order: max page order desired.
  4967. * @errcode: pointer to error code if any
  4968. * @gfp_mask: allocation mask
  4969. *
  4970. * This can be used to allocate a paged skb, given a maximal order for frags.
  4971. */
  4972. struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
  4973. unsigned long data_len,
  4974. int max_page_order,
  4975. int *errcode,
  4976. gfp_t gfp_mask)
  4977. {
  4978. int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
  4979. unsigned long chunk;
  4980. struct sk_buff *skb;
  4981. struct page *page;
  4982. int i;
  4983. *errcode = -EMSGSIZE;
  4984. /* Note this test could be relaxed, if we succeed to allocate
  4985. * high order pages...
  4986. */
  4987. if (npages > MAX_SKB_FRAGS)
  4988. return NULL;
  4989. *errcode = -ENOBUFS;
  4990. skb = alloc_skb(header_len, gfp_mask);
  4991. if (!skb)
  4992. return NULL;
  4993. skb->truesize += npages << PAGE_SHIFT;
  4994. for (i = 0; npages > 0; i++) {
  4995. int order = max_page_order;
  4996. while (order) {
  4997. if (npages >= 1 << order) {
  4998. page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
  4999. __GFP_COMP |
  5000. __GFP_NOWARN,
  5001. order);
  5002. if (page)
  5003. goto fill_page;
  5004. /* Do not retry other high order allocations */
  5005. order = 1;
  5006. max_page_order = 0;
  5007. }
  5008. order--;
  5009. }
  5010. page = alloc_page(gfp_mask);
  5011. if (!page)
  5012. goto failure;
  5013. fill_page:
  5014. chunk = min_t(unsigned long, data_len,
  5015. PAGE_SIZE << order);
  5016. skb_fill_page_desc(skb, i, page, 0, chunk);
  5017. data_len -= chunk;
  5018. npages -= 1 << order;
  5019. }
  5020. return skb;
  5021. failure:
  5022. kfree_skb(skb);
  5023. return NULL;
  5024. }
  5025. EXPORT_SYMBOL(alloc_skb_with_frags);
  5026. /* carve out the first off bytes from skb when off < headlen */
  5027. static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
  5028. const int headlen, gfp_t gfp_mask)
  5029. {
  5030. int i;
  5031. int size = skb_end_offset(skb);
  5032. int new_hlen = headlen - off;
  5033. u8 *data;
  5034. size = SKB_DATA_ALIGN(size);
  5035. if (skb_pfmemalloc(skb))
  5036. gfp_mask |= __GFP_MEMALLOC;
  5037. data = kmalloc_reserve(size +
  5038. SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
  5039. gfp_mask, NUMA_NO_NODE, NULL);
  5040. if (!data)
  5041. return -ENOMEM;
  5042. size = SKB_WITH_OVERHEAD(ksize(data));
  5043. /* Copy real data, and all frags */
  5044. skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
  5045. skb->len -= off;
  5046. memcpy((struct skb_shared_info *)(data + size),
  5047. skb_shinfo(skb),
  5048. offsetof(struct skb_shared_info,
  5049. frags[skb_shinfo(skb)->nr_frags]));
  5050. if (skb_cloned(skb)) {
  5051. /* drop the old head gracefully */
  5052. if (skb_orphan_frags(skb, gfp_mask)) {
  5053. kfree(data);
  5054. return -ENOMEM;
  5055. }
  5056. for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
  5057. skb_frag_ref(skb, i);
  5058. if (skb_has_frag_list(skb))
  5059. skb_clone_fraglist(skb);
  5060. skb_release_data(skb);
  5061. } else {
  5062. /* we can reuse existing recount- all we did was
  5063. * relocate values
  5064. */
  5065. skb_free_head(skb);
  5066. }
  5067. skb->head = data;
  5068. skb->data = data;
  5069. skb->head_frag = 0;
  5070. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  5071. skb->end = size;
  5072. #else
  5073. skb->end = skb->head + size;
  5074. #endif
  5075. skb_set_tail_pointer(skb, skb_headlen(skb));
  5076. skb_headers_offset_update(skb, 0);
  5077. skb->cloned = 0;
  5078. skb->hdr_len = 0;
  5079. skb->nohdr = 0;
  5080. atomic_set(&skb_shinfo(skb)->dataref, 1);
  5081. return 0;
  5082. }
  5083. static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
  5084. /* carve out the first eat bytes from skb's frag_list. May recurse into
  5085. * pskb_carve()
  5086. */
  5087. static int pskb_carve_frag_list(struct sk_buff *skb,
  5088. struct skb_shared_info *shinfo, int eat,
  5089. gfp_t gfp_mask)
  5090. {
  5091. struct sk_buff *list = shinfo->frag_list;
  5092. struct sk_buff *clone = NULL;
  5093. struct sk_buff *insp = NULL;
  5094. do {
  5095. if (!list) {
  5096. pr_err("Not enough bytes to eat. Want %d\n", eat);
  5097. return -EFAULT;
  5098. }
  5099. if (list->len <= eat) {
  5100. /* Eaten as whole. */
  5101. eat -= list->len;
  5102. list = list->next;
  5103. insp = list;
  5104. } else {
  5105. /* Eaten partially. */
  5106. if (skb_shared(list)) {
  5107. clone = skb_clone(list, gfp_mask);
  5108. if (!clone)
  5109. return -ENOMEM;
  5110. insp = list->next;
  5111. list = clone;
  5112. } else {
  5113. /* This may be pulled without problems. */
  5114. insp = list;
  5115. }
  5116. if (pskb_carve(list, eat, gfp_mask) < 0) {
  5117. kfree_skb(clone);
  5118. return -ENOMEM;
  5119. }
  5120. break;
  5121. }
  5122. } while (eat);
  5123. /* Free pulled out fragments. */
  5124. while ((list = shinfo->frag_list) != insp) {
  5125. shinfo->frag_list = list->next;
  5126. consume_skb(list);
  5127. }
  5128. /* And insert new clone at head. */
  5129. if (clone) {
  5130. clone->next = list;
  5131. shinfo->frag_list = clone;
  5132. }
  5133. return 0;
  5134. }
  5135. /* carve off first len bytes from skb. Split line (off) is in the
  5136. * non-linear part of skb
  5137. */
  5138. static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
  5139. int pos, gfp_t gfp_mask)
  5140. {
  5141. int i, k = 0;
  5142. int size = skb_end_offset(skb);
  5143. u8 *data;
  5144. const int nfrags = skb_shinfo(skb)->nr_frags;
  5145. struct skb_shared_info *shinfo;
  5146. size = SKB_DATA_ALIGN(size);
  5147. if (skb_pfmemalloc(skb))
  5148. gfp_mask |= __GFP_MEMALLOC;
  5149. data = kmalloc_reserve(size +
  5150. SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
  5151. gfp_mask, NUMA_NO_NODE, NULL);
  5152. if (!data)
  5153. return -ENOMEM;
  5154. size = SKB_WITH_OVERHEAD(ksize(data));
  5155. memcpy((struct skb_shared_info *)(data + size),
  5156. skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
  5157. if (skb_orphan_frags(skb, gfp_mask)) {
  5158. kfree(data);
  5159. return -ENOMEM;
  5160. }
  5161. shinfo = (struct skb_shared_info *)(data + size);
  5162. for (i = 0; i < nfrags; i++) {
  5163. int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
  5164. if (pos + fsize > off) {
  5165. shinfo->frags[k] = skb_shinfo(skb)->frags[i];
  5166. if (pos < off) {
  5167. /* Split frag.
  5168. * We have two variants in this case:
  5169. * 1. Move all the frag to the second
  5170. * part, if it is possible. F.e.
  5171. * this approach is mandatory for TUX,
  5172. * where splitting is expensive.
  5173. * 2. Split is accurately. We make this.
  5174. */
  5175. skb_frag_off_add(&shinfo->frags[0], off - pos);
  5176. skb_frag_size_sub(&shinfo->frags[0], off - pos);
  5177. }
  5178. skb_frag_ref(skb, i);
  5179. k++;
  5180. }
  5181. pos += fsize;
  5182. }
  5183. shinfo->nr_frags = k;
  5184. if (skb_has_frag_list(skb))
  5185. skb_clone_fraglist(skb);
  5186. /* split line is in frag list */
  5187. if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
  5188. /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
  5189. if (skb_has_frag_list(skb))
  5190. kfree_skb_list(skb_shinfo(skb)->frag_list);
  5191. kfree(data);
  5192. return -ENOMEM;
  5193. }
  5194. skb_release_data(skb);
  5195. skb->head = data;
  5196. skb->head_frag = 0;
  5197. skb->data = data;
  5198. #ifdef NET_SKBUFF_DATA_USES_OFFSET
  5199. skb->end = size;
  5200. #else
  5201. skb->end = skb->head + size;
  5202. #endif
  5203. skb_reset_tail_pointer(skb);
  5204. skb_headers_offset_update(skb, 0);
  5205. skb->cloned = 0;
  5206. skb->hdr_len = 0;
  5207. skb->nohdr = 0;
  5208. skb->len -= off;
  5209. skb->data_len = skb->len;
  5210. atomic_set(&skb_shinfo(skb)->dataref, 1);
  5211. return 0;
  5212. }
  5213. /* remove len bytes from the beginning of the skb */
  5214. static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
  5215. {
  5216. int headlen = skb_headlen(skb);
  5217. if (len < headlen)
  5218. return pskb_carve_inside_header(skb, len, headlen, gfp);
  5219. else
  5220. return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
  5221. }
  5222. /* Extract to_copy bytes starting at off from skb, and return this in
  5223. * a new skb
  5224. */
  5225. struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
  5226. int to_copy, gfp_t gfp)
  5227. {
  5228. struct sk_buff *clone = skb_clone(skb, gfp);
  5229. if (!clone)
  5230. return NULL;
  5231. if (pskb_carve(clone, off, gfp) < 0 ||
  5232. pskb_trim(clone, to_copy)) {
  5233. kfree_skb(clone);
  5234. return NULL;
  5235. }
  5236. return clone;
  5237. }
  5238. EXPORT_SYMBOL(pskb_extract);
  5239. /**
  5240. * skb_condense - try to get rid of fragments/frag_list if possible
  5241. * @skb: buffer
  5242. *
  5243. * Can be used to save memory before skb is added to a busy queue.
  5244. * If packet has bytes in frags and enough tail room in skb->head,
  5245. * pull all of them, so that we can free the frags right now and adjust
  5246. * truesize.
  5247. * Notes:
  5248. * We do not reallocate skb->head thus can not fail.
  5249. * Caller must re-evaluate skb->truesize if needed.
  5250. */
  5251. void skb_condense(struct sk_buff *skb)
  5252. {
  5253. if (skb->data_len) {
  5254. if (skb->data_len > skb->end - skb->tail ||
  5255. skb_cloned(skb))
  5256. return;
  5257. /* Nice, we can free page frag(s) right now */
  5258. __pskb_pull_tail(skb, skb->data_len);
  5259. }
  5260. /* At this point, skb->truesize might be over estimated,
  5261. * because skb had a fragment, and fragments do not tell
  5262. * their truesize.
  5263. * When we pulled its content into skb->head, fragment
  5264. * was freed, but __pskb_pull_tail() could not possibly
  5265. * adjust skb->truesize, not knowing the frag truesize.
  5266. */
  5267. skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
  5268. }
  5269. #ifdef CONFIG_SKB_EXTENSIONS
  5270. static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
  5271. {
  5272. return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
  5273. }
  5274. /**
  5275. * __skb_ext_alloc - allocate a new skb extensions storage
  5276. *
  5277. * @flags: See kmalloc().
  5278. *
  5279. * Returns the newly allocated pointer. The pointer can later attached to a
  5280. * skb via __skb_ext_set().
  5281. * Note: caller must handle the skb_ext as an opaque data.
  5282. */
  5283. struct skb_ext *__skb_ext_alloc(gfp_t flags)
  5284. {
  5285. struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
  5286. if (new) {
  5287. memset(new->offset, 0, sizeof(new->offset));
  5288. refcount_set(&new->refcnt, 1);
  5289. }
  5290. return new;
  5291. }
  5292. static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
  5293. unsigned int old_active)
  5294. {
  5295. struct skb_ext *new;
  5296. if (refcount_read(&old->refcnt) == 1)
  5297. return old;
  5298. new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
  5299. if (!new)
  5300. return NULL;
  5301. memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
  5302. refcount_set(&new->refcnt, 1);
  5303. #ifdef CONFIG_XFRM
  5304. if (old_active & (1 << SKB_EXT_SEC_PATH)) {
  5305. struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
  5306. unsigned int i;
  5307. for (i = 0; i < sp->len; i++)
  5308. xfrm_state_hold(sp->xvec[i]);
  5309. }
  5310. #endif
  5311. __skb_ext_put(old);
  5312. return new;
  5313. }
  5314. /**
  5315. * __skb_ext_set - attach the specified extension storage to this skb
  5316. * @skb: buffer
  5317. * @id: extension id
  5318. * @ext: extension storage previously allocated via __skb_ext_alloc()
  5319. *
  5320. * Existing extensions, if any, are cleared.
  5321. *
  5322. * Returns the pointer to the extension.
  5323. */
  5324. void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
  5325. struct skb_ext *ext)
  5326. {
  5327. unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
  5328. skb_ext_put(skb);
  5329. newlen = newoff + skb_ext_type_len[id];
  5330. ext->chunks = newlen;
  5331. ext->offset[id] = newoff;
  5332. skb->extensions = ext;
  5333. skb->active_extensions = 1 << id;
  5334. return skb_ext_get_ptr(ext, id);
  5335. }
  5336. /**
  5337. * skb_ext_add - allocate space for given extension, COW if needed
  5338. * @skb: buffer
  5339. * @id: extension to allocate space for
  5340. *
  5341. * Allocates enough space for the given extension.
  5342. * If the extension is already present, a pointer to that extension
  5343. * is returned.
  5344. *
  5345. * If the skb was cloned, COW applies and the returned memory can be
  5346. * modified without changing the extension space of clones buffers.
  5347. *
  5348. * Returns pointer to the extension or NULL on allocation failure.
  5349. */
  5350. void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
  5351. {
  5352. struct skb_ext *new, *old = NULL;
  5353. unsigned int newlen, newoff;
  5354. if (skb->active_extensions) {
  5355. old = skb->extensions;
  5356. new = skb_ext_maybe_cow(old, skb->active_extensions);
  5357. if (!new)
  5358. return NULL;
  5359. if (__skb_ext_exist(new, id))
  5360. goto set_active;
  5361. newoff = new->chunks;
  5362. } else {
  5363. newoff = SKB_EXT_CHUNKSIZEOF(*new);
  5364. new = __skb_ext_alloc(GFP_ATOMIC);
  5365. if (!new)
  5366. return NULL;
  5367. }
  5368. newlen = newoff + skb_ext_type_len[id];
  5369. new->chunks = newlen;
  5370. new->offset[id] = newoff;
  5371. set_active:
  5372. skb->extensions = new;
  5373. skb->active_extensions |= 1 << id;
  5374. return skb_ext_get_ptr(new, id);
  5375. }
  5376. EXPORT_SYMBOL(skb_ext_add);
  5377. #ifdef CONFIG_XFRM
  5378. static void skb_ext_put_sp(struct sec_path *sp)
  5379. {
  5380. unsigned int i;
  5381. for (i = 0; i < sp->len; i++)
  5382. xfrm_state_put(sp->xvec[i]);
  5383. }
  5384. #endif
  5385. void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
  5386. {
  5387. struct skb_ext *ext = skb->extensions;
  5388. skb->active_extensions &= ~(1 << id);
  5389. if (skb->active_extensions == 0) {
  5390. skb->extensions = NULL;
  5391. __skb_ext_put(ext);
  5392. #ifdef CONFIG_XFRM
  5393. } else if (id == SKB_EXT_SEC_PATH &&
  5394. refcount_read(&ext->refcnt) == 1) {
  5395. struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
  5396. skb_ext_put_sp(sp);
  5397. sp->len = 0;
  5398. #endif
  5399. }
  5400. }
  5401. EXPORT_SYMBOL(__skb_ext_del);
  5402. void __skb_ext_put(struct skb_ext *ext)
  5403. {
  5404. /* If this is last clone, nothing can increment
  5405. * it after check passes. Avoids one atomic op.
  5406. */
  5407. if (refcount_read(&ext->refcnt) == 1)
  5408. goto free_now;
  5409. if (!refcount_dec_and_test(&ext->refcnt))
  5410. return;
  5411. free_now:
  5412. #ifdef CONFIG_XFRM
  5413. if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
  5414. skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
  5415. #endif
  5416. kmem_cache_free(skbuff_ext_cache, ext);
  5417. }
  5418. EXPORT_SYMBOL(__skb_ext_put);
  5419. #endif /* CONFIG_SKB_EXTENSIONS */