tcp_input.c 197 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * INET An implementation of the TCP/IP protocol suite for the LINUX
  4. * operating system. INET is implemented using the BSD Socket
  5. * interface as the means of communication with the user level.
  6. *
  7. * Implementation of the Transmission Control Protocol(TCP).
  8. *
  9. * Authors: Ross Biro
  10. * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11. * Mark Evans, <evansmp@uhura.aston.ac.uk>
  12. * Corey Minyard <wf-rch!minyard@relay.EU.net>
  13. * Florian La Roche, <flla@stud.uni-sb.de>
  14. * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  15. * Linus Torvalds, <torvalds@cs.helsinki.fi>
  16. * Alan Cox, <gw4pts@gw4pts.ampr.org>
  17. * Matthew Dillon, <dillon@apollo.west.oic.com>
  18. * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  19. * Jorge Cwik, <jorge@laser.satlink.net>
  20. */
  21. /*
  22. * Changes:
  23. * Pedro Roque : Fast Retransmit/Recovery.
  24. * Two receive queues.
  25. * Retransmit queue handled by TCP.
  26. * Better retransmit timer handling.
  27. * New congestion avoidance.
  28. * Header prediction.
  29. * Variable renaming.
  30. *
  31. * Eric : Fast Retransmit.
  32. * Randy Scott : MSS option defines.
  33. * Eric Schenk : Fixes to slow start algorithm.
  34. * Eric Schenk : Yet another double ACK bug.
  35. * Eric Schenk : Delayed ACK bug fixes.
  36. * Eric Schenk : Floyd style fast retrans war avoidance.
  37. * David S. Miller : Don't allow zero congestion window.
  38. * Eric Schenk : Fix retransmitter so that it sends
  39. * next packet on ack of previous packet.
  40. * Andi Kleen : Moved open_request checking here
  41. * and process RSTs for open_requests.
  42. * Andi Kleen : Better prune_queue, and other fixes.
  43. * Andrey Savochkin: Fix RTT measurements in the presence of
  44. * timestamps.
  45. * Andrey Savochkin: Check sequence numbers correctly when
  46. * removing SACKs due to in sequence incoming
  47. * data segments.
  48. * Andi Kleen: Make sure we never ack data there is not
  49. * enough room for. Also make this condition
  50. * a fatal error if it might still happen.
  51. * Andi Kleen: Add tcp_measure_rcv_mss to make
  52. * connections with MSS<min(MTU,ann. MSS)
  53. * work without delayed acks.
  54. * Andi Kleen: Process packets with PSH set in the
  55. * fast path.
  56. * J Hadi Salim: ECN support
  57. * Andrei Gurtov,
  58. * Pasi Sarolahti,
  59. * Panu Kuhlberg: Experimental audit of TCP (re)transmission
  60. * engine. Lots of bugs are found.
  61. * Pasi Sarolahti: F-RTO for dealing with spurious RTOs
  62. */
  63. #define pr_fmt(fmt) "TCP: " fmt
  64. #include <linux/mm.h>
  65. #include <linux/slab.h>
  66. #include <linux/module.h>
  67. #include <linux/sysctl.h>
  68. #include <linux/kernel.h>
  69. #include <linux/prefetch.h>
  70. #include <net/dst.h>
  71. #include <net/tcp.h>
  72. #include <net/inet_common.h>
  73. #include <linux/ipsec.h>
  74. #include <asm/unaligned.h>
  75. #include <linux/errqueue.h>
  76. #include <trace/events/tcp.h>
  77. #include <linux/jump_label_ratelimit.h>
  78. #include <net/busy_poll.h>
  79. #include <net/mptcp.h>
  80. #include <trace/hooks/net.h>
  81. int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
  82. #define FLAG_DATA 0x01 /* Incoming frame contained data. */
  83. #define FLAG_WIN_UPDATE 0x02 /* Incoming ACK was a window update. */
  84. #define FLAG_DATA_ACKED 0x04 /* This ACK acknowledged new data. */
  85. #define FLAG_RETRANS_DATA_ACKED 0x08 /* "" "" some of which was retransmitted. */
  86. #define FLAG_SYN_ACKED 0x10 /* This ACK acknowledged SYN. */
  87. #define FLAG_DATA_SACKED 0x20 /* New SACK. */
  88. #define FLAG_ECE 0x40 /* ECE in this ACK */
  89. #define FLAG_LOST_RETRANS 0x80 /* This ACK marks some retransmission lost */
  90. #define FLAG_SLOWPATH 0x100 /* Do not skip RFC checks for window update.*/
  91. #define FLAG_ORIG_SACK_ACKED 0x200 /* Never retransmitted data are (s)acked */
  92. #define FLAG_SND_UNA_ADVANCED 0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
  93. #define FLAG_DSACKING_ACK 0x800 /* SACK blocks contained D-SACK info */
  94. #define FLAG_SET_XMIT_TIMER 0x1000 /* Set TLP or RTO timer */
  95. #define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */
  96. #define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */
  97. #define FLAG_NO_CHALLENGE_ACK 0x8000 /* do not call tcp_send_challenge_ack() */
  98. #define FLAG_ACK_MAYBE_DELAYED 0x10000 /* Likely a delayed ACK */
  99. #define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED)
  100. #define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED)
  101. #define FLAG_CA_ALERT (FLAG_DATA_SACKED|FLAG_ECE|FLAG_DSACKING_ACK)
  102. #define FLAG_FORWARD_PROGRESS (FLAG_ACKED|FLAG_DATA_SACKED)
  103. #define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
  104. #define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
  105. #define REXMIT_NONE 0 /* no loss recovery to do */
  106. #define REXMIT_LOST 1 /* retransmit packets marked lost */
  107. #define REXMIT_NEW 2 /* FRTO-style transmit of unsent/new packets */
  108. #if IS_ENABLED(CONFIG_TLS_DEVICE)
  109. static DEFINE_STATIC_KEY_DEFERRED_FALSE(clean_acked_data_enabled, HZ);
  110. void clean_acked_data_enable(struct inet_connection_sock *icsk,
  111. void (*cad)(struct sock *sk, u32 ack_seq))
  112. {
  113. icsk->icsk_clean_acked = cad;
  114. static_branch_deferred_inc(&clean_acked_data_enabled);
  115. }
  116. EXPORT_SYMBOL_GPL(clean_acked_data_enable);
  117. void clean_acked_data_disable(struct inet_connection_sock *icsk)
  118. {
  119. static_branch_slow_dec_deferred(&clean_acked_data_enabled);
  120. icsk->icsk_clean_acked = NULL;
  121. }
  122. EXPORT_SYMBOL_GPL(clean_acked_data_disable);
  123. void clean_acked_data_flush(void)
  124. {
  125. static_key_deferred_flush(&clean_acked_data_enabled);
  126. }
  127. EXPORT_SYMBOL_GPL(clean_acked_data_flush);
  128. #endif
  129. #ifdef CONFIG_CGROUP_BPF
  130. static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
  131. {
  132. bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown &&
  133. BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
  134. BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG);
  135. bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
  136. BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG);
  137. struct bpf_sock_ops_kern sock_ops;
  138. if (likely(!unknown_opt && !parse_all_opt))
  139. return;
  140. /* The skb will be handled in the
  141. * bpf_skops_established() or
  142. * bpf_skops_write_hdr_opt().
  143. */
  144. switch (sk->sk_state) {
  145. case TCP_SYN_RECV:
  146. case TCP_SYN_SENT:
  147. case TCP_LISTEN:
  148. return;
  149. }
  150. sock_owned_by_me(sk);
  151. memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
  152. sock_ops.op = BPF_SOCK_OPS_PARSE_HDR_OPT_CB;
  153. sock_ops.is_fullsock = 1;
  154. sock_ops.sk = sk;
  155. bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
  156. BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
  157. }
  158. static void bpf_skops_established(struct sock *sk, int bpf_op,
  159. struct sk_buff *skb)
  160. {
  161. struct bpf_sock_ops_kern sock_ops;
  162. sock_owned_by_me(sk);
  163. memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
  164. sock_ops.op = bpf_op;
  165. sock_ops.is_fullsock = 1;
  166. sock_ops.sk = sk;
  167. /* sk with TCP_REPAIR_ON does not have skb in tcp_finish_connect */
  168. if (skb)
  169. bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
  170. BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
  171. }
  172. #else
  173. static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
  174. {
  175. }
  176. static void bpf_skops_established(struct sock *sk, int bpf_op,
  177. struct sk_buff *skb)
  178. {
  179. }
  180. #endif
  181. static void tcp_gro_dev_warn(struct sock *sk, const struct sk_buff *skb,
  182. unsigned int len)
  183. {
  184. static bool __once __read_mostly;
  185. if (!__once) {
  186. struct net_device *dev;
  187. __once = true;
  188. rcu_read_lock();
  189. dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
  190. if (!dev || len >= dev->mtu)
  191. pr_warn("%s: Driver has suspect GRO implementation, TCP performance may be compromised.\n",
  192. dev ? dev->name : "Unknown driver");
  193. rcu_read_unlock();
  194. }
  195. }
  196. /* Adapt the MSS value used to make delayed ack decision to the
  197. * real world.
  198. */
  199. static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb)
  200. {
  201. struct inet_connection_sock *icsk = inet_csk(sk);
  202. const unsigned int lss = icsk->icsk_ack.last_seg_size;
  203. unsigned int len;
  204. icsk->icsk_ack.last_seg_size = 0;
  205. /* skb->len may jitter because of SACKs, even if peer
  206. * sends good full-sized frames.
  207. */
  208. len = skb_shinfo(skb)->gso_size ? : skb->len;
  209. if (len >= icsk->icsk_ack.rcv_mss) {
  210. icsk->icsk_ack.rcv_mss = min_t(unsigned int, len,
  211. tcp_sk(sk)->advmss);
  212. /* Account for possibly-removed options */
  213. if (unlikely(len > icsk->icsk_ack.rcv_mss +
  214. MAX_TCP_OPTION_SPACE))
  215. tcp_gro_dev_warn(sk, skb, len);
  216. } else {
  217. /* Otherwise, we make more careful check taking into account,
  218. * that SACKs block is variable.
  219. *
  220. * "len" is invariant segment length, including TCP header.
  221. */
  222. len += skb->data - skb_transport_header(skb);
  223. if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) ||
  224. /* If PSH is not set, packet should be
  225. * full sized, provided peer TCP is not badly broken.
  226. * This observation (if it is correct 8)) allows
  227. * to handle super-low mtu links fairly.
  228. */
  229. (len >= TCP_MIN_MSS + sizeof(struct tcphdr) &&
  230. !(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) {
  231. /* Subtract also invariant (if peer is RFC compliant),
  232. * tcp header plus fixed timestamp option length.
  233. * Resulting "len" is MSS free of SACK jitter.
  234. */
  235. len -= tcp_sk(sk)->tcp_header_len;
  236. icsk->icsk_ack.last_seg_size = len;
  237. if (len == lss) {
  238. icsk->icsk_ack.rcv_mss = len;
  239. return;
  240. }
  241. }
  242. if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)
  243. icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2;
  244. icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
  245. }
  246. }
  247. static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks)
  248. {
  249. struct inet_connection_sock *icsk = inet_csk(sk);
  250. unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss);
  251. if (quickacks == 0)
  252. quickacks = 2;
  253. quickacks = min(quickacks, max_quickacks);
  254. if (quickacks > icsk->icsk_ack.quick)
  255. icsk->icsk_ack.quick = quickacks;
  256. }
  257. void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks)
  258. {
  259. struct inet_connection_sock *icsk = inet_csk(sk);
  260. tcp_incr_quickack(sk, max_quickacks);
  261. inet_csk_exit_pingpong_mode(sk);
  262. icsk->icsk_ack.ato = TCP_ATO_MIN;
  263. }
  264. EXPORT_SYMBOL(tcp_enter_quickack_mode);
  265. /* Send ACKs quickly, if "quick" count is not exhausted
  266. * and the session is not interactive.
  267. */
  268. static bool tcp_in_quickack_mode(struct sock *sk)
  269. {
  270. const struct inet_connection_sock *icsk = inet_csk(sk);
  271. const struct dst_entry *dst = __sk_dst_get(sk);
  272. return (dst && dst_metric(dst, RTAX_QUICKACK)) ||
  273. (icsk->icsk_ack.quick && !inet_csk_in_pingpong_mode(sk));
  274. }
  275. static void tcp_ecn_queue_cwr(struct tcp_sock *tp)
  276. {
  277. if (tp->ecn_flags & TCP_ECN_OK)
  278. tp->ecn_flags |= TCP_ECN_QUEUE_CWR;
  279. }
  280. static void tcp_ecn_accept_cwr(struct sock *sk, const struct sk_buff *skb)
  281. {
  282. if (tcp_hdr(skb)->cwr) {
  283. tcp_sk(sk)->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
  284. /* If the sender is telling us it has entered CWR, then its
  285. * cwnd may be very low (even just 1 packet), so we should ACK
  286. * immediately.
  287. */
  288. if (TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq)
  289. inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
  290. }
  291. }
  292. static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp)
  293. {
  294. tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
  295. }
  296. static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
  297. {
  298. struct tcp_sock *tp = tcp_sk(sk);
  299. switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) {
  300. case INET_ECN_NOT_ECT:
  301. /* Funny extension: if ECT is not set on a segment,
  302. * and we already seen ECT on a previous segment,
  303. * it is probably a retransmit.
  304. */
  305. if (tp->ecn_flags & TCP_ECN_SEEN)
  306. tcp_enter_quickack_mode(sk, 2);
  307. break;
  308. case INET_ECN_CE:
  309. if (tcp_ca_needs_ecn(sk))
  310. tcp_ca_event(sk, CA_EVENT_ECN_IS_CE);
  311. if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {
  312. /* Better not delay acks, sender can have a very low cwnd */
  313. tcp_enter_quickack_mode(sk, 2);
  314. tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
  315. }
  316. tp->ecn_flags |= TCP_ECN_SEEN;
  317. break;
  318. default:
  319. if (tcp_ca_needs_ecn(sk))
  320. tcp_ca_event(sk, CA_EVENT_ECN_NO_CE);
  321. tp->ecn_flags |= TCP_ECN_SEEN;
  322. break;
  323. }
  324. }
  325. static void tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
  326. {
  327. if (tcp_sk(sk)->ecn_flags & TCP_ECN_OK)
  328. __tcp_ecn_check_ce(sk, skb);
  329. }
  330. static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th)
  331. {
  332. if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr))
  333. tp->ecn_flags &= ~TCP_ECN_OK;
  334. }
  335. static void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th)
  336. {
  337. if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr))
  338. tp->ecn_flags &= ~TCP_ECN_OK;
  339. }
  340. static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th)
  341. {
  342. if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK))
  343. return true;
  344. return false;
  345. }
  346. /* Buffer size and advertised window tuning.
  347. *
  348. * 1. Tuning sk->sk_sndbuf, when connection enters established state.
  349. */
  350. static void tcp_sndbuf_expand(struct sock *sk)
  351. {
  352. const struct tcp_sock *tp = tcp_sk(sk);
  353. const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
  354. int sndmem, per_mss;
  355. u32 nr_segs;
  356. /* Worst case is non GSO/TSO : each frame consumes one skb
  357. * and skb->head is kmalloced using power of two area of memory
  358. */
  359. per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
  360. MAX_TCP_HEADER +
  361. SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
  362. per_mss = roundup_pow_of_two(per_mss) +
  363. SKB_DATA_ALIGN(sizeof(struct sk_buff));
  364. nr_segs = max_t(u32, TCP_INIT_CWND, tp->snd_cwnd);
  365. nr_segs = max_t(u32, nr_segs, tp->reordering + 1);
  366. /* Fast Recovery (RFC 5681 3.2) :
  367. * Cubic needs 1.7 factor, rounded to 2 to include
  368. * extra cushion (application might react slowly to EPOLLOUT)
  369. */
  370. sndmem = ca_ops->sndbuf_expand ? ca_ops->sndbuf_expand(sk) : 2;
  371. sndmem *= nr_segs * per_mss;
  372. if (sk->sk_sndbuf < sndmem)
  373. WRITE_ONCE(sk->sk_sndbuf,
  374. min(sndmem, sock_net(sk)->ipv4.sysctl_tcp_wmem[2]));
  375. }
  376. /* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
  377. *
  378. * All tcp_full_space() is split to two parts: "network" buffer, allocated
  379. * forward and advertised in receiver window (tp->rcv_wnd) and
  380. * "application buffer", required to isolate scheduling/application
  381. * latencies from network.
  382. * window_clamp is maximal advertised window. It can be less than
  383. * tcp_full_space(), in this case tcp_full_space() - window_clamp
  384. * is reserved for "application" buffer. The less window_clamp is
  385. * the smoother our behaviour from viewpoint of network, but the lower
  386. * throughput and the higher sensitivity of the connection to losses. 8)
  387. *
  388. * rcv_ssthresh is more strict window_clamp used at "slow start"
  389. * phase to predict further behaviour of this connection.
  390. * It is used for two goals:
  391. * - to enforce header prediction at sender, even when application
  392. * requires some significant "application buffer". It is check #1.
  393. * - to prevent pruning of receive queue because of misprediction
  394. * of receiver window. Check #2.
  395. *
  396. * The scheme does not work when sender sends good segments opening
  397. * window and then starts to feed us spaghetti. But it should work
  398. * in common situations. Otherwise, we have to rely on queue collapsing.
  399. */
  400. /* Slow part of check#2. */
  401. static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb)
  402. {
  403. struct tcp_sock *tp = tcp_sk(sk);
  404. /* Optimize this! */
  405. int truesize = tcp_win_from_space(sk, skb->truesize) >> 1;
  406. int window = tcp_win_from_space(sk, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
  407. while (tp->rcv_ssthresh <= window) {
  408. if (truesize <= skb->len)
  409. return 2 * inet_csk(sk)->icsk_ack.rcv_mss;
  410. truesize >>= 1;
  411. window >>= 1;
  412. }
  413. return 0;
  414. }
  415. static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb)
  416. {
  417. struct tcp_sock *tp = tcp_sk(sk);
  418. int room;
  419. room = min_t(int, tp->window_clamp, tcp_space(sk)) - tp->rcv_ssthresh;
  420. /* Check #1 */
  421. if (room > 0 && !tcp_under_memory_pressure(sk)) {
  422. int incr;
  423. /* Check #2. Increase window, if skb with such overhead
  424. * will fit to rcvbuf in future.
  425. */
  426. if (tcp_win_from_space(sk, skb->truesize) <= skb->len)
  427. incr = 2 * tp->advmss;
  428. else
  429. incr = __tcp_grow_window(sk, skb);
  430. if (incr) {
  431. incr = max_t(int, incr, 2 * skb->len);
  432. tp->rcv_ssthresh += min(room, incr);
  433. inet_csk(sk)->icsk_ack.quick |= 1;
  434. }
  435. }
  436. }
  437. /* 3. Try to fixup all. It is made immediately after connection enters
  438. * established state.
  439. */
  440. static void tcp_init_buffer_space(struct sock *sk)
  441. {
  442. int tcp_app_win = sock_net(sk)->ipv4.sysctl_tcp_app_win;
  443. struct tcp_sock *tp = tcp_sk(sk);
  444. int maxwin;
  445. if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK))
  446. tcp_sndbuf_expand(sk);
  447. tcp_mstamp_refresh(tp);
  448. tp->rcvq_space.time = tp->tcp_mstamp;
  449. tp->rcvq_space.seq = tp->copied_seq;
  450. maxwin = tcp_full_space(sk);
  451. if (tp->window_clamp >= maxwin) {
  452. tp->window_clamp = maxwin;
  453. if (tcp_app_win && maxwin > 4 * tp->advmss)
  454. tp->window_clamp = max(maxwin -
  455. (maxwin >> tcp_app_win),
  456. 4 * tp->advmss);
  457. }
  458. /* Force reservation of one segment. */
  459. if (tcp_app_win &&
  460. tp->window_clamp > 2 * tp->advmss &&
  461. tp->window_clamp + tp->advmss > maxwin)
  462. tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss);
  463. tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp);
  464. tp->snd_cwnd_stamp = tcp_jiffies32;
  465. tp->rcvq_space.space = min3(tp->rcv_ssthresh, tp->rcv_wnd,
  466. (u32)TCP_INIT_CWND * tp->advmss);
  467. }
  468. /* 4. Recalculate window clamp after socket hit its memory bounds. */
  469. static void tcp_clamp_window(struct sock *sk)
  470. {
  471. struct tcp_sock *tp = tcp_sk(sk);
  472. struct inet_connection_sock *icsk = inet_csk(sk);
  473. struct net *net = sock_net(sk);
  474. icsk->icsk_ack.quick = 0;
  475. if (sk->sk_rcvbuf < net->ipv4.sysctl_tcp_rmem[2] &&
  476. !(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
  477. !tcp_under_memory_pressure(sk) &&
  478. sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)) {
  479. WRITE_ONCE(sk->sk_rcvbuf,
  480. min(atomic_read(&sk->sk_rmem_alloc),
  481. net->ipv4.sysctl_tcp_rmem[2]));
  482. }
  483. if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
  484. tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss);
  485. }
  486. /* Initialize RCV_MSS value.
  487. * RCV_MSS is an our guess about MSS used by the peer.
  488. * We haven't any direct information about the MSS.
  489. * It's better to underestimate the RCV_MSS rather than overestimate.
  490. * Overestimations make us ACKing less frequently than needed.
  491. * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
  492. */
  493. void tcp_initialize_rcv_mss(struct sock *sk)
  494. {
  495. const struct tcp_sock *tp = tcp_sk(sk);
  496. unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache);
  497. hint = min(hint, tp->rcv_wnd / 2);
  498. hint = min(hint, TCP_MSS_DEFAULT);
  499. hint = max(hint, TCP_MIN_MSS);
  500. inet_csk(sk)->icsk_ack.rcv_mss = hint;
  501. }
  502. EXPORT_SYMBOL(tcp_initialize_rcv_mss);
  503. /* Receiver "autotuning" code.
  504. *
  505. * The algorithm for RTT estimation w/o timestamps is based on
  506. * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
  507. * <https://public.lanl.gov/radiant/pubs.html#DRS>
  508. *
  509. * More detail on this code can be found at
  510. * <http://staff.psc.edu/jheffner/>,
  511. * though this reference is out of date. A new paper
  512. * is pending.
  513. */
  514. static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
  515. {
  516. u32 new_sample = tp->rcv_rtt_est.rtt_us;
  517. long m = sample;
  518. if (new_sample != 0) {
  519. /* If we sample in larger samples in the non-timestamp
  520. * case, we could grossly overestimate the RTT especially
  521. * with chatty applications or bulk transfer apps which
  522. * are stalled on filesystem I/O.
  523. *
  524. * Also, since we are only going for a minimum in the
  525. * non-timestamp case, we do not smooth things out
  526. * else with timestamps disabled convergence takes too
  527. * long.
  528. */
  529. if (!win_dep) {
  530. m -= (new_sample >> 3);
  531. new_sample += m;
  532. } else {
  533. m <<= 3;
  534. if (m < new_sample)
  535. new_sample = m;
  536. }
  537. } else {
  538. /* No previous measure. */
  539. new_sample = m << 3;
  540. }
  541. tp->rcv_rtt_est.rtt_us = new_sample;
  542. }
  543. static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp)
  544. {
  545. u32 delta_us;
  546. if (tp->rcv_rtt_est.time == 0)
  547. goto new_measure;
  548. if (before(tp->rcv_nxt, tp->rcv_rtt_est.seq))
  549. return;
  550. delta_us = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcv_rtt_est.time);
  551. if (!delta_us)
  552. delta_us = 1;
  553. tcp_rcv_rtt_update(tp, delta_us, 1);
  554. new_measure:
  555. tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd;
  556. tp->rcv_rtt_est.time = tp->tcp_mstamp;
  557. }
  558. static inline void tcp_rcv_rtt_measure_ts(struct sock *sk,
  559. const struct sk_buff *skb)
  560. {
  561. struct tcp_sock *tp = tcp_sk(sk);
  562. if (tp->rx_opt.rcv_tsecr == tp->rcv_rtt_last_tsecr)
  563. return;
  564. tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
  565. if (TCP_SKB_CB(skb)->end_seq -
  566. TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss) {
  567. u32 delta = tcp_time_stamp(tp) - tp->rx_opt.rcv_tsecr;
  568. u32 delta_us;
  569. if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
  570. if (!delta)
  571. delta = 1;
  572. delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
  573. tcp_rcv_rtt_update(tp, delta_us, 0);
  574. }
  575. }
  576. }
  577. /*
  578. * This function should be called every time data is copied to user space.
  579. * It calculates the appropriate TCP receive buffer space.
  580. */
  581. void tcp_rcv_space_adjust(struct sock *sk)
  582. {
  583. struct tcp_sock *tp = tcp_sk(sk);
  584. u32 copied;
  585. int time;
  586. trace_tcp_rcv_space_adjust(sk);
  587. tcp_mstamp_refresh(tp);
  588. time = tcp_stamp_us_delta(tp->tcp_mstamp, tp->rcvq_space.time);
  589. if (time < (tp->rcv_rtt_est.rtt_us >> 3) || tp->rcv_rtt_est.rtt_us == 0)
  590. return;
  591. /* Number of bytes copied to user in last RTT */
  592. copied = tp->copied_seq - tp->rcvq_space.seq;
  593. if (copied <= tp->rcvq_space.space)
  594. goto new_measure;
  595. /* A bit of theory :
  596. * copied = bytes received in previous RTT, our base window
  597. * To cope with packet losses, we need a 2x factor
  598. * To cope with slow start, and sender growing its cwin by 100 %
  599. * every RTT, we need a 4x factor, because the ACK we are sending
  600. * now is for the next RTT, not the current one :
  601. * <prev RTT . ><current RTT .. ><next RTT .... >
  602. */
  603. if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf &&
  604. !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
  605. int rcvmem, rcvbuf;
  606. u64 rcvwin, grow;
  607. /* minimal window to cope with packet losses, assuming
  608. * steady state. Add some cushion because of small variations.
  609. */
  610. rcvwin = ((u64)copied << 1) + 16 * tp->advmss;
  611. /* Accommodate for sender rate increase (eg. slow start) */
  612. grow = rcvwin * (copied - tp->rcvq_space.space);
  613. do_div(grow, tp->rcvq_space.space);
  614. rcvwin += (grow << 1);
  615. rcvmem = SKB_TRUESIZE(tp->advmss + MAX_TCP_HEADER);
  616. while (tcp_win_from_space(sk, rcvmem) < tp->advmss)
  617. rcvmem += 128;
  618. do_div(rcvwin, tp->advmss);
  619. rcvbuf = min_t(u64, rcvwin * rcvmem,
  620. sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
  621. if (rcvbuf > sk->sk_rcvbuf) {
  622. WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
  623. /* Make the window clamp follow along. */
  624. tp->window_clamp = tcp_win_from_space(sk, rcvbuf);
  625. }
  626. }
  627. tp->rcvq_space.space = copied;
  628. new_measure:
  629. tp->rcvq_space.seq = tp->copied_seq;
  630. tp->rcvq_space.time = tp->tcp_mstamp;
  631. }
  632. /* There is something which you must keep in mind when you analyze the
  633. * behavior of the tp->ato delayed ack timeout interval. When a
  634. * connection starts up, we want to ack as quickly as possible. The
  635. * problem is that "good" TCP's do slow start at the beginning of data
  636. * transmission. The means that until we send the first few ACK's the
  637. * sender will sit on his end and only queue most of his data, because
  638. * he can only send snd_cwnd unacked packets at any given time. For
  639. * each ACK we send, he increments snd_cwnd and transmits more of his
  640. * queue. -DaveM
  641. */
  642. static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
  643. {
  644. struct tcp_sock *tp = tcp_sk(sk);
  645. struct inet_connection_sock *icsk = inet_csk(sk);
  646. u32 now;
  647. inet_csk_schedule_ack(sk);
  648. tcp_measure_rcv_mss(sk, skb);
  649. tcp_rcv_rtt_measure(tp);
  650. now = tcp_jiffies32;
  651. if (!icsk->icsk_ack.ato) {
  652. /* The _first_ data packet received, initialize
  653. * delayed ACK engine.
  654. */
  655. tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
  656. icsk->icsk_ack.ato = TCP_ATO_MIN;
  657. } else {
  658. int m = now - icsk->icsk_ack.lrcvtime;
  659. if (m <= TCP_ATO_MIN / 2) {
  660. /* The fastest case is the first. */
  661. icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2;
  662. } else if (m < icsk->icsk_ack.ato) {
  663. icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m;
  664. if (icsk->icsk_ack.ato > icsk->icsk_rto)
  665. icsk->icsk_ack.ato = icsk->icsk_rto;
  666. } else if (m > icsk->icsk_rto) {
  667. /* Too long gap. Apparently sender failed to
  668. * restart window, so that we send ACKs quickly.
  669. */
  670. tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
  671. sk_mem_reclaim(sk);
  672. }
  673. }
  674. icsk->icsk_ack.lrcvtime = now;
  675. tcp_ecn_check_ce(sk, skb);
  676. if (skb->len >= 128)
  677. tcp_grow_window(sk, skb);
  678. }
  679. /* Called to compute a smoothed rtt estimate. The data fed to this
  680. * routine either comes from timestamps, or from segments that were
  681. * known _not_ to have been retransmitted [see Karn/Partridge
  682. * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
  683. * piece by Van Jacobson.
  684. * NOTE: the next three routines used to be one big routine.
  685. * To save cycles in the RFC 1323 implementation it was better to break
  686. * it up into three procedures. -- erics
  687. */
  688. static void tcp_rtt_estimator(struct sock *sk, long mrtt_us)
  689. {
  690. struct tcp_sock *tp = tcp_sk(sk);
  691. long m = mrtt_us; /* RTT */
  692. u32 srtt = tp->srtt_us;
  693. /* The following amusing code comes from Jacobson's
  694. * article in SIGCOMM '88. Note that rtt and mdev
  695. * are scaled versions of rtt and mean deviation.
  696. * This is designed to be as fast as possible
  697. * m stands for "measurement".
  698. *
  699. * On a 1990 paper the rto value is changed to:
  700. * RTO = rtt + 4 * mdev
  701. *
  702. * Funny. This algorithm seems to be very broken.
  703. * These formulae increase RTO, when it should be decreased, increase
  704. * too slowly, when it should be increased quickly, decrease too quickly
  705. * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
  706. * does not matter how to _calculate_ it. Seems, it was trap
  707. * that VJ failed to avoid. 8)
  708. */
  709. if (srtt != 0) {
  710. m -= (srtt >> 3); /* m is now error in rtt est */
  711. srtt += m; /* rtt = 7/8 rtt + 1/8 new */
  712. if (m < 0) {
  713. m = -m; /* m is now abs(error) */
  714. m -= (tp->mdev_us >> 2); /* similar update on mdev */
  715. /* This is similar to one of Eifel findings.
  716. * Eifel blocks mdev updates when rtt decreases.
  717. * This solution is a bit different: we use finer gain
  718. * for mdev in this case (alpha*beta).
  719. * Like Eifel it also prevents growth of rto,
  720. * but also it limits too fast rto decreases,
  721. * happening in pure Eifel.
  722. */
  723. if (m > 0)
  724. m >>= 3;
  725. } else {
  726. m -= (tp->mdev_us >> 2); /* similar update on mdev */
  727. }
  728. tp->mdev_us += m; /* mdev = 3/4 mdev + 1/4 new */
  729. if (tp->mdev_us > tp->mdev_max_us) {
  730. tp->mdev_max_us = tp->mdev_us;
  731. if (tp->mdev_max_us > tp->rttvar_us)
  732. tp->rttvar_us = tp->mdev_max_us;
  733. }
  734. if (after(tp->snd_una, tp->rtt_seq)) {
  735. if (tp->mdev_max_us < tp->rttvar_us)
  736. tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
  737. tp->rtt_seq = tp->snd_nxt;
  738. tp->mdev_max_us = tcp_rto_min_us(sk);
  739. tcp_bpf_rtt(sk);
  740. }
  741. } else {
  742. /* no previous measure. */
  743. srtt = m << 3; /* take the measured time to be rtt */
  744. tp->mdev_us = m << 1; /* make sure rto = 3*rtt */
  745. tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
  746. tp->mdev_max_us = tp->rttvar_us;
  747. tp->rtt_seq = tp->snd_nxt;
  748. tcp_bpf_rtt(sk);
  749. }
  750. tp->srtt_us = max(1U, srtt);
  751. }
  752. static void tcp_update_pacing_rate(struct sock *sk)
  753. {
  754. const struct tcp_sock *tp = tcp_sk(sk);
  755. u64 rate;
  756. /* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
  757. rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);
  758. /* current rate is (cwnd * mss) / srtt
  759. * In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
  760. * In Congestion Avoidance phase, set it to 120 % the current rate.
  761. *
  762. * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
  763. * If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
  764. * end of slow start and should slow down.
  765. */
  766. if (tp->snd_cwnd < tp->snd_ssthresh / 2)
  767. rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio;
  768. else
  769. rate *= sock_net(sk)->ipv4.sysctl_tcp_pacing_ca_ratio;
  770. rate *= max(tp->snd_cwnd, tp->packets_out);
  771. if (likely(tp->srtt_us))
  772. do_div(rate, tp->srtt_us);
  773. /* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate
  774. * without any lock. We want to make sure compiler wont store
  775. * intermediate values in this location.
  776. */
  777. WRITE_ONCE(sk->sk_pacing_rate, min_t(u64, rate,
  778. sk->sk_max_pacing_rate));
  779. }
  780. /* Calculate rto without backoff. This is the second half of Van Jacobson's
  781. * routine referred to above.
  782. */
  783. static void tcp_set_rto(struct sock *sk)
  784. {
  785. const struct tcp_sock *tp = tcp_sk(sk);
  786. /* Old crap is replaced with new one. 8)
  787. *
  788. * More seriously:
  789. * 1. If rtt variance happened to be less 50msec, it is hallucination.
  790. * It cannot be less due to utterly erratic ACK generation made
  791. * at least by solaris and freebsd. "Erratic ACKs" has _nothing_
  792. * to do with delayed acks, because at cwnd>2 true delack timeout
  793. * is invisible. Actually, Linux-2.4 also generates erratic
  794. * ACKs in some circumstances.
  795. */
  796. inet_csk(sk)->icsk_rto = __tcp_set_rto(tp);
  797. /* 2. Fixups made earlier cannot be right.
  798. * If we do not estimate RTO correctly without them,
  799. * all the algo is pure shit and should be replaced
  800. * with correct one. It is exactly, which we pretend to do.
  801. */
  802. /* NOTE: clamping at TCP_RTO_MIN is not required, current algo
  803. * guarantees that rto is higher.
  804. */
  805. tcp_bound_rto(sk);
  806. }
  807. __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst)
  808. {
  809. __u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0);
  810. if (!cwnd)
  811. cwnd = TCP_INIT_CWND;
  812. return min_t(__u32, cwnd, tp->snd_cwnd_clamp);
  813. }
  814. struct tcp_sacktag_state {
  815. /* Timestamps for earliest and latest never-retransmitted segment
  816. * that was SACKed. RTO needs the earliest RTT to stay conservative,
  817. * but congestion control should still get an accurate delay signal.
  818. */
  819. u64 first_sackt;
  820. u64 last_sackt;
  821. u32 reord;
  822. u32 sack_delivered;
  823. int flag;
  824. unsigned int mss_now;
  825. struct rate_sample *rate;
  826. };
  827. /* Take a notice that peer is sending D-SACKs. Skip update of data delivery
  828. * and spurious retransmission information if this DSACK is unlikely caused by
  829. * sender's action:
  830. * - DSACKed sequence range is larger than maximum receiver's window.
  831. * - Total no. of DSACKed segments exceed the total no. of retransmitted segs.
  832. */
  833. static u32 tcp_dsack_seen(struct tcp_sock *tp, u32 start_seq,
  834. u32 end_seq, struct tcp_sacktag_state *state)
  835. {
  836. u32 seq_len, dup_segs = 1;
  837. if (!before(start_seq, end_seq))
  838. return 0;
  839. seq_len = end_seq - start_seq;
  840. /* Dubious DSACK: DSACKed range greater than maximum advertised rwnd */
  841. if (seq_len > tp->max_window)
  842. return 0;
  843. if (seq_len > tp->mss_cache)
  844. dup_segs = DIV_ROUND_UP(seq_len, tp->mss_cache);
  845. tp->dsack_dups += dup_segs;
  846. /* Skip the DSACK if dup segs weren't retransmitted by sender */
  847. if (tp->dsack_dups > tp->total_retrans)
  848. return 0;
  849. tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
  850. tp->rack.dsack_seen = 1;
  851. state->flag |= FLAG_DSACKING_ACK;
  852. /* A spurious retransmission is delivered */
  853. state->sack_delivered += dup_segs;
  854. return dup_segs;
  855. }
  856. /* It's reordering when higher sequence was delivered (i.e. sacked) before
  857. * some lower never-retransmitted sequence ("low_seq"). The maximum reordering
  858. * distance is approximated in full-mss packet distance ("reordering").
  859. */
  860. static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq,
  861. const int ts)
  862. {
  863. struct tcp_sock *tp = tcp_sk(sk);
  864. const u32 mss = tp->mss_cache;
  865. u32 fack, metric;
  866. fack = tcp_highest_sack_seq(tp);
  867. if (!before(low_seq, fack))
  868. return;
  869. metric = fack - low_seq;
  870. if ((metric > tp->reordering * mss) && mss) {
  871. #if FASTRETRANS_DEBUG > 1
  872. pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
  873. tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state,
  874. tp->reordering,
  875. 0,
  876. tp->sacked_out,
  877. tp->undo_marker ? tp->undo_retrans : 0);
  878. #endif
  879. tp->reordering = min_t(u32, (metric + mss - 1) / mss,
  880. sock_net(sk)->ipv4.sysctl_tcp_max_reordering);
  881. }
  882. /* This exciting event is worth to be remembered. 8) */
  883. tp->reord_seen++;
  884. NET_INC_STATS(sock_net(sk),
  885. ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER);
  886. }
  887. /* This must be called before lost_out or retrans_out are updated
  888. * on a new loss, because we want to know if all skbs previously
  889. * known to be lost have already been retransmitted, indicating
  890. * that this newly lost skb is our next skb to retransmit.
  891. */
  892. static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
  893. {
  894. if ((!tp->retransmit_skb_hint && tp->retrans_out >= tp->lost_out) ||
  895. (tp->retransmit_skb_hint &&
  896. before(TCP_SKB_CB(skb)->seq,
  897. TCP_SKB_CB(tp->retransmit_skb_hint)->seq)))
  898. tp->retransmit_skb_hint = skb;
  899. }
  900. /* Sum the number of packets on the wire we have marked as lost, and
  901. * notify the congestion control module that the given skb was marked lost.
  902. */
  903. static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb)
  904. {
  905. tp->lost += tcp_skb_pcount(skb);
  906. }
  907. void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)
  908. {
  909. __u8 sacked = TCP_SKB_CB(skb)->sacked;
  910. struct tcp_sock *tp = tcp_sk(sk);
  911. if (sacked & TCPCB_SACKED_ACKED)
  912. return;
  913. tcp_verify_retransmit_hint(tp, skb);
  914. if (sacked & TCPCB_LOST) {
  915. if (sacked & TCPCB_SACKED_RETRANS) {
  916. /* Account for retransmits that are lost again */
  917. TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
  918. tp->retrans_out -= tcp_skb_pcount(skb);
  919. NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPLOSTRETRANSMIT,
  920. tcp_skb_pcount(skb));
  921. tcp_notify_skb_loss_event(tp, skb);
  922. }
  923. } else {
  924. tp->lost_out += tcp_skb_pcount(skb);
  925. TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
  926. tcp_notify_skb_loss_event(tp, skb);
  927. }
  928. }
  929. /* Updates the delivered and delivered_ce counts */
  930. static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered,
  931. bool ece_ack)
  932. {
  933. tp->delivered += delivered;
  934. if (ece_ack)
  935. tp->delivered_ce += delivered;
  936. }
  937. /* This procedure tags the retransmission queue when SACKs arrive.
  938. *
  939. * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
  940. * Packets in queue with these bits set are counted in variables
  941. * sacked_out, retrans_out and lost_out, correspondingly.
  942. *
  943. * Valid combinations are:
  944. * Tag InFlight Description
  945. * 0 1 - orig segment is in flight.
  946. * S 0 - nothing flies, orig reached receiver.
  947. * L 0 - nothing flies, orig lost by net.
  948. * R 2 - both orig and retransmit are in flight.
  949. * L|R 1 - orig is lost, retransmit is in flight.
  950. * S|R 1 - orig reached receiver, retrans is still in flight.
  951. * (L|S|R is logically valid, it could occur when L|R is sacked,
  952. * but it is equivalent to plain S and code short-curcuits it to S.
  953. * L|S is logically invalid, it would mean -1 packet in flight 8))
  954. *
  955. * These 6 states form finite state machine, controlled by the following events:
  956. * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
  957. * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
  958. * 3. Loss detection event of two flavors:
  959. * A. Scoreboard estimator decided the packet is lost.
  960. * A'. Reno "three dupacks" marks head of queue lost.
  961. * B. SACK arrives sacking SND.NXT at the moment, when the
  962. * segment was retransmitted.
  963. * 4. D-SACK added new rule: D-SACK changes any tag to S.
  964. *
  965. * It is pleasant to note, that state diagram turns out to be commutative,
  966. * so that we are allowed not to be bothered by order of our actions,
  967. * when multiple events arrive simultaneously. (see the function below).
  968. *
  969. * Reordering detection.
  970. * --------------------
  971. * Reordering metric is maximal distance, which a packet can be displaced
  972. * in packet stream. With SACKs we can estimate it:
  973. *
  974. * 1. SACK fills old hole and the corresponding segment was not
  975. * ever retransmitted -> reordering. Alas, we cannot use it
  976. * when segment was retransmitted.
  977. * 2. The last flaw is solved with D-SACK. D-SACK arrives
  978. * for retransmitted and already SACKed segment -> reordering..
  979. * Both of these heuristics are not used in Loss state, when we cannot
  980. * account for retransmits accurately.
  981. *
  982. * SACK block validation.
  983. * ----------------------
  984. *
  985. * SACK block range validation checks that the received SACK block fits to
  986. * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
  987. * Note that SND.UNA is not included to the range though being valid because
  988. * it means that the receiver is rather inconsistent with itself reporting
  989. * SACK reneging when it should advance SND.UNA. Such SACK block this is
  990. * perfectly valid, however, in light of RFC2018 which explicitly states
  991. * that "SACK block MUST reflect the newest segment. Even if the newest
  992. * segment is going to be discarded ...", not that it looks very clever
  993. * in case of head skb. Due to potentional receiver driven attacks, we
  994. * choose to avoid immediate execution of a walk in write queue due to
  995. * reneging and defer head skb's loss recovery to standard loss recovery
  996. * procedure that will eventually trigger (nothing forbids us doing this).
  997. *
  998. * Implements also blockage to start_seq wrap-around. Problem lies in the
  999. * fact that though start_seq (s) is before end_seq (i.e., not reversed),
  1000. * there's no guarantee that it will be before snd_nxt (n). The problem
  1001. * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
  1002. * wrap (s_w):
  1003. *
  1004. * <- outs wnd -> <- wrapzone ->
  1005. * u e n u_w e_w s n_w
  1006. * | | | | | | |
  1007. * |<------------+------+----- TCP seqno space --------------+---------->|
  1008. * ...-- <2^31 ->| |<--------...
  1009. * ...---- >2^31 ------>| |<--------...
  1010. *
  1011. * Current code wouldn't be vulnerable but it's better still to discard such
  1012. * crazy SACK blocks. Doing this check for start_seq alone closes somewhat
  1013. * similar case (end_seq after snd_nxt wrap) as earlier reversed check in
  1014. * snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
  1015. * equal to the ideal case (infinite seqno space without wrap caused issues).
  1016. *
  1017. * With D-SACK the lower bound is extended to cover sequence space below
  1018. * SND.UNA down to undo_marker, which is the last point of interest. Yet
  1019. * again, D-SACK block must not to go across snd_una (for the same reason as
  1020. * for the normal SACK blocks, explained above). But there all simplicity
  1021. * ends, TCP might receive valid D-SACKs below that. As long as they reside
  1022. * fully below undo_marker they do not affect behavior in anyway and can
  1023. * therefore be safely ignored. In rare cases (which are more or less
  1024. * theoretical ones), the D-SACK will nicely cross that boundary due to skb
  1025. * fragmentation and packet reordering past skb's retransmission. To consider
  1026. * them correctly, the acceptable range must be extended even more though
  1027. * the exact amount is rather hard to quantify. However, tp->max_window can
  1028. * be used as an exaggerated estimate.
  1029. */
  1030. static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack,
  1031. u32 start_seq, u32 end_seq)
  1032. {
  1033. /* Too far in future, or reversed (interpretation is ambiguous) */
  1034. if (after(end_seq, tp->snd_nxt) || !before(start_seq, end_seq))
  1035. return false;
  1036. /* Nasty start_seq wrap-around check (see comments above) */
  1037. if (!before(start_seq, tp->snd_nxt))
  1038. return false;
  1039. /* In outstanding window? ...This is valid exit for D-SACKs too.
  1040. * start_seq == snd_una is non-sensical (see comments above)
  1041. */
  1042. if (after(start_seq, tp->snd_una))
  1043. return true;
  1044. if (!is_dsack || !tp->undo_marker)
  1045. return false;
  1046. /* ...Then it's D-SACK, and must reside below snd_una completely */
  1047. if (after(end_seq, tp->snd_una))
  1048. return false;
  1049. if (!before(start_seq, tp->undo_marker))
  1050. return true;
  1051. /* Too old */
  1052. if (!after(end_seq, tp->undo_marker))
  1053. return false;
  1054. /* Undo_marker boundary crossing (overestimates a lot). Known already:
  1055. * start_seq < undo_marker and end_seq >= undo_marker.
  1056. */
  1057. return !before(start_seq, end_seq - tp->max_window);
  1058. }
  1059. static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb,
  1060. struct tcp_sack_block_wire *sp, int num_sacks,
  1061. u32 prior_snd_una, struct tcp_sacktag_state *state)
  1062. {
  1063. struct tcp_sock *tp = tcp_sk(sk);
  1064. u32 start_seq_0 = get_unaligned_be32(&sp[0].start_seq);
  1065. u32 end_seq_0 = get_unaligned_be32(&sp[0].end_seq);
  1066. u32 dup_segs;
  1067. if (before(start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) {
  1068. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECV);
  1069. } else if (num_sacks > 1) {
  1070. u32 end_seq_1 = get_unaligned_be32(&sp[1].end_seq);
  1071. u32 start_seq_1 = get_unaligned_be32(&sp[1].start_seq);
  1072. if (after(end_seq_0, end_seq_1) || before(start_seq_0, start_seq_1))
  1073. return false;
  1074. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKOFORECV);
  1075. } else {
  1076. return false;
  1077. }
  1078. dup_segs = tcp_dsack_seen(tp, start_seq_0, end_seq_0, state);
  1079. if (!dup_segs) { /* Skip dubious DSACK */
  1080. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKIGNOREDDUBIOUS);
  1081. return false;
  1082. }
  1083. NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECVSEGS, dup_segs);
  1084. /* D-SACK for already forgotten data... Do dumb counting. */
  1085. if (tp->undo_marker && tp->undo_retrans > 0 &&
  1086. !after(end_seq_0, prior_snd_una) &&
  1087. after(end_seq_0, tp->undo_marker))
  1088. tp->undo_retrans = max_t(int, 0, tp->undo_retrans - dup_segs);
  1089. return true;
  1090. }
  1091. /* Check if skb is fully within the SACK block. In presence of GSO skbs,
  1092. * the incoming SACK may not exactly match but we can find smaller MSS
  1093. * aligned portion of it that matches. Therefore we might need to fragment
  1094. * which may fail and creates some hassle (caller must handle error case
  1095. * returns).
  1096. *
  1097. * FIXME: this could be merged to shift decision code
  1098. */
  1099. static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb,
  1100. u32 start_seq, u32 end_seq)
  1101. {
  1102. int err;
  1103. bool in_sack;
  1104. unsigned int pkt_len;
  1105. unsigned int mss;
  1106. in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
  1107. !before(end_seq, TCP_SKB_CB(skb)->end_seq);
  1108. if (tcp_skb_pcount(skb) > 1 && !in_sack &&
  1109. after(TCP_SKB_CB(skb)->end_seq, start_seq)) {
  1110. mss = tcp_skb_mss(skb);
  1111. in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
  1112. if (!in_sack) {
  1113. pkt_len = start_seq - TCP_SKB_CB(skb)->seq;
  1114. if (pkt_len < mss)
  1115. pkt_len = mss;
  1116. } else {
  1117. pkt_len = end_seq - TCP_SKB_CB(skb)->seq;
  1118. if (pkt_len < mss)
  1119. return -EINVAL;
  1120. }
  1121. /* Round if necessary so that SACKs cover only full MSSes
  1122. * and/or the remaining small portion (if present)
  1123. */
  1124. if (pkt_len > mss) {
  1125. unsigned int new_len = (pkt_len / mss) * mss;
  1126. if (!in_sack && new_len < pkt_len)
  1127. new_len += mss;
  1128. pkt_len = new_len;
  1129. }
  1130. if (pkt_len >= skb->len && !in_sack)
  1131. return 0;
  1132. err = tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
  1133. pkt_len, mss, GFP_ATOMIC);
  1134. if (err < 0)
  1135. return err;
  1136. }
  1137. return in_sack;
  1138. }
  1139. /* Mark the given newly-SACKed range as such, adjusting counters and hints. */
  1140. static u8 tcp_sacktag_one(struct sock *sk,
  1141. struct tcp_sacktag_state *state, u8 sacked,
  1142. u32 start_seq, u32 end_seq,
  1143. int dup_sack, int pcount,
  1144. u64 xmit_time)
  1145. {
  1146. struct tcp_sock *tp = tcp_sk(sk);
  1147. /* Account D-SACK for retransmitted packet. */
  1148. if (dup_sack && (sacked & TCPCB_RETRANS)) {
  1149. if (tp->undo_marker && tp->undo_retrans > 0 &&
  1150. after(end_seq, tp->undo_marker))
  1151. tp->undo_retrans = max_t(int, 0, tp->undo_retrans - pcount);
  1152. if ((sacked & TCPCB_SACKED_ACKED) &&
  1153. before(start_seq, state->reord))
  1154. state->reord = start_seq;
  1155. }
  1156. /* Nothing to do; acked frame is about to be dropped (was ACKed). */
  1157. if (!after(end_seq, tp->snd_una))
  1158. return sacked;
  1159. if (!(sacked & TCPCB_SACKED_ACKED)) {
  1160. tcp_rack_advance(tp, sacked, end_seq, xmit_time);
  1161. if (sacked & TCPCB_SACKED_RETRANS) {
  1162. /* If the segment is not tagged as lost,
  1163. * we do not clear RETRANS, believing
  1164. * that retransmission is still in flight.
  1165. */
  1166. if (sacked & TCPCB_LOST) {
  1167. sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS);
  1168. tp->lost_out -= pcount;
  1169. tp->retrans_out -= pcount;
  1170. }
  1171. } else {
  1172. if (!(sacked & TCPCB_RETRANS)) {
  1173. /* New sack for not retransmitted frame,
  1174. * which was in hole. It is reordering.
  1175. */
  1176. if (before(start_seq,
  1177. tcp_highest_sack_seq(tp)) &&
  1178. before(start_seq, state->reord))
  1179. state->reord = start_seq;
  1180. if (!after(end_seq, tp->high_seq))
  1181. state->flag |= FLAG_ORIG_SACK_ACKED;
  1182. if (state->first_sackt == 0)
  1183. state->first_sackt = xmit_time;
  1184. state->last_sackt = xmit_time;
  1185. }
  1186. if (sacked & TCPCB_LOST) {
  1187. sacked &= ~TCPCB_LOST;
  1188. tp->lost_out -= pcount;
  1189. }
  1190. }
  1191. sacked |= TCPCB_SACKED_ACKED;
  1192. state->flag |= FLAG_DATA_SACKED;
  1193. tp->sacked_out += pcount;
  1194. /* Out-of-order packets delivered */
  1195. state->sack_delivered += pcount;
  1196. /* Lost marker hint past SACKed? Tweak RFC3517 cnt */
  1197. if (tp->lost_skb_hint &&
  1198. before(start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq))
  1199. tp->lost_cnt_hint += pcount;
  1200. }
  1201. /* D-SACK. We can detect redundant retransmission in S|R and plain R
  1202. * frames and clear it. undo_retrans is decreased above, L|R frames
  1203. * are accounted above as well.
  1204. */
  1205. if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) {
  1206. sacked &= ~TCPCB_SACKED_RETRANS;
  1207. tp->retrans_out -= pcount;
  1208. }
  1209. return sacked;
  1210. }
  1211. /* Shift newly-SACKed bytes from this skb to the immediately previous
  1212. * already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
  1213. */
  1214. static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,
  1215. struct sk_buff *skb,
  1216. struct tcp_sacktag_state *state,
  1217. unsigned int pcount, int shifted, int mss,
  1218. bool dup_sack)
  1219. {
  1220. struct tcp_sock *tp = tcp_sk(sk);
  1221. u32 start_seq = TCP_SKB_CB(skb)->seq; /* start of newly-SACKed */
  1222. u32 end_seq = start_seq + shifted; /* end of newly-SACKed */
  1223. BUG_ON(!pcount);
  1224. /* Adjust counters and hints for the newly sacked sequence
  1225. * range but discard the return value since prev is already
  1226. * marked. We must tag the range first because the seq
  1227. * advancement below implicitly advances
  1228. * tcp_highest_sack_seq() when skb is highest_sack.
  1229. */
  1230. tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked,
  1231. start_seq, end_seq, dup_sack, pcount,
  1232. tcp_skb_timestamp_us(skb));
  1233. tcp_rate_skb_delivered(sk, skb, state->rate);
  1234. if (skb == tp->lost_skb_hint)
  1235. tp->lost_cnt_hint += pcount;
  1236. TCP_SKB_CB(prev)->end_seq += shifted;
  1237. TCP_SKB_CB(skb)->seq += shifted;
  1238. tcp_skb_pcount_add(prev, pcount);
  1239. WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount);
  1240. tcp_skb_pcount_add(skb, -pcount);
  1241. /* When we're adding to gso_segs == 1, gso_size will be zero,
  1242. * in theory this shouldn't be necessary but as long as DSACK
  1243. * code can come after this skb later on it's better to keep
  1244. * setting gso_size to something.
  1245. */
  1246. if (!TCP_SKB_CB(prev)->tcp_gso_size)
  1247. TCP_SKB_CB(prev)->tcp_gso_size = mss;
  1248. /* CHECKME: To clear or not to clear? Mimics normal skb currently */
  1249. if (tcp_skb_pcount(skb) <= 1)
  1250. TCP_SKB_CB(skb)->tcp_gso_size = 0;
  1251. /* Difference in this won't matter, both ACKed by the same cumul. ACK */
  1252. TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS);
  1253. if (skb->len > 0) {
  1254. BUG_ON(!tcp_skb_pcount(skb));
  1255. NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTED);
  1256. return false;
  1257. }
  1258. /* Whole SKB was eaten :-) */
  1259. if (skb == tp->retransmit_skb_hint)
  1260. tp->retransmit_skb_hint = prev;
  1261. if (skb == tp->lost_skb_hint) {
  1262. tp->lost_skb_hint = prev;
  1263. tp->lost_cnt_hint -= tcp_skb_pcount(prev);
  1264. }
  1265. TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
  1266. TCP_SKB_CB(prev)->eor = TCP_SKB_CB(skb)->eor;
  1267. if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
  1268. TCP_SKB_CB(prev)->end_seq++;
  1269. if (skb == tcp_highest_sack(sk))
  1270. tcp_advance_highest_sack(sk, skb);
  1271. tcp_skb_collapse_tstamp(prev, skb);
  1272. if (unlikely(TCP_SKB_CB(prev)->tx.delivered_mstamp))
  1273. TCP_SKB_CB(prev)->tx.delivered_mstamp = 0;
  1274. tcp_rtx_queue_unlink_and_free(skb, sk);
  1275. NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKMERGED);
  1276. return true;
  1277. }
  1278. /* I wish gso_size would have a bit more sane initialization than
  1279. * something-or-zero which complicates things
  1280. */
  1281. static int tcp_skb_seglen(const struct sk_buff *skb)
  1282. {
  1283. return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb);
  1284. }
  1285. /* Shifting pages past head area doesn't work */
  1286. static int skb_can_shift(const struct sk_buff *skb)
  1287. {
  1288. return !skb_headlen(skb) && skb_is_nonlinear(skb);
  1289. }
  1290. int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from,
  1291. int pcount, int shiftlen)
  1292. {
  1293. /* TCP min gso_size is 8 bytes (TCP_MIN_GSO_SIZE)
  1294. * Since TCP_SKB_CB(skb)->tcp_gso_segs is 16 bits, we need
  1295. * to make sure not storing more than 65535 * 8 bytes per skb,
  1296. * even if current MSS is bigger.
  1297. */
  1298. if (unlikely(to->len + shiftlen >= 65535 * TCP_MIN_GSO_SIZE))
  1299. return 0;
  1300. if (unlikely(tcp_skb_pcount(to) + pcount > 65535))
  1301. return 0;
  1302. return skb_shift(to, from, shiftlen);
  1303. }
  1304. /* Try collapsing SACK blocks spanning across multiple skbs to a single
  1305. * skb.
  1306. */
  1307. static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
  1308. struct tcp_sacktag_state *state,
  1309. u32 start_seq, u32 end_seq,
  1310. bool dup_sack)
  1311. {
  1312. struct tcp_sock *tp = tcp_sk(sk);
  1313. struct sk_buff *prev;
  1314. int mss;
  1315. int pcount = 0;
  1316. int len;
  1317. int in_sack;
  1318. /* Normally R but no L won't result in plain S */
  1319. if (!dup_sack &&
  1320. (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS)
  1321. goto fallback;
  1322. if (!skb_can_shift(skb))
  1323. goto fallback;
  1324. /* This frame is about to be dropped (was ACKed). */
  1325. if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
  1326. goto fallback;
  1327. /* Can only happen with delayed DSACK + discard craziness */
  1328. prev = skb_rb_prev(skb);
  1329. if (!prev)
  1330. goto fallback;
  1331. if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED)
  1332. goto fallback;
  1333. if (!tcp_skb_can_collapse(prev, skb))
  1334. goto fallback;
  1335. in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
  1336. !before(end_seq, TCP_SKB_CB(skb)->end_seq);
  1337. if (in_sack) {
  1338. len = skb->len;
  1339. pcount = tcp_skb_pcount(skb);
  1340. mss = tcp_skb_seglen(skb);
  1341. /* TODO: Fix DSACKs to not fragment already SACKed and we can
  1342. * drop this restriction as unnecessary
  1343. */
  1344. if (mss != tcp_skb_seglen(prev))
  1345. goto fallback;
  1346. } else {
  1347. if (!after(TCP_SKB_CB(skb)->end_seq, start_seq))
  1348. goto noop;
  1349. /* CHECKME: This is non-MSS split case only?, this will
  1350. * cause skipped skbs due to advancing loop btw, original
  1351. * has that feature too
  1352. */
  1353. if (tcp_skb_pcount(skb) <= 1)
  1354. goto noop;
  1355. in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
  1356. if (!in_sack) {
  1357. /* TODO: head merge to next could be attempted here
  1358. * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
  1359. * though it might not be worth of the additional hassle
  1360. *
  1361. * ...we can probably just fallback to what was done
  1362. * previously. We could try merging non-SACKed ones
  1363. * as well but it probably isn't going to buy off
  1364. * because later SACKs might again split them, and
  1365. * it would make skb timestamp tracking considerably
  1366. * harder problem.
  1367. */
  1368. goto fallback;
  1369. }
  1370. len = end_seq - TCP_SKB_CB(skb)->seq;
  1371. BUG_ON(len < 0);
  1372. BUG_ON(len > skb->len);
  1373. /* MSS boundaries should be honoured or else pcount will
  1374. * severely break even though it makes things bit trickier.
  1375. * Optimize common case to avoid most of the divides
  1376. */
  1377. mss = tcp_skb_mss(skb);
  1378. /* TODO: Fix DSACKs to not fragment already SACKed and we can
  1379. * drop this restriction as unnecessary
  1380. */
  1381. if (mss != tcp_skb_seglen(prev))
  1382. goto fallback;
  1383. if (len == mss) {
  1384. pcount = 1;
  1385. } else if (len < mss) {
  1386. goto noop;
  1387. } else {
  1388. pcount = len / mss;
  1389. len = pcount * mss;
  1390. }
  1391. }
  1392. /* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
  1393. if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una))
  1394. goto fallback;
  1395. if (!tcp_skb_shift(prev, skb, pcount, len))
  1396. goto fallback;
  1397. if (!tcp_shifted_skb(sk, prev, skb, state, pcount, len, mss, dup_sack))
  1398. goto out;
  1399. /* Hole filled allows collapsing with the next as well, this is very
  1400. * useful when hole on every nth skb pattern happens
  1401. */
  1402. skb = skb_rb_next(prev);
  1403. if (!skb)
  1404. goto out;
  1405. if (!skb_can_shift(skb) ||
  1406. ((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) ||
  1407. (mss != tcp_skb_seglen(skb)))
  1408. goto out;
  1409. if (!tcp_skb_can_collapse(prev, skb))
  1410. goto out;
  1411. len = skb->len;
  1412. pcount = tcp_skb_pcount(skb);
  1413. if (tcp_skb_shift(prev, skb, pcount, len))
  1414. tcp_shifted_skb(sk, prev, skb, state, pcount,
  1415. len, mss, 0);
  1416. out:
  1417. return prev;
  1418. noop:
  1419. return skb;
  1420. fallback:
  1421. NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK);
  1422. return NULL;
  1423. }
  1424. static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk,
  1425. struct tcp_sack_block *next_dup,
  1426. struct tcp_sacktag_state *state,
  1427. u32 start_seq, u32 end_seq,
  1428. bool dup_sack_in)
  1429. {
  1430. struct tcp_sock *tp = tcp_sk(sk);
  1431. struct sk_buff *tmp;
  1432. skb_rbtree_walk_from(skb) {
  1433. int in_sack = 0;
  1434. bool dup_sack = dup_sack_in;
  1435. /* queue is in-order => we can short-circuit the walk early */
  1436. if (!before(TCP_SKB_CB(skb)->seq, end_seq))
  1437. break;
  1438. if (next_dup &&
  1439. before(TCP_SKB_CB(skb)->seq, next_dup->end_seq)) {
  1440. in_sack = tcp_match_skb_to_sack(sk, skb,
  1441. next_dup->start_seq,
  1442. next_dup->end_seq);
  1443. if (in_sack > 0)
  1444. dup_sack = true;
  1445. }
  1446. /* skb reference here is a bit tricky to get right, since
  1447. * shifting can eat and free both this skb and the next,
  1448. * so not even _safe variant of the loop is enough.
  1449. */
  1450. if (in_sack <= 0) {
  1451. tmp = tcp_shift_skb_data(sk, skb, state,
  1452. start_seq, end_seq, dup_sack);
  1453. if (tmp) {
  1454. if (tmp != skb) {
  1455. skb = tmp;
  1456. continue;
  1457. }
  1458. in_sack = 0;
  1459. } else {
  1460. in_sack = tcp_match_skb_to_sack(sk, skb,
  1461. start_seq,
  1462. end_seq);
  1463. }
  1464. }
  1465. if (unlikely(in_sack < 0))
  1466. break;
  1467. if (in_sack) {
  1468. TCP_SKB_CB(skb)->sacked =
  1469. tcp_sacktag_one(sk,
  1470. state,
  1471. TCP_SKB_CB(skb)->sacked,
  1472. TCP_SKB_CB(skb)->seq,
  1473. TCP_SKB_CB(skb)->end_seq,
  1474. dup_sack,
  1475. tcp_skb_pcount(skb),
  1476. tcp_skb_timestamp_us(skb));
  1477. tcp_rate_skb_delivered(sk, skb, state->rate);
  1478. if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
  1479. list_del_init(&skb->tcp_tsorted_anchor);
  1480. if (!before(TCP_SKB_CB(skb)->seq,
  1481. tcp_highest_sack_seq(tp)))
  1482. tcp_advance_highest_sack(sk, skb);
  1483. }
  1484. }
  1485. return skb;
  1486. }
  1487. static struct sk_buff *tcp_sacktag_bsearch(struct sock *sk, u32 seq)
  1488. {
  1489. struct rb_node *parent, **p = &sk->tcp_rtx_queue.rb_node;
  1490. struct sk_buff *skb;
  1491. while (*p) {
  1492. parent = *p;
  1493. skb = rb_to_skb(parent);
  1494. if (before(seq, TCP_SKB_CB(skb)->seq)) {
  1495. p = &parent->rb_left;
  1496. continue;
  1497. }
  1498. if (!before(seq, TCP_SKB_CB(skb)->end_seq)) {
  1499. p = &parent->rb_right;
  1500. continue;
  1501. }
  1502. return skb;
  1503. }
  1504. return NULL;
  1505. }
  1506. static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk,
  1507. u32 skip_to_seq)
  1508. {
  1509. if (skb && after(TCP_SKB_CB(skb)->seq, skip_to_seq))
  1510. return skb;
  1511. return tcp_sacktag_bsearch(sk, skip_to_seq);
  1512. }
  1513. static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb,
  1514. struct sock *sk,
  1515. struct tcp_sack_block *next_dup,
  1516. struct tcp_sacktag_state *state,
  1517. u32 skip_to_seq)
  1518. {
  1519. if (!next_dup)
  1520. return skb;
  1521. if (before(next_dup->start_seq, skip_to_seq)) {
  1522. skb = tcp_sacktag_skip(skb, sk, next_dup->start_seq);
  1523. skb = tcp_sacktag_walk(skb, sk, NULL, state,
  1524. next_dup->start_seq, next_dup->end_seq,
  1525. 1);
  1526. }
  1527. return skb;
  1528. }
  1529. static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache)
  1530. {
  1531. return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
  1532. }
  1533. static int
  1534. tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb,
  1535. u32 prior_snd_una, struct tcp_sacktag_state *state)
  1536. {
  1537. struct tcp_sock *tp = tcp_sk(sk);
  1538. const unsigned char *ptr = (skb_transport_header(ack_skb) +
  1539. TCP_SKB_CB(ack_skb)->sacked);
  1540. struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2);
  1541. struct tcp_sack_block sp[TCP_NUM_SACKS];
  1542. struct tcp_sack_block *cache;
  1543. struct sk_buff *skb;
  1544. int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3);
  1545. int used_sacks;
  1546. bool found_dup_sack = false;
  1547. int i, j;
  1548. int first_sack_index;
  1549. state->flag = 0;
  1550. state->reord = tp->snd_nxt;
  1551. if (!tp->sacked_out)
  1552. tcp_highest_sack_reset(sk);
  1553. found_dup_sack = tcp_check_dsack(sk, ack_skb, sp_wire,
  1554. num_sacks, prior_snd_una, state);
  1555. /* Eliminate too old ACKs, but take into
  1556. * account more or less fresh ones, they can
  1557. * contain valid SACK info.
  1558. */
  1559. if (before(TCP_SKB_CB(ack_skb)->ack_seq, prior_snd_una - tp->max_window))
  1560. return 0;
  1561. if (!tp->packets_out)
  1562. goto out;
  1563. used_sacks = 0;
  1564. first_sack_index = 0;
  1565. for (i = 0; i < num_sacks; i++) {
  1566. bool dup_sack = !i && found_dup_sack;
  1567. sp[used_sacks].start_seq = get_unaligned_be32(&sp_wire[i].start_seq);
  1568. sp[used_sacks].end_seq = get_unaligned_be32(&sp_wire[i].end_seq);
  1569. if (!tcp_is_sackblock_valid(tp, dup_sack,
  1570. sp[used_sacks].start_seq,
  1571. sp[used_sacks].end_seq)) {
  1572. int mib_idx;
  1573. if (dup_sack) {
  1574. if (!tp->undo_marker)
  1575. mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO;
  1576. else
  1577. mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD;
  1578. } else {
  1579. /* Don't count olds caused by ACK reordering */
  1580. if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) &&
  1581. !after(sp[used_sacks].end_seq, tp->snd_una))
  1582. continue;
  1583. mib_idx = LINUX_MIB_TCPSACKDISCARD;
  1584. }
  1585. NET_INC_STATS(sock_net(sk), mib_idx);
  1586. if (i == 0)
  1587. first_sack_index = -1;
  1588. continue;
  1589. }
  1590. /* Ignore very old stuff early */
  1591. if (!after(sp[used_sacks].end_seq, prior_snd_una)) {
  1592. if (i == 0)
  1593. first_sack_index = -1;
  1594. continue;
  1595. }
  1596. used_sacks++;
  1597. }
  1598. /* order SACK blocks to allow in order walk of the retrans queue */
  1599. for (i = used_sacks - 1; i > 0; i--) {
  1600. for (j = 0; j < i; j++) {
  1601. if (after(sp[j].start_seq, sp[j + 1].start_seq)) {
  1602. swap(sp[j], sp[j + 1]);
  1603. /* Track where the first SACK block goes to */
  1604. if (j == first_sack_index)
  1605. first_sack_index = j + 1;
  1606. }
  1607. }
  1608. }
  1609. state->mss_now = tcp_current_mss(sk);
  1610. skb = NULL;
  1611. i = 0;
  1612. if (!tp->sacked_out) {
  1613. /* It's already past, so skip checking against it */
  1614. cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
  1615. } else {
  1616. cache = tp->recv_sack_cache;
  1617. /* Skip empty blocks in at head of the cache */
  1618. while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq &&
  1619. !cache->end_seq)
  1620. cache++;
  1621. }
  1622. while (i < used_sacks) {
  1623. u32 start_seq = sp[i].start_seq;
  1624. u32 end_seq = sp[i].end_seq;
  1625. bool dup_sack = (found_dup_sack && (i == first_sack_index));
  1626. struct tcp_sack_block *next_dup = NULL;
  1627. if (found_dup_sack && ((i + 1) == first_sack_index))
  1628. next_dup = &sp[i + 1];
  1629. /* Skip too early cached blocks */
  1630. while (tcp_sack_cache_ok(tp, cache) &&
  1631. !before(start_seq, cache->end_seq))
  1632. cache++;
  1633. /* Can skip some work by looking recv_sack_cache? */
  1634. if (tcp_sack_cache_ok(tp, cache) && !dup_sack &&
  1635. after(end_seq, cache->start_seq)) {
  1636. /* Head todo? */
  1637. if (before(start_seq, cache->start_seq)) {
  1638. skb = tcp_sacktag_skip(skb, sk, start_seq);
  1639. skb = tcp_sacktag_walk(skb, sk, next_dup,
  1640. state,
  1641. start_seq,
  1642. cache->start_seq,
  1643. dup_sack);
  1644. }
  1645. /* Rest of the block already fully processed? */
  1646. if (!after(end_seq, cache->end_seq))
  1647. goto advance_sp;
  1648. skb = tcp_maybe_skipping_dsack(skb, sk, next_dup,
  1649. state,
  1650. cache->end_seq);
  1651. /* ...tail remains todo... */
  1652. if (tcp_highest_sack_seq(tp) == cache->end_seq) {
  1653. /* ...but better entrypoint exists! */
  1654. skb = tcp_highest_sack(sk);
  1655. if (!skb)
  1656. break;
  1657. cache++;
  1658. goto walk;
  1659. }
  1660. skb = tcp_sacktag_skip(skb, sk, cache->end_seq);
  1661. /* Check overlap against next cached too (past this one already) */
  1662. cache++;
  1663. continue;
  1664. }
  1665. if (!before(start_seq, tcp_highest_sack_seq(tp))) {
  1666. skb = tcp_highest_sack(sk);
  1667. if (!skb)
  1668. break;
  1669. }
  1670. skb = tcp_sacktag_skip(skb, sk, start_seq);
  1671. walk:
  1672. skb = tcp_sacktag_walk(skb, sk, next_dup, state,
  1673. start_seq, end_seq, dup_sack);
  1674. advance_sp:
  1675. i++;
  1676. }
  1677. /* Clear the head of the cache sack blocks so we can skip it next time */
  1678. for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) {
  1679. tp->recv_sack_cache[i].start_seq = 0;
  1680. tp->recv_sack_cache[i].end_seq = 0;
  1681. }
  1682. for (j = 0; j < used_sacks; j++)
  1683. tp->recv_sack_cache[i++] = sp[j];
  1684. if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss || tp->undo_marker)
  1685. tcp_check_sack_reordering(sk, state->reord, 0);
  1686. tcp_verify_left_out(tp);
  1687. out:
  1688. #if FASTRETRANS_DEBUG > 0
  1689. WARN_ON((int)tp->sacked_out < 0);
  1690. WARN_ON((int)tp->lost_out < 0);
  1691. WARN_ON((int)tp->retrans_out < 0);
  1692. WARN_ON((int)tcp_packets_in_flight(tp) < 0);
  1693. #endif
  1694. return state->flag;
  1695. }
  1696. /* Limits sacked_out so that sum with lost_out isn't ever larger than
  1697. * packets_out. Returns false if sacked_out adjustement wasn't necessary.
  1698. */
  1699. static bool tcp_limit_reno_sacked(struct tcp_sock *tp)
  1700. {
  1701. u32 holes;
  1702. holes = max(tp->lost_out, 1U);
  1703. holes = min(holes, tp->packets_out);
  1704. if ((tp->sacked_out + holes) > tp->packets_out) {
  1705. tp->sacked_out = tp->packets_out - holes;
  1706. return true;
  1707. }
  1708. return false;
  1709. }
  1710. /* If we receive more dupacks than we expected counting segments
  1711. * in assumption of absent reordering, interpret this as reordering.
  1712. * The only another reason could be bug in receiver TCP.
  1713. */
  1714. static void tcp_check_reno_reordering(struct sock *sk, const int addend)
  1715. {
  1716. struct tcp_sock *tp = tcp_sk(sk);
  1717. if (!tcp_limit_reno_sacked(tp))
  1718. return;
  1719. tp->reordering = min_t(u32, tp->packets_out + addend,
  1720. sock_net(sk)->ipv4.sysctl_tcp_max_reordering);
  1721. tp->reord_seen++;
  1722. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER);
  1723. }
  1724. /* Emulate SACKs for SACKless connection: account for a new dupack. */
  1725. static void tcp_add_reno_sack(struct sock *sk, int num_dupack, bool ece_ack)
  1726. {
  1727. if (num_dupack) {
  1728. struct tcp_sock *tp = tcp_sk(sk);
  1729. u32 prior_sacked = tp->sacked_out;
  1730. s32 delivered;
  1731. tp->sacked_out += num_dupack;
  1732. tcp_check_reno_reordering(sk, 0);
  1733. delivered = tp->sacked_out - prior_sacked;
  1734. if (delivered > 0)
  1735. tcp_count_delivered(tp, delivered, ece_ack);
  1736. tcp_verify_left_out(tp);
  1737. }
  1738. }
  1739. /* Account for ACK, ACKing some data in Reno Recovery phase. */
  1740. static void tcp_remove_reno_sacks(struct sock *sk, int acked, bool ece_ack)
  1741. {
  1742. struct tcp_sock *tp = tcp_sk(sk);
  1743. if (acked > 0) {
  1744. /* One ACK acked hole. The rest eat duplicate ACKs. */
  1745. tcp_count_delivered(tp, max_t(int, acked - tp->sacked_out, 1),
  1746. ece_ack);
  1747. if (acked - 1 >= tp->sacked_out)
  1748. tp->sacked_out = 0;
  1749. else
  1750. tp->sacked_out -= acked - 1;
  1751. }
  1752. tcp_check_reno_reordering(sk, acked);
  1753. tcp_verify_left_out(tp);
  1754. }
  1755. static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
  1756. {
  1757. tp->sacked_out = 0;
  1758. }
  1759. void tcp_clear_retrans(struct tcp_sock *tp)
  1760. {
  1761. tp->retrans_out = 0;
  1762. tp->lost_out = 0;
  1763. tp->undo_marker = 0;
  1764. tp->undo_retrans = -1;
  1765. tp->sacked_out = 0;
  1766. }
  1767. static inline void tcp_init_undo(struct tcp_sock *tp)
  1768. {
  1769. tp->undo_marker = tp->snd_una;
  1770. /* Retransmission still in flight may cause DSACKs later. */
  1771. tp->undo_retrans = tp->retrans_out ? : -1;
  1772. }
  1773. static bool tcp_is_rack(const struct sock *sk)
  1774. {
  1775. return sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_LOSS_DETECTION;
  1776. }
  1777. /* If we detect SACK reneging, forget all SACK information
  1778. * and reset tags completely, otherwise preserve SACKs. If receiver
  1779. * dropped its ofo queue, we will know this due to reneging detection.
  1780. */
  1781. static void tcp_timeout_mark_lost(struct sock *sk)
  1782. {
  1783. struct tcp_sock *tp = tcp_sk(sk);
  1784. struct sk_buff *skb, *head;
  1785. bool is_reneg; /* is receiver reneging on SACKs? */
  1786. head = tcp_rtx_queue_head(sk);
  1787. is_reneg = head && (TCP_SKB_CB(head)->sacked & TCPCB_SACKED_ACKED);
  1788. if (is_reneg) {
  1789. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
  1790. tp->sacked_out = 0;
  1791. /* Mark SACK reneging until we recover from this loss event. */
  1792. tp->is_sack_reneg = 1;
  1793. } else if (tcp_is_reno(tp)) {
  1794. tcp_reset_reno_sack(tp);
  1795. }
  1796. skb = head;
  1797. skb_rbtree_walk_from(skb) {
  1798. if (is_reneg)
  1799. TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
  1800. else if (tcp_is_rack(sk) && skb != head &&
  1801. tcp_rack_skb_timeout(tp, skb, 0) > 0)
  1802. continue; /* Don't mark recently sent ones lost yet */
  1803. tcp_mark_skb_lost(sk, skb);
  1804. }
  1805. tcp_verify_left_out(tp);
  1806. tcp_clear_all_retrans_hints(tp);
  1807. }
  1808. /* Enter Loss state. */
  1809. void tcp_enter_loss(struct sock *sk)
  1810. {
  1811. const struct inet_connection_sock *icsk = inet_csk(sk);
  1812. struct tcp_sock *tp = tcp_sk(sk);
  1813. struct net *net = sock_net(sk);
  1814. bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery;
  1815. tcp_timeout_mark_lost(sk);
  1816. /* Reduce ssthresh if it has not yet been made inside this window. */
  1817. if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
  1818. !after(tp->high_seq, tp->snd_una) ||
  1819. (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
  1820. tp->prior_ssthresh = tcp_current_ssthresh(sk);
  1821. tp->prior_cwnd = tp->snd_cwnd;
  1822. tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
  1823. tcp_ca_event(sk, CA_EVENT_LOSS);
  1824. tcp_init_undo(tp);
  1825. }
  1826. tp->snd_cwnd = tcp_packets_in_flight(tp) + 1;
  1827. tp->snd_cwnd_cnt = 0;
  1828. tp->snd_cwnd_stamp = tcp_jiffies32;
  1829. /* Timeout in disordered state after receiving substantial DUPACKs
  1830. * suggests that the degree of reordering is over-estimated.
  1831. */
  1832. if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
  1833. tp->sacked_out >= net->ipv4.sysctl_tcp_reordering)
  1834. tp->reordering = min_t(unsigned int, tp->reordering,
  1835. net->ipv4.sysctl_tcp_reordering);
  1836. tcp_set_ca_state(sk, TCP_CA_Loss);
  1837. tp->high_seq = tp->snd_nxt;
  1838. tcp_ecn_queue_cwr(tp);
  1839. /* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
  1840. * loss recovery is underway except recurring timeout(s) on
  1841. * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
  1842. */
  1843. tp->frto = net->ipv4.sysctl_tcp_frto &&
  1844. (new_recovery || icsk->icsk_retransmits) &&
  1845. !inet_csk(sk)->icsk_mtup.probe_size;
  1846. }
  1847. /* If ACK arrived pointing to a remembered SACK, it means that our
  1848. * remembered SACKs do not reflect real state of receiver i.e.
  1849. * receiver _host_ is heavily congested (or buggy).
  1850. *
  1851. * To avoid big spurious retransmission bursts due to transient SACK
  1852. * scoreboard oddities that look like reneging, we give the receiver a
  1853. * little time (max(RTT/2, 10ms)) to send us some more ACKs that will
  1854. * restore sanity to the SACK scoreboard. If the apparent reneging
  1855. * persists until this RTO then we'll clear the SACK scoreboard.
  1856. */
  1857. static bool tcp_check_sack_reneging(struct sock *sk, int flag)
  1858. {
  1859. if (flag & FLAG_SACK_RENEGING) {
  1860. struct tcp_sock *tp = tcp_sk(sk);
  1861. unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4),
  1862. msecs_to_jiffies(10));
  1863. inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
  1864. delay, TCP_RTO_MAX);
  1865. return true;
  1866. }
  1867. return false;
  1868. }
  1869. /* Heurestics to calculate number of duplicate ACKs. There's no dupACKs
  1870. * counter when SACK is enabled (without SACK, sacked_out is used for
  1871. * that purpose).
  1872. *
  1873. * With reordering, holes may still be in flight, so RFC3517 recovery
  1874. * uses pure sacked_out (total number of SACKed segments) even though
  1875. * it violates the RFC that uses duplicate ACKs, often these are equal
  1876. * but when e.g. out-of-window ACKs or packet duplication occurs,
  1877. * they differ. Since neither occurs due to loss, TCP should really
  1878. * ignore them.
  1879. */
  1880. static inline int tcp_dupack_heuristics(const struct tcp_sock *tp)
  1881. {
  1882. return tp->sacked_out + 1;
  1883. }
  1884. /* Linux NewReno/SACK/ECN state machine.
  1885. * --------------------------------------
  1886. *
  1887. * "Open" Normal state, no dubious events, fast path.
  1888. * "Disorder" In all the respects it is "Open",
  1889. * but requires a bit more attention. It is entered when
  1890. * we see some SACKs or dupacks. It is split of "Open"
  1891. * mainly to move some processing from fast path to slow one.
  1892. * "CWR" CWND was reduced due to some Congestion Notification event.
  1893. * It can be ECN, ICMP source quench, local device congestion.
  1894. * "Recovery" CWND was reduced, we are fast-retransmitting.
  1895. * "Loss" CWND was reduced due to RTO timeout or SACK reneging.
  1896. *
  1897. * tcp_fastretrans_alert() is entered:
  1898. * - each incoming ACK, if state is not "Open"
  1899. * - when arrived ACK is unusual, namely:
  1900. * * SACK
  1901. * * Duplicate ACK.
  1902. * * ECN ECE.
  1903. *
  1904. * Counting packets in flight is pretty simple.
  1905. *
  1906. * in_flight = packets_out - left_out + retrans_out
  1907. *
  1908. * packets_out is SND.NXT-SND.UNA counted in packets.
  1909. *
  1910. * retrans_out is number of retransmitted segments.
  1911. *
  1912. * left_out is number of segments left network, but not ACKed yet.
  1913. *
  1914. * left_out = sacked_out + lost_out
  1915. *
  1916. * sacked_out: Packets, which arrived to receiver out of order
  1917. * and hence not ACKed. With SACKs this number is simply
  1918. * amount of SACKed data. Even without SACKs
  1919. * it is easy to give pretty reliable estimate of this number,
  1920. * counting duplicate ACKs.
  1921. *
  1922. * lost_out: Packets lost by network. TCP has no explicit
  1923. * "loss notification" feedback from network (for now).
  1924. * It means that this number can be only _guessed_.
  1925. * Actually, it is the heuristics to predict lossage that
  1926. * distinguishes different algorithms.
  1927. *
  1928. * F.e. after RTO, when all the queue is considered as lost,
  1929. * lost_out = packets_out and in_flight = retrans_out.
  1930. *
  1931. * Essentially, we have now a few algorithms detecting
  1932. * lost packets.
  1933. *
  1934. * If the receiver supports SACK:
  1935. *
  1936. * RFC6675/3517: It is the conventional algorithm. A packet is
  1937. * considered lost if the number of higher sequence packets
  1938. * SACKed is greater than or equal the DUPACK thoreshold
  1939. * (reordering). This is implemented in tcp_mark_head_lost and
  1940. * tcp_update_scoreboard.
  1941. *
  1942. * RACK (draft-ietf-tcpm-rack-01): it is a newer algorithm
  1943. * (2017-) that checks timing instead of counting DUPACKs.
  1944. * Essentially a packet is considered lost if it's not S/ACKed
  1945. * after RTT + reordering_window, where both metrics are
  1946. * dynamically measured and adjusted. This is implemented in
  1947. * tcp_rack_mark_lost.
  1948. *
  1949. * If the receiver does not support SACK:
  1950. *
  1951. * NewReno (RFC6582): in Recovery we assume that one segment
  1952. * is lost (classic Reno). While we are in Recovery and
  1953. * a partial ACK arrives, we assume that one more packet
  1954. * is lost (NewReno). This heuristics are the same in NewReno
  1955. * and SACK.
  1956. *
  1957. * Really tricky (and requiring careful tuning) part of algorithm
  1958. * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue().
  1959. * The first determines the moment _when_ we should reduce CWND and,
  1960. * hence, slow down forward transmission. In fact, it determines the moment
  1961. * when we decide that hole is caused by loss, rather than by a reorder.
  1962. *
  1963. * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
  1964. * holes, caused by lost packets.
  1965. *
  1966. * And the most logically complicated part of algorithm is undo
  1967. * heuristics. We detect false retransmits due to both too early
  1968. * fast retransmit (reordering) and underestimated RTO, analyzing
  1969. * timestamps and D-SACKs. When we detect that some segments were
  1970. * retransmitted by mistake and CWND reduction was wrong, we undo
  1971. * window reduction and abort recovery phase. This logic is hidden
  1972. * inside several functions named tcp_try_undo_<something>.
  1973. */
  1974. /* This function decides, when we should leave Disordered state
  1975. * and enter Recovery phase, reducing congestion window.
  1976. *
  1977. * Main question: may we further continue forward transmission
  1978. * with the same cwnd?
  1979. */
  1980. static bool tcp_time_to_recover(struct sock *sk, int flag)
  1981. {
  1982. struct tcp_sock *tp = tcp_sk(sk);
  1983. /* Trick#1: The loss is proven. */
  1984. if (tp->lost_out)
  1985. return true;
  1986. /* Not-A-Trick#2 : Classic rule... */
  1987. if (!tcp_is_rack(sk) && tcp_dupack_heuristics(tp) > tp->reordering)
  1988. return true;
  1989. return false;
  1990. }
  1991. /* Detect loss in event "A" above by marking head of queue up as lost.
  1992. * For RFC3517 SACK, a segment is considered lost if it
  1993. * has at least tp->reordering SACKed seqments above it; "packets" refers to
  1994. * the maximum SACKed segments to pass before reaching this limit.
  1995. */
  1996. static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head)
  1997. {
  1998. struct tcp_sock *tp = tcp_sk(sk);
  1999. struct sk_buff *skb;
  2000. int cnt;
  2001. /* Use SACK to deduce losses of new sequences sent during recovery */
  2002. const u32 loss_high = tp->snd_nxt;
  2003. WARN_ON(packets > tp->packets_out);
  2004. skb = tp->lost_skb_hint;
  2005. if (skb) {
  2006. /* Head already handled? */
  2007. if (mark_head && after(TCP_SKB_CB(skb)->seq, tp->snd_una))
  2008. return;
  2009. cnt = tp->lost_cnt_hint;
  2010. } else {
  2011. skb = tcp_rtx_queue_head(sk);
  2012. cnt = 0;
  2013. }
  2014. skb_rbtree_walk_from(skb) {
  2015. /* TODO: do this better */
  2016. /* this is not the most efficient way to do this... */
  2017. tp->lost_skb_hint = skb;
  2018. tp->lost_cnt_hint = cnt;
  2019. if (after(TCP_SKB_CB(skb)->end_seq, loss_high))
  2020. break;
  2021. if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
  2022. cnt += tcp_skb_pcount(skb);
  2023. if (cnt > packets)
  2024. break;
  2025. if (!(TCP_SKB_CB(skb)->sacked & TCPCB_LOST))
  2026. tcp_mark_skb_lost(sk, skb);
  2027. if (mark_head)
  2028. break;
  2029. }
  2030. tcp_verify_left_out(tp);
  2031. }
  2032. /* Account newly detected lost packet(s) */
  2033. static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
  2034. {
  2035. struct tcp_sock *tp = tcp_sk(sk);
  2036. if (tcp_is_sack(tp)) {
  2037. int sacked_upto = tp->sacked_out - tp->reordering;
  2038. if (sacked_upto >= 0)
  2039. tcp_mark_head_lost(sk, sacked_upto, 0);
  2040. else if (fast_rexmit)
  2041. tcp_mark_head_lost(sk, 1, 1);
  2042. }
  2043. }
  2044. static bool tcp_tsopt_ecr_before(const struct tcp_sock *tp, u32 when)
  2045. {
  2046. return tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
  2047. before(tp->rx_opt.rcv_tsecr, when);
  2048. }
  2049. /* skb is spurious retransmitted if the returned timestamp echo
  2050. * reply is prior to the skb transmission time
  2051. */
  2052. static bool tcp_skb_spurious_retrans(const struct tcp_sock *tp,
  2053. const struct sk_buff *skb)
  2054. {
  2055. return (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) &&
  2056. tcp_tsopt_ecr_before(tp, tcp_skb_timestamp(skb));
  2057. }
  2058. /* Nothing was retransmitted or returned timestamp is less
  2059. * than timestamp of the first retransmission.
  2060. */
  2061. static inline bool tcp_packet_delayed(const struct tcp_sock *tp)
  2062. {
  2063. return tp->retrans_stamp &&
  2064. tcp_tsopt_ecr_before(tp, tp->retrans_stamp);
  2065. }
  2066. /* Undo procedures. */
  2067. /* We can clear retrans_stamp when there are no retransmissions in the
  2068. * window. It would seem that it is trivially available for us in
  2069. * tp->retrans_out, however, that kind of assumptions doesn't consider
  2070. * what will happen if errors occur when sending retransmission for the
  2071. * second time. ...It could the that such segment has only
  2072. * TCPCB_EVER_RETRANS set at the present time. It seems that checking
  2073. * the head skb is enough except for some reneging corner cases that
  2074. * are not worth the effort.
  2075. *
  2076. * Main reason for all this complexity is the fact that connection dying
  2077. * time now depends on the validity of the retrans_stamp, in particular,
  2078. * that successive retransmissions of a segment must not advance
  2079. * retrans_stamp under any conditions.
  2080. */
  2081. static bool tcp_any_retrans_done(const struct sock *sk)
  2082. {
  2083. const struct tcp_sock *tp = tcp_sk(sk);
  2084. struct sk_buff *skb;
  2085. if (tp->retrans_out)
  2086. return true;
  2087. skb = tcp_rtx_queue_head(sk);
  2088. if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS))
  2089. return true;
  2090. return false;
  2091. }
  2092. static void DBGUNDO(struct sock *sk, const char *msg)
  2093. {
  2094. #if FASTRETRANS_DEBUG > 1
  2095. struct tcp_sock *tp = tcp_sk(sk);
  2096. struct inet_sock *inet = inet_sk(sk);
  2097. if (sk->sk_family == AF_INET) {
  2098. pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
  2099. msg,
  2100. &inet->inet_daddr, ntohs(inet->inet_dport),
  2101. tp->snd_cwnd, tcp_left_out(tp),
  2102. tp->snd_ssthresh, tp->prior_ssthresh,
  2103. tp->packets_out);
  2104. }
  2105. #if IS_ENABLED(CONFIG_IPV6)
  2106. else if (sk->sk_family == AF_INET6) {
  2107. pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
  2108. msg,
  2109. &sk->sk_v6_daddr, ntohs(inet->inet_dport),
  2110. tp->snd_cwnd, tcp_left_out(tp),
  2111. tp->snd_ssthresh, tp->prior_ssthresh,
  2112. tp->packets_out);
  2113. }
  2114. #endif
  2115. #endif
  2116. }
  2117. static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss)
  2118. {
  2119. struct tcp_sock *tp = tcp_sk(sk);
  2120. if (unmark_loss) {
  2121. struct sk_buff *skb;
  2122. skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
  2123. TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
  2124. }
  2125. tp->lost_out = 0;
  2126. tcp_clear_all_retrans_hints(tp);
  2127. }
  2128. if (tp->prior_ssthresh) {
  2129. const struct inet_connection_sock *icsk = inet_csk(sk);
  2130. tp->snd_cwnd = icsk->icsk_ca_ops->undo_cwnd(sk);
  2131. if (tp->prior_ssthresh > tp->snd_ssthresh) {
  2132. tp->snd_ssthresh = tp->prior_ssthresh;
  2133. tcp_ecn_withdraw_cwr(tp);
  2134. }
  2135. }
  2136. tp->snd_cwnd_stamp = tcp_jiffies32;
  2137. tp->undo_marker = 0;
  2138. tp->rack.advanced = 1; /* Force RACK to re-exam losses */
  2139. }
  2140. static inline bool tcp_may_undo(const struct tcp_sock *tp)
  2141. {
  2142. return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp));
  2143. }
  2144. /* People celebrate: "We love our President!" */
  2145. static bool tcp_try_undo_recovery(struct sock *sk)
  2146. {
  2147. struct tcp_sock *tp = tcp_sk(sk);
  2148. if (tcp_may_undo(tp)) {
  2149. int mib_idx;
  2150. /* Happy end! We did not retransmit anything
  2151. * or our original transmission succeeded.
  2152. */
  2153. DBGUNDO(sk, inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans");
  2154. tcp_undo_cwnd_reduction(sk, false);
  2155. if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
  2156. mib_idx = LINUX_MIB_TCPLOSSUNDO;
  2157. else
  2158. mib_idx = LINUX_MIB_TCPFULLUNDO;
  2159. NET_INC_STATS(sock_net(sk), mib_idx);
  2160. } else if (tp->rack.reo_wnd_persist) {
  2161. tp->rack.reo_wnd_persist--;
  2162. }
  2163. if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) {
  2164. /* Hold old state until something *above* high_seq
  2165. * is ACKed. For Reno it is MUST to prevent false
  2166. * fast retransmits (RFC2582). SACK TCP is safe. */
  2167. if (!tcp_any_retrans_done(sk))
  2168. tp->retrans_stamp = 0;
  2169. return true;
  2170. }
  2171. tcp_set_ca_state(sk, TCP_CA_Open);
  2172. tp->is_sack_reneg = 0;
  2173. return false;
  2174. }
  2175. /* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
  2176. static bool tcp_try_undo_dsack(struct sock *sk)
  2177. {
  2178. struct tcp_sock *tp = tcp_sk(sk);
  2179. if (tp->undo_marker && !tp->undo_retrans) {
  2180. tp->rack.reo_wnd_persist = min(TCP_RACK_RECOVERY_THRESH,
  2181. tp->rack.reo_wnd_persist + 1);
  2182. DBGUNDO(sk, "D-SACK");
  2183. tcp_undo_cwnd_reduction(sk, false);
  2184. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKUNDO);
  2185. return true;
  2186. }
  2187. return false;
  2188. }
  2189. /* Undo during loss recovery after partial ACK or using F-RTO. */
  2190. static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo)
  2191. {
  2192. struct tcp_sock *tp = tcp_sk(sk);
  2193. if (frto_undo || tcp_may_undo(tp)) {
  2194. tcp_undo_cwnd_reduction(sk, true);
  2195. DBGUNDO(sk, "partial loss");
  2196. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSUNDO);
  2197. if (frto_undo)
  2198. NET_INC_STATS(sock_net(sk),
  2199. LINUX_MIB_TCPSPURIOUSRTOS);
  2200. inet_csk(sk)->icsk_retransmits = 0;
  2201. if (frto_undo || tcp_is_sack(tp)) {
  2202. tcp_set_ca_state(sk, TCP_CA_Open);
  2203. tp->is_sack_reneg = 0;
  2204. }
  2205. return true;
  2206. }
  2207. return false;
  2208. }
  2209. /* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937.
  2210. * It computes the number of packets to send (sndcnt) based on packets newly
  2211. * delivered:
  2212. * 1) If the packets in flight is larger than ssthresh, PRR spreads the
  2213. * cwnd reductions across a full RTT.
  2214. * 2) Otherwise PRR uses packet conservation to send as much as delivered.
  2215. * But when the retransmits are acked without further losses, PRR
  2216. * slow starts cwnd up to ssthresh to speed up the recovery.
  2217. */
  2218. static void tcp_init_cwnd_reduction(struct sock *sk)
  2219. {
  2220. struct tcp_sock *tp = tcp_sk(sk);
  2221. tp->high_seq = tp->snd_nxt;
  2222. tp->tlp_high_seq = 0;
  2223. tp->snd_cwnd_cnt = 0;
  2224. tp->prior_cwnd = tp->snd_cwnd;
  2225. tp->prr_delivered = 0;
  2226. tp->prr_out = 0;
  2227. tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk);
  2228. tcp_ecn_queue_cwr(tp);
  2229. }
  2230. void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag)
  2231. {
  2232. struct tcp_sock *tp = tcp_sk(sk);
  2233. int sndcnt = 0;
  2234. int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp);
  2235. if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd))
  2236. return;
  2237. tp->prr_delivered += newly_acked_sacked;
  2238. if (delta < 0) {
  2239. u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered +
  2240. tp->prior_cwnd - 1;
  2241. sndcnt = div_u64(dividend, tp->prior_cwnd) - tp->prr_out;
  2242. } else if ((flag & (FLAG_RETRANS_DATA_ACKED | FLAG_LOST_RETRANS)) ==
  2243. FLAG_RETRANS_DATA_ACKED) {
  2244. sndcnt = min_t(int, delta,
  2245. max_t(int, tp->prr_delivered - tp->prr_out,
  2246. newly_acked_sacked) + 1);
  2247. } else {
  2248. sndcnt = min(delta, newly_acked_sacked);
  2249. }
  2250. /* Force a fast retransmit upon entering fast recovery */
  2251. sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
  2252. tp->snd_cwnd = tcp_packets_in_flight(tp) + sndcnt;
  2253. }
  2254. static inline void tcp_end_cwnd_reduction(struct sock *sk)
  2255. {
  2256. struct tcp_sock *tp = tcp_sk(sk);
  2257. if (inet_csk(sk)->icsk_ca_ops->cong_control)
  2258. return;
  2259. /* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
  2260. if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
  2261. (inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || tp->undo_marker)) {
  2262. tp->snd_cwnd = tp->snd_ssthresh;
  2263. tp->snd_cwnd_stamp = tcp_jiffies32;
  2264. }
  2265. tcp_ca_event(sk, CA_EVENT_COMPLETE_CWR);
  2266. }
  2267. /* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
  2268. void tcp_enter_cwr(struct sock *sk)
  2269. {
  2270. struct tcp_sock *tp = tcp_sk(sk);
  2271. tp->prior_ssthresh = 0;
  2272. if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
  2273. tp->undo_marker = 0;
  2274. tcp_init_cwnd_reduction(sk);
  2275. tcp_set_ca_state(sk, TCP_CA_CWR);
  2276. }
  2277. }
  2278. EXPORT_SYMBOL(tcp_enter_cwr);
  2279. static void tcp_try_keep_open(struct sock *sk)
  2280. {
  2281. struct tcp_sock *tp = tcp_sk(sk);
  2282. int state = TCP_CA_Open;
  2283. if (tcp_left_out(tp) || tcp_any_retrans_done(sk))
  2284. state = TCP_CA_Disorder;
  2285. if (inet_csk(sk)->icsk_ca_state != state) {
  2286. tcp_set_ca_state(sk, state);
  2287. tp->high_seq = tp->snd_nxt;
  2288. }
  2289. }
  2290. static void tcp_try_to_open(struct sock *sk, int flag)
  2291. {
  2292. struct tcp_sock *tp = tcp_sk(sk);
  2293. tcp_verify_left_out(tp);
  2294. if (!tcp_any_retrans_done(sk))
  2295. tp->retrans_stamp = 0;
  2296. if (flag & FLAG_ECE)
  2297. tcp_enter_cwr(sk);
  2298. if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) {
  2299. tcp_try_keep_open(sk);
  2300. }
  2301. }
  2302. static void tcp_mtup_probe_failed(struct sock *sk)
  2303. {
  2304. struct inet_connection_sock *icsk = inet_csk(sk);
  2305. icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1;
  2306. icsk->icsk_mtup.probe_size = 0;
  2307. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPFAIL);
  2308. }
  2309. static void tcp_mtup_probe_success(struct sock *sk)
  2310. {
  2311. struct tcp_sock *tp = tcp_sk(sk);
  2312. struct inet_connection_sock *icsk = inet_csk(sk);
  2313. /* FIXME: breaks with very large cwnd */
  2314. tp->prior_ssthresh = tcp_current_ssthresh(sk);
  2315. tp->snd_cwnd = tp->snd_cwnd *
  2316. tcp_mss_to_mtu(sk, tp->mss_cache) /
  2317. icsk->icsk_mtup.probe_size;
  2318. tp->snd_cwnd_cnt = 0;
  2319. tp->snd_cwnd_stamp = tcp_jiffies32;
  2320. tp->snd_ssthresh = tcp_current_ssthresh(sk);
  2321. icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size;
  2322. icsk->icsk_mtup.probe_size = 0;
  2323. tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
  2324. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPSUCCESS);
  2325. }
  2326. /* Do a simple retransmit without using the backoff mechanisms in
  2327. * tcp_timer. This is used for path mtu discovery.
  2328. * The socket is already locked here.
  2329. */
  2330. void tcp_simple_retransmit(struct sock *sk)
  2331. {
  2332. const struct inet_connection_sock *icsk = inet_csk(sk);
  2333. struct tcp_sock *tp = tcp_sk(sk);
  2334. struct sk_buff *skb;
  2335. unsigned int mss = tcp_current_mss(sk);
  2336. skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
  2337. if (tcp_skb_seglen(skb) > mss)
  2338. tcp_mark_skb_lost(sk, skb);
  2339. }
  2340. tcp_clear_retrans_hints_partial(tp);
  2341. if (!tp->lost_out)
  2342. return;
  2343. if (tcp_is_reno(tp))
  2344. tcp_limit_reno_sacked(tp);
  2345. tcp_verify_left_out(tp);
  2346. /* Don't muck with the congestion window here.
  2347. * Reason is that we do not increase amount of _data_
  2348. * in network, but units changed and effective
  2349. * cwnd/ssthresh really reduced now.
  2350. */
  2351. if (icsk->icsk_ca_state != TCP_CA_Loss) {
  2352. tp->high_seq = tp->snd_nxt;
  2353. tp->snd_ssthresh = tcp_current_ssthresh(sk);
  2354. tp->prior_ssthresh = 0;
  2355. tp->undo_marker = 0;
  2356. tcp_set_ca_state(sk, TCP_CA_Loss);
  2357. }
  2358. tcp_xmit_retransmit_queue(sk);
  2359. }
  2360. EXPORT_SYMBOL(tcp_simple_retransmit);
  2361. void tcp_enter_recovery(struct sock *sk, bool ece_ack)
  2362. {
  2363. struct tcp_sock *tp = tcp_sk(sk);
  2364. int mib_idx;
  2365. if (tcp_is_reno(tp))
  2366. mib_idx = LINUX_MIB_TCPRENORECOVERY;
  2367. else
  2368. mib_idx = LINUX_MIB_TCPSACKRECOVERY;
  2369. NET_INC_STATS(sock_net(sk), mib_idx);
  2370. tp->prior_ssthresh = 0;
  2371. tcp_init_undo(tp);
  2372. if (!tcp_in_cwnd_reduction(sk)) {
  2373. if (!ece_ack)
  2374. tp->prior_ssthresh = tcp_current_ssthresh(sk);
  2375. tcp_init_cwnd_reduction(sk);
  2376. }
  2377. tcp_set_ca_state(sk, TCP_CA_Recovery);
  2378. }
  2379. /* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
  2380. * recovered or spurious. Otherwise retransmits more on partial ACKs.
  2381. */
  2382. static void tcp_process_loss(struct sock *sk, int flag, int num_dupack,
  2383. int *rexmit)
  2384. {
  2385. struct tcp_sock *tp = tcp_sk(sk);
  2386. bool recovered = !before(tp->snd_una, tp->high_seq);
  2387. if ((flag & FLAG_SND_UNA_ADVANCED || rcu_access_pointer(tp->fastopen_rsk)) &&
  2388. tcp_try_undo_loss(sk, false))
  2389. return;
  2390. if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
  2391. /* Step 3.b. A timeout is spurious if not all data are
  2392. * lost, i.e., never-retransmitted data are (s)acked.
  2393. */
  2394. if ((flag & FLAG_ORIG_SACK_ACKED) &&
  2395. tcp_try_undo_loss(sk, true))
  2396. return;
  2397. if (after(tp->snd_nxt, tp->high_seq)) {
  2398. if (flag & FLAG_DATA_SACKED || num_dupack)
  2399. tp->frto = 0; /* Step 3.a. loss was real */
  2400. } else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) {
  2401. tp->high_seq = tp->snd_nxt;
  2402. /* Step 2.b. Try send new data (but deferred until cwnd
  2403. * is updated in tcp_ack()). Otherwise fall back to
  2404. * the conventional recovery.
  2405. */
  2406. if (!tcp_write_queue_empty(sk) &&
  2407. after(tcp_wnd_end(tp), tp->snd_nxt)) {
  2408. *rexmit = REXMIT_NEW;
  2409. return;
  2410. }
  2411. tp->frto = 0;
  2412. }
  2413. }
  2414. if (recovered) {
  2415. /* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
  2416. tcp_try_undo_recovery(sk);
  2417. return;
  2418. }
  2419. if (tcp_is_reno(tp)) {
  2420. /* A Reno DUPACK means new data in F-RTO step 2.b above are
  2421. * delivered. Lower inflight to clock out (re)tranmissions.
  2422. */
  2423. if (after(tp->snd_nxt, tp->high_seq) && num_dupack)
  2424. tcp_add_reno_sack(sk, num_dupack, flag & FLAG_ECE);
  2425. else if (flag & FLAG_SND_UNA_ADVANCED)
  2426. tcp_reset_reno_sack(tp);
  2427. }
  2428. *rexmit = REXMIT_LOST;
  2429. }
  2430. static bool tcp_force_fast_retransmit(struct sock *sk)
  2431. {
  2432. struct tcp_sock *tp = tcp_sk(sk);
  2433. return after(tcp_highest_sack_seq(tp),
  2434. tp->snd_una + tp->reordering * tp->mss_cache);
  2435. }
  2436. /* Undo during fast recovery after partial ACK. */
  2437. static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una,
  2438. bool *do_lost)
  2439. {
  2440. struct tcp_sock *tp = tcp_sk(sk);
  2441. if (tp->undo_marker && tcp_packet_delayed(tp)) {
  2442. /* Plain luck! Hole if filled with delayed
  2443. * packet, rather than with a retransmit. Check reordering.
  2444. */
  2445. tcp_check_sack_reordering(sk, prior_snd_una, 1);
  2446. /* We are getting evidence that the reordering degree is higher
  2447. * than we realized. If there are no retransmits out then we
  2448. * can undo. Otherwise we clock out new packets but do not
  2449. * mark more packets lost or retransmit more.
  2450. */
  2451. if (tp->retrans_out)
  2452. return true;
  2453. if (!tcp_any_retrans_done(sk))
  2454. tp->retrans_stamp = 0;
  2455. DBGUNDO(sk, "partial recovery");
  2456. tcp_undo_cwnd_reduction(sk, true);
  2457. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO);
  2458. tcp_try_keep_open(sk);
  2459. } else {
  2460. /* Partial ACK arrived. Force fast retransmit. */
  2461. *do_lost = tcp_force_fast_retransmit(sk);
  2462. }
  2463. return false;
  2464. }
  2465. static void tcp_identify_packet_loss(struct sock *sk, int *ack_flag)
  2466. {
  2467. struct tcp_sock *tp = tcp_sk(sk);
  2468. if (tcp_rtx_queue_empty(sk))
  2469. return;
  2470. if (unlikely(tcp_is_reno(tp))) {
  2471. tcp_newreno_mark_lost(sk, *ack_flag & FLAG_SND_UNA_ADVANCED);
  2472. } else if (tcp_is_rack(sk)) {
  2473. u32 prior_retrans = tp->retrans_out;
  2474. if (tcp_rack_mark_lost(sk))
  2475. *ack_flag &= ~FLAG_SET_XMIT_TIMER;
  2476. if (prior_retrans > tp->retrans_out)
  2477. *ack_flag |= FLAG_LOST_RETRANS;
  2478. }
  2479. }
  2480. /* Process an event, which can update packets-in-flight not trivially.
  2481. * Main goal of this function is to calculate new estimate for left_out,
  2482. * taking into account both packets sitting in receiver's buffer and
  2483. * packets lost by network.
  2484. *
  2485. * Besides that it updates the congestion state when packet loss or ECN
  2486. * is detected. But it does not reduce the cwnd, it is done by the
  2487. * congestion control later.
  2488. *
  2489. * It does _not_ decide what to send, it is made in function
  2490. * tcp_xmit_retransmit_queue().
  2491. */
  2492. static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
  2493. int num_dupack, int *ack_flag, int *rexmit)
  2494. {
  2495. struct inet_connection_sock *icsk = inet_csk(sk);
  2496. struct tcp_sock *tp = tcp_sk(sk);
  2497. int fast_rexmit = 0, flag = *ack_flag;
  2498. bool ece_ack = flag & FLAG_ECE;
  2499. bool do_lost = num_dupack || ((flag & FLAG_DATA_SACKED) &&
  2500. tcp_force_fast_retransmit(sk));
  2501. if (!tp->packets_out && tp->sacked_out)
  2502. tp->sacked_out = 0;
  2503. /* Now state machine starts.
  2504. * A. ECE, hence prohibit cwnd undoing, the reduction is required. */
  2505. if (ece_ack)
  2506. tp->prior_ssthresh = 0;
  2507. /* B. In all the states check for reneging SACKs. */
  2508. if (tcp_check_sack_reneging(sk, flag))
  2509. return;
  2510. /* C. Check consistency of the current state. */
  2511. tcp_verify_left_out(tp);
  2512. /* D. Check state exit conditions. State can be terminated
  2513. * when high_seq is ACKed. */
  2514. if (icsk->icsk_ca_state == TCP_CA_Open) {
  2515. WARN_ON(tp->retrans_out != 0);
  2516. tp->retrans_stamp = 0;
  2517. } else if (!before(tp->snd_una, tp->high_seq)) {
  2518. switch (icsk->icsk_ca_state) {
  2519. case TCP_CA_CWR:
  2520. /* CWR is to be held something *above* high_seq
  2521. * is ACKed for CWR bit to reach receiver. */
  2522. if (tp->snd_una != tp->high_seq) {
  2523. tcp_end_cwnd_reduction(sk);
  2524. tcp_set_ca_state(sk, TCP_CA_Open);
  2525. }
  2526. break;
  2527. case TCP_CA_Recovery:
  2528. if (tcp_is_reno(tp))
  2529. tcp_reset_reno_sack(tp);
  2530. if (tcp_try_undo_recovery(sk))
  2531. return;
  2532. tcp_end_cwnd_reduction(sk);
  2533. break;
  2534. }
  2535. }
  2536. /* E. Process state. */
  2537. switch (icsk->icsk_ca_state) {
  2538. case TCP_CA_Recovery:
  2539. if (!(flag & FLAG_SND_UNA_ADVANCED)) {
  2540. if (tcp_is_reno(tp))
  2541. tcp_add_reno_sack(sk, num_dupack, ece_ack);
  2542. } else if (tcp_try_undo_partial(sk, prior_snd_una, &do_lost))
  2543. return;
  2544. if (tcp_try_undo_dsack(sk))
  2545. tcp_try_keep_open(sk);
  2546. tcp_identify_packet_loss(sk, ack_flag);
  2547. if (icsk->icsk_ca_state != TCP_CA_Recovery) {
  2548. if (!tcp_time_to_recover(sk, flag))
  2549. return;
  2550. /* Undo reverts the recovery state. If loss is evident,
  2551. * starts a new recovery (e.g. reordering then loss);
  2552. */
  2553. tcp_enter_recovery(sk, ece_ack);
  2554. }
  2555. break;
  2556. case TCP_CA_Loss:
  2557. tcp_process_loss(sk, flag, num_dupack, rexmit);
  2558. tcp_identify_packet_loss(sk, ack_flag);
  2559. if (!(icsk->icsk_ca_state == TCP_CA_Open ||
  2560. (*ack_flag & FLAG_LOST_RETRANS)))
  2561. return;
  2562. /* Change state if cwnd is undone or retransmits are lost */
  2563. fallthrough;
  2564. default:
  2565. if (tcp_is_reno(tp)) {
  2566. if (flag & FLAG_SND_UNA_ADVANCED)
  2567. tcp_reset_reno_sack(tp);
  2568. tcp_add_reno_sack(sk, num_dupack, ece_ack);
  2569. }
  2570. if (icsk->icsk_ca_state <= TCP_CA_Disorder)
  2571. tcp_try_undo_dsack(sk);
  2572. tcp_identify_packet_loss(sk, ack_flag);
  2573. if (!tcp_time_to_recover(sk, flag)) {
  2574. tcp_try_to_open(sk, flag);
  2575. return;
  2576. }
  2577. /* MTU probe failure: don't reduce cwnd */
  2578. if (icsk->icsk_ca_state < TCP_CA_CWR &&
  2579. icsk->icsk_mtup.probe_size &&
  2580. tp->snd_una == tp->mtu_probe.probe_seq_start) {
  2581. tcp_mtup_probe_failed(sk);
  2582. /* Restores the reduction we did in tcp_mtup_probe() */
  2583. tp->snd_cwnd++;
  2584. tcp_simple_retransmit(sk);
  2585. return;
  2586. }
  2587. /* Otherwise enter Recovery state */
  2588. tcp_enter_recovery(sk, ece_ack);
  2589. fast_rexmit = 1;
  2590. }
  2591. if (!tcp_is_rack(sk) && do_lost)
  2592. tcp_update_scoreboard(sk, fast_rexmit);
  2593. *rexmit = REXMIT_LOST;
  2594. }
  2595. static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag)
  2596. {
  2597. u32 wlen = sock_net(sk)->ipv4.sysctl_tcp_min_rtt_wlen * HZ;
  2598. struct tcp_sock *tp = tcp_sk(sk);
  2599. if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) {
  2600. /* If the remote keeps returning delayed ACKs, eventually
  2601. * the min filter would pick it up and overestimate the
  2602. * prop. delay when it expires. Skip suspected delayed ACKs.
  2603. */
  2604. return;
  2605. }
  2606. minmax_running_min(&tp->rtt_min, wlen, tcp_jiffies32,
  2607. rtt_us ? : jiffies_to_usecs(1));
  2608. }
  2609. static bool tcp_ack_update_rtt(struct sock *sk, const int flag,
  2610. long seq_rtt_us, long sack_rtt_us,
  2611. long ca_rtt_us, struct rate_sample *rs)
  2612. {
  2613. const struct tcp_sock *tp = tcp_sk(sk);
  2614. /* Prefer RTT measured from ACK's timing to TS-ECR. This is because
  2615. * broken middle-boxes or peers may corrupt TS-ECR fields. But
  2616. * Karn's algorithm forbids taking RTT if some retransmitted data
  2617. * is acked (RFC6298).
  2618. */
  2619. if (seq_rtt_us < 0)
  2620. seq_rtt_us = sack_rtt_us;
  2621. /* RTTM Rule: A TSecr value received in a segment is used to
  2622. * update the averaged RTT measurement only if the segment
  2623. * acknowledges some new data, i.e., only if it advances the
  2624. * left edge of the send window.
  2625. * See draft-ietf-tcplw-high-performance-00, section 3.3.
  2626. */
  2627. if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
  2628. flag & FLAG_ACKED) {
  2629. u32 delta = tcp_time_stamp(tp) - tp->rx_opt.rcv_tsecr;
  2630. if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
  2631. if (!delta)
  2632. delta = 1;
  2633. seq_rtt_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
  2634. ca_rtt_us = seq_rtt_us;
  2635. }
  2636. }
  2637. rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */
  2638. if (seq_rtt_us < 0)
  2639. return false;
  2640. /* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is
  2641. * always taken together with ACK, SACK, or TS-opts. Any negative
  2642. * values will be skipped with the seq_rtt_us < 0 check above.
  2643. */
  2644. tcp_update_rtt_min(sk, ca_rtt_us, flag);
  2645. tcp_rtt_estimator(sk, seq_rtt_us);
  2646. tcp_set_rto(sk);
  2647. /* RFC6298: only reset backoff on valid RTT measurement. */
  2648. inet_csk(sk)->icsk_backoff = 0;
  2649. return true;
  2650. }
  2651. /* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
  2652. void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req)
  2653. {
  2654. struct rate_sample rs;
  2655. long rtt_us = -1L;
  2656. if (req && !req->num_retrans && tcp_rsk(req)->snt_synack)
  2657. rtt_us = tcp_stamp_us_delta(tcp_clock_us(), tcp_rsk(req)->snt_synack);
  2658. tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, rtt_us, -1L, rtt_us, &rs);
  2659. }
  2660. static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
  2661. {
  2662. const struct inet_connection_sock *icsk = inet_csk(sk);
  2663. icsk->icsk_ca_ops->cong_avoid(sk, ack, acked);
  2664. tcp_sk(sk)->snd_cwnd_stamp = tcp_jiffies32;
  2665. }
  2666. /* Restart timer after forward progress on connection.
  2667. * RFC2988 recommends to restart timer to now+rto.
  2668. */
  2669. void tcp_rearm_rto(struct sock *sk)
  2670. {
  2671. const struct inet_connection_sock *icsk = inet_csk(sk);
  2672. struct tcp_sock *tp = tcp_sk(sk);
  2673. /* If the retrans timer is currently being used by Fast Open
  2674. * for SYN-ACK retrans purpose, stay put.
  2675. */
  2676. if (rcu_access_pointer(tp->fastopen_rsk))
  2677. return;
  2678. if (!tp->packets_out) {
  2679. inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
  2680. } else {
  2681. u32 rto = inet_csk(sk)->icsk_rto;
  2682. /* Offset the time elapsed after installing regular RTO */
  2683. if (icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
  2684. icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
  2685. s64 delta_us = tcp_rto_delta_us(sk);
  2686. /* delta_us may not be positive if the socket is locked
  2687. * when the retrans timer fires and is rescheduled.
  2688. */
  2689. rto = usecs_to_jiffies(max_t(int, delta_us, 1));
  2690. }
  2691. tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, rto,
  2692. TCP_RTO_MAX);
  2693. }
  2694. }
  2695. /* Try to schedule a loss probe; if that doesn't work, then schedule an RTO. */
  2696. static void tcp_set_xmit_timer(struct sock *sk)
  2697. {
  2698. if (!tcp_schedule_loss_probe(sk, true))
  2699. tcp_rearm_rto(sk);
  2700. }
  2701. /* If we get here, the whole TSO packet has not been acked. */
  2702. static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb)
  2703. {
  2704. struct tcp_sock *tp = tcp_sk(sk);
  2705. u32 packets_acked;
  2706. BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una));
  2707. packets_acked = tcp_skb_pcount(skb);
  2708. if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
  2709. return 0;
  2710. packets_acked -= tcp_skb_pcount(skb);
  2711. if (packets_acked) {
  2712. BUG_ON(tcp_skb_pcount(skb) == 0);
  2713. BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq));
  2714. }
  2715. return packets_acked;
  2716. }
  2717. static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb,
  2718. u32 prior_snd_una)
  2719. {
  2720. const struct skb_shared_info *shinfo;
  2721. /* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
  2722. if (likely(!TCP_SKB_CB(skb)->txstamp_ack))
  2723. return;
  2724. shinfo = skb_shinfo(skb);
  2725. if (!before(shinfo->tskey, prior_snd_una) &&
  2726. before(shinfo->tskey, tcp_sk(sk)->snd_una)) {
  2727. tcp_skb_tsorted_save(skb) {
  2728. __skb_tstamp_tx(skb, NULL, sk, SCM_TSTAMP_ACK);
  2729. } tcp_skb_tsorted_restore(skb);
  2730. }
  2731. }
  2732. /* Remove acknowledged frames from the retransmission queue. If our packet
  2733. * is before the ack sequence we can discard it as it's confirmed to have
  2734. * arrived at the other end.
  2735. */
  2736. static int tcp_clean_rtx_queue(struct sock *sk, u32 prior_fack,
  2737. u32 prior_snd_una,
  2738. struct tcp_sacktag_state *sack, bool ece_ack)
  2739. {
  2740. const struct inet_connection_sock *icsk = inet_csk(sk);
  2741. u64 first_ackt, last_ackt;
  2742. struct tcp_sock *tp = tcp_sk(sk);
  2743. u32 prior_sacked = tp->sacked_out;
  2744. u32 reord = tp->snd_nxt; /* lowest acked un-retx un-sacked seq */
  2745. struct sk_buff *skb, *next;
  2746. bool fully_acked = true;
  2747. long sack_rtt_us = -1L;
  2748. long seq_rtt_us = -1L;
  2749. long ca_rtt_us = -1L;
  2750. u32 pkts_acked = 0;
  2751. u32 last_in_flight = 0;
  2752. bool rtt_update;
  2753. int flag = 0;
  2754. first_ackt = 0;
  2755. for (skb = skb_rb_first(&sk->tcp_rtx_queue); skb; skb = next) {
  2756. struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
  2757. const u32 start_seq = scb->seq;
  2758. u8 sacked = scb->sacked;
  2759. u32 acked_pcount;
  2760. /* Determine how many packets and what bytes were acked, tso and else */
  2761. if (after(scb->end_seq, tp->snd_una)) {
  2762. if (tcp_skb_pcount(skb) == 1 ||
  2763. !after(tp->snd_una, scb->seq))
  2764. break;
  2765. acked_pcount = tcp_tso_acked(sk, skb);
  2766. if (!acked_pcount)
  2767. break;
  2768. fully_acked = false;
  2769. } else {
  2770. acked_pcount = tcp_skb_pcount(skb);
  2771. }
  2772. if (unlikely(sacked & TCPCB_RETRANS)) {
  2773. if (sacked & TCPCB_SACKED_RETRANS)
  2774. tp->retrans_out -= acked_pcount;
  2775. flag |= FLAG_RETRANS_DATA_ACKED;
  2776. } else if (!(sacked & TCPCB_SACKED_ACKED)) {
  2777. last_ackt = tcp_skb_timestamp_us(skb);
  2778. WARN_ON_ONCE(last_ackt == 0);
  2779. if (!first_ackt)
  2780. first_ackt = last_ackt;
  2781. last_in_flight = TCP_SKB_CB(skb)->tx.in_flight;
  2782. if (before(start_seq, reord))
  2783. reord = start_seq;
  2784. if (!after(scb->end_seq, tp->high_seq))
  2785. flag |= FLAG_ORIG_SACK_ACKED;
  2786. }
  2787. if (sacked & TCPCB_SACKED_ACKED) {
  2788. tp->sacked_out -= acked_pcount;
  2789. } else if (tcp_is_sack(tp)) {
  2790. tcp_count_delivered(tp, acked_pcount, ece_ack);
  2791. if (!tcp_skb_spurious_retrans(tp, skb))
  2792. tcp_rack_advance(tp, sacked, scb->end_seq,
  2793. tcp_skb_timestamp_us(skb));
  2794. }
  2795. if (sacked & TCPCB_LOST)
  2796. tp->lost_out -= acked_pcount;
  2797. tp->packets_out -= acked_pcount;
  2798. pkts_acked += acked_pcount;
  2799. tcp_rate_skb_delivered(sk, skb, sack->rate);
  2800. /* Initial outgoing SYN's get put onto the write_queue
  2801. * just like anything else we transmit. It is not
  2802. * true data, and if we misinform our callers that
  2803. * this ACK acks real data, we will erroneously exit
  2804. * connection startup slow start one packet too
  2805. * quickly. This is severely frowned upon behavior.
  2806. */
  2807. if (likely(!(scb->tcp_flags & TCPHDR_SYN))) {
  2808. flag |= FLAG_DATA_ACKED;
  2809. } else {
  2810. flag |= FLAG_SYN_ACKED;
  2811. tp->retrans_stamp = 0;
  2812. }
  2813. if (!fully_acked)
  2814. break;
  2815. tcp_ack_tstamp(sk, skb, prior_snd_una);
  2816. next = skb_rb_next(skb);
  2817. if (unlikely(skb == tp->retransmit_skb_hint))
  2818. tp->retransmit_skb_hint = NULL;
  2819. if (unlikely(skb == tp->lost_skb_hint))
  2820. tp->lost_skb_hint = NULL;
  2821. tcp_highest_sack_replace(sk, skb, next);
  2822. tcp_rtx_queue_unlink_and_free(skb, sk);
  2823. }
  2824. if (!skb)
  2825. tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
  2826. if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una)))
  2827. tp->snd_up = tp->snd_una;
  2828. if (skb) {
  2829. tcp_ack_tstamp(sk, skb, prior_snd_una);
  2830. if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
  2831. flag |= FLAG_SACK_RENEGING;
  2832. }
  2833. if (likely(first_ackt) && !(flag & FLAG_RETRANS_DATA_ACKED)) {
  2834. seq_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, first_ackt);
  2835. ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, last_ackt);
  2836. if (pkts_acked == 1 && last_in_flight < tp->mss_cache &&
  2837. last_in_flight && !prior_sacked && fully_acked &&
  2838. sack->rate->prior_delivered + 1 == tp->delivered &&
  2839. !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) {
  2840. /* Conservatively mark a delayed ACK. It's typically
  2841. * from a lone runt packet over the round trip to
  2842. * a receiver w/o out-of-order or CE events.
  2843. */
  2844. flag |= FLAG_ACK_MAYBE_DELAYED;
  2845. }
  2846. }
  2847. if (sack->first_sackt) {
  2848. sack_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->first_sackt);
  2849. ca_rtt_us = tcp_stamp_us_delta(tp->tcp_mstamp, sack->last_sackt);
  2850. }
  2851. rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us,
  2852. ca_rtt_us, sack->rate);
  2853. if (flag & FLAG_ACKED) {
  2854. flag |= FLAG_SET_XMIT_TIMER; /* set TLP or RTO timer */
  2855. if (unlikely(icsk->icsk_mtup.probe_size &&
  2856. !after(tp->mtu_probe.probe_seq_end, tp->snd_una))) {
  2857. tcp_mtup_probe_success(sk);
  2858. }
  2859. if (tcp_is_reno(tp)) {
  2860. tcp_remove_reno_sacks(sk, pkts_acked, ece_ack);
  2861. /* If any of the cumulatively ACKed segments was
  2862. * retransmitted, non-SACK case cannot confirm that
  2863. * progress was due to original transmission due to
  2864. * lack of TCPCB_SACKED_ACKED bits even if some of
  2865. * the packets may have been never retransmitted.
  2866. */
  2867. if (flag & FLAG_RETRANS_DATA_ACKED)
  2868. flag &= ~FLAG_ORIG_SACK_ACKED;
  2869. } else {
  2870. int delta;
  2871. /* Non-retransmitted hole got filled? That's reordering */
  2872. if (before(reord, prior_fack))
  2873. tcp_check_sack_reordering(sk, reord, 0);
  2874. delta = prior_sacked - tp->sacked_out;
  2875. tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta);
  2876. }
  2877. } else if (skb && rtt_update && sack_rtt_us >= 0 &&
  2878. sack_rtt_us > tcp_stamp_us_delta(tp->tcp_mstamp,
  2879. tcp_skb_timestamp_us(skb))) {
  2880. /* Do not re-arm RTO if the sack RTT is measured from data sent
  2881. * after when the head was last (re)transmitted. Otherwise the
  2882. * timeout may continue to extend in loss recovery.
  2883. */
  2884. flag |= FLAG_SET_XMIT_TIMER; /* set TLP or RTO timer */
  2885. }
  2886. if (icsk->icsk_ca_ops->pkts_acked) {
  2887. struct ack_sample sample = { .pkts_acked = pkts_acked,
  2888. .rtt_us = sack->rate->rtt_us,
  2889. .in_flight = last_in_flight };
  2890. icsk->icsk_ca_ops->pkts_acked(sk, &sample);
  2891. }
  2892. #if FASTRETRANS_DEBUG > 0
  2893. WARN_ON((int)tp->sacked_out < 0);
  2894. WARN_ON((int)tp->lost_out < 0);
  2895. WARN_ON((int)tp->retrans_out < 0);
  2896. if (!tp->packets_out && tcp_is_sack(tp)) {
  2897. icsk = inet_csk(sk);
  2898. if (tp->lost_out) {
  2899. pr_debug("Leak l=%u %d\n",
  2900. tp->lost_out, icsk->icsk_ca_state);
  2901. tp->lost_out = 0;
  2902. }
  2903. if (tp->sacked_out) {
  2904. pr_debug("Leak s=%u %d\n",
  2905. tp->sacked_out, icsk->icsk_ca_state);
  2906. tp->sacked_out = 0;
  2907. }
  2908. if (tp->retrans_out) {
  2909. pr_debug("Leak r=%u %d\n",
  2910. tp->retrans_out, icsk->icsk_ca_state);
  2911. tp->retrans_out = 0;
  2912. }
  2913. }
  2914. #endif
  2915. return flag;
  2916. }
  2917. static void tcp_ack_probe(struct sock *sk)
  2918. {
  2919. struct inet_connection_sock *icsk = inet_csk(sk);
  2920. struct sk_buff *head = tcp_send_head(sk);
  2921. const struct tcp_sock *tp = tcp_sk(sk);
  2922. /* Was it a usable window open? */
  2923. if (!head)
  2924. return;
  2925. if (!after(TCP_SKB_CB(head)->end_seq, tcp_wnd_end(tp))) {
  2926. icsk->icsk_backoff = 0;
  2927. icsk->icsk_probes_tstamp = 0;
  2928. inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
  2929. /* Socket must be waked up by subsequent tcp_data_snd_check().
  2930. * This function is not for random using!
  2931. */
  2932. } else {
  2933. unsigned long when = tcp_probe0_when(sk, TCP_RTO_MAX);
  2934. when = tcp_clamp_probe0_to_user_timeout(sk, when);
  2935. tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, when, TCP_RTO_MAX);
  2936. }
  2937. }
  2938. static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag)
  2939. {
  2940. return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) ||
  2941. inet_csk(sk)->icsk_ca_state != TCP_CA_Open;
  2942. }
  2943. /* Decide wheather to run the increase function of congestion control. */
  2944. static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
  2945. {
  2946. /* If reordering is high then always grow cwnd whenever data is
  2947. * delivered regardless of its ordering. Otherwise stay conservative
  2948. * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
  2949. * new SACK or ECE mark may first advance cwnd here and later reduce
  2950. * cwnd in tcp_fastretrans_alert() based on more states.
  2951. */
  2952. if (tcp_sk(sk)->reordering > sock_net(sk)->ipv4.sysctl_tcp_reordering)
  2953. return flag & FLAG_FORWARD_PROGRESS;
  2954. return flag & FLAG_DATA_ACKED;
  2955. }
  2956. /* The "ultimate" congestion control function that aims to replace the rigid
  2957. * cwnd increase and decrease control (tcp_cong_avoid,tcp_*cwnd_reduction).
  2958. * It's called toward the end of processing an ACK with precise rate
  2959. * information. All transmission or retransmission are delayed afterwards.
  2960. */
  2961. static void tcp_cong_control(struct sock *sk, u32 ack, u32 acked_sacked,
  2962. int flag, const struct rate_sample *rs)
  2963. {
  2964. const struct inet_connection_sock *icsk = inet_csk(sk);
  2965. if (icsk->icsk_ca_ops->cong_control) {
  2966. icsk->icsk_ca_ops->cong_control(sk, rs);
  2967. return;
  2968. }
  2969. if (tcp_in_cwnd_reduction(sk)) {
  2970. /* Reduce cwnd if state mandates */
  2971. tcp_cwnd_reduction(sk, acked_sacked, flag);
  2972. } else if (tcp_may_raise_cwnd(sk, flag)) {
  2973. /* Advance cwnd if state allows */
  2974. tcp_cong_avoid(sk, ack, acked_sacked);
  2975. }
  2976. tcp_update_pacing_rate(sk);
  2977. }
  2978. /* Check that window update is acceptable.
  2979. * The function assumes that snd_una<=ack<=snd_next.
  2980. */
  2981. static inline bool tcp_may_update_window(const struct tcp_sock *tp,
  2982. const u32 ack, const u32 ack_seq,
  2983. const u32 nwin)
  2984. {
  2985. return after(ack, tp->snd_una) ||
  2986. after(ack_seq, tp->snd_wl1) ||
  2987. (ack_seq == tp->snd_wl1 && nwin > tp->snd_wnd);
  2988. }
  2989. /* If we update tp->snd_una, also update tp->bytes_acked */
  2990. static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack)
  2991. {
  2992. u32 delta = ack - tp->snd_una;
  2993. sock_owned_by_me((struct sock *)tp);
  2994. tp->bytes_acked += delta;
  2995. tp->snd_una = ack;
  2996. }
  2997. /* If we update tp->rcv_nxt, also update tp->bytes_received */
  2998. static void tcp_rcv_nxt_update(struct tcp_sock *tp, u32 seq)
  2999. {
  3000. u32 delta = seq - tp->rcv_nxt;
  3001. sock_owned_by_me((struct sock *)tp);
  3002. tp->bytes_received += delta;
  3003. WRITE_ONCE(tp->rcv_nxt, seq);
  3004. }
  3005. /* Update our send window.
  3006. *
  3007. * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
  3008. * and in FreeBSD. NetBSD's one is even worse.) is wrong.
  3009. */
  3010. static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack,
  3011. u32 ack_seq)
  3012. {
  3013. struct tcp_sock *tp = tcp_sk(sk);
  3014. int flag = 0;
  3015. u32 nwin = ntohs(tcp_hdr(skb)->window);
  3016. if (likely(!tcp_hdr(skb)->syn))
  3017. nwin <<= tp->rx_opt.snd_wscale;
  3018. if (tcp_may_update_window(tp, ack, ack_seq, nwin)) {
  3019. flag |= FLAG_WIN_UPDATE;
  3020. tcp_update_wl(tp, ack_seq);
  3021. if (tp->snd_wnd != nwin) {
  3022. tp->snd_wnd = nwin;
  3023. /* Note, it is the only place, where
  3024. * fast path is recovered for sending TCP.
  3025. */
  3026. tp->pred_flags = 0;
  3027. tcp_fast_path_check(sk);
  3028. if (!tcp_write_queue_empty(sk))
  3029. tcp_slow_start_after_idle_check(sk);
  3030. if (nwin > tp->max_window) {
  3031. tp->max_window = nwin;
  3032. tcp_sync_mss(sk, inet_csk(sk)->icsk_pmtu_cookie);
  3033. }
  3034. }
  3035. }
  3036. tcp_snd_una_update(tp, ack);
  3037. return flag;
  3038. }
  3039. static bool __tcp_oow_rate_limited(struct net *net, int mib_idx,
  3040. u32 *last_oow_ack_time)
  3041. {
  3042. if (*last_oow_ack_time) {
  3043. s32 elapsed = (s32)(tcp_jiffies32 - *last_oow_ack_time);
  3044. if (0 <= elapsed && elapsed < net->ipv4.sysctl_tcp_invalid_ratelimit) {
  3045. NET_INC_STATS(net, mib_idx);
  3046. return true; /* rate-limited: don't send yet! */
  3047. }
  3048. }
  3049. *last_oow_ack_time = tcp_jiffies32;
  3050. return false; /* not rate-limited: go ahead, send dupack now! */
  3051. }
  3052. /* Return true if we're currently rate-limiting out-of-window ACKs and
  3053. * thus shouldn't send a dupack right now. We rate-limit dupacks in
  3054. * response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS
  3055. * attacks that send repeated SYNs or ACKs for the same connection. To
  3056. * do this, we do not send a duplicate SYNACK or ACK if the remote
  3057. * endpoint is sending out-of-window SYNs or pure ACKs at a high rate.
  3058. */
  3059. bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
  3060. int mib_idx, u32 *last_oow_ack_time)
  3061. {
  3062. /* Data packets without SYNs are not likely part of an ACK loop. */
  3063. if ((TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq) &&
  3064. !tcp_hdr(skb)->syn)
  3065. return false;
  3066. return __tcp_oow_rate_limited(net, mib_idx, last_oow_ack_time);
  3067. }
  3068. /* RFC 5961 7 [ACK Throttling] */
  3069. static void tcp_send_challenge_ack(struct sock *sk, const struct sk_buff *skb)
  3070. {
  3071. /* unprotected vars, we dont care of overwrites */
  3072. static u32 challenge_timestamp;
  3073. static unsigned int challenge_count;
  3074. struct tcp_sock *tp = tcp_sk(sk);
  3075. struct net *net = sock_net(sk);
  3076. u32 count, now;
  3077. /* First check our per-socket dupack rate limit. */
  3078. if (__tcp_oow_rate_limited(net,
  3079. LINUX_MIB_TCPACKSKIPPEDCHALLENGE,
  3080. &tp->last_oow_ack_time))
  3081. return;
  3082. /* Then check host-wide RFC 5961 rate limit. */
  3083. now = jiffies / HZ;
  3084. if (now != challenge_timestamp) {
  3085. u32 ack_limit = net->ipv4.sysctl_tcp_challenge_ack_limit;
  3086. u32 half = (ack_limit + 1) >> 1;
  3087. challenge_timestamp = now;
  3088. WRITE_ONCE(challenge_count, half + prandom_u32_max(ack_limit));
  3089. }
  3090. count = READ_ONCE(challenge_count);
  3091. if (count > 0) {
  3092. WRITE_ONCE(challenge_count, count - 1);
  3093. NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK);
  3094. tcp_send_ack(sk);
  3095. }
  3096. }
  3097. static void tcp_store_ts_recent(struct tcp_sock *tp)
  3098. {
  3099. tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval;
  3100. tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
  3101. }
  3102. static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq)
  3103. {
  3104. if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) {
  3105. /* PAWS bug workaround wrt. ACK frames, the PAWS discard
  3106. * extra check below makes sure this can only happen
  3107. * for pure ACK frames. -DaveM
  3108. *
  3109. * Not only, also it occurs for expired timestamps.
  3110. */
  3111. if (tcp_paws_check(&tp->rx_opt, 0))
  3112. tcp_store_ts_recent(tp);
  3113. }
  3114. }
  3115. /* This routine deals with acks during a TLP episode and ends an episode by
  3116. * resetting tlp_high_seq. Ref: TLP algorithm in draft-ietf-tcpm-rack
  3117. */
  3118. static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag)
  3119. {
  3120. struct tcp_sock *tp = tcp_sk(sk);
  3121. if (before(ack, tp->tlp_high_seq))
  3122. return;
  3123. if (!tp->tlp_retrans) {
  3124. /* TLP of new data has been acknowledged */
  3125. tp->tlp_high_seq = 0;
  3126. } else if (flag & FLAG_DSACKING_ACK) {
  3127. /* This DSACK means original and TLP probe arrived; no loss */
  3128. tp->tlp_high_seq = 0;
  3129. } else if (after(ack, tp->tlp_high_seq)) {
  3130. /* ACK advances: there was a loss, so reduce cwnd. Reset
  3131. * tlp_high_seq in tcp_init_cwnd_reduction()
  3132. */
  3133. tcp_init_cwnd_reduction(sk);
  3134. tcp_set_ca_state(sk, TCP_CA_CWR);
  3135. tcp_end_cwnd_reduction(sk);
  3136. tcp_try_keep_open(sk);
  3137. NET_INC_STATS(sock_net(sk),
  3138. LINUX_MIB_TCPLOSSPROBERECOVERY);
  3139. } else if (!(flag & (FLAG_SND_UNA_ADVANCED |
  3140. FLAG_NOT_DUP | FLAG_DATA_SACKED))) {
  3141. /* Pure dupack: original and TLP probe arrived; no loss */
  3142. tp->tlp_high_seq = 0;
  3143. }
  3144. }
  3145. static inline void tcp_in_ack_event(struct sock *sk, u32 flags)
  3146. {
  3147. const struct inet_connection_sock *icsk = inet_csk(sk);
  3148. if (icsk->icsk_ca_ops->in_ack_event)
  3149. icsk->icsk_ca_ops->in_ack_event(sk, flags);
  3150. }
  3151. /* Congestion control has updated the cwnd already. So if we're in
  3152. * loss recovery then now we do any new sends (for FRTO) or
  3153. * retransmits (for CA_Loss or CA_recovery) that make sense.
  3154. */
  3155. static void tcp_xmit_recovery(struct sock *sk, int rexmit)
  3156. {
  3157. struct tcp_sock *tp = tcp_sk(sk);
  3158. if (rexmit == REXMIT_NONE || sk->sk_state == TCP_SYN_SENT)
  3159. return;
  3160. if (unlikely(rexmit == REXMIT_NEW)) {
  3161. __tcp_push_pending_frames(sk, tcp_current_mss(sk),
  3162. TCP_NAGLE_OFF);
  3163. if (after(tp->snd_nxt, tp->high_seq))
  3164. return;
  3165. tp->frto = 0;
  3166. }
  3167. tcp_xmit_retransmit_queue(sk);
  3168. }
  3169. /* Returns the number of packets newly acked or sacked by the current ACK */
  3170. static u32 tcp_newly_delivered(struct sock *sk, u32 prior_delivered, int flag)
  3171. {
  3172. const struct net *net = sock_net(sk);
  3173. struct tcp_sock *tp = tcp_sk(sk);
  3174. u32 delivered;
  3175. delivered = tp->delivered - prior_delivered;
  3176. NET_ADD_STATS(net, LINUX_MIB_TCPDELIVERED, delivered);
  3177. if (flag & FLAG_ECE)
  3178. NET_ADD_STATS(net, LINUX_MIB_TCPDELIVEREDCE, delivered);
  3179. return delivered;
  3180. }
  3181. /* This routine deals with incoming acks, but not outgoing ones. */
  3182. static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
  3183. {
  3184. struct inet_connection_sock *icsk = inet_csk(sk);
  3185. struct tcp_sock *tp = tcp_sk(sk);
  3186. struct tcp_sacktag_state sack_state;
  3187. struct rate_sample rs = { .prior_delivered = 0 };
  3188. u32 prior_snd_una = tp->snd_una;
  3189. bool is_sack_reneg = tp->is_sack_reneg;
  3190. u32 ack_seq = TCP_SKB_CB(skb)->seq;
  3191. u32 ack = TCP_SKB_CB(skb)->ack_seq;
  3192. int num_dupack = 0;
  3193. int prior_packets = tp->packets_out;
  3194. u32 delivered = tp->delivered;
  3195. u32 lost = tp->lost;
  3196. int rexmit = REXMIT_NONE; /* Flag to (re)transmit to recover losses */
  3197. u32 prior_fack;
  3198. sack_state.first_sackt = 0;
  3199. sack_state.rate = &rs;
  3200. sack_state.sack_delivered = 0;
  3201. /* We very likely will need to access rtx queue. */
  3202. prefetch(sk->tcp_rtx_queue.rb_node);
  3203. /* If the ack is older than previous acks
  3204. * then we can probably ignore it.
  3205. */
  3206. if (before(ack, prior_snd_una)) {
  3207. /* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
  3208. if (before(ack, prior_snd_una - tp->max_window)) {
  3209. if (!(flag & FLAG_NO_CHALLENGE_ACK))
  3210. tcp_send_challenge_ack(sk, skb);
  3211. return -1;
  3212. }
  3213. goto old_ack;
  3214. }
  3215. /* If the ack includes data we haven't sent yet, discard
  3216. * this segment (RFC793 Section 3.9).
  3217. */
  3218. if (after(ack, tp->snd_nxt))
  3219. return -1;
  3220. if (after(ack, prior_snd_una)) {
  3221. flag |= FLAG_SND_UNA_ADVANCED;
  3222. icsk->icsk_retransmits = 0;
  3223. #if IS_ENABLED(CONFIG_TLS_DEVICE)
  3224. if (static_branch_unlikely(&clean_acked_data_enabled.key))
  3225. if (icsk->icsk_clean_acked)
  3226. icsk->icsk_clean_acked(sk, ack);
  3227. #endif
  3228. }
  3229. prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;
  3230. rs.prior_in_flight = tcp_packets_in_flight(tp);
  3231. /* ts_recent update must be made after we are sure that the packet
  3232. * is in window.
  3233. */
  3234. if (flag & FLAG_UPDATE_TS_RECENT)
  3235. tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);
  3236. if ((flag & (FLAG_SLOWPATH | FLAG_SND_UNA_ADVANCED)) ==
  3237. FLAG_SND_UNA_ADVANCED) {
  3238. /* Window is constant, pure forward advance.
  3239. * No more checks are required.
  3240. * Note, we use the fact that SND.UNA>=SND.WL2.
  3241. */
  3242. tcp_update_wl(tp, ack_seq);
  3243. tcp_snd_una_update(tp, ack);
  3244. flag |= FLAG_WIN_UPDATE;
  3245. tcp_in_ack_event(sk, CA_ACK_WIN_UPDATE);
  3246. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPACKS);
  3247. } else {
  3248. u32 ack_ev_flags = CA_ACK_SLOWPATH;
  3249. if (ack_seq != TCP_SKB_CB(skb)->end_seq)
  3250. flag |= FLAG_DATA;
  3251. else
  3252. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPUREACKS);
  3253. flag |= tcp_ack_update_window(sk, skb, ack, ack_seq);
  3254. if (TCP_SKB_CB(skb)->sacked)
  3255. flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
  3256. &sack_state);
  3257. if (tcp_ecn_rcv_ecn_echo(tp, tcp_hdr(skb))) {
  3258. flag |= FLAG_ECE;
  3259. ack_ev_flags |= CA_ACK_ECE;
  3260. }
  3261. if (sack_state.sack_delivered)
  3262. tcp_count_delivered(tp, sack_state.sack_delivered,
  3263. flag & FLAG_ECE);
  3264. if (flag & FLAG_WIN_UPDATE)
  3265. ack_ev_flags |= CA_ACK_WIN_UPDATE;
  3266. tcp_in_ack_event(sk, ack_ev_flags);
  3267. }
  3268. /* This is a deviation from RFC3168 since it states that:
  3269. * "When the TCP data sender is ready to set the CWR bit after reducing
  3270. * the congestion window, it SHOULD set the CWR bit only on the first
  3271. * new data packet that it transmits."
  3272. * We accept CWR on pure ACKs to be more robust
  3273. * with widely-deployed TCP implementations that do this.
  3274. */
  3275. tcp_ecn_accept_cwr(sk, skb);
  3276. /* We passed data and got it acked, remove any soft error
  3277. * log. Something worked...
  3278. */
  3279. sk->sk_err_soft = 0;
  3280. icsk->icsk_probes_out = 0;
  3281. tp->rcv_tstamp = tcp_jiffies32;
  3282. if (!prior_packets)
  3283. goto no_queue;
  3284. /* See if we can take anything off of the retransmit queue. */
  3285. flag |= tcp_clean_rtx_queue(sk, prior_fack, prior_snd_una, &sack_state,
  3286. flag & FLAG_ECE);
  3287. tcp_rack_update_reo_wnd(sk, &rs);
  3288. if (tp->tlp_high_seq)
  3289. tcp_process_tlp_ack(sk, ack, flag);
  3290. if (tcp_ack_is_dubious(sk, flag)) {
  3291. if (!(flag & (FLAG_SND_UNA_ADVANCED | FLAG_NOT_DUP))) {
  3292. num_dupack = 1;
  3293. /* Consider if pure acks were aggregated in tcp_add_backlog() */
  3294. if (!(flag & FLAG_DATA))
  3295. num_dupack = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
  3296. }
  3297. tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
  3298. &rexmit);
  3299. }
  3300. /* If needed, reset TLP/RTO timer when RACK doesn't set. */
  3301. if (flag & FLAG_SET_XMIT_TIMER)
  3302. tcp_set_xmit_timer(sk);
  3303. if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP))
  3304. sk_dst_confirm(sk);
  3305. delivered = tcp_newly_delivered(sk, delivered, flag);
  3306. lost = tp->lost - lost; /* freshly marked lost */
  3307. rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);
  3308. tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate);
  3309. tcp_cong_control(sk, ack, delivered, flag, sack_state.rate);
  3310. tcp_xmit_recovery(sk, rexmit);
  3311. return 1;
  3312. no_queue:
  3313. /* If data was DSACKed, see if we can undo a cwnd reduction. */
  3314. if (flag & FLAG_DSACKING_ACK) {
  3315. tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
  3316. &rexmit);
  3317. tcp_newly_delivered(sk, delivered, flag);
  3318. }
  3319. /* If this ack opens up a zero window, clear backoff. It was
  3320. * being used to time the probes, and is probably far higher than
  3321. * it needs to be for normal retransmission.
  3322. */
  3323. tcp_ack_probe(sk);
  3324. if (tp->tlp_high_seq)
  3325. tcp_process_tlp_ack(sk, ack, flag);
  3326. return 1;
  3327. old_ack:
  3328. /* If data was SACKed, tag it and see if we should send more data.
  3329. * If data was DSACKed, see if we can undo a cwnd reduction.
  3330. */
  3331. if (TCP_SKB_CB(skb)->sacked) {
  3332. flag |= tcp_sacktag_write_queue(sk, skb, prior_snd_una,
  3333. &sack_state);
  3334. tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, &flag,
  3335. &rexmit);
  3336. tcp_newly_delivered(sk, delivered, flag);
  3337. tcp_xmit_recovery(sk, rexmit);
  3338. }
  3339. return 0;
  3340. }
  3341. static void tcp_parse_fastopen_option(int len, const unsigned char *cookie,
  3342. bool syn, struct tcp_fastopen_cookie *foc,
  3343. bool exp_opt)
  3344. {
  3345. /* Valid only in SYN or SYN-ACK with an even length. */
  3346. if (!foc || !syn || len < 0 || (len & 1))
  3347. return;
  3348. if (len >= TCP_FASTOPEN_COOKIE_MIN &&
  3349. len <= TCP_FASTOPEN_COOKIE_MAX)
  3350. memcpy(foc->val, cookie, len);
  3351. else if (len != 0)
  3352. len = -1;
  3353. foc->len = len;
  3354. foc->exp = exp_opt;
  3355. }
  3356. static bool smc_parse_options(const struct tcphdr *th,
  3357. struct tcp_options_received *opt_rx,
  3358. const unsigned char *ptr,
  3359. int opsize)
  3360. {
  3361. #if IS_ENABLED(CONFIG_SMC)
  3362. if (static_branch_unlikely(&tcp_have_smc)) {
  3363. if (th->syn && !(opsize & 1) &&
  3364. opsize >= TCPOLEN_EXP_SMC_BASE &&
  3365. get_unaligned_be32(ptr) == TCPOPT_SMC_MAGIC) {
  3366. opt_rx->smc_ok = 1;
  3367. return true;
  3368. }
  3369. }
  3370. #endif
  3371. return false;
  3372. }
  3373. /* Try to parse the MSS option from the TCP header. Return 0 on failure, clamped
  3374. * value on success.
  3375. */
  3376. static u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss)
  3377. {
  3378. const unsigned char *ptr = (const unsigned char *)(th + 1);
  3379. int length = (th->doff * 4) - sizeof(struct tcphdr);
  3380. u16 mss = 0;
  3381. while (length > 0) {
  3382. int opcode = *ptr++;
  3383. int opsize;
  3384. switch (opcode) {
  3385. case TCPOPT_EOL:
  3386. return mss;
  3387. case TCPOPT_NOP: /* Ref: RFC 793 section 3.1 */
  3388. length--;
  3389. continue;
  3390. default:
  3391. if (length < 2)
  3392. return mss;
  3393. opsize = *ptr++;
  3394. if (opsize < 2) /* "silly options" */
  3395. return mss;
  3396. if (opsize > length)
  3397. return mss; /* fail on partial options */
  3398. if (opcode == TCPOPT_MSS && opsize == TCPOLEN_MSS) {
  3399. u16 in_mss = get_unaligned_be16(ptr);
  3400. if (in_mss) {
  3401. if (user_mss && user_mss < in_mss)
  3402. in_mss = user_mss;
  3403. mss = in_mss;
  3404. }
  3405. }
  3406. ptr += opsize - 2;
  3407. length -= opsize;
  3408. }
  3409. }
  3410. return mss;
  3411. }
  3412. /* Look for tcp options. Normally only called on SYN and SYNACK packets.
  3413. * But, this can also be called on packets in the established flow when
  3414. * the fast version below fails.
  3415. */
  3416. void tcp_parse_options(const struct net *net,
  3417. const struct sk_buff *skb,
  3418. struct tcp_options_received *opt_rx, int estab,
  3419. struct tcp_fastopen_cookie *foc)
  3420. {
  3421. const unsigned char *ptr;
  3422. const struct tcphdr *th = tcp_hdr(skb);
  3423. int length = (th->doff * 4) - sizeof(struct tcphdr);
  3424. ptr = (const unsigned char *)(th + 1);
  3425. opt_rx->saw_tstamp = 0;
  3426. opt_rx->saw_unknown = 0;
  3427. while (length > 0) {
  3428. int opcode = *ptr++;
  3429. int opsize;
  3430. switch (opcode) {
  3431. case TCPOPT_EOL:
  3432. return;
  3433. case TCPOPT_NOP: /* Ref: RFC 793 section 3.1 */
  3434. length--;
  3435. continue;
  3436. default:
  3437. if (length < 2)
  3438. return;
  3439. opsize = *ptr++;
  3440. if (opsize < 2) /* "silly options" */
  3441. return;
  3442. if (opsize > length)
  3443. return; /* don't parse partial options */
  3444. switch (opcode) {
  3445. case TCPOPT_MSS:
  3446. if (opsize == TCPOLEN_MSS && th->syn && !estab) {
  3447. u16 in_mss = get_unaligned_be16(ptr);
  3448. if (in_mss) {
  3449. if (opt_rx->user_mss &&
  3450. opt_rx->user_mss < in_mss)
  3451. in_mss = opt_rx->user_mss;
  3452. opt_rx->mss_clamp = in_mss;
  3453. }
  3454. }
  3455. break;
  3456. case TCPOPT_WINDOW:
  3457. if (opsize == TCPOLEN_WINDOW && th->syn &&
  3458. !estab && net->ipv4.sysctl_tcp_window_scaling) {
  3459. __u8 snd_wscale = *(__u8 *)ptr;
  3460. opt_rx->wscale_ok = 1;
  3461. if (snd_wscale > TCP_MAX_WSCALE) {
  3462. net_info_ratelimited("%s: Illegal window scaling value %d > %u received\n",
  3463. __func__,
  3464. snd_wscale,
  3465. TCP_MAX_WSCALE);
  3466. snd_wscale = TCP_MAX_WSCALE;
  3467. }
  3468. opt_rx->snd_wscale = snd_wscale;
  3469. }
  3470. break;
  3471. case TCPOPT_TIMESTAMP:
  3472. if ((opsize == TCPOLEN_TIMESTAMP) &&
  3473. ((estab && opt_rx->tstamp_ok) ||
  3474. (!estab && net->ipv4.sysctl_tcp_timestamps))) {
  3475. opt_rx->saw_tstamp = 1;
  3476. opt_rx->rcv_tsval = get_unaligned_be32(ptr);
  3477. opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4);
  3478. }
  3479. break;
  3480. case TCPOPT_SACK_PERM:
  3481. if (opsize == TCPOLEN_SACK_PERM && th->syn &&
  3482. !estab && net->ipv4.sysctl_tcp_sack) {
  3483. opt_rx->sack_ok = TCP_SACK_SEEN;
  3484. tcp_sack_reset(opt_rx);
  3485. }
  3486. break;
  3487. case TCPOPT_SACK:
  3488. if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) &&
  3489. !((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) &&
  3490. opt_rx->sack_ok) {
  3491. TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th;
  3492. }
  3493. break;
  3494. #ifdef CONFIG_TCP_MD5SIG
  3495. case TCPOPT_MD5SIG:
  3496. /*
  3497. * The MD5 Hash has already been
  3498. * checked (see tcp_v{4,6}_do_rcv()).
  3499. */
  3500. break;
  3501. #endif
  3502. case TCPOPT_FASTOPEN:
  3503. tcp_parse_fastopen_option(
  3504. opsize - TCPOLEN_FASTOPEN_BASE,
  3505. ptr, th->syn, foc, false);
  3506. break;
  3507. case TCPOPT_EXP:
  3508. /* Fast Open option shares code 254 using a
  3509. * 16 bits magic number.
  3510. */
  3511. if (opsize >= TCPOLEN_EXP_FASTOPEN_BASE &&
  3512. get_unaligned_be16(ptr) ==
  3513. TCPOPT_FASTOPEN_MAGIC) {
  3514. tcp_parse_fastopen_option(opsize -
  3515. TCPOLEN_EXP_FASTOPEN_BASE,
  3516. ptr + 2, th->syn, foc, true);
  3517. break;
  3518. }
  3519. if (smc_parse_options(th, opt_rx, ptr, opsize))
  3520. break;
  3521. opt_rx->saw_unknown = 1;
  3522. break;
  3523. default:
  3524. opt_rx->saw_unknown = 1;
  3525. }
  3526. ptr += opsize-2;
  3527. length -= opsize;
  3528. }
  3529. }
  3530. }
  3531. EXPORT_SYMBOL(tcp_parse_options);
  3532. static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th)
  3533. {
  3534. const __be32 *ptr = (const __be32 *)(th + 1);
  3535. if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
  3536. | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
  3537. tp->rx_opt.saw_tstamp = 1;
  3538. ++ptr;
  3539. tp->rx_opt.rcv_tsval = ntohl(*ptr);
  3540. ++ptr;
  3541. if (*ptr)
  3542. tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset;
  3543. else
  3544. tp->rx_opt.rcv_tsecr = 0;
  3545. return true;
  3546. }
  3547. return false;
  3548. }
  3549. /* Fast parse options. This hopes to only see timestamps.
  3550. * If it is wrong it falls back on tcp_parse_options().
  3551. */
  3552. static bool tcp_fast_parse_options(const struct net *net,
  3553. const struct sk_buff *skb,
  3554. const struct tcphdr *th, struct tcp_sock *tp)
  3555. {
  3556. /* In the spirit of fast parsing, compare doff directly to constant
  3557. * values. Because equality is used, short doff can be ignored here.
  3558. */
  3559. if (th->doff == (sizeof(*th) / 4)) {
  3560. tp->rx_opt.saw_tstamp = 0;
  3561. return false;
  3562. } else if (tp->rx_opt.tstamp_ok &&
  3563. th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) {
  3564. if (tcp_parse_aligned_timestamp(tp, th))
  3565. return true;
  3566. }
  3567. tcp_parse_options(net, skb, &tp->rx_opt, 1, NULL);
  3568. if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
  3569. tp->rx_opt.rcv_tsecr -= tp->tsoffset;
  3570. return true;
  3571. }
  3572. #ifdef CONFIG_TCP_MD5SIG
  3573. /*
  3574. * Parse MD5 Signature option
  3575. */
  3576. const u8 *tcp_parse_md5sig_option(const struct tcphdr *th)
  3577. {
  3578. int length = (th->doff << 2) - sizeof(*th);
  3579. const u8 *ptr = (const u8 *)(th + 1);
  3580. /* If not enough data remaining, we can short cut */
  3581. while (length >= TCPOLEN_MD5SIG) {
  3582. int opcode = *ptr++;
  3583. int opsize;
  3584. switch (opcode) {
  3585. case TCPOPT_EOL:
  3586. return NULL;
  3587. case TCPOPT_NOP:
  3588. length--;
  3589. continue;
  3590. default:
  3591. opsize = *ptr++;
  3592. if (opsize < 2 || opsize > length)
  3593. return NULL;
  3594. if (opcode == TCPOPT_MD5SIG)
  3595. return opsize == TCPOLEN_MD5SIG ? ptr : NULL;
  3596. }
  3597. ptr += opsize - 2;
  3598. length -= opsize;
  3599. }
  3600. return NULL;
  3601. }
  3602. EXPORT_SYMBOL(tcp_parse_md5sig_option);
  3603. #endif
  3604. /* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
  3605. *
  3606. * It is not fatal. If this ACK does _not_ change critical state (seqs, window)
  3607. * it can pass through stack. So, the following predicate verifies that
  3608. * this segment is not used for anything but congestion avoidance or
  3609. * fast retransmit. Moreover, we even are able to eliminate most of such
  3610. * second order effects, if we apply some small "replay" window (~RTO)
  3611. * to timestamp space.
  3612. *
  3613. * All these measures still do not guarantee that we reject wrapped ACKs
  3614. * on networks with high bandwidth, when sequence space is recycled fastly,
  3615. * but it guarantees that such events will be very rare and do not affect
  3616. * connection seriously. This doesn't look nice, but alas, PAWS is really
  3617. * buggy extension.
  3618. *
  3619. * [ Later note. Even worse! It is buggy for segments _with_ data. RFC
  3620. * states that events when retransmit arrives after original data are rare.
  3621. * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
  3622. * the biggest problem on large power networks even with minor reordering.
  3623. * OK, let's give it small replay window. If peer clock is even 1hz, it is safe
  3624. * up to bandwidth of 18Gigabit/sec. 8) ]
  3625. */
  3626. static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb)
  3627. {
  3628. const struct tcp_sock *tp = tcp_sk(sk);
  3629. const struct tcphdr *th = tcp_hdr(skb);
  3630. u32 seq = TCP_SKB_CB(skb)->seq;
  3631. u32 ack = TCP_SKB_CB(skb)->ack_seq;
  3632. return (/* 1. Pure ACK with correct sequence number. */
  3633. (th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) &&
  3634. /* 2. ... and duplicate ACK. */
  3635. ack == tp->snd_una &&
  3636. /* 3. ... and does not update window. */
  3637. !tcp_may_update_window(tp, ack, seq, ntohs(th->window) << tp->rx_opt.snd_wscale) &&
  3638. /* 4. ... and sits in replay window. */
  3639. (s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <= (inet_csk(sk)->icsk_rto * 1024) / HZ);
  3640. }
  3641. static inline bool tcp_paws_discard(const struct sock *sk,
  3642. const struct sk_buff *skb)
  3643. {
  3644. const struct tcp_sock *tp = tcp_sk(sk);
  3645. return !tcp_paws_check(&tp->rx_opt, TCP_PAWS_WINDOW) &&
  3646. !tcp_disordered_ack(sk, skb);
  3647. }
  3648. /* Check segment sequence number for validity.
  3649. *
  3650. * Segment controls are considered valid, if the segment
  3651. * fits to the window after truncation to the window. Acceptability
  3652. * of data (and SYN, FIN, of course) is checked separately.
  3653. * See tcp_data_queue(), for example.
  3654. *
  3655. * Also, controls (RST is main one) are accepted using RCV.WUP instead
  3656. * of RCV.NXT. Peer still did not advance his SND.UNA when we
  3657. * delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
  3658. * (borrowed from freebsd)
  3659. */
  3660. static inline bool tcp_sequence(const struct tcp_sock *tp, u32 seq, u32 end_seq)
  3661. {
  3662. return !before(end_seq, tp->rcv_wup) &&
  3663. !after(seq, tp->rcv_nxt + tcp_receive_window(tp));
  3664. }
  3665. /* When we get a reset we do this. */
  3666. void tcp_reset(struct sock *sk)
  3667. {
  3668. trace_tcp_receive_reset(sk);
  3669. /* We want the right error as BSD sees it (and indeed as we do). */
  3670. switch (sk->sk_state) {
  3671. case TCP_SYN_SENT:
  3672. sk->sk_err = ECONNREFUSED;
  3673. break;
  3674. case TCP_CLOSE_WAIT:
  3675. sk->sk_err = EPIPE;
  3676. break;
  3677. case TCP_CLOSE:
  3678. return;
  3679. default:
  3680. sk->sk_err = ECONNRESET;
  3681. }
  3682. /* This barrier is coupled with smp_rmb() in tcp_poll() */
  3683. smp_wmb();
  3684. tcp_write_queue_purge(sk);
  3685. tcp_done(sk);
  3686. if (!sock_flag(sk, SOCK_DEAD))
  3687. sk->sk_error_report(sk);
  3688. }
  3689. /*
  3690. * Process the FIN bit. This now behaves as it is supposed to work
  3691. * and the FIN takes effect when it is validly part of sequence
  3692. * space. Not before when we get holes.
  3693. *
  3694. * If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
  3695. * (and thence onto LAST-ACK and finally, CLOSE, we never enter
  3696. * TIME-WAIT)
  3697. *
  3698. * If we are in FINWAIT-1, a received FIN indicates simultaneous
  3699. * close and we go into CLOSING (and later onto TIME-WAIT)
  3700. *
  3701. * If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
  3702. */
  3703. void tcp_fin(struct sock *sk)
  3704. {
  3705. struct tcp_sock *tp = tcp_sk(sk);
  3706. inet_csk_schedule_ack(sk);
  3707. sk->sk_shutdown |= RCV_SHUTDOWN;
  3708. sock_set_flag(sk, SOCK_DONE);
  3709. switch (sk->sk_state) {
  3710. case TCP_SYN_RECV:
  3711. case TCP_ESTABLISHED:
  3712. /* Move to CLOSE_WAIT */
  3713. tcp_set_state(sk, TCP_CLOSE_WAIT);
  3714. inet_csk_enter_pingpong_mode(sk);
  3715. break;
  3716. case TCP_CLOSE_WAIT:
  3717. case TCP_CLOSING:
  3718. /* Received a retransmission of the FIN, do
  3719. * nothing.
  3720. */
  3721. break;
  3722. case TCP_LAST_ACK:
  3723. /* RFC793: Remain in the LAST-ACK state. */
  3724. break;
  3725. case TCP_FIN_WAIT1:
  3726. /* This case occurs when a simultaneous close
  3727. * happens, we must ack the received FIN and
  3728. * enter the CLOSING state.
  3729. */
  3730. tcp_send_ack(sk);
  3731. tcp_set_state(sk, TCP_CLOSING);
  3732. break;
  3733. case TCP_FIN_WAIT2:
  3734. /* Received a FIN -- send ACK and enter TIME_WAIT. */
  3735. tcp_send_ack(sk);
  3736. tcp_time_wait(sk, TCP_TIME_WAIT, 0);
  3737. break;
  3738. default:
  3739. /* Only TCP_LISTEN and TCP_CLOSE are left, in these
  3740. * cases we should never reach this piece of code.
  3741. */
  3742. pr_err("%s: Impossible, sk->sk_state=%d\n",
  3743. __func__, sk->sk_state);
  3744. break;
  3745. }
  3746. /* It _is_ possible, that we have something out-of-order _after_ FIN.
  3747. * Probably, we should reset in this case. For now drop them.
  3748. */
  3749. skb_rbtree_purge(&tp->out_of_order_queue);
  3750. if (tcp_is_sack(tp))
  3751. tcp_sack_reset(&tp->rx_opt);
  3752. sk_mem_reclaim(sk);
  3753. if (!sock_flag(sk, SOCK_DEAD)) {
  3754. sk->sk_state_change(sk);
  3755. /* Do not send POLL_HUP for half duplex close. */
  3756. if (sk->sk_shutdown == SHUTDOWN_MASK ||
  3757. sk->sk_state == TCP_CLOSE)
  3758. sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
  3759. else
  3760. sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
  3761. }
  3762. }
  3763. static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq,
  3764. u32 end_seq)
  3765. {
  3766. if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) {
  3767. if (before(seq, sp->start_seq))
  3768. sp->start_seq = seq;
  3769. if (after(end_seq, sp->end_seq))
  3770. sp->end_seq = end_seq;
  3771. return true;
  3772. }
  3773. return false;
  3774. }
  3775. static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq)
  3776. {
  3777. struct tcp_sock *tp = tcp_sk(sk);
  3778. if (tcp_is_sack(tp) && sock_net(sk)->ipv4.sysctl_tcp_dsack) {
  3779. int mib_idx;
  3780. if (before(seq, tp->rcv_nxt))
  3781. mib_idx = LINUX_MIB_TCPDSACKOLDSENT;
  3782. else
  3783. mib_idx = LINUX_MIB_TCPDSACKOFOSENT;
  3784. NET_INC_STATS(sock_net(sk), mib_idx);
  3785. tp->rx_opt.dsack = 1;
  3786. tp->duplicate_sack[0].start_seq = seq;
  3787. tp->duplicate_sack[0].end_seq = end_seq;
  3788. }
  3789. }
  3790. static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq)
  3791. {
  3792. struct tcp_sock *tp = tcp_sk(sk);
  3793. if (!tp->rx_opt.dsack)
  3794. tcp_dsack_set(sk, seq, end_seq);
  3795. else
  3796. tcp_sack_extend(tp->duplicate_sack, seq, end_seq);
  3797. }
  3798. static void tcp_rcv_spurious_retrans(struct sock *sk, const struct sk_buff *skb)
  3799. {
  3800. /* When the ACK path fails or drops most ACKs, the sender would
  3801. * timeout and spuriously retransmit the same segment repeatedly.
  3802. * The receiver remembers and reflects via DSACKs. Leverage the
  3803. * DSACK state and change the txhash to re-route speculatively.
  3804. */
  3805. if (TCP_SKB_CB(skb)->seq == tcp_sk(sk)->duplicate_sack[0].start_seq &&
  3806. sk_rethink_txhash(sk))
  3807. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDUPLICATEDATAREHASH);
  3808. }
  3809. static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb)
  3810. {
  3811. struct tcp_sock *tp = tcp_sk(sk);
  3812. if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
  3813. before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
  3814. NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
  3815. tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
  3816. if (tcp_is_sack(tp) && sock_net(sk)->ipv4.sysctl_tcp_dsack) {
  3817. u32 end_seq = TCP_SKB_CB(skb)->end_seq;
  3818. tcp_rcv_spurious_retrans(sk, skb);
  3819. if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))
  3820. end_seq = tp->rcv_nxt;
  3821. tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq);
  3822. }
  3823. }
  3824. tcp_send_ack(sk);
  3825. }
  3826. /* These routines update the SACK block as out-of-order packets arrive or
  3827. * in-order packets close up the sequence space.
  3828. */
  3829. static void tcp_sack_maybe_coalesce(struct tcp_sock *tp)
  3830. {
  3831. int this_sack;
  3832. struct tcp_sack_block *sp = &tp->selective_acks[0];
  3833. struct tcp_sack_block *swalk = sp + 1;
  3834. /* See if the recent change to the first SACK eats into
  3835. * or hits the sequence space of other SACK blocks, if so coalesce.
  3836. */
  3837. for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) {
  3838. if (tcp_sack_extend(sp, swalk->start_seq, swalk->end_seq)) {
  3839. int i;
  3840. /* Zap SWALK, by moving every further SACK up by one slot.
  3841. * Decrease num_sacks.
  3842. */
  3843. tp->rx_opt.num_sacks--;
  3844. for (i = this_sack; i < tp->rx_opt.num_sacks; i++)
  3845. sp[i] = sp[i + 1];
  3846. continue;
  3847. }
  3848. this_sack++;
  3849. swalk++;
  3850. }
  3851. }
  3852. static void tcp_sack_compress_send_ack(struct sock *sk)
  3853. {
  3854. struct tcp_sock *tp = tcp_sk(sk);
  3855. if (!tp->compressed_ack)
  3856. return;
  3857. if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
  3858. __sock_put(sk);
  3859. /* Since we have to send one ack finally,
  3860. * substract one from tp->compressed_ack to keep
  3861. * LINUX_MIB_TCPACKCOMPRESSED accurate.
  3862. */
  3863. NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
  3864. tp->compressed_ack - 1);
  3865. tp->compressed_ack = 0;
  3866. tcp_send_ack(sk);
  3867. }
  3868. /* Reasonable amount of sack blocks included in TCP SACK option
  3869. * The max is 4, but this becomes 3 if TCP timestamps are there.
  3870. * Given that SACK packets might be lost, be conservative and use 2.
  3871. */
  3872. #define TCP_SACK_BLOCKS_EXPECTED 2
  3873. static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq)
  3874. {
  3875. struct tcp_sock *tp = tcp_sk(sk);
  3876. struct tcp_sack_block *sp = &tp->selective_acks[0];
  3877. int cur_sacks = tp->rx_opt.num_sacks;
  3878. int this_sack;
  3879. if (!cur_sacks)
  3880. goto new_sack;
  3881. for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) {
  3882. if (tcp_sack_extend(sp, seq, end_seq)) {
  3883. if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
  3884. tcp_sack_compress_send_ack(sk);
  3885. /* Rotate this_sack to the first one. */
  3886. for (; this_sack > 0; this_sack--, sp--)
  3887. swap(*sp, *(sp - 1));
  3888. if (cur_sacks > 1)
  3889. tcp_sack_maybe_coalesce(tp);
  3890. return;
  3891. }
  3892. }
  3893. if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
  3894. tcp_sack_compress_send_ack(sk);
  3895. /* Could not find an adjacent existing SACK, build a new one,
  3896. * put it at the front, and shift everyone else down. We
  3897. * always know there is at least one SACK present already here.
  3898. *
  3899. * If the sack array is full, forget about the last one.
  3900. */
  3901. if (this_sack >= TCP_NUM_SACKS) {
  3902. this_sack--;
  3903. tp->rx_opt.num_sacks--;
  3904. sp--;
  3905. }
  3906. for (; this_sack > 0; this_sack--, sp--)
  3907. *sp = *(sp - 1);
  3908. new_sack:
  3909. /* Build the new head SACK, and we're done. */
  3910. sp->start_seq = seq;
  3911. sp->end_seq = end_seq;
  3912. tp->rx_opt.num_sacks++;
  3913. }
  3914. /* RCV.NXT advances, some SACKs should be eaten. */
  3915. static void tcp_sack_remove(struct tcp_sock *tp)
  3916. {
  3917. struct tcp_sack_block *sp = &tp->selective_acks[0];
  3918. int num_sacks = tp->rx_opt.num_sacks;
  3919. int this_sack;
  3920. /* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
  3921. if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
  3922. tp->rx_opt.num_sacks = 0;
  3923. return;
  3924. }
  3925. for (this_sack = 0; this_sack < num_sacks;) {
  3926. /* Check if the start of the sack is covered by RCV.NXT. */
  3927. if (!before(tp->rcv_nxt, sp->start_seq)) {
  3928. int i;
  3929. /* RCV.NXT must cover all the block! */
  3930. WARN_ON(before(tp->rcv_nxt, sp->end_seq));
  3931. /* Zap this SACK, by moving forward any other SACKS. */
  3932. for (i = this_sack+1; i < num_sacks; i++)
  3933. tp->selective_acks[i-1] = tp->selective_acks[i];
  3934. num_sacks--;
  3935. continue;
  3936. }
  3937. this_sack++;
  3938. sp++;
  3939. }
  3940. tp->rx_opt.num_sacks = num_sacks;
  3941. }
  3942. /**
  3943. * tcp_try_coalesce - try to merge skb to prior one
  3944. * @sk: socket
  3945. * @to: prior buffer
  3946. * @from: buffer to add in queue
  3947. * @fragstolen: pointer to boolean
  3948. *
  3949. * Before queueing skb @from after @to, try to merge them
  3950. * to reduce overall memory use and queue lengths, if cost is small.
  3951. * Packets in ofo or receive queues can stay a long time.
  3952. * Better try to coalesce them right now to avoid future collapses.
  3953. * Returns true if caller should free @from instead of queueing it
  3954. */
  3955. static bool tcp_try_coalesce(struct sock *sk,
  3956. struct sk_buff *to,
  3957. struct sk_buff *from,
  3958. bool *fragstolen)
  3959. {
  3960. int delta;
  3961. *fragstolen = false;
  3962. /* Its possible this segment overlaps with prior segment in queue */
  3963. if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq)
  3964. return false;
  3965. if (!mptcp_skb_can_collapse(to, from))
  3966. return false;
  3967. #ifdef CONFIG_TLS_DEVICE
  3968. if (from->decrypted != to->decrypted)
  3969. return false;
  3970. #endif
  3971. if (!skb_try_coalesce(to, from, fragstolen, &delta))
  3972. return false;
  3973. atomic_add(delta, &sk->sk_rmem_alloc);
  3974. sk_mem_charge(sk, delta);
  3975. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE);
  3976. TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq;
  3977. TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq;
  3978. TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags;
  3979. if (TCP_SKB_CB(from)->has_rxtstamp) {
  3980. TCP_SKB_CB(to)->has_rxtstamp = true;
  3981. to->tstamp = from->tstamp;
  3982. skb_hwtstamps(to)->hwtstamp = skb_hwtstamps(from)->hwtstamp;
  3983. }
  3984. return true;
  3985. }
  3986. static bool tcp_ooo_try_coalesce(struct sock *sk,
  3987. struct sk_buff *to,
  3988. struct sk_buff *from,
  3989. bool *fragstolen)
  3990. {
  3991. bool res = tcp_try_coalesce(sk, to, from, fragstolen);
  3992. /* In case tcp_drop() is called later, update to->gso_segs */
  3993. if (res) {
  3994. u32 gso_segs = max_t(u16, 1, skb_shinfo(to)->gso_segs) +
  3995. max_t(u16, 1, skb_shinfo(from)->gso_segs);
  3996. skb_shinfo(to)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
  3997. }
  3998. return res;
  3999. }
  4000. static void tcp_drop(struct sock *sk, struct sk_buff *skb)
  4001. {
  4002. trace_android_vh_kfree_skb(skb);
  4003. sk_drops_add(sk, skb);
  4004. __kfree_skb(skb);
  4005. }
  4006. /* This one checks to see if we can put data from the
  4007. * out_of_order queue into the receive_queue.
  4008. */
  4009. static void tcp_ofo_queue(struct sock *sk)
  4010. {
  4011. struct tcp_sock *tp = tcp_sk(sk);
  4012. __u32 dsack_high = tp->rcv_nxt;
  4013. bool fin, fragstolen, eaten;
  4014. struct sk_buff *skb, *tail;
  4015. struct rb_node *p;
  4016. p = rb_first(&tp->out_of_order_queue);
  4017. while (p) {
  4018. skb = rb_to_skb(p);
  4019. if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
  4020. break;
  4021. if (before(TCP_SKB_CB(skb)->seq, dsack_high)) {
  4022. __u32 dsack = dsack_high;
  4023. if (before(TCP_SKB_CB(skb)->end_seq, dsack_high))
  4024. dsack_high = TCP_SKB_CB(skb)->end_seq;
  4025. tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, dsack);
  4026. }
  4027. p = rb_next(p);
  4028. rb_erase(&skb->rbnode, &tp->out_of_order_queue);
  4029. if (unlikely(!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))) {
  4030. tcp_drop(sk, skb);
  4031. continue;
  4032. }
  4033. tail = skb_peek_tail(&sk->sk_receive_queue);
  4034. eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen);
  4035. tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq);
  4036. fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
  4037. if (!eaten)
  4038. __skb_queue_tail(&sk->sk_receive_queue, skb);
  4039. else
  4040. kfree_skb_partial(skb, fragstolen);
  4041. if (unlikely(fin)) {
  4042. tcp_fin(sk);
  4043. /* tcp_fin() purges tp->out_of_order_queue,
  4044. * so we must end this loop right now.
  4045. */
  4046. break;
  4047. }
  4048. }
  4049. }
  4050. static bool tcp_prune_ofo_queue(struct sock *sk);
  4051. static int tcp_prune_queue(struct sock *sk);
  4052. static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb,
  4053. unsigned int size)
  4054. {
  4055. if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
  4056. !sk_rmem_schedule(sk, skb, size)) {
  4057. if (tcp_prune_queue(sk) < 0)
  4058. return -1;
  4059. while (!sk_rmem_schedule(sk, skb, size)) {
  4060. if (!tcp_prune_ofo_queue(sk))
  4061. return -1;
  4062. }
  4063. }
  4064. return 0;
  4065. }
  4066. static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
  4067. {
  4068. struct tcp_sock *tp = tcp_sk(sk);
  4069. struct rb_node **p, *parent;
  4070. struct sk_buff *skb1;
  4071. u32 seq, end_seq;
  4072. bool fragstolen;
  4073. tcp_ecn_check_ce(sk, skb);
  4074. if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
  4075. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP);
  4076. sk->sk_data_ready(sk);
  4077. tcp_drop(sk, skb);
  4078. return;
  4079. }
  4080. /* Disable header prediction. */
  4081. tp->pred_flags = 0;
  4082. inet_csk_schedule_ack(sk);
  4083. tp->rcv_ooopack += max_t(u16, 1, skb_shinfo(skb)->gso_segs);
  4084. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOQUEUE);
  4085. seq = TCP_SKB_CB(skb)->seq;
  4086. end_seq = TCP_SKB_CB(skb)->end_seq;
  4087. p = &tp->out_of_order_queue.rb_node;
  4088. if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
  4089. /* Initial out of order segment, build 1 SACK. */
  4090. if (tcp_is_sack(tp)) {
  4091. tp->rx_opt.num_sacks = 1;
  4092. tp->selective_acks[0].start_seq = seq;
  4093. tp->selective_acks[0].end_seq = end_seq;
  4094. }
  4095. rb_link_node(&skb->rbnode, NULL, p);
  4096. rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
  4097. tp->ooo_last_skb = skb;
  4098. goto end;
  4099. }
  4100. /* In the typical case, we are adding an skb to the end of the list.
  4101. * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
  4102. */
  4103. if (tcp_ooo_try_coalesce(sk, tp->ooo_last_skb,
  4104. skb, &fragstolen)) {
  4105. coalesce_done:
  4106. /* For non sack flows, do not grow window to force DUPACK
  4107. * and trigger fast retransmit.
  4108. */
  4109. if (tcp_is_sack(tp))
  4110. tcp_grow_window(sk, skb);
  4111. kfree_skb_partial(skb, fragstolen);
  4112. skb = NULL;
  4113. goto add_sack;
  4114. }
  4115. /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
  4116. if (!before(seq, TCP_SKB_CB(tp->ooo_last_skb)->end_seq)) {
  4117. parent = &tp->ooo_last_skb->rbnode;
  4118. p = &parent->rb_right;
  4119. goto insert;
  4120. }
  4121. /* Find place to insert this segment. Handle overlaps on the way. */
  4122. parent = NULL;
  4123. while (*p) {
  4124. parent = *p;
  4125. skb1 = rb_to_skb(parent);
  4126. if (before(seq, TCP_SKB_CB(skb1)->seq)) {
  4127. p = &parent->rb_left;
  4128. continue;
  4129. }
  4130. if (before(seq, TCP_SKB_CB(skb1)->end_seq)) {
  4131. if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
  4132. /* All the bits are present. Drop. */
  4133. NET_INC_STATS(sock_net(sk),
  4134. LINUX_MIB_TCPOFOMERGE);
  4135. tcp_drop(sk, skb);
  4136. skb = NULL;
  4137. tcp_dsack_set(sk, seq, end_seq);
  4138. goto add_sack;
  4139. }
  4140. if (after(seq, TCP_SKB_CB(skb1)->seq)) {
  4141. /* Partial overlap. */
  4142. tcp_dsack_set(sk, seq, TCP_SKB_CB(skb1)->end_seq);
  4143. } else {
  4144. /* skb's seq == skb1's seq and skb covers skb1.
  4145. * Replace skb1 with skb.
  4146. */
  4147. rb_replace_node(&skb1->rbnode, &skb->rbnode,
  4148. &tp->out_of_order_queue);
  4149. tcp_dsack_extend(sk,
  4150. TCP_SKB_CB(skb1)->seq,
  4151. TCP_SKB_CB(skb1)->end_seq);
  4152. NET_INC_STATS(sock_net(sk),
  4153. LINUX_MIB_TCPOFOMERGE);
  4154. tcp_drop(sk, skb1);
  4155. goto merge_right;
  4156. }
  4157. } else if (tcp_ooo_try_coalesce(sk, skb1,
  4158. skb, &fragstolen)) {
  4159. goto coalesce_done;
  4160. }
  4161. p = &parent->rb_right;
  4162. }
  4163. insert:
  4164. /* Insert segment into RB tree. */
  4165. rb_link_node(&skb->rbnode, parent, p);
  4166. rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
  4167. merge_right:
  4168. /* Remove other segments covered by skb. */
  4169. while ((skb1 = skb_rb_next(skb)) != NULL) {
  4170. if (!after(end_seq, TCP_SKB_CB(skb1)->seq))
  4171. break;
  4172. if (before(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
  4173. tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
  4174. end_seq);
  4175. break;
  4176. }
  4177. rb_erase(&skb1->rbnode, &tp->out_of_order_queue);
  4178. tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
  4179. TCP_SKB_CB(skb1)->end_seq);
  4180. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
  4181. tcp_drop(sk, skb1);
  4182. }
  4183. /* If there is no skb after us, we are the last_skb ! */
  4184. if (!skb1)
  4185. tp->ooo_last_skb = skb;
  4186. add_sack:
  4187. if (tcp_is_sack(tp))
  4188. tcp_sack_new_ofo_skb(sk, seq, end_seq);
  4189. end:
  4190. if (skb) {
  4191. /* For non sack flows, do not grow window to force DUPACK
  4192. * and trigger fast retransmit.
  4193. */
  4194. if (tcp_is_sack(tp))
  4195. tcp_grow_window(sk, skb);
  4196. skb_condense(skb);
  4197. skb_set_owner_r(skb, sk);
  4198. }
  4199. }
  4200. static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb,
  4201. bool *fragstolen)
  4202. {
  4203. int eaten;
  4204. struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue);
  4205. eaten = (tail &&
  4206. tcp_try_coalesce(sk, tail,
  4207. skb, fragstolen)) ? 1 : 0;
  4208. tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq);
  4209. if (!eaten) {
  4210. __skb_queue_tail(&sk->sk_receive_queue, skb);
  4211. skb_set_owner_r(skb, sk);
  4212. }
  4213. return eaten;
  4214. }
  4215. int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
  4216. {
  4217. struct sk_buff *skb;
  4218. int err = -ENOMEM;
  4219. int data_len = 0;
  4220. bool fragstolen;
  4221. if (size == 0)
  4222. return 0;
  4223. if (size > PAGE_SIZE) {
  4224. int npages = min_t(size_t, size >> PAGE_SHIFT, MAX_SKB_FRAGS);
  4225. data_len = npages << PAGE_SHIFT;
  4226. size = data_len + (size & ~PAGE_MASK);
  4227. }
  4228. skb = alloc_skb_with_frags(size - data_len, data_len,
  4229. PAGE_ALLOC_COSTLY_ORDER,
  4230. &err, sk->sk_allocation);
  4231. if (!skb)
  4232. goto err;
  4233. skb_put(skb, size - data_len);
  4234. skb->data_len = data_len;
  4235. skb->len = size;
  4236. if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) {
  4237. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
  4238. goto err_free;
  4239. }
  4240. err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size);
  4241. if (err)
  4242. goto err_free;
  4243. TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt;
  4244. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size;
  4245. TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1;
  4246. if (tcp_queue_rcv(sk, skb, &fragstolen)) {
  4247. WARN_ON_ONCE(fragstolen); /* should not happen */
  4248. __kfree_skb(skb);
  4249. }
  4250. return size;
  4251. err_free:
  4252. kfree_skb(skb);
  4253. err:
  4254. return err;
  4255. }
  4256. void tcp_data_ready(struct sock *sk)
  4257. {
  4258. const struct tcp_sock *tp = tcp_sk(sk);
  4259. int avail = tp->rcv_nxt - tp->copied_seq;
  4260. if (avail < sk->sk_rcvlowat && !tcp_rmem_pressure(sk) &&
  4261. !sock_flag(sk, SOCK_DONE) &&
  4262. tcp_receive_window(tp) > inet_csk(sk)->icsk_ack.rcv_mss)
  4263. return;
  4264. sk->sk_data_ready(sk);
  4265. }
  4266. static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
  4267. {
  4268. struct tcp_sock *tp = tcp_sk(sk);
  4269. bool fragstolen;
  4270. int eaten;
  4271. if (sk_is_mptcp(sk))
  4272. mptcp_incoming_options(sk, skb);
  4273. if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
  4274. __kfree_skb(skb);
  4275. return;
  4276. }
  4277. skb_dst_drop(skb);
  4278. __skb_pull(skb, tcp_hdr(skb)->doff * 4);
  4279. tp->rx_opt.dsack = 0;
  4280. /* Queue data for delivery to the user.
  4281. * Packets in sequence go to the receive queue.
  4282. * Out of sequence packets to the out_of_order_queue.
  4283. */
  4284. if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
  4285. if (tcp_receive_window(tp) == 0) {
  4286. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
  4287. goto out_of_window;
  4288. }
  4289. /* Ok. In sequence. In window. */
  4290. queue_and_out:
  4291. if (skb_queue_len(&sk->sk_receive_queue) == 0)
  4292. sk_forced_mem_schedule(sk, skb->truesize);
  4293. else if (tcp_try_rmem_schedule(sk, skb, skb->truesize)) {
  4294. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
  4295. sk->sk_data_ready(sk);
  4296. goto drop;
  4297. }
  4298. eaten = tcp_queue_rcv(sk, skb, &fragstolen);
  4299. if (skb->len)
  4300. tcp_event_data_recv(sk, skb);
  4301. if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
  4302. tcp_fin(sk);
  4303. if (!RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
  4304. tcp_ofo_queue(sk);
  4305. /* RFC5681. 4.2. SHOULD send immediate ACK, when
  4306. * gap in queue is filled.
  4307. */
  4308. if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
  4309. inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
  4310. }
  4311. if (tp->rx_opt.num_sacks)
  4312. tcp_sack_remove(tp);
  4313. tcp_fast_path_check(sk);
  4314. if (eaten > 0)
  4315. kfree_skb_partial(skb, fragstolen);
  4316. if (!sock_flag(sk, SOCK_DEAD))
  4317. tcp_data_ready(sk);
  4318. return;
  4319. }
  4320. if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
  4321. tcp_rcv_spurious_retrans(sk, skb);
  4322. /* A retransmit, 2nd most common case. Force an immediate ack. */
  4323. NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
  4324. tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
  4325. out_of_window:
  4326. tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
  4327. inet_csk_schedule_ack(sk);
  4328. drop:
  4329. tcp_drop(sk, skb);
  4330. return;
  4331. }
  4332. /* Out of window. F.e. zero window probe. */
  4333. if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt + tcp_receive_window(tp)))
  4334. goto out_of_window;
  4335. if (before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
  4336. /* Partial packet, seq < rcv_next < end_seq */
  4337. tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, tp->rcv_nxt);
  4338. /* If window is closed, drop tail of packet. But after
  4339. * remembering D-SACK for its head made in previous line.
  4340. */
  4341. if (!tcp_receive_window(tp)) {
  4342. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
  4343. goto out_of_window;
  4344. }
  4345. goto queue_and_out;
  4346. }
  4347. tcp_data_queue_ofo(sk, skb);
  4348. }
  4349. static struct sk_buff *tcp_skb_next(struct sk_buff *skb, struct sk_buff_head *list)
  4350. {
  4351. if (list)
  4352. return !skb_queue_is_last(list, skb) ? skb->next : NULL;
  4353. return skb_rb_next(skb);
  4354. }
  4355. static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb,
  4356. struct sk_buff_head *list,
  4357. struct rb_root *root)
  4358. {
  4359. struct sk_buff *next = tcp_skb_next(skb, list);
  4360. if (list)
  4361. __skb_unlink(skb, list);
  4362. else
  4363. rb_erase(&skb->rbnode, root);
  4364. __kfree_skb(skb);
  4365. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED);
  4366. return next;
  4367. }
  4368. /* Insert skb into rb tree, ordered by TCP_SKB_CB(skb)->seq */
  4369. void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb)
  4370. {
  4371. struct rb_node **p = &root->rb_node;
  4372. struct rb_node *parent = NULL;
  4373. struct sk_buff *skb1;
  4374. while (*p) {
  4375. parent = *p;
  4376. skb1 = rb_to_skb(parent);
  4377. if (before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb1)->seq))
  4378. p = &parent->rb_left;
  4379. else
  4380. p = &parent->rb_right;
  4381. }
  4382. rb_link_node(&skb->rbnode, parent, p);
  4383. rb_insert_color(&skb->rbnode, root);
  4384. }
  4385. /* Collapse contiguous sequence of skbs head..tail with
  4386. * sequence numbers start..end.
  4387. *
  4388. * If tail is NULL, this means until the end of the queue.
  4389. *
  4390. * Segments with FIN/SYN are not collapsed (only because this
  4391. * simplifies code)
  4392. */
  4393. static void
  4394. tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
  4395. struct sk_buff *head, struct sk_buff *tail, u32 start, u32 end)
  4396. {
  4397. struct sk_buff *skb = head, *n;
  4398. struct sk_buff_head tmp;
  4399. bool end_of_skbs;
  4400. /* First, check that queue is collapsible and find
  4401. * the point where collapsing can be useful.
  4402. */
  4403. restart:
  4404. for (end_of_skbs = true; skb != NULL && skb != tail; skb = n) {
  4405. n = tcp_skb_next(skb, list);
  4406. /* No new bits? It is possible on ofo queue. */
  4407. if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
  4408. skb = tcp_collapse_one(sk, skb, list, root);
  4409. if (!skb)
  4410. break;
  4411. goto restart;
  4412. }
  4413. /* The first skb to collapse is:
  4414. * - not SYN/FIN and
  4415. * - bloated or contains data before "start" or
  4416. * overlaps to the next one and mptcp allow collapsing.
  4417. */
  4418. if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
  4419. (tcp_win_from_space(sk, skb->truesize) > skb->len ||
  4420. before(TCP_SKB_CB(skb)->seq, start))) {
  4421. end_of_skbs = false;
  4422. break;
  4423. }
  4424. if (n && n != tail && mptcp_skb_can_collapse(skb, n) &&
  4425. TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(n)->seq) {
  4426. end_of_skbs = false;
  4427. break;
  4428. }
  4429. /* Decided to skip this, advance start seq. */
  4430. start = TCP_SKB_CB(skb)->end_seq;
  4431. }
  4432. if (end_of_skbs ||
  4433. (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
  4434. return;
  4435. __skb_queue_head_init(&tmp);
  4436. while (before(start, end)) {
  4437. int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start);
  4438. struct sk_buff *nskb;
  4439. nskb = alloc_skb(copy, GFP_ATOMIC);
  4440. if (!nskb)
  4441. break;
  4442. memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
  4443. #ifdef CONFIG_TLS_DEVICE
  4444. nskb->decrypted = skb->decrypted;
  4445. #endif
  4446. TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
  4447. if (list)
  4448. __skb_queue_before(list, skb, nskb);
  4449. else
  4450. __skb_queue_tail(&tmp, nskb); /* defer rbtree insertion */
  4451. skb_set_owner_r(nskb, sk);
  4452. mptcp_skb_ext_move(nskb, skb);
  4453. /* Copy data, releasing collapsed skbs. */
  4454. while (copy > 0) {
  4455. int offset = start - TCP_SKB_CB(skb)->seq;
  4456. int size = TCP_SKB_CB(skb)->end_seq - start;
  4457. BUG_ON(offset < 0);
  4458. if (size > 0) {
  4459. size = min(copy, size);
  4460. if (skb_copy_bits(skb, offset, skb_put(nskb, size), size))
  4461. BUG();
  4462. TCP_SKB_CB(nskb)->end_seq += size;
  4463. copy -= size;
  4464. start += size;
  4465. }
  4466. if (!before(start, TCP_SKB_CB(skb)->end_seq)) {
  4467. skb = tcp_collapse_one(sk, skb, list, root);
  4468. if (!skb ||
  4469. skb == tail ||
  4470. !mptcp_skb_can_collapse(nskb, skb) ||
  4471. (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
  4472. goto end;
  4473. #ifdef CONFIG_TLS_DEVICE
  4474. if (skb->decrypted != nskb->decrypted)
  4475. goto end;
  4476. #endif
  4477. }
  4478. }
  4479. }
  4480. end:
  4481. skb_queue_walk_safe(&tmp, skb, n)
  4482. tcp_rbtree_insert(root, skb);
  4483. }
  4484. /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
  4485. * and tcp_collapse() them until all the queue is collapsed.
  4486. */
  4487. static void tcp_collapse_ofo_queue(struct sock *sk)
  4488. {
  4489. struct tcp_sock *tp = tcp_sk(sk);
  4490. u32 range_truesize, sum_tiny = 0;
  4491. struct sk_buff *skb, *head;
  4492. u32 start, end;
  4493. skb = skb_rb_first(&tp->out_of_order_queue);
  4494. new_range:
  4495. if (!skb) {
  4496. tp->ooo_last_skb = skb_rb_last(&tp->out_of_order_queue);
  4497. return;
  4498. }
  4499. start = TCP_SKB_CB(skb)->seq;
  4500. end = TCP_SKB_CB(skb)->end_seq;
  4501. range_truesize = skb->truesize;
  4502. for (head = skb;;) {
  4503. skb = skb_rb_next(skb);
  4504. /* Range is terminated when we see a gap or when
  4505. * we are at the queue end.
  4506. */
  4507. if (!skb ||
  4508. after(TCP_SKB_CB(skb)->seq, end) ||
  4509. before(TCP_SKB_CB(skb)->end_seq, start)) {
  4510. /* Do not attempt collapsing tiny skbs */
  4511. if (range_truesize != head->truesize ||
  4512. end - start >= SKB_WITH_OVERHEAD(SK_MEM_QUANTUM)) {
  4513. tcp_collapse(sk, NULL, &tp->out_of_order_queue,
  4514. head, skb, start, end);
  4515. } else {
  4516. sum_tiny += range_truesize;
  4517. if (sum_tiny > sk->sk_rcvbuf >> 3)
  4518. return;
  4519. }
  4520. goto new_range;
  4521. }
  4522. range_truesize += skb->truesize;
  4523. if (unlikely(before(TCP_SKB_CB(skb)->seq, start)))
  4524. start = TCP_SKB_CB(skb)->seq;
  4525. if (after(TCP_SKB_CB(skb)->end_seq, end))
  4526. end = TCP_SKB_CB(skb)->end_seq;
  4527. }
  4528. }
  4529. /*
  4530. * Clean the out-of-order queue to make room.
  4531. * We drop high sequences packets to :
  4532. * 1) Let a chance for holes to be filled.
  4533. * 2) not add too big latencies if thousands of packets sit there.
  4534. * (But if application shrinks SO_RCVBUF, we could still end up
  4535. * freeing whole queue here)
  4536. * 3) Drop at least 12.5 % of sk_rcvbuf to avoid malicious attacks.
  4537. *
  4538. * Return true if queue has shrunk.
  4539. */
  4540. static bool tcp_prune_ofo_queue(struct sock *sk)
  4541. {
  4542. struct tcp_sock *tp = tcp_sk(sk);
  4543. struct rb_node *node, *prev;
  4544. int goal;
  4545. if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
  4546. return false;
  4547. NET_INC_STATS(sock_net(sk), LINUX_MIB_OFOPRUNED);
  4548. goal = sk->sk_rcvbuf >> 3;
  4549. node = &tp->ooo_last_skb->rbnode;
  4550. do {
  4551. prev = rb_prev(node);
  4552. rb_erase(node, &tp->out_of_order_queue);
  4553. goal -= rb_to_skb(node)->truesize;
  4554. tcp_drop(sk, rb_to_skb(node));
  4555. if (!prev || goal <= 0) {
  4556. sk_mem_reclaim(sk);
  4557. if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf &&
  4558. !tcp_under_memory_pressure(sk))
  4559. break;
  4560. goal = sk->sk_rcvbuf >> 3;
  4561. }
  4562. node = prev;
  4563. } while (node);
  4564. tp->ooo_last_skb = rb_to_skb(prev);
  4565. /* Reset SACK state. A conforming SACK implementation will
  4566. * do the same at a timeout based retransmit. When a connection
  4567. * is in a sad state like this, we care only about integrity
  4568. * of the connection not performance.
  4569. */
  4570. if (tp->rx_opt.sack_ok)
  4571. tcp_sack_reset(&tp->rx_opt);
  4572. return true;
  4573. }
  4574. /* Reduce allocated memory if we can, trying to get
  4575. * the socket within its memory limits again.
  4576. *
  4577. * Return less than zero if we should start dropping frames
  4578. * until the socket owning process reads some of the data
  4579. * to stabilize the situation.
  4580. */
  4581. static int tcp_prune_queue(struct sock *sk)
  4582. {
  4583. struct tcp_sock *tp = tcp_sk(sk);
  4584. NET_INC_STATS(sock_net(sk), LINUX_MIB_PRUNECALLED);
  4585. if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
  4586. tcp_clamp_window(sk);
  4587. else if (tcp_under_memory_pressure(sk))
  4588. tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss);
  4589. if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
  4590. return 0;
  4591. tcp_collapse_ofo_queue(sk);
  4592. if (!skb_queue_empty(&sk->sk_receive_queue))
  4593. tcp_collapse(sk, &sk->sk_receive_queue, NULL,
  4594. skb_peek(&sk->sk_receive_queue),
  4595. NULL,
  4596. tp->copied_seq, tp->rcv_nxt);
  4597. sk_mem_reclaim(sk);
  4598. if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
  4599. return 0;
  4600. /* Collapsing did not help, destructive actions follow.
  4601. * This must not ever occur. */
  4602. tcp_prune_ofo_queue(sk);
  4603. if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
  4604. return 0;
  4605. /* If we are really being abused, tell the caller to silently
  4606. * drop receive data on the floor. It will get retransmitted
  4607. * and hopefully then we'll have sufficient space.
  4608. */
  4609. NET_INC_STATS(sock_net(sk), LINUX_MIB_RCVPRUNED);
  4610. /* Massive buffer overcommit. */
  4611. tp->pred_flags = 0;
  4612. return -1;
  4613. }
  4614. static bool tcp_should_expand_sndbuf(const struct sock *sk)
  4615. {
  4616. const struct tcp_sock *tp = tcp_sk(sk);
  4617. /* If the user specified a specific send buffer setting, do
  4618. * not modify it.
  4619. */
  4620. if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
  4621. return false;
  4622. /* If we are under global TCP memory pressure, do not expand. */
  4623. if (tcp_under_memory_pressure(sk))
  4624. return false;
  4625. /* If we are under soft global TCP memory pressure, do not expand. */
  4626. if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, 0))
  4627. return false;
  4628. /* If we filled the congestion window, do not expand. */
  4629. if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
  4630. return false;
  4631. return true;
  4632. }
  4633. static void tcp_new_space(struct sock *sk)
  4634. {
  4635. struct tcp_sock *tp = tcp_sk(sk);
  4636. if (tcp_should_expand_sndbuf(sk)) {
  4637. tcp_sndbuf_expand(sk);
  4638. tp->snd_cwnd_stamp = tcp_jiffies32;
  4639. }
  4640. sk->sk_write_space(sk);
  4641. }
  4642. static void tcp_check_space(struct sock *sk)
  4643. {
  4644. /* pairs with tcp_poll() */
  4645. smp_mb();
  4646. if (sk->sk_socket &&
  4647. test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
  4648. tcp_new_space(sk);
  4649. if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
  4650. tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
  4651. }
  4652. }
  4653. static inline void tcp_data_snd_check(struct sock *sk)
  4654. {
  4655. tcp_push_pending_frames(sk);
  4656. tcp_check_space(sk);
  4657. }
  4658. /*
  4659. * Check if sending an ack is needed.
  4660. */
  4661. static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
  4662. {
  4663. struct tcp_sock *tp = tcp_sk(sk);
  4664. unsigned long rtt, delay;
  4665. /* More than one full frame received... */
  4666. if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
  4667. /* ... and right edge of window advances far enough.
  4668. * (tcp_recvmsg() will send ACK otherwise).
  4669. * If application uses SO_RCVLOWAT, we want send ack now if
  4670. * we have not received enough bytes to satisfy the condition.
  4671. */
  4672. (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
  4673. __tcp_select_window(sk) >= tp->rcv_wnd)) ||
  4674. /* We ACK each frame or... */
  4675. tcp_in_quickack_mode(sk) ||
  4676. /* Protocol state mandates a one-time immediate ACK */
  4677. inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOW) {
  4678. send_now:
  4679. tcp_send_ack(sk);
  4680. return;
  4681. }
  4682. if (!ofo_possible || RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
  4683. tcp_send_delayed_ack(sk);
  4684. return;
  4685. }
  4686. if (!tcp_is_sack(tp) ||
  4687. tp->compressed_ack >= sock_net(sk)->ipv4.sysctl_tcp_comp_sack_nr)
  4688. goto send_now;
  4689. if (tp->compressed_ack_rcv_nxt != tp->rcv_nxt) {
  4690. tp->compressed_ack_rcv_nxt = tp->rcv_nxt;
  4691. tp->dup_ack_counter = 0;
  4692. }
  4693. if (tp->dup_ack_counter < TCP_FASTRETRANS_THRESH) {
  4694. tp->dup_ack_counter++;
  4695. goto send_now;
  4696. }
  4697. tp->compressed_ack++;
  4698. if (hrtimer_is_queued(&tp->compressed_ack_timer))
  4699. return;
  4700. /* compress ack timer : 5 % of rtt, but no more than tcp_comp_sack_delay_ns */
  4701. rtt = tp->rcv_rtt_est.rtt_us;
  4702. if (tp->srtt_us && tp->srtt_us < rtt)
  4703. rtt = tp->srtt_us;
  4704. delay = min_t(unsigned long, sock_net(sk)->ipv4.sysctl_tcp_comp_sack_delay_ns,
  4705. rtt * (NSEC_PER_USEC >> 3)/20);
  4706. sock_hold(sk);
  4707. hrtimer_start_range_ns(&tp->compressed_ack_timer, ns_to_ktime(delay),
  4708. sock_net(sk)->ipv4.sysctl_tcp_comp_sack_slack_ns,
  4709. HRTIMER_MODE_REL_PINNED_SOFT);
  4710. }
  4711. static inline void tcp_ack_snd_check(struct sock *sk)
  4712. {
  4713. if (!inet_csk_ack_scheduled(sk)) {
  4714. /* We sent a data segment already. */
  4715. return;
  4716. }
  4717. __tcp_ack_snd_check(sk, 1);
  4718. }
  4719. /*
  4720. * This routine is only called when we have urgent data
  4721. * signaled. Its the 'slow' part of tcp_urg. It could be
  4722. * moved inline now as tcp_urg is only called from one
  4723. * place. We handle URGent data wrong. We have to - as
  4724. * BSD still doesn't use the correction from RFC961.
  4725. * For 1003.1g we should support a new option TCP_STDURG to permit
  4726. * either form (or just set the sysctl tcp_stdurg).
  4727. */
  4728. static void tcp_check_urg(struct sock *sk, const struct tcphdr *th)
  4729. {
  4730. struct tcp_sock *tp = tcp_sk(sk);
  4731. u32 ptr = ntohs(th->urg_ptr);
  4732. if (ptr && !sock_net(sk)->ipv4.sysctl_tcp_stdurg)
  4733. ptr--;
  4734. ptr += ntohl(th->seq);
  4735. /* Ignore urgent data that we've already seen and read. */
  4736. if (after(tp->copied_seq, ptr))
  4737. return;
  4738. /* Do not replay urg ptr.
  4739. *
  4740. * NOTE: interesting situation not covered by specs.
  4741. * Misbehaving sender may send urg ptr, pointing to segment,
  4742. * which we already have in ofo queue. We are not able to fetch
  4743. * such data and will stay in TCP_URG_NOTYET until will be eaten
  4744. * by recvmsg(). Seems, we are not obliged to handle such wicked
  4745. * situations. But it is worth to think about possibility of some
  4746. * DoSes using some hypothetical application level deadlock.
  4747. */
  4748. if (before(ptr, tp->rcv_nxt))
  4749. return;
  4750. /* Do we already have a newer (or duplicate) urgent pointer? */
  4751. if (tp->urg_data && !after(ptr, tp->urg_seq))
  4752. return;
  4753. /* Tell the world about our new urgent pointer. */
  4754. sk_send_sigurg(sk);
  4755. /* We may be adding urgent data when the last byte read was
  4756. * urgent. To do this requires some care. We cannot just ignore
  4757. * tp->copied_seq since we would read the last urgent byte again
  4758. * as data, nor can we alter copied_seq until this data arrives
  4759. * or we break the semantics of SIOCATMARK (and thus sockatmark())
  4760. *
  4761. * NOTE. Double Dutch. Rendering to plain English: author of comment
  4762. * above did something sort of send("A", MSG_OOB); send("B", MSG_OOB);
  4763. * and expect that both A and B disappear from stream. This is _wrong_.
  4764. * Though this happens in BSD with high probability, this is occasional.
  4765. * Any application relying on this is buggy. Note also, that fix "works"
  4766. * only in this artificial test. Insert some normal data between A and B and we will
  4767. * decline of BSD again. Verdict: it is better to remove to trap
  4768. * buggy users.
  4769. */
  4770. if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
  4771. !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
  4772. struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
  4773. tp->copied_seq++;
  4774. if (skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) {
  4775. __skb_unlink(skb, &sk->sk_receive_queue);
  4776. __kfree_skb(skb);
  4777. }
  4778. }
  4779. tp->urg_data = TCP_URG_NOTYET;
  4780. WRITE_ONCE(tp->urg_seq, ptr);
  4781. /* Disable header prediction. */
  4782. tp->pred_flags = 0;
  4783. }
  4784. /* This is the 'fast' part of urgent handling. */
  4785. static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th)
  4786. {
  4787. struct tcp_sock *tp = tcp_sk(sk);
  4788. /* Check if we get a new urgent pointer - normally not. */
  4789. if (th->urg)
  4790. tcp_check_urg(sk, th);
  4791. /* Do we wait for any urgent data? - normally not... */
  4792. if (tp->urg_data == TCP_URG_NOTYET) {
  4793. u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
  4794. th->syn;
  4795. /* Is the urgent pointer pointing into this packet? */
  4796. if (ptr < skb->len) {
  4797. u8 tmp;
  4798. if (skb_copy_bits(skb, ptr, &tmp, 1))
  4799. BUG();
  4800. tp->urg_data = TCP_URG_VALID | tmp;
  4801. if (!sock_flag(sk, SOCK_DEAD))
  4802. sk->sk_data_ready(sk);
  4803. }
  4804. }
  4805. }
  4806. /* Accept RST for rcv_nxt - 1 after a FIN.
  4807. * When tcp connections are abruptly terminated from Mac OSX (via ^C), a
  4808. * FIN is sent followed by a RST packet. The RST is sent with the same
  4809. * sequence number as the FIN, and thus according to RFC 5961 a challenge
  4810. * ACK should be sent. However, Mac OSX rate limits replies to challenge
  4811. * ACKs on the closed socket. In addition middleboxes can drop either the
  4812. * challenge ACK or a subsequent RST.
  4813. */
  4814. static bool tcp_reset_check(const struct sock *sk, const struct sk_buff *skb)
  4815. {
  4816. struct tcp_sock *tp = tcp_sk(sk);
  4817. return unlikely(TCP_SKB_CB(skb)->seq == (tp->rcv_nxt - 1) &&
  4818. (1 << sk->sk_state) & (TCPF_CLOSE_WAIT | TCPF_LAST_ACK |
  4819. TCPF_CLOSING));
  4820. }
  4821. /* Does PAWS and seqno based validation of an incoming segment, flags will
  4822. * play significant role here.
  4823. */
  4824. static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
  4825. const struct tcphdr *th, int syn_inerr)
  4826. {
  4827. struct tcp_sock *tp = tcp_sk(sk);
  4828. bool rst_seq_match = false;
  4829. /* RFC1323: H1. Apply PAWS check first. */
  4830. if (tcp_fast_parse_options(sock_net(sk), skb, th, tp) &&
  4831. tp->rx_opt.saw_tstamp &&
  4832. tcp_paws_discard(sk, skb)) {
  4833. if (!th->rst) {
  4834. NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
  4835. if (!tcp_oow_rate_limited(sock_net(sk), skb,
  4836. LINUX_MIB_TCPACKSKIPPEDPAWS,
  4837. &tp->last_oow_ack_time))
  4838. tcp_send_dupack(sk, skb);
  4839. goto discard;
  4840. }
  4841. /* Reset is accepted even if it did not pass PAWS. */
  4842. }
  4843. /* Step 1: check sequence number */
  4844. if (!tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq)) {
  4845. /* RFC793, page 37: "In all states except SYN-SENT, all reset
  4846. * (RST) segments are validated by checking their SEQ-fields."
  4847. * And page 69: "If an incoming segment is not acceptable,
  4848. * an acknowledgment should be sent in reply (unless the RST
  4849. * bit is set, if so drop the segment and return)".
  4850. */
  4851. if (!th->rst) {
  4852. if (th->syn)
  4853. goto syn_challenge;
  4854. if (!tcp_oow_rate_limited(sock_net(sk), skb,
  4855. LINUX_MIB_TCPACKSKIPPEDSEQ,
  4856. &tp->last_oow_ack_time))
  4857. tcp_send_dupack(sk, skb);
  4858. } else if (tcp_reset_check(sk, skb)) {
  4859. tcp_reset(sk);
  4860. }
  4861. goto discard;
  4862. }
  4863. /* Step 2: check RST bit */
  4864. if (th->rst) {
  4865. /* RFC 5961 3.2 (extend to match against (RCV.NXT - 1) after a
  4866. * FIN and SACK too if available):
  4867. * If seq num matches RCV.NXT or (RCV.NXT - 1) after a FIN, or
  4868. * the right-most SACK block,
  4869. * then
  4870. * RESET the connection
  4871. * else
  4872. * Send a challenge ACK
  4873. */
  4874. if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt ||
  4875. tcp_reset_check(sk, skb)) {
  4876. rst_seq_match = true;
  4877. } else if (tcp_is_sack(tp) && tp->rx_opt.num_sacks > 0) {
  4878. struct tcp_sack_block *sp = &tp->selective_acks[0];
  4879. int max_sack = sp[0].end_seq;
  4880. int this_sack;
  4881. for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;
  4882. ++this_sack) {
  4883. max_sack = after(sp[this_sack].end_seq,
  4884. max_sack) ?
  4885. sp[this_sack].end_seq : max_sack;
  4886. }
  4887. if (TCP_SKB_CB(skb)->seq == max_sack)
  4888. rst_seq_match = true;
  4889. }
  4890. if (rst_seq_match)
  4891. tcp_reset(sk);
  4892. else {
  4893. /* Disable TFO if RST is out-of-order
  4894. * and no data has been received
  4895. * for current active TFO socket
  4896. */
  4897. if (tp->syn_fastopen && !tp->data_segs_in &&
  4898. sk->sk_state == TCP_ESTABLISHED)
  4899. tcp_fastopen_active_disable(sk);
  4900. tcp_send_challenge_ack(sk, skb);
  4901. }
  4902. goto discard;
  4903. }
  4904. /* step 3: check security and precedence [ignored] */
  4905. /* step 4: Check for a SYN
  4906. * RFC 5961 4.2 : Send a challenge ack
  4907. */
  4908. if (th->syn) {
  4909. syn_challenge:
  4910. if (syn_inerr)
  4911. TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
  4912. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE);
  4913. tcp_send_challenge_ack(sk, skb);
  4914. goto discard;
  4915. }
  4916. bpf_skops_parse_hdr(sk, skb);
  4917. return true;
  4918. discard:
  4919. tcp_drop(sk, skb);
  4920. return false;
  4921. }
  4922. /*
  4923. * TCP receive function for the ESTABLISHED state.
  4924. *
  4925. * It is split into a fast path and a slow path. The fast path is
  4926. * disabled when:
  4927. * - A zero window was announced from us - zero window probing
  4928. * is only handled properly in the slow path.
  4929. * - Out of order segments arrived.
  4930. * - Urgent data is expected.
  4931. * - There is no buffer space left
  4932. * - Unexpected TCP flags/window values/header lengths are received
  4933. * (detected by checking the TCP header against pred_flags)
  4934. * - Data is sent in both directions. Fast path only supports pure senders
  4935. * or pure receivers (this means either the sequence number or the ack
  4936. * value must stay constant)
  4937. * - Unexpected TCP option.
  4938. *
  4939. * When these conditions are not satisfied it drops into a standard
  4940. * receive procedure patterned after RFC793 to handle all cases.
  4941. * The first three cases are guaranteed by proper pred_flags setting,
  4942. * the rest is checked inline. Fast processing is turned on in
  4943. * tcp_data_queue when everything is OK.
  4944. */
  4945. void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
  4946. {
  4947. const struct tcphdr *th = (const struct tcphdr *)skb->data;
  4948. struct tcp_sock *tp = tcp_sk(sk);
  4949. unsigned int len = skb->len;
  4950. /* TCP congestion window tracking */
  4951. trace_tcp_probe(sk, skb);
  4952. tcp_mstamp_refresh(tp);
  4953. if (unlikely(!sk->sk_rx_dst))
  4954. inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
  4955. /*
  4956. * Header prediction.
  4957. * The code loosely follows the one in the famous
  4958. * "30 instruction TCP receive" Van Jacobson mail.
  4959. *
  4960. * Van's trick is to deposit buffers into socket queue
  4961. * on a device interrupt, to call tcp_recv function
  4962. * on the receive process context and checksum and copy
  4963. * the buffer to user space. smart...
  4964. *
  4965. * Our current scheme is not silly either but we take the
  4966. * extra cost of the net_bh soft interrupt processing...
  4967. * We do checksum and copy also but from device to kernel.
  4968. */
  4969. tp->rx_opt.saw_tstamp = 0;
  4970. /* pred_flags is 0xS?10 << 16 + snd_wnd
  4971. * if header_prediction is to be made
  4972. * 'S' will always be tp->tcp_header_len >> 2
  4973. * '?' will be 0 for the fast path, otherwise pred_flags is 0 to
  4974. * turn it off (when there are holes in the receive
  4975. * space for instance)
  4976. * PSH flag is ignored.
  4977. */
  4978. if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags &&
  4979. TCP_SKB_CB(skb)->seq == tp->rcv_nxt &&
  4980. !after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
  4981. int tcp_header_len = tp->tcp_header_len;
  4982. /* Timestamp header prediction: tcp_header_len
  4983. * is automatically equal to th->doff*4 due to pred_flags
  4984. * match.
  4985. */
  4986. /* Check timestamp */
  4987. if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) {
  4988. /* No? Slow path! */
  4989. if (!tcp_parse_aligned_timestamp(tp, th))
  4990. goto slow_path;
  4991. /* If PAWS failed, check it more carefully in slow path */
  4992. if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0)
  4993. goto slow_path;
  4994. /* DO NOT update ts_recent here, if checksum fails
  4995. * and timestamp was corrupted part, it will result
  4996. * in a hung connection since we will drop all
  4997. * future packets due to the PAWS test.
  4998. */
  4999. }
  5000. if (len <= tcp_header_len) {
  5001. /* Bulk data transfer: sender */
  5002. if (len == tcp_header_len) {
  5003. /* Predicted packet is in window by definition.
  5004. * seq == rcv_nxt and rcv_wup <= rcv_nxt.
  5005. * Hence, check seq<=rcv_wup reduces to:
  5006. */
  5007. if (tcp_header_len ==
  5008. (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
  5009. tp->rcv_nxt == tp->rcv_wup)
  5010. tcp_store_ts_recent(tp);
  5011. /* We know that such packets are checksummed
  5012. * on entry.
  5013. */
  5014. tcp_ack(sk, skb, 0);
  5015. __kfree_skb(skb);
  5016. tcp_data_snd_check(sk);
  5017. /* When receiving pure ack in fast path, update
  5018. * last ts ecr directly instead of calling
  5019. * tcp_rcv_rtt_measure_ts()
  5020. */
  5021. tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
  5022. return;
  5023. } else { /* Header too small */
  5024. TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
  5025. goto discard;
  5026. }
  5027. } else {
  5028. int eaten = 0;
  5029. bool fragstolen = false;
  5030. if (tcp_checksum_complete(skb))
  5031. goto csum_error;
  5032. if ((int)skb->truesize > sk->sk_forward_alloc)
  5033. goto step5;
  5034. /* Predicted packet is in window by definition.
  5035. * seq == rcv_nxt and rcv_wup <= rcv_nxt.
  5036. * Hence, check seq<=rcv_wup reduces to:
  5037. */
  5038. if (tcp_header_len ==
  5039. (sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
  5040. tp->rcv_nxt == tp->rcv_wup)
  5041. tcp_store_ts_recent(tp);
  5042. tcp_rcv_rtt_measure_ts(sk, skb);
  5043. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPHITS);
  5044. /* Bulk data transfer: receiver */
  5045. __skb_pull(skb, tcp_header_len);
  5046. eaten = tcp_queue_rcv(sk, skb, &fragstolen);
  5047. tcp_event_data_recv(sk, skb);
  5048. if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) {
  5049. /* Well, only one small jumplet in fast path... */
  5050. tcp_ack(sk, skb, FLAG_DATA);
  5051. tcp_data_snd_check(sk);
  5052. if (!inet_csk_ack_scheduled(sk))
  5053. goto no_ack;
  5054. } else {
  5055. tcp_update_wl(tp, TCP_SKB_CB(skb)->seq);
  5056. }
  5057. __tcp_ack_snd_check(sk, 0);
  5058. no_ack:
  5059. if (eaten)
  5060. kfree_skb_partial(skb, fragstolen);
  5061. tcp_data_ready(sk);
  5062. return;
  5063. }
  5064. }
  5065. slow_path:
  5066. if (len < (th->doff << 2) || tcp_checksum_complete(skb))
  5067. goto csum_error;
  5068. if (!th->ack && !th->rst && !th->syn)
  5069. goto discard;
  5070. /*
  5071. * Standard slow path.
  5072. */
  5073. if (!tcp_validate_incoming(sk, skb, th, 1))
  5074. return;
  5075. step5:
  5076. if (tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT) < 0)
  5077. goto discard;
  5078. tcp_rcv_rtt_measure_ts(sk, skb);
  5079. /* Process urgent data. */
  5080. tcp_urg(sk, skb, th);
  5081. /* step 7: process the segment text */
  5082. tcp_data_queue(sk, skb);
  5083. tcp_data_snd_check(sk);
  5084. tcp_ack_snd_check(sk);
  5085. return;
  5086. csum_error:
  5087. TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
  5088. TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
  5089. discard:
  5090. tcp_drop(sk, skb);
  5091. }
  5092. EXPORT_SYMBOL(tcp_rcv_established);
  5093. void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb)
  5094. {
  5095. struct inet_connection_sock *icsk = inet_csk(sk);
  5096. struct tcp_sock *tp = tcp_sk(sk);
  5097. tcp_mtup_init(sk);
  5098. icsk->icsk_af_ops->rebuild_header(sk);
  5099. tcp_init_metrics(sk);
  5100. /* Initialize the congestion window to start the transfer.
  5101. * Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
  5102. * retransmitted. In light of RFC6298 more aggressive 1sec
  5103. * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
  5104. * retransmission has occurred.
  5105. */
  5106. if (tp->total_retrans > 1 && tp->undo_marker)
  5107. tp->snd_cwnd = 1;
  5108. else
  5109. tp->snd_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
  5110. tp->snd_cwnd_stamp = tcp_jiffies32;
  5111. bpf_skops_established(sk, bpf_op, skb);
  5112. /* Initialize congestion control unless BPF initialized it already: */
  5113. if (!icsk->icsk_ca_initialized)
  5114. tcp_init_congestion_control(sk);
  5115. tcp_init_buffer_space(sk);
  5116. }
  5117. void tcp_finish_connect(struct sock *sk, struct sk_buff *skb)
  5118. {
  5119. struct tcp_sock *tp = tcp_sk(sk);
  5120. struct inet_connection_sock *icsk = inet_csk(sk);
  5121. tcp_set_state(sk, TCP_ESTABLISHED);
  5122. icsk->icsk_ack.lrcvtime = tcp_jiffies32;
  5123. if (skb) {
  5124. icsk->icsk_af_ops->sk_rx_dst_set(sk, skb);
  5125. security_inet_conn_established(sk, skb);
  5126. sk_mark_napi_id(sk, skb);
  5127. }
  5128. tcp_init_transfer(sk, BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, skb);
  5129. /* Prevent spurious tcp_cwnd_restart() on first data
  5130. * packet.
  5131. */
  5132. tp->lsndtime = tcp_jiffies32;
  5133. if (sock_flag(sk, SOCK_KEEPOPEN))
  5134. inet_csk_reset_keepalive_timer(sk, keepalive_time_when(tp));
  5135. if (!tp->rx_opt.snd_wscale)
  5136. __tcp_fast_path_on(tp, tp->snd_wnd);
  5137. else
  5138. tp->pred_flags = 0;
  5139. }
  5140. static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack,
  5141. struct tcp_fastopen_cookie *cookie)
  5142. {
  5143. struct tcp_sock *tp = tcp_sk(sk);
  5144. struct sk_buff *data = tp->syn_data ? tcp_rtx_queue_head(sk) : NULL;
  5145. u16 mss = tp->rx_opt.mss_clamp, try_exp = 0;
  5146. bool syn_drop = false;
  5147. if (mss == tp->rx_opt.user_mss) {
  5148. struct tcp_options_received opt;
  5149. /* Get original SYNACK MSS value if user MSS sets mss_clamp */
  5150. tcp_clear_options(&opt);
  5151. opt.user_mss = opt.mss_clamp = 0;
  5152. tcp_parse_options(sock_net(sk), synack, &opt, 0, NULL);
  5153. mss = opt.mss_clamp;
  5154. }
  5155. if (!tp->syn_fastopen) {
  5156. /* Ignore an unsolicited cookie */
  5157. cookie->len = -1;
  5158. } else if (tp->total_retrans) {
  5159. /* SYN timed out and the SYN-ACK neither has a cookie nor
  5160. * acknowledges data. Presumably the remote received only
  5161. * the retransmitted (regular) SYNs: either the original
  5162. * SYN-data or the corresponding SYN-ACK was dropped.
  5163. */
  5164. syn_drop = (cookie->len < 0 && data);
  5165. } else if (cookie->len < 0 && !tp->syn_data) {
  5166. /* We requested a cookie but didn't get it. If we did not use
  5167. * the (old) exp opt format then try so next time (try_exp=1).
  5168. * Otherwise we go back to use the RFC7413 opt (try_exp=2).
  5169. */
  5170. try_exp = tp->syn_fastopen_exp ? 2 : 1;
  5171. }
  5172. tcp_fastopen_cache_set(sk, mss, cookie, syn_drop, try_exp);
  5173. if (data) { /* Retransmit unacked data in SYN */
  5174. if (tp->total_retrans)
  5175. tp->fastopen_client_fail = TFO_SYN_RETRANSMITTED;
  5176. else
  5177. tp->fastopen_client_fail = TFO_DATA_NOT_ACKED;
  5178. skb_rbtree_walk_from(data) {
  5179. if (__tcp_retransmit_skb(sk, data, 1))
  5180. break;
  5181. }
  5182. tcp_rearm_rto(sk);
  5183. NET_INC_STATS(sock_net(sk),
  5184. LINUX_MIB_TCPFASTOPENACTIVEFAIL);
  5185. return true;
  5186. }
  5187. tp->syn_data_acked = tp->syn_data;
  5188. if (tp->syn_data_acked) {
  5189. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
  5190. /* SYN-data is counted as two separate packets in tcp_ack() */
  5191. if (tp->delivered > 1)
  5192. --tp->delivered;
  5193. }
  5194. tcp_fastopen_add_skb(sk, synack);
  5195. return false;
  5196. }
  5197. static void smc_check_reset_syn(struct tcp_sock *tp)
  5198. {
  5199. #if IS_ENABLED(CONFIG_SMC)
  5200. if (static_branch_unlikely(&tcp_have_smc)) {
  5201. if (tp->syn_smc && !tp->rx_opt.smc_ok)
  5202. tp->syn_smc = 0;
  5203. }
  5204. #endif
  5205. }
  5206. static void tcp_try_undo_spurious_syn(struct sock *sk)
  5207. {
  5208. struct tcp_sock *tp = tcp_sk(sk);
  5209. u32 syn_stamp;
  5210. /* undo_marker is set when SYN or SYNACK times out. The timeout is
  5211. * spurious if the ACK's timestamp option echo value matches the
  5212. * original SYN timestamp.
  5213. */
  5214. syn_stamp = tp->retrans_stamp;
  5215. if (tp->undo_marker && syn_stamp && tp->rx_opt.saw_tstamp &&
  5216. syn_stamp == tp->rx_opt.rcv_tsecr)
  5217. tp->undo_marker = 0;
  5218. }
  5219. static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
  5220. const struct tcphdr *th)
  5221. {
  5222. struct inet_connection_sock *icsk = inet_csk(sk);
  5223. struct tcp_sock *tp = tcp_sk(sk);
  5224. struct tcp_fastopen_cookie foc = { .len = -1 };
  5225. int saved_clamp = tp->rx_opt.mss_clamp;
  5226. bool fastopen_fail;
  5227. tcp_parse_options(sock_net(sk), skb, &tp->rx_opt, 0, &foc);
  5228. if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
  5229. tp->rx_opt.rcv_tsecr -= tp->tsoffset;
  5230. if (th->ack) {
  5231. /* rfc793:
  5232. * "If the state is SYN-SENT then
  5233. * first check the ACK bit
  5234. * If the ACK bit is set
  5235. * If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
  5236. * a reset (unless the RST bit is set, if so drop
  5237. * the segment and return)"
  5238. */
  5239. if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) ||
  5240. after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
  5241. /* Previous FIN/ACK or RST/ACK might be ignored. */
  5242. if (icsk->icsk_retransmits == 0)
  5243. inet_csk_reset_xmit_timer(sk,
  5244. ICSK_TIME_RETRANS,
  5245. TCP_TIMEOUT_MIN, TCP_RTO_MAX);
  5246. goto reset_and_undo;
  5247. }
  5248. if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
  5249. !between(tp->rx_opt.rcv_tsecr, tp->retrans_stamp,
  5250. tcp_time_stamp(tp))) {
  5251. NET_INC_STATS(sock_net(sk),
  5252. LINUX_MIB_PAWSACTIVEREJECTED);
  5253. goto reset_and_undo;
  5254. }
  5255. /* Now ACK is acceptable.
  5256. *
  5257. * "If the RST bit is set
  5258. * If the ACK was acceptable then signal the user "error:
  5259. * connection reset", drop the segment, enter CLOSED state,
  5260. * delete TCB, and return."
  5261. */
  5262. if (th->rst) {
  5263. tcp_reset(sk);
  5264. goto discard;
  5265. }
  5266. /* rfc793:
  5267. * "fifth, if neither of the SYN or RST bits is set then
  5268. * drop the segment and return."
  5269. *
  5270. * See note below!
  5271. * --ANK(990513)
  5272. */
  5273. if (!th->syn)
  5274. goto discard_and_undo;
  5275. /* rfc793:
  5276. * "If the SYN bit is on ...
  5277. * are acceptable then ...
  5278. * (our SYN has been ACKed), change the connection
  5279. * state to ESTABLISHED..."
  5280. */
  5281. tcp_ecn_rcv_synack(tp, th);
  5282. tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
  5283. tcp_try_undo_spurious_syn(sk);
  5284. tcp_ack(sk, skb, FLAG_SLOWPATH);
  5285. /* Ok.. it's good. Set up sequence numbers and
  5286. * move to established.
  5287. */
  5288. WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
  5289. tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
  5290. /* RFC1323: The window in SYN & SYN/ACK segments is
  5291. * never scaled.
  5292. */
  5293. tp->snd_wnd = ntohs(th->window);
  5294. if (!tp->rx_opt.wscale_ok) {
  5295. tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0;
  5296. tp->window_clamp = min(tp->window_clamp, 65535U);
  5297. }
  5298. if (tp->rx_opt.saw_tstamp) {
  5299. tp->rx_opt.tstamp_ok = 1;
  5300. tp->tcp_header_len =
  5301. sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
  5302. tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
  5303. tcp_store_ts_recent(tp);
  5304. } else {
  5305. tp->tcp_header_len = sizeof(struct tcphdr);
  5306. }
  5307. tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
  5308. tcp_initialize_rcv_mss(sk);
  5309. /* Remember, tcp_poll() does not lock socket!
  5310. * Change state from SYN-SENT only after copied_seq
  5311. * is initialized. */
  5312. WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
  5313. smc_check_reset_syn(tp);
  5314. smp_mb();
  5315. tcp_finish_connect(sk, skb);
  5316. fastopen_fail = (tp->syn_fastopen || tp->syn_data) &&
  5317. tcp_rcv_fastopen_synack(sk, skb, &foc);
  5318. if (!sock_flag(sk, SOCK_DEAD)) {
  5319. sk->sk_state_change(sk);
  5320. sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
  5321. }
  5322. if (fastopen_fail)
  5323. return -1;
  5324. if (sk->sk_write_pending ||
  5325. icsk->icsk_accept_queue.rskq_defer_accept ||
  5326. inet_csk_in_pingpong_mode(sk)) {
  5327. /* Save one ACK. Data will be ready after
  5328. * several ticks, if write_pending is set.
  5329. *
  5330. * It may be deleted, but with this feature tcpdumps
  5331. * look so _wonderfully_ clever, that I was not able
  5332. * to stand against the temptation 8) --ANK
  5333. */
  5334. inet_csk_schedule_ack(sk);
  5335. tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
  5336. inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
  5337. TCP_DELACK_MAX, TCP_RTO_MAX);
  5338. discard:
  5339. tcp_drop(sk, skb);
  5340. return 0;
  5341. } else {
  5342. tcp_send_ack(sk);
  5343. }
  5344. return -1;
  5345. }
  5346. /* No ACK in the segment */
  5347. if (th->rst) {
  5348. /* rfc793:
  5349. * "If the RST bit is set
  5350. *
  5351. * Otherwise (no ACK) drop the segment and return."
  5352. */
  5353. goto discard_and_undo;
  5354. }
  5355. /* PAWS check. */
  5356. if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp &&
  5357. tcp_paws_reject(&tp->rx_opt, 0))
  5358. goto discard_and_undo;
  5359. if (th->syn) {
  5360. /* We see SYN without ACK. It is attempt of
  5361. * simultaneous connect with crossed SYNs.
  5362. * Particularly, it can be connect to self.
  5363. */
  5364. tcp_set_state(sk, TCP_SYN_RECV);
  5365. if (tp->rx_opt.saw_tstamp) {
  5366. tp->rx_opt.tstamp_ok = 1;
  5367. tcp_store_ts_recent(tp);
  5368. tp->tcp_header_len =
  5369. sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
  5370. } else {
  5371. tp->tcp_header_len = sizeof(struct tcphdr);
  5372. }
  5373. WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
  5374. WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
  5375. tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
  5376. /* RFC1323: The window in SYN & SYN/ACK segments is
  5377. * never scaled.
  5378. */
  5379. tp->snd_wnd = ntohs(th->window);
  5380. tp->snd_wl1 = TCP_SKB_CB(skb)->seq;
  5381. tp->max_window = tp->snd_wnd;
  5382. tcp_ecn_rcv_syn(tp, th);
  5383. tcp_mtup_init(sk);
  5384. tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
  5385. tcp_initialize_rcv_mss(sk);
  5386. tcp_send_synack(sk);
  5387. #if 0
  5388. /* Note, we could accept data and URG from this segment.
  5389. * There are no obstacles to make this (except that we must
  5390. * either change tcp_recvmsg() to prevent it from returning data
  5391. * before 3WHS completes per RFC793, or employ TCP Fast Open).
  5392. *
  5393. * However, if we ignore data in ACKless segments sometimes,
  5394. * we have no reasons to accept it sometimes.
  5395. * Also, seems the code doing it in step6 of tcp_rcv_state_process
  5396. * is not flawless. So, discard packet for sanity.
  5397. * Uncomment this return to process the data.
  5398. */
  5399. return -1;
  5400. #else
  5401. goto discard;
  5402. #endif
  5403. }
  5404. /* "fifth, if neither of the SYN or RST bits is set then
  5405. * drop the segment and return."
  5406. */
  5407. discard_and_undo:
  5408. tcp_clear_options(&tp->rx_opt);
  5409. tp->rx_opt.mss_clamp = saved_clamp;
  5410. goto discard;
  5411. reset_and_undo:
  5412. tcp_clear_options(&tp->rx_opt);
  5413. tp->rx_opt.mss_clamp = saved_clamp;
  5414. return 1;
  5415. }
  5416. static void tcp_rcv_synrecv_state_fastopen(struct sock *sk)
  5417. {
  5418. struct request_sock *req;
  5419. /* If we are still handling the SYNACK RTO, see if timestamp ECR allows
  5420. * undo. If peer SACKs triggered fast recovery, we can't undo here.
  5421. */
  5422. if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
  5423. tcp_try_undo_loss(sk, false);
  5424. /* Reset rtx states to prevent spurious retransmits_timed_out() */
  5425. tcp_sk(sk)->retrans_stamp = 0;
  5426. inet_csk(sk)->icsk_retransmits = 0;
  5427. /* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1,
  5428. * we no longer need req so release it.
  5429. */
  5430. req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
  5431. lockdep_sock_is_held(sk));
  5432. reqsk_fastopen_remove(sk, req, false);
  5433. /* Re-arm the timer because data may have been sent out.
  5434. * This is similar to the regular data transmission case
  5435. * when new data has just been ack'ed.
  5436. *
  5437. * (TFO) - we could try to be more aggressive and
  5438. * retransmitting any data sooner based on when they
  5439. * are sent out.
  5440. */
  5441. tcp_rearm_rto(sk);
  5442. }
  5443. /*
  5444. * This function implements the receiving procedure of RFC 793 for
  5445. * all states except ESTABLISHED and TIME_WAIT.
  5446. * It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
  5447. * address independent.
  5448. */
  5449. int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
  5450. {
  5451. struct tcp_sock *tp = tcp_sk(sk);
  5452. struct inet_connection_sock *icsk = inet_csk(sk);
  5453. const struct tcphdr *th = tcp_hdr(skb);
  5454. struct request_sock *req;
  5455. int queued = 0;
  5456. bool acceptable;
  5457. switch (sk->sk_state) {
  5458. case TCP_CLOSE:
  5459. goto discard;
  5460. case TCP_LISTEN:
  5461. if (th->ack)
  5462. return 1;
  5463. if (th->rst)
  5464. goto discard;
  5465. if (th->syn) {
  5466. if (th->fin)
  5467. goto discard;
  5468. /* It is possible that we process SYN packets from backlog,
  5469. * so we need to make sure to disable BH and RCU right there.
  5470. */
  5471. rcu_read_lock();
  5472. local_bh_disable();
  5473. acceptable = icsk->icsk_af_ops->conn_request(sk, skb) >= 0;
  5474. local_bh_enable();
  5475. rcu_read_unlock();
  5476. if (!acceptable)
  5477. return 1;
  5478. consume_skb(skb);
  5479. return 0;
  5480. }
  5481. goto discard;
  5482. case TCP_SYN_SENT:
  5483. tp->rx_opt.saw_tstamp = 0;
  5484. tcp_mstamp_refresh(tp);
  5485. queued = tcp_rcv_synsent_state_process(sk, skb, th);
  5486. if (queued >= 0)
  5487. return queued;
  5488. /* Do step6 onward by hand. */
  5489. tcp_urg(sk, skb, th);
  5490. __kfree_skb(skb);
  5491. tcp_data_snd_check(sk);
  5492. return 0;
  5493. }
  5494. tcp_mstamp_refresh(tp);
  5495. tp->rx_opt.saw_tstamp = 0;
  5496. req = rcu_dereference_protected(tp->fastopen_rsk,
  5497. lockdep_sock_is_held(sk));
  5498. if (req) {
  5499. bool req_stolen;
  5500. WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
  5501. sk->sk_state != TCP_FIN_WAIT1);
  5502. if (!tcp_check_req(sk, skb, req, true, &req_stolen))
  5503. goto discard;
  5504. }
  5505. if (!th->ack && !th->rst && !th->syn)
  5506. goto discard;
  5507. if (!tcp_validate_incoming(sk, skb, th, 0))
  5508. return 0;
  5509. /* step 5: check the ACK field */
  5510. acceptable = tcp_ack(sk, skb, FLAG_SLOWPATH |
  5511. FLAG_UPDATE_TS_RECENT |
  5512. FLAG_NO_CHALLENGE_ACK) > 0;
  5513. if (!acceptable) {
  5514. if (sk->sk_state == TCP_SYN_RECV)
  5515. return 1; /* send one RST */
  5516. tcp_send_challenge_ack(sk, skb);
  5517. goto discard;
  5518. }
  5519. switch (sk->sk_state) {
  5520. case TCP_SYN_RECV:
  5521. tp->delivered++; /* SYN-ACK delivery isn't tracked in tcp_ack */
  5522. if (!tp->srtt_us)
  5523. tcp_synack_rtt_meas(sk, req);
  5524. if (req) {
  5525. tcp_rcv_synrecv_state_fastopen(sk);
  5526. } else {
  5527. tcp_try_undo_spurious_syn(sk);
  5528. tp->retrans_stamp = 0;
  5529. tcp_init_transfer(sk, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB,
  5530. skb);
  5531. WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
  5532. }
  5533. smp_mb();
  5534. tcp_set_state(sk, TCP_ESTABLISHED);
  5535. sk->sk_state_change(sk);
  5536. /* Note, that this wakeup is only for marginal crossed SYN case.
  5537. * Passively open sockets are not waked up, because
  5538. * sk->sk_sleep == NULL and sk->sk_socket == NULL.
  5539. */
  5540. if (sk->sk_socket)
  5541. sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
  5542. tp->snd_una = TCP_SKB_CB(skb)->ack_seq;
  5543. tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
  5544. tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
  5545. if (tp->rx_opt.tstamp_ok)
  5546. tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
  5547. if (!inet_csk(sk)->icsk_ca_ops->cong_control)
  5548. tcp_update_pacing_rate(sk);
  5549. /* Prevent spurious tcp_cwnd_restart() on first data packet */
  5550. tp->lsndtime = tcp_jiffies32;
  5551. tcp_initialize_rcv_mss(sk);
  5552. tcp_fast_path_on(tp);
  5553. break;
  5554. case TCP_FIN_WAIT1: {
  5555. int tmo;
  5556. if (req)
  5557. tcp_rcv_synrecv_state_fastopen(sk);
  5558. if (tp->snd_una != tp->write_seq)
  5559. break;
  5560. tcp_set_state(sk, TCP_FIN_WAIT2);
  5561. sk->sk_shutdown |= SEND_SHUTDOWN;
  5562. sk_dst_confirm(sk);
  5563. if (!sock_flag(sk, SOCK_DEAD)) {
  5564. /* Wake up lingering close() */
  5565. sk->sk_state_change(sk);
  5566. break;
  5567. }
  5568. if (tp->linger2 < 0) {
  5569. tcp_done(sk);
  5570. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
  5571. return 1;
  5572. }
  5573. if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
  5574. after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
  5575. /* Receive out of order FIN after close() */
  5576. if (tp->syn_fastopen && th->fin)
  5577. tcp_fastopen_active_disable(sk);
  5578. tcp_done(sk);
  5579. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
  5580. return 1;
  5581. }
  5582. tmo = tcp_fin_time(sk);
  5583. if (tmo > TCP_TIMEWAIT_LEN) {
  5584. inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN);
  5585. } else if (th->fin || sock_owned_by_user(sk)) {
  5586. /* Bad case. We could lose such FIN otherwise.
  5587. * It is not a big problem, but it looks confusing
  5588. * and not so rare event. We still can lose it now,
  5589. * if it spins in bh_lock_sock(), but it is really
  5590. * marginal case.
  5591. */
  5592. inet_csk_reset_keepalive_timer(sk, tmo);
  5593. } else {
  5594. tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
  5595. goto discard;
  5596. }
  5597. break;
  5598. }
  5599. case TCP_CLOSING:
  5600. if (tp->snd_una == tp->write_seq) {
  5601. tcp_time_wait(sk, TCP_TIME_WAIT, 0);
  5602. goto discard;
  5603. }
  5604. break;
  5605. case TCP_LAST_ACK:
  5606. if (tp->snd_una == tp->write_seq) {
  5607. tcp_update_metrics(sk);
  5608. tcp_done(sk);
  5609. goto discard;
  5610. }
  5611. break;
  5612. }
  5613. /* step 6: check the URG bit */
  5614. tcp_urg(sk, skb, th);
  5615. /* step 7: process the segment text */
  5616. switch (sk->sk_state) {
  5617. case TCP_CLOSE_WAIT:
  5618. case TCP_CLOSING:
  5619. case TCP_LAST_ACK:
  5620. if (!before(TCP_SKB_CB(skb)->seq, tp->rcv_nxt)) {
  5621. if (sk_is_mptcp(sk))
  5622. mptcp_incoming_options(sk, skb);
  5623. break;
  5624. }
  5625. fallthrough;
  5626. case TCP_FIN_WAIT1:
  5627. case TCP_FIN_WAIT2:
  5628. /* RFC 793 says to queue data in these states,
  5629. * RFC 1122 says we MUST send a reset.
  5630. * BSD 4.4 also does reset.
  5631. */
  5632. if (sk->sk_shutdown & RCV_SHUTDOWN) {
  5633. if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
  5634. after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
  5635. NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
  5636. tcp_reset(sk);
  5637. return 1;
  5638. }
  5639. }
  5640. fallthrough;
  5641. case TCP_ESTABLISHED:
  5642. tcp_data_queue(sk, skb);
  5643. queued = 1;
  5644. break;
  5645. }
  5646. /* tcp_data could move socket to TIME-WAIT */
  5647. if (sk->sk_state != TCP_CLOSE) {
  5648. tcp_data_snd_check(sk);
  5649. tcp_ack_snd_check(sk);
  5650. }
  5651. if (!queued) {
  5652. discard:
  5653. tcp_drop(sk, skb);
  5654. }
  5655. return 0;
  5656. }
  5657. EXPORT_SYMBOL(tcp_rcv_state_process);
  5658. static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
  5659. {
  5660. struct inet_request_sock *ireq = inet_rsk(req);
  5661. if (family == AF_INET)
  5662. net_dbg_ratelimited("drop open request from %pI4/%u\n",
  5663. &ireq->ir_rmt_addr, port);
  5664. #if IS_ENABLED(CONFIG_IPV6)
  5665. else if (family == AF_INET6)
  5666. net_dbg_ratelimited("drop open request from %pI6/%u\n",
  5667. &ireq->ir_v6_rmt_addr, port);
  5668. #endif
  5669. }
  5670. /* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
  5671. *
  5672. * If we receive a SYN packet with these bits set, it means a
  5673. * network is playing bad games with TOS bits. In order to
  5674. * avoid possible false congestion notifications, we disable
  5675. * TCP ECN negotiation.
  5676. *
  5677. * Exception: tcp_ca wants ECN. This is required for DCTCP
  5678. * congestion control: Linux DCTCP asserts ECT on all packets,
  5679. * including SYN, which is most optimal solution; however,
  5680. * others, such as FreeBSD do not.
  5681. *
  5682. * Exception: At least one of the reserved bits of the TCP header (th->res1) is
  5683. * set, indicating the use of a future TCP extension (such as AccECN). See
  5684. * RFC8311 §4.3 which updates RFC3168 to allow the development of such
  5685. * extensions.
  5686. */
  5687. static void tcp_ecn_create_request(struct request_sock *req,
  5688. const struct sk_buff *skb,
  5689. const struct sock *listen_sk,
  5690. const struct dst_entry *dst)
  5691. {
  5692. const struct tcphdr *th = tcp_hdr(skb);
  5693. const struct net *net = sock_net(listen_sk);
  5694. bool th_ecn = th->ece && th->cwr;
  5695. bool ect, ecn_ok;
  5696. u32 ecn_ok_dst;
  5697. if (!th_ecn)
  5698. return;
  5699. ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield);
  5700. ecn_ok_dst = dst_feature(dst, DST_FEATURE_ECN_MASK);
  5701. ecn_ok = net->ipv4.sysctl_tcp_ecn || ecn_ok_dst;
  5702. if (((!ect || th->res1) && ecn_ok) || tcp_ca_needs_ecn(listen_sk) ||
  5703. (ecn_ok_dst & DST_FEATURE_ECN_CA) ||
  5704. tcp_bpf_ca_needs_ecn((struct sock *)req))
  5705. inet_rsk(req)->ecn_ok = 1;
  5706. }
  5707. static void tcp_openreq_init(struct request_sock *req,
  5708. const struct tcp_options_received *rx_opt,
  5709. struct sk_buff *skb, const struct sock *sk)
  5710. {
  5711. struct inet_request_sock *ireq = inet_rsk(req);
  5712. req->rsk_rcv_wnd = 0; /* So that tcp_send_synack() knows! */
  5713. tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
  5714. tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
  5715. tcp_rsk(req)->snt_synack = 0;
  5716. tcp_rsk(req)->last_oow_ack_time = 0;
  5717. req->mss = rx_opt->mss_clamp;
  5718. req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
  5719. ireq->tstamp_ok = rx_opt->tstamp_ok;
  5720. ireq->sack_ok = rx_opt->sack_ok;
  5721. ireq->snd_wscale = rx_opt->snd_wscale;
  5722. ireq->wscale_ok = rx_opt->wscale_ok;
  5723. ireq->acked = 0;
  5724. ireq->ecn_ok = 0;
  5725. ireq->ir_rmt_port = tcp_hdr(skb)->source;
  5726. ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
  5727. ireq->ir_mark = inet_request_mark(sk, skb);
  5728. #if IS_ENABLED(CONFIG_SMC)
  5729. ireq->smc_ok = rx_opt->smc_ok;
  5730. #endif
  5731. }
  5732. struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
  5733. struct sock *sk_listener,
  5734. bool attach_listener)
  5735. {
  5736. struct request_sock *req = reqsk_alloc(ops, sk_listener,
  5737. attach_listener);
  5738. if (req) {
  5739. struct inet_request_sock *ireq = inet_rsk(req);
  5740. ireq->ireq_opt = NULL;
  5741. #if IS_ENABLED(CONFIG_IPV6)
  5742. ireq->pktopts = NULL;
  5743. #endif
  5744. atomic64_set(&ireq->ir_cookie, 0);
  5745. ireq->ireq_state = TCP_NEW_SYN_RECV;
  5746. write_pnet(&ireq->ireq_net, sock_net(sk_listener));
  5747. ireq->ireq_family = sk_listener->sk_family;
  5748. }
  5749. return req;
  5750. }
  5751. EXPORT_SYMBOL(inet_reqsk_alloc);
  5752. /*
  5753. * Return true if a syncookie should be sent
  5754. */
  5755. static bool tcp_syn_flood_action(const struct sock *sk, const char *proto)
  5756. {
  5757. struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
  5758. const char *msg = "Dropping request";
  5759. bool want_cookie = false;
  5760. struct net *net = sock_net(sk);
  5761. #ifdef CONFIG_SYN_COOKIES
  5762. if (net->ipv4.sysctl_tcp_syncookies) {
  5763. msg = "Sending cookies";
  5764. want_cookie = true;
  5765. __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES);
  5766. } else
  5767. #endif
  5768. __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP);
  5769. if (!queue->synflood_warned &&
  5770. net->ipv4.sysctl_tcp_syncookies != 2 &&
  5771. xchg(&queue->synflood_warned, 1) == 0)
  5772. net_info_ratelimited("%s: Possible SYN flooding on port %d. %s. Check SNMP counters.\n",
  5773. proto, sk->sk_num, msg);
  5774. return want_cookie;
  5775. }
  5776. static void tcp_reqsk_record_syn(const struct sock *sk,
  5777. struct request_sock *req,
  5778. const struct sk_buff *skb)
  5779. {
  5780. if (tcp_sk(sk)->save_syn) {
  5781. u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb);
  5782. struct saved_syn *saved_syn;
  5783. u32 mac_hdrlen;
  5784. void *base;
  5785. if (tcp_sk(sk)->save_syn == 2) { /* Save full header. */
  5786. base = skb_mac_header(skb);
  5787. mac_hdrlen = skb_mac_header_len(skb);
  5788. len += mac_hdrlen;
  5789. } else {
  5790. base = skb_network_header(skb);
  5791. mac_hdrlen = 0;
  5792. }
  5793. saved_syn = kmalloc(struct_size(saved_syn, data, len),
  5794. GFP_ATOMIC);
  5795. if (saved_syn) {
  5796. saved_syn->mac_hdrlen = mac_hdrlen;
  5797. saved_syn->network_hdrlen = skb_network_header_len(skb);
  5798. saved_syn->tcp_hdrlen = tcp_hdrlen(skb);
  5799. memcpy(saved_syn->data, base, len);
  5800. req->saved_syn = saved_syn;
  5801. }
  5802. }
  5803. }
  5804. /* If a SYN cookie is required and supported, returns a clamped MSS value to be
  5805. * used for SYN cookie generation.
  5806. */
  5807. u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
  5808. const struct tcp_request_sock_ops *af_ops,
  5809. struct sock *sk, struct tcphdr *th)
  5810. {
  5811. struct tcp_sock *tp = tcp_sk(sk);
  5812. u16 mss;
  5813. if (sock_net(sk)->ipv4.sysctl_tcp_syncookies != 2 &&
  5814. !inet_csk_reqsk_queue_is_full(sk))
  5815. return 0;
  5816. if (!tcp_syn_flood_action(sk, rsk_ops->slab_name))
  5817. return 0;
  5818. if (sk_acceptq_is_full(sk)) {
  5819. NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
  5820. return 0;
  5821. }
  5822. mss = tcp_parse_mss_option(th, tp->rx_opt.user_mss);
  5823. if (!mss)
  5824. mss = af_ops->mss_clamp;
  5825. return mss;
  5826. }
  5827. EXPORT_SYMBOL_GPL(tcp_get_syncookie_mss);
  5828. int tcp_conn_request(struct request_sock_ops *rsk_ops,
  5829. const struct tcp_request_sock_ops *af_ops,
  5830. struct sock *sk, struct sk_buff *skb)
  5831. {
  5832. struct tcp_fastopen_cookie foc = { .len = -1 };
  5833. __u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
  5834. struct tcp_options_received tmp_opt;
  5835. struct tcp_sock *tp = tcp_sk(sk);
  5836. struct net *net = sock_net(sk);
  5837. struct sock *fastopen_sk = NULL;
  5838. struct request_sock *req;
  5839. bool want_cookie = false;
  5840. struct dst_entry *dst;
  5841. struct flowi fl;
  5842. /* TW buckets are converted to open requests without
  5843. * limitations, they conserve resources and peer is
  5844. * evidently real one.
  5845. */
  5846. if ((net->ipv4.sysctl_tcp_syncookies == 2 ||
  5847. inet_csk_reqsk_queue_is_full(sk)) && !isn) {
  5848. want_cookie = tcp_syn_flood_action(sk, rsk_ops->slab_name);
  5849. if (!want_cookie)
  5850. goto drop;
  5851. }
  5852. if (sk_acceptq_is_full(sk)) {
  5853. NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
  5854. goto drop;
  5855. }
  5856. req = inet_reqsk_alloc(rsk_ops, sk, !want_cookie);
  5857. if (!req)
  5858. goto drop;
  5859. req->syncookie = want_cookie;
  5860. tcp_rsk(req)->af_specific = af_ops;
  5861. tcp_rsk(req)->ts_off = 0;
  5862. #if IS_ENABLED(CONFIG_MPTCP)
  5863. tcp_rsk(req)->is_mptcp = 0;
  5864. #endif
  5865. tcp_clear_options(&tmp_opt);
  5866. tmp_opt.mss_clamp = af_ops->mss_clamp;
  5867. tmp_opt.user_mss = tp->rx_opt.user_mss;
  5868. tcp_parse_options(sock_net(sk), skb, &tmp_opt, 0,
  5869. want_cookie ? NULL : &foc);
  5870. if (want_cookie && !tmp_opt.saw_tstamp)
  5871. tcp_clear_options(&tmp_opt);
  5872. if (IS_ENABLED(CONFIG_SMC) && want_cookie)
  5873. tmp_opt.smc_ok = 0;
  5874. tmp_opt.tstamp_ok = tmp_opt.saw_tstamp;
  5875. tcp_openreq_init(req, &tmp_opt, skb, sk);
  5876. inet_rsk(req)->no_srccheck = inet_sk(sk)->transparent;
  5877. /* Note: tcp_v6_init_req() might override ir_iif for link locals */
  5878. inet_rsk(req)->ir_iif = inet_request_bound_dev_if(sk, skb);
  5879. af_ops->init_req(req, sk, skb);
  5880. if (security_inet_conn_request(sk, skb, req))
  5881. goto drop_and_free;
  5882. if (tmp_opt.tstamp_ok)
  5883. tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb);
  5884. dst = af_ops->route_req(sk, &fl, req);
  5885. if (!dst)
  5886. goto drop_and_free;
  5887. if (!want_cookie && !isn) {
  5888. /* Kill the following clause, if you dislike this way. */
  5889. if (!net->ipv4.sysctl_tcp_syncookies &&
  5890. (net->ipv4.sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
  5891. (net->ipv4.sysctl_max_syn_backlog >> 2)) &&
  5892. !tcp_peer_is_proven(req, dst)) {
  5893. /* Without syncookies last quarter of
  5894. * backlog is filled with destinations,
  5895. * proven to be alive.
  5896. * It means that we continue to communicate
  5897. * to destinations, already remembered
  5898. * to the moment of synflood.
  5899. */
  5900. pr_drop_req(req, ntohs(tcp_hdr(skb)->source),
  5901. rsk_ops->family);
  5902. goto drop_and_release;
  5903. }
  5904. isn = af_ops->init_seq(skb);
  5905. }
  5906. tcp_ecn_create_request(req, skb, sk, dst);
  5907. if (want_cookie) {
  5908. isn = cookie_init_sequence(af_ops, sk, skb, &req->mss);
  5909. if (!tmp_opt.tstamp_ok)
  5910. inet_rsk(req)->ecn_ok = 0;
  5911. }
  5912. tcp_rsk(req)->snt_isn = isn;
  5913. tcp_rsk(req)->txhash = net_tx_rndhash();
  5914. tcp_rsk(req)->syn_tos = TCP_SKB_CB(skb)->ip_dsfield;
  5915. tcp_openreq_init_rwin(req, sk, dst);
  5916. sk_rx_queue_set(req_to_sk(req), skb);
  5917. if (!want_cookie) {
  5918. tcp_reqsk_record_syn(sk, req, skb);
  5919. fastopen_sk = tcp_try_fastopen(sk, skb, req, &foc, dst);
  5920. }
  5921. if (fastopen_sk) {
  5922. af_ops->send_synack(fastopen_sk, dst, &fl, req,
  5923. &foc, TCP_SYNACK_FASTOPEN, skb);
  5924. /* Add the child socket directly into the accept queue */
  5925. if (!inet_csk_reqsk_queue_add(sk, req, fastopen_sk)) {
  5926. reqsk_fastopen_remove(fastopen_sk, req, false);
  5927. bh_unlock_sock(fastopen_sk);
  5928. sock_put(fastopen_sk);
  5929. goto drop_and_free;
  5930. }
  5931. sk->sk_data_ready(sk);
  5932. bh_unlock_sock(fastopen_sk);
  5933. sock_put(fastopen_sk);
  5934. } else {
  5935. tcp_rsk(req)->tfo_listener = false;
  5936. if (!want_cookie)
  5937. inet_csk_reqsk_queue_hash_add(sk, req,
  5938. tcp_timeout_init((struct sock *)req));
  5939. af_ops->send_synack(sk, dst, &fl, req, &foc,
  5940. !want_cookie ? TCP_SYNACK_NORMAL :
  5941. TCP_SYNACK_COOKIE,
  5942. skb);
  5943. if (want_cookie) {
  5944. reqsk_free(req);
  5945. return 0;
  5946. }
  5947. }
  5948. reqsk_put(req);
  5949. return 0;
  5950. drop_and_release:
  5951. dst_release(dst);
  5952. drop_and_free:
  5953. __reqsk_free(req);
  5954. drop:
  5955. tcp_listendrop(sk);
  5956. return 0;
  5957. }
  5958. EXPORT_SYMBOL(tcp_conn_request);