tcp_output.c 72 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529
  1. /*
  2. * INET An implementation of the TCP/IP protocol suite for the LINUX
  3. * operating system. INET is implemented using the BSD Socket
  4. * interface as the means of communication with the user level.
  5. *
  6. * Implementation of the Transmission Control Protocol(TCP).
  7. *
  8. * Version: $Id: tcp_output.c,v 1.1.1.1 2007/06/12 07:27:14 eyryu Exp $
  9. *
  10. * Authors: Ross Biro
  11. * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  12. * Mark Evans, <evansmp@uhura.aston.ac.uk>
  13. * Corey Minyard <wf-rch!minyard@relay.EU.net>
  14. * Florian La Roche, <flla@stud.uni-sb.de>
  15. * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  16. * Linus Torvalds, <torvalds@cs.helsinki.fi>
  17. * Alan Cox, <gw4pts@gw4pts.ampr.org>
  18. * Matthew Dillon, <dillon@apollo.west.oic.com>
  19. * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  20. * Jorge Cwik, <jorge@laser.satlink.net>
  21. */
  22. /*
  23. * Changes: Pedro Roque : Retransmit queue handled by TCP.
  24. * : Fragmentation on mtu decrease
  25. * : Segment collapse on retransmit
  26. * : AF independence
  27. *
  28. * Linus Torvalds : send_delayed_ack
  29. * David S. Miller : Charge memory using the right skb
  30. * during syn/ack processing.
  31. * David S. Miller : Output engine completely rewritten.
  32. * Andrea Arcangeli: SYNACK carry ts_recent in tsecr.
  33. * Cacophonix Gaul : draft-minshall-nagle-01
  34. * J Hadi Salim : ECN support
  35. *
  36. */
  37. #include <net/tcp.h>
  38. #include <linux/compiler.h>
  39. #include <linux/module.h>
  40. #include <linux/smp_lock.h>
  41. /* People can turn this off for buggy TCP's found in printers etc. */
  42. int sysctl_tcp_retrans_collapse __read_mostly = 1;
  43. /* People can turn this on to work with those rare, broken TCPs that
  44. * interpret the window field as a signed quantity.
  45. */
  46. int sysctl_tcp_workaround_signed_windows __read_mostly = 0;
  47. /* This limits the percentage of the congestion window which we
  48. * will allow a single TSO frame to consume. Building TSO frames
  49. * which are too large can cause TCP streams to be bursty.
  50. */
  51. int sysctl_tcp_tso_win_divisor __read_mostly = 3;
  52. int sysctl_tcp_mtu_probing __read_mostly = 0;
  53. int sysctl_tcp_base_mss __read_mostly = 512;
  54. /* By default, RFC2861 behavior. */
  55. int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
  56. static void update_send_head(struct sock *sk, struct tcp_sock *tp,
  57. struct sk_buff *skb)
  58. {
  59. sk->sk_send_head = skb->next;
  60. if (sk->sk_send_head == (struct sk_buff *)&sk->sk_write_queue)
  61. sk->sk_send_head = NULL;
  62. tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
  63. tcp_packets_out_inc(sk, tp, skb);
  64. }
  65. /* SND.NXT, if window was not shrunk.
  66. * If window has been shrunk, what should we make? It is not clear at all.
  67. * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
  68. * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
  69. * invalid. OK, let's make this for now:
  70. */
  71. static inline __u32 tcp_acceptable_seq(struct sock *sk, struct tcp_sock *tp)
  72. {
  73. if (!before(tp->snd_una+tp->snd_wnd, tp->snd_nxt))
  74. return tp->snd_nxt;
  75. else
  76. return tp->snd_una+tp->snd_wnd;
  77. }
  78. /* Calculate mss to advertise in SYN segment.
  79. * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
  80. *
  81. * 1. It is independent of path mtu.
  82. * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
  83. * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
  84. * attached devices, because some buggy hosts are confused by
  85. * large MSS.
  86. * 4. We do not make 3, we advertise MSS, calculated from first
  87. * hop device mtu, but allow to raise it to ip_rt_min_advmss.
  88. * This may be overridden via information stored in routing table.
  89. * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
  90. * probably even Jumbo".
  91. */
  92. static __u16 tcp_advertise_mss(struct sock *sk)
  93. {
  94. struct tcp_sock *tp = tcp_sk(sk);
  95. struct dst_entry *dst = __sk_dst_get(sk);
  96. int mss = tp->advmss;
  97. if (dst && dst_metric(dst, RTAX_ADVMSS) < mss) {
  98. mss = dst_metric(dst, RTAX_ADVMSS);
  99. tp->advmss = mss;
  100. }
  101. return (__u16)mss;
  102. }
  103. /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
  104. * This is the first part of cwnd validation mechanism. */
  105. static void tcp_cwnd_restart(struct sock *sk, struct dst_entry *dst)
  106. {
  107. struct tcp_sock *tp = tcp_sk(sk);
  108. s32 delta = tcp_time_stamp - tp->lsndtime;
  109. u32 restart_cwnd = tcp_init_cwnd(tp, dst);
  110. u32 cwnd = tp->snd_cwnd;
  111. tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
  112. tp->snd_ssthresh = tcp_current_ssthresh(sk);
  113. restart_cwnd = min(restart_cwnd, cwnd);
  114. while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
  115. cwnd >>= 1;
  116. tp->snd_cwnd = max(cwnd, restart_cwnd);
  117. tp->snd_cwnd_stamp = tcp_time_stamp;
  118. tp->snd_cwnd_used = 0;
  119. }
  120. static void tcp_event_data_sent(struct tcp_sock *tp,
  121. struct sk_buff *skb, struct sock *sk)
  122. {
  123. struct inet_connection_sock *icsk = inet_csk(sk);
  124. const u32 now = tcp_time_stamp;
  125. if (sysctl_tcp_slow_start_after_idle &&
  126. (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto))
  127. tcp_cwnd_restart(sk, __sk_dst_get(sk));
  128. tp->lsndtime = now;
  129. /* If it is a reply for ato after last received
  130. * packet, enter pingpong mode.
  131. */
  132. if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
  133. icsk->icsk_ack.pingpong = 1;
  134. }
  135. static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
  136. {
  137. tcp_dec_quickack_mode(sk, pkts);
  138. inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
  139. }
  140. /* Determine a window scaling and initial window to offer.
  141. * Based on the assumption that the given amount of space
  142. * will be offered. Store the results in the tp structure.
  143. * NOTE: for smooth operation initial space offering should
  144. * be a multiple of mss if possible. We assume here that mss >= 1.
  145. * This MUST be enforced by all callers.
  146. */
  147. void tcp_select_initial_window(int __space, __u32 mss,
  148. __u32 *rcv_wnd, __u32 *window_clamp,
  149. int wscale_ok, __u8 *rcv_wscale)
  150. {
  151. unsigned int space = (__space < 0 ? 0 : __space);
  152. /* If no clamp set the clamp to the max possible scaled window */
  153. if (*window_clamp == 0)
  154. (*window_clamp) = (65535 << 14);
  155. space = min(*window_clamp, space);
  156. /* Quantize space offering to a multiple of mss if possible. */
  157. if (space > mss)
  158. space = (space / mss) * mss;
  159. /* NOTE: offering an initial window larger than 32767
  160. * will break some buggy TCP stacks. If the admin tells us
  161. * it is likely we could be speaking with such a buggy stack
  162. * we will truncate our initial window offering to 32K-1
  163. * unless the remote has sent us a window scaling option,
  164. * which we interpret as a sign the remote TCP is not
  165. * misinterpreting the window field as a signed quantity.
  166. */
  167. if (sysctl_tcp_workaround_signed_windows)
  168. (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
  169. else
  170. (*rcv_wnd) = space;
  171. (*rcv_wscale) = 0;
  172. if (wscale_ok) {
  173. /* Set window scaling on max possible window
  174. * See RFC1323 for an explanation of the limit to 14
  175. */
  176. space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max);
  177. space = min_t(u32, space, *window_clamp);
  178. while (space > 65535 && (*rcv_wscale) < 14) {
  179. space >>= 1;
  180. (*rcv_wscale)++;
  181. }
  182. }
  183. /* Set initial window to value enough for senders,
  184. * following RFC2414. Senders, not following this RFC,
  185. * will be satisfied with 2.
  186. */
  187. if (mss > (1<<*rcv_wscale)) {
  188. int init_cwnd = 4;
  189. if (mss > 1460*3)
  190. init_cwnd = 2;
  191. else if (mss > 1460)
  192. init_cwnd = 3;
  193. if (*rcv_wnd > init_cwnd*mss)
  194. *rcv_wnd = init_cwnd*mss;
  195. }
  196. /* Set the clamp no higher than max representable value */
  197. (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp);
  198. }
  199. /* Chose a new window to advertise, update state in tcp_sock for the
  200. * socket, and return result with RFC1323 scaling applied. The return
  201. * value can be stuffed directly into th->window for an outgoing
  202. * frame.
  203. */
  204. static u16 tcp_select_window(struct sock *sk)
  205. {
  206. struct tcp_sock *tp = tcp_sk(sk);
  207. u32 cur_win = tcp_receive_window(tp);
  208. u32 new_win = __tcp_select_window(sk);
  209. /* Never shrink the offered window */
  210. if(new_win < cur_win) {
  211. /* Danger Will Robinson!
  212. * Don't update rcv_wup/rcv_wnd here or else
  213. * we will not be able to advertise a zero
  214. * window in time. --DaveM
  215. *
  216. * Relax Will Robinson.
  217. */
  218. new_win = cur_win;
  219. }
  220. tp->rcv_wnd = new_win;
  221. tp->rcv_wup = tp->rcv_nxt;
  222. /* Make sure we do not exceed the maximum possible
  223. * scaled window.
  224. */
  225. if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows)
  226. new_win = min(new_win, MAX_TCP_WINDOW);
  227. else
  228. new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
  229. /* RFC1323 scaling applied */
  230. new_win >>= tp->rx_opt.rcv_wscale;
  231. /* If we advertise zero window, disable fast path. */
  232. if (new_win == 0)
  233. tp->pred_flags = 0;
  234. return new_win;
  235. }
  236. static void tcp_build_and_update_options(__be32 *ptr, struct tcp_sock *tp,
  237. __u32 tstamp, __u8 **md5_hash)
  238. {
  239. if (tp->rx_opt.tstamp_ok) {
  240. *ptr++ = htonl((TCPOPT_NOP << 24) |
  241. (TCPOPT_NOP << 16) |
  242. (TCPOPT_TIMESTAMP << 8) |
  243. TCPOLEN_TIMESTAMP);
  244. *ptr++ = htonl(tstamp);
  245. *ptr++ = htonl(tp->rx_opt.ts_recent);
  246. }
  247. if (tp->rx_opt.eff_sacks) {
  248. struct tcp_sack_block *sp = tp->rx_opt.dsack ? tp->duplicate_sack : tp->selective_acks;
  249. int this_sack;
  250. *ptr++ = htonl((TCPOPT_NOP << 24) |
  251. (TCPOPT_NOP << 16) |
  252. (TCPOPT_SACK << 8) |
  253. (TCPOLEN_SACK_BASE + (tp->rx_opt.eff_sacks *
  254. TCPOLEN_SACK_PERBLOCK)));
  255. for(this_sack = 0; this_sack < tp->rx_opt.eff_sacks; this_sack++) {
  256. *ptr++ = htonl(sp[this_sack].start_seq);
  257. *ptr++ = htonl(sp[this_sack].end_seq);
  258. }
  259. if (tp->rx_opt.dsack) {
  260. tp->rx_opt.dsack = 0;
  261. tp->rx_opt.eff_sacks--;
  262. }
  263. }
  264. #ifdef CONFIG_TCP_MD5SIG
  265. if (md5_hash) {
  266. *ptr++ = htonl((TCPOPT_NOP << 24) |
  267. (TCPOPT_NOP << 16) |
  268. (TCPOPT_MD5SIG << 8) |
  269. TCPOLEN_MD5SIG);
  270. *md5_hash = (__u8 *)ptr;
  271. }
  272. #endif
  273. }
  274. /* Construct a tcp options header for a SYN or SYN_ACK packet.
  275. * If this is every changed make sure to change the definition of
  276. * MAX_SYN_SIZE to match the new maximum number of options that you
  277. * can generate.
  278. *
  279. * Note - that with the RFC2385 TCP option, we make room for the
  280. * 16 byte MD5 hash. This will be filled in later, so the pointer for the
  281. * location to be filled is passed back up.
  282. */
  283. static void tcp_syn_build_options(__be32 *ptr, int mss, int ts, int sack,
  284. int offer_wscale, int wscale, __u32 tstamp,
  285. __u32 ts_recent, __u8 **md5_hash)
  286. {
  287. /* We always get an MSS option.
  288. * The option bytes which will be seen in normal data
  289. * packets should timestamps be used, must be in the MSS
  290. * advertised. But we subtract them from tp->mss_cache so
  291. * that calculations in tcp_sendmsg are simpler etc.
  292. * So account for this fact here if necessary. If we
  293. * don't do this correctly, as a receiver we won't
  294. * recognize data packets as being full sized when we
  295. * should, and thus we won't abide by the delayed ACK
  296. * rules correctly.
  297. * SACKs don't matter, we never delay an ACK when we
  298. * have any of those going out.
  299. */
  300. *ptr++ = htonl((TCPOPT_MSS << 24) | (TCPOLEN_MSS << 16) | mss);
  301. if (ts) {
  302. if(sack)
  303. *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
  304. (TCPOLEN_SACK_PERM << 16) |
  305. (TCPOPT_TIMESTAMP << 8) |
  306. TCPOLEN_TIMESTAMP);
  307. else
  308. *ptr++ = htonl((TCPOPT_NOP << 24) |
  309. (TCPOPT_NOP << 16) |
  310. (TCPOPT_TIMESTAMP << 8) |
  311. TCPOLEN_TIMESTAMP);
  312. *ptr++ = htonl(tstamp); /* TSVAL */
  313. *ptr++ = htonl(ts_recent); /* TSECR */
  314. } else if(sack)
  315. *ptr++ = htonl((TCPOPT_NOP << 24) |
  316. (TCPOPT_NOP << 16) |
  317. (TCPOPT_SACK_PERM << 8) |
  318. TCPOLEN_SACK_PERM);
  319. if (offer_wscale)
  320. *ptr++ = htonl((TCPOPT_NOP << 24) |
  321. (TCPOPT_WINDOW << 16) |
  322. (TCPOLEN_WINDOW << 8) |
  323. (wscale));
  324. #ifdef CONFIG_TCP_MD5SIG
  325. /*
  326. * If MD5 is enabled, then we set the option, and include the size
  327. * (always 18). The actual MD5 hash is added just before the
  328. * packet is sent.
  329. */
  330. if (md5_hash) {
  331. *ptr++ = htonl((TCPOPT_NOP << 24) |
  332. (TCPOPT_NOP << 16) |
  333. (TCPOPT_MD5SIG << 8) |
  334. TCPOLEN_MD5SIG);
  335. *md5_hash = (__u8 *) ptr;
  336. }
  337. #endif
  338. }
  339. /* This routine actually transmits TCP packets queued in by
  340. * tcp_do_sendmsg(). This is used by both the initial
  341. * transmission and possible later retransmissions.
  342. * All SKB's seen here are completely headerless. It is our
  343. * job to build the TCP header, and pass the packet down to
  344. * IP so it can do the same plus pass the packet off to the
  345. * device.
  346. *
  347. * We are working here with either a clone of the original
  348. * SKB, or a fresh unique copy made by the retransmit engine.
  349. */
  350. static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, gfp_t gfp_mask)
  351. {
  352. const struct inet_connection_sock *icsk = inet_csk(sk);
  353. struct inet_sock *inet;
  354. struct tcp_sock *tp;
  355. struct tcp_skb_cb *tcb;
  356. int tcp_header_size;
  357. #ifdef CONFIG_TCP_MD5SIG
  358. struct tcp_md5sig_key *md5;
  359. __u8 *md5_hash_location;
  360. #endif
  361. struct tcphdr *th;
  362. int sysctl_flags;
  363. int err;
  364. BUG_ON(!skb || !tcp_skb_pcount(skb));
  365. /* If congestion control is doing timestamping, we must
  366. * take such a timestamp before we potentially clone/copy.
  367. */
  368. if (icsk->icsk_ca_ops->rtt_sample)
  369. __net_timestamp(skb);
  370. if (likely(clone_it)) {
  371. if (unlikely(skb_cloned(skb)))
  372. skb = pskb_copy(skb, gfp_mask);
  373. else
  374. skb = skb_clone(skb, gfp_mask);
  375. if (unlikely(!skb))
  376. return -ENOBUFS;
  377. }
  378. inet = inet_sk(sk);
  379. tp = tcp_sk(sk);
  380. tcb = TCP_SKB_CB(skb);
  381. tcp_header_size = tp->tcp_header_len;
  382. #define SYSCTL_FLAG_TSTAMPS 0x1
  383. #define SYSCTL_FLAG_WSCALE 0x2
  384. #define SYSCTL_FLAG_SACK 0x4
  385. sysctl_flags = 0;
  386. if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) {
  387. tcp_header_size = sizeof(struct tcphdr) + TCPOLEN_MSS;
  388. if(sysctl_tcp_timestamps) {
  389. tcp_header_size += TCPOLEN_TSTAMP_ALIGNED;
  390. sysctl_flags |= SYSCTL_FLAG_TSTAMPS;
  391. }
  392. if (sysctl_tcp_window_scaling) {
  393. tcp_header_size += TCPOLEN_WSCALE_ALIGNED;
  394. sysctl_flags |= SYSCTL_FLAG_WSCALE;
  395. }
  396. if (sysctl_tcp_sack) {
  397. sysctl_flags |= SYSCTL_FLAG_SACK;
  398. if (!(sysctl_flags & SYSCTL_FLAG_TSTAMPS))
  399. tcp_header_size += TCPOLEN_SACKPERM_ALIGNED;
  400. }
  401. } else if (unlikely(tp->rx_opt.eff_sacks)) {
  402. /* A SACK is 2 pad bytes, a 2 byte header, plus
  403. * 2 32-bit sequence numbers for each SACK block.
  404. */
  405. tcp_header_size += (TCPOLEN_SACK_BASE_ALIGNED +
  406. (tp->rx_opt.eff_sacks *
  407. TCPOLEN_SACK_PERBLOCK));
  408. }
  409. if (tcp_packets_in_flight(tp) == 0)
  410. tcp_ca_event(sk, CA_EVENT_TX_START);
  411. #ifdef CONFIG_TCP_MD5SIG
  412. /*
  413. * Are we doing MD5 on this segment? If so - make
  414. * room for it.
  415. */
  416. md5 = tp->af_specific->md5_lookup(sk, sk);
  417. if (md5)
  418. tcp_header_size += TCPOLEN_MD5SIG_ALIGNED;
  419. #endif
  420. th = (struct tcphdr *) skb_push(skb, tcp_header_size);
  421. skb->h.th = th;
  422. skb_set_owner_w(skb, sk);
  423. /* Build TCP header and checksum it. */
  424. th->source = inet->sport;
  425. th->dest = inet->dport;
  426. th->seq = htonl(tcb->seq);
  427. th->ack_seq = htonl(tp->rcv_nxt);
  428. *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) |
  429. tcb->flags);
  430. if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) {
  431. /* RFC1323: The window in SYN & SYN/ACK segments
  432. * is never scaled.
  433. */
  434. th->window = htons(min(tp->rcv_wnd, 65535U));
  435. } else {
  436. th->window = htons(tcp_select_window(sk));
  437. }
  438. th->check = 0;
  439. th->urg_ptr = 0;
  440. if (unlikely(tp->urg_mode &&
  441. between(tp->snd_up, tcb->seq+1, tcb->seq+0xFFFF))) {
  442. th->urg_ptr = htons(tp->snd_up-tcb->seq);
  443. th->urg = 1;
  444. }
  445. if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) {
  446. tcp_syn_build_options((__be32 *)(th + 1),
  447. tcp_advertise_mss(sk),
  448. (sysctl_flags & SYSCTL_FLAG_TSTAMPS),
  449. (sysctl_flags & SYSCTL_FLAG_SACK),
  450. (sysctl_flags & SYSCTL_FLAG_WSCALE),
  451. tp->rx_opt.rcv_wscale,
  452. tcb->when,
  453. tp->rx_opt.ts_recent,
  454. #ifdef CONFIG_TCP_MD5SIG
  455. md5 ? &md5_hash_location :
  456. #endif
  457. NULL);
  458. } else {
  459. tcp_build_and_update_options((__be32 *)(th + 1),
  460. tp, tcb->when,
  461. #ifdef CONFIG_TCP_MD5SIG
  462. md5 ? &md5_hash_location :
  463. #endif
  464. NULL);
  465. TCP_ECN_send(sk, tp, skb, tcp_header_size);
  466. }
  467. #ifdef CONFIG_TCP_MD5SIG
  468. /* Calculate the MD5 hash, as we have all we need now */
  469. if (md5) {
  470. tp->af_specific->calc_md5_hash(md5_hash_location,
  471. md5,
  472. sk, NULL, NULL,
  473. skb->h.th,
  474. sk->sk_protocol,
  475. skb->len);
  476. }
  477. #endif
  478. icsk->icsk_af_ops->send_check(sk, skb->len, skb);
  479. if (likely(tcb->flags & TCPCB_FLAG_ACK))
  480. tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
  481. if (skb->len != tcp_header_size)
  482. tcp_event_data_sent(tp, skb, sk);
  483. if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
  484. TCP_INC_STATS(TCP_MIB_OUTSEGS);
  485. err = icsk->icsk_af_ops->queue_xmit(skb, 0);
  486. if (likely(err <= 0))
  487. return err;
  488. tcp_enter_cwr(sk);
  489. return net_xmit_eval(err);
  490. #undef SYSCTL_FLAG_TSTAMPS
  491. #undef SYSCTL_FLAG_WSCALE
  492. #undef SYSCTL_FLAG_SACK
  493. }
  494. /* This routine just queue's the buffer
  495. *
  496. * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
  497. * otherwise socket can stall.
  498. */
  499. static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
  500. {
  501. struct tcp_sock *tp = tcp_sk(sk);
  502. /* Advance write_seq and place onto the write_queue. */
  503. tp->write_seq = TCP_SKB_CB(skb)->end_seq;
  504. skb_header_release(skb);
  505. __skb_queue_tail(&sk->sk_write_queue, skb);
  506. sk_charge_skb(sk, skb);
  507. /* Queue it, remembering where we must start sending. */
  508. if (sk->sk_send_head == NULL)
  509. sk->sk_send_head = skb;
  510. }
  511. static void tcp_set_skb_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now)
  512. {
  513. if (skb->len <= mss_now || !sk_can_gso(sk)) {
  514. /* Avoid the costly divide in the normal
  515. * non-TSO case.
  516. */
  517. skb_shinfo(skb)->gso_segs = 1;
  518. skb_shinfo(skb)->gso_size = 0;
  519. skb_shinfo(skb)->gso_type = 0;
  520. } else {
  521. unsigned int factor;
  522. factor = skb->len + (mss_now - 1);
  523. factor /= mss_now;
  524. skb_shinfo(skb)->gso_segs = factor;
  525. skb_shinfo(skb)->gso_size = mss_now;
  526. skb_shinfo(skb)->gso_type = sk->sk_gso_type;
  527. }
  528. }
  529. /* Function to create two new TCP segments. Shrinks the given segment
  530. * to the specified size and appends a new segment with the rest of the
  531. * packet to the list. This won't be called frequently, I hope.
  532. * Remember, these are still headerless SKBs at this point.
  533. */
  534. int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len, unsigned int mss_now)
  535. {
  536. struct tcp_sock *tp = tcp_sk(sk);
  537. struct sk_buff *buff;
  538. int nsize, old_factor;
  539. int nlen;
  540. u16 flags;
  541. BUG_ON(len > skb->len);
  542. clear_all_retrans_hints(tp);
  543. nsize = skb_headlen(skb) - len;
  544. if (nsize < 0)
  545. nsize = 0;
  546. if (skb_cloned(skb) &&
  547. skb_is_nonlinear(skb) &&
  548. pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
  549. return -ENOMEM;
  550. /* Get a new skb... force flag on. */
  551. buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC);
  552. if (buff == NULL)
  553. return -ENOMEM; /* We'll just try again later. */
  554. sk_charge_skb(sk, buff);
  555. nlen = skb->len - len - nsize;
  556. buff->truesize += nlen;
  557. skb->truesize -= nlen;
  558. /* Correct the sequence numbers. */
  559. TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
  560. TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
  561. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
  562. /* PSH and FIN should only be set in the second packet. */
  563. flags = TCP_SKB_CB(skb)->flags;
  564. TCP_SKB_CB(skb)->flags = flags & ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH);
  565. TCP_SKB_CB(buff)->flags = flags;
  566. TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
  567. TCP_SKB_CB(skb)->sacked &= ~TCPCB_AT_TAIL;
  568. if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
  569. /* Copy and checksum data tail into the new buffer. */
  570. buff->csum = csum_partial_copy_nocheck(skb->data + len, skb_put(buff, nsize),
  571. nsize, 0);
  572. skb_trim(skb, len);
  573. skb->csum = csum_block_sub(skb->csum, buff->csum, len);
  574. } else {
  575. skb->ip_summed = CHECKSUM_PARTIAL;
  576. skb_split(skb, buff, len);
  577. }
  578. buff->ip_summed = skb->ip_summed;
  579. /* Looks stupid, but our code really uses when of
  580. * skbs, which it never sent before. --ANK
  581. */
  582. TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when;
  583. buff->tstamp = skb->tstamp;
  584. old_factor = tcp_skb_pcount(skb);
  585. /* Fix up tso_factor for both original and new SKB. */
  586. tcp_set_skb_tso_segs(sk, skb, mss_now);
  587. tcp_set_skb_tso_segs(sk, buff, mss_now);
  588. /* If this packet has been sent out already, we must
  589. * adjust the various packet counters.
  590. */
  591. if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
  592. int diff = old_factor - tcp_skb_pcount(skb) -
  593. tcp_skb_pcount(buff);
  594. tp->packets_out -= diff;
  595. if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
  596. tp->sacked_out -= diff;
  597. if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
  598. tp->retrans_out -= diff;
  599. if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST) {
  600. tp->lost_out -= diff;
  601. tp->left_out -= diff;
  602. }
  603. if (diff > 0) {
  604. /* Adjust Reno SACK estimate. */
  605. if (!tp->rx_opt.sack_ok) {
  606. tp->sacked_out -= diff;
  607. if ((int)tp->sacked_out < 0)
  608. tp->sacked_out = 0;
  609. tcp_sync_left_out(tp);
  610. }
  611. tp->fackets_out -= diff;
  612. if ((int)tp->fackets_out < 0)
  613. tp->fackets_out = 0;
  614. }
  615. }
  616. /* Link BUFF into the send queue. */
  617. skb_header_release(buff);
  618. __skb_append(skb, buff, &sk->sk_write_queue);
  619. return 0;
  620. }
  621. /* This is similar to __pskb_pull_head() (it will go to core/skbuff.c
  622. * eventually). The difference is that pulled data not copied, but
  623. * immediately discarded.
  624. */
  625. static void __pskb_trim_head(struct sk_buff *skb, int len)
  626. {
  627. int i, k, eat;
  628. eat = len;
  629. k = 0;
  630. for (i=0; i<skb_shinfo(skb)->nr_frags; i++) {
  631. if (skb_shinfo(skb)->frags[i].size <= eat) {
  632. put_page(skb_shinfo(skb)->frags[i].page);
  633. eat -= skb_shinfo(skb)->frags[i].size;
  634. } else {
  635. skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
  636. if (eat) {
  637. skb_shinfo(skb)->frags[k].page_offset += eat;
  638. skb_shinfo(skb)->frags[k].size -= eat;
  639. eat = 0;
  640. }
  641. k++;
  642. }
  643. }
  644. skb_shinfo(skb)->nr_frags = k;
  645. skb->tail = skb->data;
  646. skb->data_len -= len;
  647. skb->len = skb->data_len;
  648. }
  649. int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
  650. {
  651. if (skb_cloned(skb) &&
  652. pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
  653. return -ENOMEM;
  654. /* If len == headlen, we avoid __skb_pull to preserve alignment. */
  655. if (unlikely(len < skb_headlen(skb)))
  656. __skb_pull(skb, len);
  657. else
  658. __pskb_trim_head(skb, len - skb_headlen(skb));
  659. TCP_SKB_CB(skb)->seq += len;
  660. skb->ip_summed = CHECKSUM_PARTIAL;
  661. skb->truesize -= len;
  662. sk->sk_wmem_queued -= len;
  663. sk->sk_forward_alloc += len;
  664. sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
  665. /* Any change of skb->len requires recalculation of tso
  666. * factor and mss.
  667. */
  668. if (tcp_skb_pcount(skb) > 1)
  669. tcp_set_skb_tso_segs(sk, skb, tcp_current_mss(sk, 1));
  670. return 0;
  671. }
  672. /* Not accounting for SACKs here. */
  673. int tcp_mtu_to_mss(struct sock *sk, int pmtu)
  674. {
  675. struct tcp_sock *tp = tcp_sk(sk);
  676. struct inet_connection_sock *icsk = inet_csk(sk);
  677. int mss_now;
  678. /* Calculate base mss without TCP options:
  679. It is MMS_S - sizeof(tcphdr) of rfc1122
  680. */
  681. mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
  682. /* Clamp it (mss_clamp does not include tcp options) */
  683. if (mss_now > tp->rx_opt.mss_clamp)
  684. mss_now = tp->rx_opt.mss_clamp;
  685. /* Now subtract optional transport overhead */
  686. mss_now -= icsk->icsk_ext_hdr_len;
  687. /* Then reserve room for full set of TCP options and 8 bytes of data */
  688. if (mss_now < 48)
  689. mss_now = 48;
  690. /* Now subtract TCP options size, not including SACKs */
  691. mss_now -= tp->tcp_header_len - sizeof(struct tcphdr);
  692. return mss_now;
  693. }
  694. /* Inverse of above */
  695. int tcp_mss_to_mtu(struct sock *sk, int mss)
  696. {
  697. struct tcp_sock *tp = tcp_sk(sk);
  698. struct inet_connection_sock *icsk = inet_csk(sk);
  699. int mtu;
  700. mtu = mss +
  701. tp->tcp_header_len +
  702. icsk->icsk_ext_hdr_len +
  703. icsk->icsk_af_ops->net_header_len;
  704. return mtu;
  705. }
  706. void tcp_mtup_init(struct sock *sk)
  707. {
  708. struct tcp_sock *tp = tcp_sk(sk);
  709. struct inet_connection_sock *icsk = inet_csk(sk);
  710. icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1;
  711. icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
  712. icsk->icsk_af_ops->net_header_len;
  713. icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss);
  714. icsk->icsk_mtup.probe_size = 0;
  715. }
  716. /* This function synchronize snd mss to current pmtu/exthdr set.
  717. tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
  718. for TCP options, but includes only bare TCP header.
  719. tp->rx_opt.mss_clamp is mss negotiated at connection setup.
  720. It is minimum of user_mss and mss received with SYN.
  721. It also does not include TCP options.
  722. inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
  723. tp->mss_cache is current effective sending mss, including
  724. all tcp options except for SACKs. It is evaluated,
  725. taking into account current pmtu, but never exceeds
  726. tp->rx_opt.mss_clamp.
  727. NOTE1. rfc1122 clearly states that advertised MSS
  728. DOES NOT include either tcp or ip options.
  729. NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
  730. are READ ONLY outside this function. --ANK (980731)
  731. */
  732. unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
  733. {
  734. struct tcp_sock *tp = tcp_sk(sk);
  735. struct inet_connection_sock *icsk = inet_csk(sk);
  736. int mss_now;
  737. if (icsk->icsk_mtup.search_high > pmtu)
  738. icsk->icsk_mtup.search_high = pmtu;
  739. mss_now = tcp_mtu_to_mss(sk, pmtu);
  740. /* Bound mss with half of window */
  741. if (tp->max_window && mss_now > (tp->max_window>>1))
  742. mss_now = max((tp->max_window>>1), 68U - tp->tcp_header_len);
  743. /* And store cached results */
  744. icsk->icsk_pmtu_cookie = pmtu;
  745. if (icsk->icsk_mtup.enabled)
  746. mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
  747. tp->mss_cache = mss_now;
  748. return mss_now;
  749. }
  750. /* Compute the current effective MSS, taking SACKs and IP options,
  751. * and even PMTU discovery events into account.
  752. *
  753. * LARGESEND note: !urg_mode is overkill, only frames up to snd_up
  754. * cannot be large. However, taking into account rare use of URG, this
  755. * is not a big flaw.
  756. */
  757. unsigned int tcp_current_mss(struct sock *sk, int large_allowed)
  758. {
  759. struct tcp_sock *tp = tcp_sk(sk);
  760. struct dst_entry *dst = __sk_dst_get(sk);
  761. u32 mss_now;
  762. u16 xmit_size_goal;
  763. int doing_tso = 0;
  764. mss_now = tp->mss_cache;
  765. if (large_allowed && sk_can_gso(sk) && !tp->urg_mode)
  766. doing_tso = 1;
  767. if (dst) {
  768. u32 mtu = dst_mtu(dst);
  769. if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
  770. mss_now = tcp_sync_mss(sk, mtu);
  771. }
  772. if (tp->rx_opt.eff_sacks)
  773. mss_now -= (TCPOLEN_SACK_BASE_ALIGNED +
  774. (tp->rx_opt.eff_sacks * TCPOLEN_SACK_PERBLOCK));
  775. #ifdef CONFIG_TCP_MD5SIG
  776. if (tp->af_specific->md5_lookup(sk, sk))
  777. mss_now -= TCPOLEN_MD5SIG_ALIGNED;
  778. #endif
  779. xmit_size_goal = mss_now;
  780. if (doing_tso) {
  781. xmit_size_goal = (65535 -
  782. inet_csk(sk)->icsk_af_ops->net_header_len -
  783. inet_csk(sk)->icsk_ext_hdr_len -
  784. tp->tcp_header_len);
  785. if (tp->max_window &&
  786. (xmit_size_goal > (tp->max_window >> 1)))
  787. xmit_size_goal = max((tp->max_window >> 1),
  788. 68U - tp->tcp_header_len);
  789. xmit_size_goal -= (xmit_size_goal % mss_now);
  790. }
  791. tp->xmit_size_goal = xmit_size_goal;
  792. return mss_now;
  793. }
  794. /* Congestion window validation. (RFC2861) */
  795. static void tcp_cwnd_validate(struct sock *sk, struct tcp_sock *tp)
  796. {
  797. __u32 packets_out = tp->packets_out;
  798. if (packets_out >= tp->snd_cwnd) {
  799. /* Network is feed fully. */
  800. tp->snd_cwnd_used = 0;
  801. tp->snd_cwnd_stamp = tcp_time_stamp;
  802. } else {
  803. /* Network starves. */
  804. if (tp->packets_out > tp->snd_cwnd_used)
  805. tp->snd_cwnd_used = tp->packets_out;
  806. if (sysctl_tcp_slow_start_after_idle &&
  807. (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto)
  808. tcp_cwnd_application_limited(sk);
  809. }
  810. }
  811. static unsigned int tcp_window_allows(struct tcp_sock *tp, struct sk_buff *skb, unsigned int mss_now, unsigned int cwnd)
  812. {
  813. u32 window, cwnd_len;
  814. window = (tp->snd_una + tp->snd_wnd - TCP_SKB_CB(skb)->seq);
  815. cwnd_len = mss_now * cwnd;
  816. return min(window, cwnd_len);
  817. }
  818. /* Can at least one segment of SKB be sent right now, according to the
  819. * congestion window rules? If so, return how many segments are allowed.
  820. */
  821. static inline unsigned int tcp_cwnd_test(struct tcp_sock *tp, struct sk_buff *skb)
  822. {
  823. u32 in_flight, cwnd;
  824. /* Don't be strict about the congestion window for the final FIN. */
  825. if ((TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) &&
  826. tcp_skb_pcount(skb) == 1)
  827. return 1;
  828. in_flight = tcp_packets_in_flight(tp);
  829. cwnd = tp->snd_cwnd;
  830. if (in_flight < cwnd)
  831. return (cwnd - in_flight);
  832. return 0;
  833. }
  834. /* This must be invoked the first time we consider transmitting
  835. * SKB onto the wire.
  836. */
  837. static int tcp_init_tso_segs(struct sock *sk, struct sk_buff *skb, unsigned int mss_now)
  838. {
  839. int tso_segs = tcp_skb_pcount(skb);
  840. if (!tso_segs ||
  841. (tso_segs > 1 &&
  842. tcp_skb_mss(skb) != mss_now)) {
  843. tcp_set_skb_tso_segs(sk, skb, mss_now);
  844. tso_segs = tcp_skb_pcount(skb);
  845. }
  846. return tso_segs;
  847. }
  848. static inline int tcp_minshall_check(const struct tcp_sock *tp)
  849. {
  850. return after(tp->snd_sml,tp->snd_una) &&
  851. !after(tp->snd_sml, tp->snd_nxt);
  852. }
  853. /* Return 0, if packet can be sent now without violation Nagle's rules:
  854. * 1. It is full sized.
  855. * 2. Or it contains FIN. (already checked by caller)
  856. * 3. Or TCP_NODELAY was set.
  857. * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
  858. * With Minshall's modification: all sent small packets are ACKed.
  859. */
  860. static inline int tcp_nagle_check(const struct tcp_sock *tp,
  861. const struct sk_buff *skb,
  862. unsigned mss_now, int nonagle)
  863. {
  864. return (skb->len < mss_now &&
  865. ((nonagle&TCP_NAGLE_CORK) ||
  866. (!nonagle &&
  867. tp->packets_out &&
  868. tcp_minshall_check(tp))));
  869. }
  870. /* Return non-zero if the Nagle test allows this packet to be
  871. * sent now.
  872. */
  873. static inline int tcp_nagle_test(struct tcp_sock *tp, struct sk_buff *skb,
  874. unsigned int cur_mss, int nonagle)
  875. {
  876. /* Nagle rule does not apply to frames, which sit in the middle of the
  877. * write_queue (they have no chances to get new data).
  878. *
  879. * This is implemented in the callers, where they modify the 'nonagle'
  880. * argument based upon the location of SKB in the send queue.
  881. */
  882. if (nonagle & TCP_NAGLE_PUSH)
  883. return 1;
  884. /* Don't use the nagle rule for urgent data (or for the final FIN). */
  885. if (tp->urg_mode ||
  886. (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN))
  887. return 1;
  888. if (!tcp_nagle_check(tp, skb, cur_mss, nonagle))
  889. return 1;
  890. return 0;
  891. }
  892. /* Does at least the first segment of SKB fit into the send window? */
  893. static inline int tcp_snd_wnd_test(struct tcp_sock *tp, struct sk_buff *skb, unsigned int cur_mss)
  894. {
  895. u32 end_seq = TCP_SKB_CB(skb)->end_seq;
  896. if (skb->len > cur_mss)
  897. end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
  898. return !after(end_seq, tp->snd_una + tp->snd_wnd);
  899. }
  900. /* This checks if the data bearing packet SKB (usually sk->sk_send_head)
  901. * should be put on the wire right now. If so, it returns the number of
  902. * packets allowed by the congestion window.
  903. */
  904. static unsigned int tcp_snd_test(struct sock *sk, struct sk_buff *skb,
  905. unsigned int cur_mss, int nonagle)
  906. {
  907. struct tcp_sock *tp = tcp_sk(sk);
  908. unsigned int cwnd_quota;
  909. tcp_init_tso_segs(sk, skb, cur_mss);
  910. if (!tcp_nagle_test(tp, skb, cur_mss, nonagle))
  911. return 0;
  912. cwnd_quota = tcp_cwnd_test(tp, skb);
  913. if (cwnd_quota &&
  914. !tcp_snd_wnd_test(tp, skb, cur_mss))
  915. cwnd_quota = 0;
  916. return cwnd_quota;
  917. }
  918. static inline int tcp_skb_is_last(const struct sock *sk,
  919. const struct sk_buff *skb)
  920. {
  921. return skb->next == (struct sk_buff *)&sk->sk_write_queue;
  922. }
  923. int tcp_may_send_now(struct sock *sk, struct tcp_sock *tp)
  924. {
  925. struct sk_buff *skb = sk->sk_send_head;
  926. return (skb &&
  927. tcp_snd_test(sk, skb, tcp_current_mss(sk, 1),
  928. (tcp_skb_is_last(sk, skb) ?
  929. TCP_NAGLE_PUSH :
  930. tp->nonagle)));
  931. }
  932. /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
  933. * which is put after SKB on the list. It is very much like
  934. * tcp_fragment() except that it may make several kinds of assumptions
  935. * in order to speed up the splitting operation. In particular, we
  936. * know that all the data is in scatter-gather pages, and that the
  937. * packet has never been sent out before (and thus is not cloned).
  938. */
  939. static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len, unsigned int mss_now)
  940. {
  941. struct sk_buff *buff;
  942. int nlen = skb->len - len;
  943. u16 flags;
  944. /* All of a TSO frame must be composed of paged data. */
  945. if (skb->len != skb->data_len)
  946. return tcp_fragment(sk, skb, len, mss_now);
  947. buff = sk_stream_alloc_pskb(sk, 0, 0, GFP_ATOMIC);
  948. if (unlikely(buff == NULL))
  949. return -ENOMEM;
  950. sk_charge_skb(sk, buff);
  951. buff->truesize += nlen;
  952. skb->truesize -= nlen;
  953. /* Correct the sequence numbers. */
  954. TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
  955. TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
  956. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
  957. /* PSH and FIN should only be set in the second packet. */
  958. flags = TCP_SKB_CB(skb)->flags;
  959. TCP_SKB_CB(skb)->flags = flags & ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH);
  960. TCP_SKB_CB(buff)->flags = flags;
  961. /* This packet was never sent out yet, so no SACK bits. */
  962. TCP_SKB_CB(buff)->sacked = 0;
  963. buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
  964. skb_split(skb, buff, len);
  965. /* Fix up tso_factor for both original and new SKB. */
  966. tcp_set_skb_tso_segs(sk, skb, mss_now);
  967. tcp_set_skb_tso_segs(sk, buff, mss_now);
  968. /* Link BUFF into the send queue. */
  969. skb_header_release(buff);
  970. __skb_append(skb, buff, &sk->sk_write_queue);
  971. return 0;
  972. }
  973. /* Try to defer sending, if possible, in order to minimize the amount
  974. * of TSO splitting we do. View it as a kind of TSO Nagle test.
  975. *
  976. * This algorithm is from John Heffner.
  977. */
  978. static int tcp_tso_should_defer(struct sock *sk, struct tcp_sock *tp, struct sk_buff *skb)
  979. {
  980. const struct inet_connection_sock *icsk = inet_csk(sk);
  981. u32 send_win, cong_win, limit, in_flight;
  982. if (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN)
  983. goto send_now;
  984. if (icsk->icsk_ca_state != TCP_CA_Open)
  985. goto send_now;
  986. /* Defer for less than two clock ticks. */
  987. if (!tp->tso_deferred && ((jiffies<<1)>>1) - (tp->tso_deferred>>1) > 1)
  988. goto send_now;
  989. in_flight = tcp_packets_in_flight(tp);
  990. BUG_ON(tcp_skb_pcount(skb) <= 1 ||
  991. (tp->snd_cwnd <= in_flight));
  992. send_win = (tp->snd_una + tp->snd_wnd) - TCP_SKB_CB(skb)->seq;
  993. /* From in_flight test above, we know that cwnd > in_flight. */
  994. cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
  995. limit = min(send_win, cong_win);
  996. /* If a full-sized TSO skb can be sent, do it. */
  997. if (limit >= 65536)
  998. goto send_now;
  999. if (sysctl_tcp_tso_win_divisor) {
  1000. u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
  1001. /* If at least some fraction of a window is available,
  1002. * just use it.
  1003. */
  1004. chunk /= sysctl_tcp_tso_win_divisor;
  1005. if (limit >= chunk)
  1006. goto send_now;
  1007. } else {
  1008. /* Different approach, try not to defer past a single
  1009. * ACK. Receiver should ACK every other full sized
  1010. * frame, so if we have space for more than 3 frames
  1011. * then send now.
  1012. */
  1013. if (limit > tcp_max_burst(tp) * tp->mss_cache)
  1014. goto send_now;
  1015. }
  1016. /* Ok, it looks like it is advisable to defer. */
  1017. tp->tso_deferred = 1 | (jiffies<<1);
  1018. return 1;
  1019. send_now:
  1020. tp->tso_deferred = 0;
  1021. return 0;
  1022. }
  1023. /* Create a new MTU probe if we are ready.
  1024. * Returns 0 if we should wait to probe (no cwnd available),
  1025. * 1 if a probe was sent,
  1026. * -1 otherwise */
  1027. static int tcp_mtu_probe(struct sock *sk)
  1028. {
  1029. struct tcp_sock *tp = tcp_sk(sk);
  1030. struct inet_connection_sock *icsk = inet_csk(sk);
  1031. struct sk_buff *skb, *nskb, *next;
  1032. int len;
  1033. int probe_size;
  1034. unsigned int pif;
  1035. int copy;
  1036. int mss_now;
  1037. /* Not currently probing/verifying,
  1038. * not in recovery,
  1039. * have enough cwnd, and
  1040. * not SACKing (the variable headers throw things off) */
  1041. if (!icsk->icsk_mtup.enabled ||
  1042. icsk->icsk_mtup.probe_size ||
  1043. inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
  1044. tp->snd_cwnd < 11 ||
  1045. tp->rx_opt.eff_sacks)
  1046. return -1;
  1047. /* Very simple search strategy: just double the MSS. */
  1048. mss_now = tcp_current_mss(sk, 0);
  1049. probe_size = 2*tp->mss_cache;
  1050. if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) {
  1051. /* TODO: set timer for probe_converge_event */
  1052. return -1;
  1053. }
  1054. /* Have enough data in the send queue to probe? */
  1055. len = 0;
  1056. if ((skb = sk->sk_send_head) == NULL)
  1057. return -1;
  1058. while ((len += skb->len) < probe_size && !tcp_skb_is_last(sk, skb))
  1059. skb = skb->next;
  1060. if (len < probe_size)
  1061. return -1;
  1062. /* Receive window check. */
  1063. if (after(TCP_SKB_CB(skb)->seq + probe_size, tp->snd_una + tp->snd_wnd)) {
  1064. if (tp->snd_wnd < probe_size)
  1065. return -1;
  1066. else
  1067. return 0;
  1068. }
  1069. /* Do we need to wait to drain cwnd? */
  1070. pif = tcp_packets_in_flight(tp);
  1071. if (pif + 2 > tp->snd_cwnd) {
  1072. /* With no packets in flight, don't stall. */
  1073. if (pif == 0)
  1074. return -1;
  1075. else
  1076. return 0;
  1077. }
  1078. /* We're allowed to probe. Build it now. */
  1079. if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL)
  1080. return -1;
  1081. sk_charge_skb(sk, nskb);
  1082. skb = sk->sk_send_head;
  1083. __skb_insert(nskb, skb->prev, skb, &sk->sk_write_queue);
  1084. sk->sk_send_head = nskb;
  1085. TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
  1086. TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
  1087. TCP_SKB_CB(nskb)->flags = TCPCB_FLAG_ACK;
  1088. TCP_SKB_CB(nskb)->sacked = 0;
  1089. nskb->csum = 0;
  1090. nskb->ip_summed = skb->ip_summed;
  1091. len = 0;
  1092. while (len < probe_size) {
  1093. next = skb->next;
  1094. copy = min_t(int, skb->len, probe_size - len);
  1095. if (nskb->ip_summed)
  1096. skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
  1097. else
  1098. nskb->csum = skb_copy_and_csum_bits(skb, 0,
  1099. skb_put(nskb, copy), copy, nskb->csum);
  1100. if (skb->len <= copy) {
  1101. /* We've eaten all the data from this skb.
  1102. * Throw it away. */
  1103. TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags;
  1104. __skb_unlink(skb, &sk->sk_write_queue);
  1105. sk_stream_free_skb(sk, skb);
  1106. } else {
  1107. TCP_SKB_CB(nskb)->flags |= TCP_SKB_CB(skb)->flags &
  1108. ~(TCPCB_FLAG_FIN|TCPCB_FLAG_PSH);
  1109. if (!skb_shinfo(skb)->nr_frags) {
  1110. skb_pull(skb, copy);
  1111. if (skb->ip_summed != CHECKSUM_PARTIAL)
  1112. skb->csum = csum_partial(skb->data, skb->len, 0);
  1113. } else {
  1114. __pskb_trim_head(skb, copy);
  1115. tcp_set_skb_tso_segs(sk, skb, mss_now);
  1116. }
  1117. TCP_SKB_CB(skb)->seq += copy;
  1118. }
  1119. len += copy;
  1120. skb = next;
  1121. }
  1122. tcp_init_tso_segs(sk, nskb, nskb->len);
  1123. /* We're ready to send. If this fails, the probe will
  1124. * be resegmented into mss-sized pieces by tcp_write_xmit(). */
  1125. TCP_SKB_CB(nskb)->when = tcp_time_stamp;
  1126. if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
  1127. /* Decrement cwnd here because we are sending
  1128. * effectively two packets. */
  1129. tp->snd_cwnd--;
  1130. update_send_head(sk, tp, nskb);
  1131. icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
  1132. tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
  1133. tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
  1134. return 1;
  1135. }
  1136. return -1;
  1137. }
  1138. /* This routine writes packets to the network. It advances the
  1139. * send_head. This happens as incoming acks open up the remote
  1140. * window for us.
  1141. *
  1142. * Returns 1, if no segments are in flight and we have queued segments, but
  1143. * cannot send anything now because of SWS or another problem.
  1144. */
  1145. static int tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle)
  1146. {
  1147. struct tcp_sock *tp = tcp_sk(sk);
  1148. struct sk_buff *skb;
  1149. unsigned int tso_segs, sent_pkts;
  1150. int cwnd_quota;
  1151. int result;
  1152. /* If we are closed, the bytes will have to remain here.
  1153. * In time closedown will finish, we empty the write queue and all
  1154. * will be happy.
  1155. */
  1156. if (unlikely(sk->sk_state == TCP_CLOSE))
  1157. return 0;
  1158. sent_pkts = 0;
  1159. /* Do MTU probing. */
  1160. if ((result = tcp_mtu_probe(sk)) == 0) {
  1161. return 0;
  1162. } else if (result > 0) {
  1163. sent_pkts = 1;
  1164. }
  1165. while ((skb = sk->sk_send_head)) {
  1166. unsigned int limit;
  1167. tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
  1168. BUG_ON(!tso_segs);
  1169. cwnd_quota = tcp_cwnd_test(tp, skb);
  1170. if (!cwnd_quota)
  1171. break;
  1172. if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
  1173. break;
  1174. if (tso_segs == 1) {
  1175. if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
  1176. (tcp_skb_is_last(sk, skb) ?
  1177. nonagle : TCP_NAGLE_PUSH))))
  1178. break;
  1179. } else {
  1180. if (tcp_tso_should_defer(sk, tp, skb))
  1181. break;
  1182. }
  1183. limit = mss_now;
  1184. if (tso_segs > 1) {
  1185. limit = tcp_window_allows(tp, skb,
  1186. mss_now, cwnd_quota);
  1187. if (skb->len < limit) {
  1188. unsigned int trim = skb->len % mss_now;
  1189. if (trim)
  1190. limit = skb->len - trim;
  1191. }
  1192. }
  1193. if (skb->len > limit &&
  1194. unlikely(tso_fragment(sk, skb, limit, mss_now)))
  1195. break;
  1196. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  1197. if (unlikely(tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC)))
  1198. break;
  1199. /* Advance the send_head. This one is sent out.
  1200. * This call will increment packets_out.
  1201. */
  1202. update_send_head(sk, tp, skb);
  1203. tcp_minshall_update(tp, mss_now, skb);
  1204. sent_pkts++;
  1205. }
  1206. if (likely(sent_pkts)) {
  1207. tcp_cwnd_validate(sk, tp);
  1208. return 0;
  1209. }
  1210. return !tp->packets_out && sk->sk_send_head;
  1211. }
  1212. /* Push out any pending frames which were held back due to
  1213. * TCP_CORK or attempt at coalescing tiny packets.
  1214. * The socket must be locked by the caller.
  1215. */
  1216. void __tcp_push_pending_frames(struct sock *sk, struct tcp_sock *tp,
  1217. unsigned int cur_mss, int nonagle)
  1218. {
  1219. struct sk_buff *skb = sk->sk_send_head;
  1220. if (skb) {
  1221. if (tcp_write_xmit(sk, cur_mss, nonagle))
  1222. tcp_check_probe_timer(sk, tp);
  1223. }
  1224. }
  1225. /* Send _single_ skb sitting at the send head. This function requires
  1226. * true push pending frames to setup probe timer etc.
  1227. */
  1228. void tcp_push_one(struct sock *sk, unsigned int mss_now)
  1229. {
  1230. struct tcp_sock *tp = tcp_sk(sk);
  1231. struct sk_buff *skb = sk->sk_send_head;
  1232. unsigned int tso_segs, cwnd_quota;
  1233. BUG_ON(!skb || skb->len < mss_now);
  1234. tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
  1235. cwnd_quota = tcp_snd_test(sk, skb, mss_now, TCP_NAGLE_PUSH);
  1236. if (likely(cwnd_quota)) {
  1237. unsigned int limit;
  1238. BUG_ON(!tso_segs);
  1239. limit = mss_now;
  1240. if (tso_segs > 1) {
  1241. limit = tcp_window_allows(tp, skb,
  1242. mss_now, cwnd_quota);
  1243. if (skb->len < limit) {
  1244. unsigned int trim = skb->len % mss_now;
  1245. if (trim)
  1246. limit = skb->len - trim;
  1247. }
  1248. }
  1249. if (skb->len > limit &&
  1250. unlikely(tso_fragment(sk, skb, limit, mss_now)))
  1251. return;
  1252. /* Send it out now. */
  1253. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  1254. if (likely(!tcp_transmit_skb(sk, skb, 1, sk->sk_allocation))) {
  1255. update_send_head(sk, tp, skb);
  1256. tcp_cwnd_validate(sk, tp);
  1257. return;
  1258. }
  1259. }
  1260. }
  1261. /* This function returns the amount that we can raise the
  1262. * usable window based on the following constraints
  1263. *
  1264. * 1. The window can never be shrunk once it is offered (RFC 793)
  1265. * 2. We limit memory per socket
  1266. *
  1267. * RFC 1122:
  1268. * "the suggested [SWS] avoidance algorithm for the receiver is to keep
  1269. * RECV.NEXT + RCV.WIN fixed until:
  1270. * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
  1271. *
  1272. * i.e. don't raise the right edge of the window until you can raise
  1273. * it at least MSS bytes.
  1274. *
  1275. * Unfortunately, the recommended algorithm breaks header prediction,
  1276. * since header prediction assumes th->window stays fixed.
  1277. *
  1278. * Strictly speaking, keeping th->window fixed violates the receiver
  1279. * side SWS prevention criteria. The problem is that under this rule
  1280. * a stream of single byte packets will cause the right side of the
  1281. * window to always advance by a single byte.
  1282. *
  1283. * Of course, if the sender implements sender side SWS prevention
  1284. * then this will not be a problem.
  1285. *
  1286. * BSD seems to make the following compromise:
  1287. *
  1288. * If the free space is less than the 1/4 of the maximum
  1289. * space available and the free space is less than 1/2 mss,
  1290. * then set the window to 0.
  1291. * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
  1292. * Otherwise, just prevent the window from shrinking
  1293. * and from being larger than the largest representable value.
  1294. *
  1295. * This prevents incremental opening of the window in the regime
  1296. * where TCP is limited by the speed of the reader side taking
  1297. * data out of the TCP receive queue. It does nothing about
  1298. * those cases where the window is constrained on the sender side
  1299. * because the pipeline is full.
  1300. *
  1301. * BSD also seems to "accidentally" limit itself to windows that are a
  1302. * multiple of MSS, at least until the free space gets quite small.
  1303. * This would appear to be a side effect of the mbuf implementation.
  1304. * Combining these two algorithms results in the observed behavior
  1305. * of having a fixed window size at almost all times.
  1306. *
  1307. * Below we obtain similar behavior by forcing the offered window to
  1308. * a multiple of the mss when it is feasible to do so.
  1309. *
  1310. * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
  1311. * Regular options like TIMESTAMP are taken into account.
  1312. */
  1313. u32 __tcp_select_window(struct sock *sk)
  1314. {
  1315. struct inet_connection_sock *icsk = inet_csk(sk);
  1316. struct tcp_sock *tp = tcp_sk(sk);
  1317. /* MSS for the peer's data. Previous versions used mss_clamp
  1318. * here. I don't know if the value based on our guesses
  1319. * of peer's MSS is better for the performance. It's more correct
  1320. * but may be worse for the performance because of rcv_mss
  1321. * fluctuations. --SAW 1998/11/1
  1322. */
  1323. int mss = icsk->icsk_ack.rcv_mss;
  1324. int free_space = tcp_space(sk);
  1325. int full_space = min_t(int, tp->window_clamp, tcp_full_space(sk));
  1326. int window;
  1327. if (mss > full_space)
  1328. mss = full_space;
  1329. if (free_space < full_space/2) {
  1330. icsk->icsk_ack.quick = 0;
  1331. if (tcp_memory_pressure)
  1332. tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U*tp->advmss);
  1333. if (free_space < mss)
  1334. return 0;
  1335. }
  1336. if (free_space > tp->rcv_ssthresh)
  1337. free_space = tp->rcv_ssthresh;
  1338. /* Don't do rounding if we are using window scaling, since the
  1339. * scaled window will not line up with the MSS boundary anyway.
  1340. */
  1341. window = tp->rcv_wnd;
  1342. if (tp->rx_opt.rcv_wscale) {
  1343. window = free_space;
  1344. /* Advertise enough space so that it won't get scaled away.
  1345. * Import case: prevent zero window announcement if
  1346. * 1<<rcv_wscale > mss.
  1347. */
  1348. if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window)
  1349. window = (((window >> tp->rx_opt.rcv_wscale) + 1)
  1350. << tp->rx_opt.rcv_wscale);
  1351. } else {
  1352. /* Get the largest window that is a nice multiple of mss.
  1353. * Window clamp already applied above.
  1354. * If our current window offering is within 1 mss of the
  1355. * free space we just keep it. This prevents the divide
  1356. * and multiply from happening most of the time.
  1357. * We also don't do any window rounding when the free space
  1358. * is too small.
  1359. */
  1360. if (window <= free_space - mss || window > free_space)
  1361. window = (free_space/mss)*mss;
  1362. else if (mss == full_space &&
  1363. free_space > window + full_space/2)
  1364. window = free_space;
  1365. }
  1366. return window;
  1367. }
  1368. /* Attempt to collapse two adjacent SKB's during retransmission. */
  1369. static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *skb, int mss_now)
  1370. {
  1371. struct tcp_sock *tp = tcp_sk(sk);
  1372. struct sk_buff *next_skb = skb->next;
  1373. /* The first test we must make is that neither of these two
  1374. * SKB's are still referenced by someone else.
  1375. */
  1376. if (!skb_cloned(skb) && !skb_cloned(next_skb)) {
  1377. int skb_size = skb->len, next_skb_size = next_skb->len;
  1378. u16 flags = TCP_SKB_CB(skb)->flags;
  1379. /* Also punt if next skb has been SACK'd. */
  1380. if(TCP_SKB_CB(next_skb)->sacked & TCPCB_SACKED_ACKED)
  1381. return;
  1382. /* Next skb is out of window. */
  1383. if (after(TCP_SKB_CB(next_skb)->end_seq, tp->snd_una+tp->snd_wnd))
  1384. return;
  1385. /* Punt if not enough space exists in the first SKB for
  1386. * the data in the second, or the total combined payload
  1387. * would exceed the MSS.
  1388. */
  1389. if ((next_skb_size > skb_tailroom(skb)) ||
  1390. ((skb_size + next_skb_size) > mss_now))
  1391. return;
  1392. BUG_ON(tcp_skb_pcount(skb) != 1 ||
  1393. tcp_skb_pcount(next_skb) != 1);
  1394. /* changing transmit queue under us so clear hints */
  1395. clear_all_retrans_hints(tp);
  1396. /* Ok. We will be able to collapse the packet. */
  1397. __skb_unlink(next_skb, &sk->sk_write_queue);
  1398. memcpy(skb_put(skb, next_skb_size), next_skb->data, next_skb_size);
  1399. if (next_skb->ip_summed == CHECKSUM_PARTIAL)
  1400. skb->ip_summed = CHECKSUM_PARTIAL;
  1401. if (skb->ip_summed != CHECKSUM_PARTIAL)
  1402. skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
  1403. /* Update sequence range on original skb. */
  1404. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
  1405. /* Merge over control information. */
  1406. flags |= TCP_SKB_CB(next_skb)->flags; /* This moves PSH/FIN etc. over */
  1407. TCP_SKB_CB(skb)->flags = flags;
  1408. /* All done, get rid of second SKB and account for it so
  1409. * packet counting does not break.
  1410. */
  1411. TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked&(TCPCB_EVER_RETRANS|TCPCB_AT_TAIL);
  1412. if (TCP_SKB_CB(next_skb)->sacked&TCPCB_SACKED_RETRANS)
  1413. tp->retrans_out -= tcp_skb_pcount(next_skb);
  1414. if (TCP_SKB_CB(next_skb)->sacked&TCPCB_LOST) {
  1415. tp->lost_out -= tcp_skb_pcount(next_skb);
  1416. tp->left_out -= tcp_skb_pcount(next_skb);
  1417. }
  1418. /* Reno case is special. Sigh... */
  1419. if (!tp->rx_opt.sack_ok && tp->sacked_out) {
  1420. tcp_dec_pcount_approx(&tp->sacked_out, next_skb);
  1421. tp->left_out -= tcp_skb_pcount(next_skb);
  1422. }
  1423. /* Not quite right: it can be > snd.fack, but
  1424. * it is better to underestimate fackets.
  1425. */
  1426. tcp_dec_pcount_approx(&tp->fackets_out, next_skb);
  1427. tcp_packets_out_dec(tp, next_skb);
  1428. sk_stream_free_skb(sk, next_skb);
  1429. }
  1430. }
  1431. /* Do a simple retransmit without using the backoff mechanisms in
  1432. * tcp_timer. This is used for path mtu discovery.
  1433. * The socket is already locked here.
  1434. */
  1435. void tcp_simple_retransmit(struct sock *sk)
  1436. {
  1437. const struct inet_connection_sock *icsk = inet_csk(sk);
  1438. struct tcp_sock *tp = tcp_sk(sk);
  1439. struct sk_buff *skb;
  1440. unsigned int mss = tcp_current_mss(sk, 0);
  1441. int lost = 0;
  1442. sk_stream_for_retrans_queue(skb, sk) {
  1443. if (skb->len > mss &&
  1444. !(TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_ACKED)) {
  1445. if (TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_RETRANS) {
  1446. TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
  1447. tp->retrans_out -= tcp_skb_pcount(skb);
  1448. }
  1449. if (!(TCP_SKB_CB(skb)->sacked&TCPCB_LOST)) {
  1450. TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
  1451. tp->lost_out += tcp_skb_pcount(skb);
  1452. lost = 1;
  1453. }
  1454. }
  1455. }
  1456. clear_all_retrans_hints(tp);
  1457. if (!lost)
  1458. return;
  1459. tcp_sync_left_out(tp);
  1460. /* Don't muck with the congestion window here.
  1461. * Reason is that we do not increase amount of _data_
  1462. * in network, but units changed and effective
  1463. * cwnd/ssthresh really reduced now.
  1464. */
  1465. if (icsk->icsk_ca_state != TCP_CA_Loss) {
  1466. tp->high_seq = tp->snd_nxt;
  1467. tp->snd_ssthresh = tcp_current_ssthresh(sk);
  1468. tp->prior_ssthresh = 0;
  1469. tp->undo_marker = 0;
  1470. tcp_set_ca_state(sk, TCP_CA_Loss);
  1471. }
  1472. tcp_xmit_retransmit_queue(sk);
  1473. }
  1474. /* This retransmits one SKB. Policy decisions and retransmit queue
  1475. * state updates are done by the caller. Returns non-zero if an
  1476. * error occurred which prevented the send.
  1477. */
  1478. int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
  1479. {
  1480. struct tcp_sock *tp = tcp_sk(sk);
  1481. struct inet_connection_sock *icsk = inet_csk(sk);
  1482. unsigned int cur_mss = tcp_current_mss(sk, 0);
  1483. int err;
  1484. /* Inconslusive MTU probe */
  1485. if (icsk->icsk_mtup.probe_size) {
  1486. icsk->icsk_mtup.probe_size = 0;
  1487. }
  1488. /* Do not sent more than we queued. 1/4 is reserved for possible
  1489. * copying overhead: fragmentation, tunneling, mangling etc.
  1490. */
  1491. if (atomic_read(&sk->sk_wmem_alloc) >
  1492. min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf))
  1493. return -EAGAIN;
  1494. if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
  1495. if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
  1496. BUG();
  1497. if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
  1498. return -ENOMEM;
  1499. }
  1500. /* If receiver has shrunk his window, and skb is out of
  1501. * new window, do not retransmit it. The exception is the
  1502. * case, when window is shrunk to zero. In this case
  1503. * our retransmit serves as a zero window probe.
  1504. */
  1505. if (!before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)
  1506. && TCP_SKB_CB(skb)->seq != tp->snd_una)
  1507. return -EAGAIN;
  1508. if (skb->len > cur_mss) {
  1509. if (tcp_fragment(sk, skb, cur_mss, cur_mss))
  1510. return -ENOMEM; /* We'll try again later. */
  1511. }
  1512. /* Collapse two adjacent packets if worthwhile and we can. */
  1513. if(!(TCP_SKB_CB(skb)->flags & TCPCB_FLAG_SYN) &&
  1514. (skb->len < (cur_mss >> 1)) &&
  1515. (skb->next != sk->sk_send_head) &&
  1516. (skb->next != (struct sk_buff *)&sk->sk_write_queue) &&
  1517. (skb_shinfo(skb)->nr_frags == 0 && skb_shinfo(skb->next)->nr_frags == 0) &&
  1518. (tcp_skb_pcount(skb) == 1 && tcp_skb_pcount(skb->next) == 1) &&
  1519. (sysctl_tcp_retrans_collapse != 0))
  1520. tcp_retrans_try_collapse(sk, skb, cur_mss);
  1521. if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
  1522. return -EHOSTUNREACH; /* Routing failure or similar. */
  1523. /* Some Solaris stacks overoptimize and ignore the FIN on a
  1524. * retransmit when old data is attached. So strip it off
  1525. * since it is cheap to do so and saves bytes on the network.
  1526. */
  1527. if(skb->len > 0 &&
  1528. (TCP_SKB_CB(skb)->flags & TCPCB_FLAG_FIN) &&
  1529. tp->snd_una == (TCP_SKB_CB(skb)->end_seq - 1)) {
  1530. if (!pskb_trim(skb, 0)) {
  1531. TCP_SKB_CB(skb)->seq = TCP_SKB_CB(skb)->end_seq - 1;
  1532. skb_shinfo(skb)->gso_segs = 1;
  1533. skb_shinfo(skb)->gso_size = 0;
  1534. skb_shinfo(skb)->gso_type = 0;
  1535. skb->ip_summed = CHECKSUM_NONE;
  1536. skb->csum = 0;
  1537. }
  1538. }
  1539. /* Make a copy, if the first transmission SKB clone we made
  1540. * is still in somebody's hands, else make a clone.
  1541. */
  1542. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  1543. err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
  1544. if (err == 0) {
  1545. /* Update global TCP statistics. */
  1546. TCP_INC_STATS(TCP_MIB_RETRANSSEGS);
  1547. tp->total_retrans++;
  1548. #if FASTRETRANS_DEBUG > 0
  1549. if (TCP_SKB_CB(skb)->sacked&TCPCB_SACKED_RETRANS) {
  1550. if (net_ratelimit())
  1551. printk(KERN_DEBUG "retrans_out leaked.\n");
  1552. }
  1553. #endif
  1554. TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
  1555. tp->retrans_out += tcp_skb_pcount(skb);
  1556. /* Save stamp of the first retransmit. */
  1557. if (!tp->retrans_stamp)
  1558. tp->retrans_stamp = TCP_SKB_CB(skb)->when;
  1559. tp->undo_retrans++;
  1560. /* snd_nxt is stored to detect loss of retransmitted segment,
  1561. * see tcp_input.c tcp_sacktag_write_queue().
  1562. */
  1563. TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt;
  1564. }
  1565. return err;
  1566. }
  1567. /* This gets called after a retransmit timeout, and the initially
  1568. * retransmitted data is acknowledged. It tries to continue
  1569. * resending the rest of the retransmit queue, until either
  1570. * we've sent it all or the congestion window limit is reached.
  1571. * If doing SACK, the first ACK which comes back for a timeout
  1572. * based retransmit packet might feed us FACK information again.
  1573. * If so, we use it to avoid unnecessarily retransmissions.
  1574. */
  1575. void tcp_xmit_retransmit_queue(struct sock *sk)
  1576. {
  1577. const struct inet_connection_sock *icsk = inet_csk(sk);
  1578. struct tcp_sock *tp = tcp_sk(sk);
  1579. struct sk_buff *skb;
  1580. int packet_cnt;
  1581. if (tp->retransmit_skb_hint) {
  1582. skb = tp->retransmit_skb_hint;
  1583. packet_cnt = tp->retransmit_cnt_hint;
  1584. }else{
  1585. skb = sk->sk_write_queue.next;
  1586. packet_cnt = 0;
  1587. }
  1588. /* First pass: retransmit lost packets. */
  1589. if (tp->lost_out) {
  1590. sk_stream_for_retrans_queue_from(skb, sk) {
  1591. __u8 sacked = TCP_SKB_CB(skb)->sacked;
  1592. /* we could do better than to assign each time */
  1593. tp->retransmit_skb_hint = skb;
  1594. tp->retransmit_cnt_hint = packet_cnt;
  1595. /* Assume this retransmit will generate
  1596. * only one packet for congestion window
  1597. * calculation purposes. This works because
  1598. * tcp_retransmit_skb() will chop up the
  1599. * packet to be MSS sized and all the
  1600. * packet counting works out.
  1601. */
  1602. if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
  1603. return;
  1604. if (sacked & TCPCB_LOST) {
  1605. if (!(sacked&(TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))) {
  1606. if (tcp_retransmit_skb(sk, skb)) {
  1607. tp->retransmit_skb_hint = NULL;
  1608. return;
  1609. }
  1610. if (icsk->icsk_ca_state != TCP_CA_Loss)
  1611. NET_INC_STATS_BH(LINUX_MIB_TCPFASTRETRANS);
  1612. else
  1613. NET_INC_STATS_BH(LINUX_MIB_TCPSLOWSTARTRETRANS);
  1614. if (skb ==
  1615. skb_peek(&sk->sk_write_queue))
  1616. inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
  1617. inet_csk(sk)->icsk_rto,
  1618. TCP_RTO_MAX);
  1619. }
  1620. packet_cnt += tcp_skb_pcount(skb);
  1621. if (packet_cnt >= tp->lost_out)
  1622. break;
  1623. }
  1624. }
  1625. }
  1626. /* OK, demanded retransmission is finished. */
  1627. /* Forward retransmissions are possible only during Recovery. */
  1628. if (icsk->icsk_ca_state != TCP_CA_Recovery)
  1629. return;
  1630. /* No forward retransmissions in Reno are possible. */
  1631. if (!tp->rx_opt.sack_ok)
  1632. return;
  1633. /* Yeah, we have to make difficult choice between forward transmission
  1634. * and retransmission... Both ways have their merits...
  1635. *
  1636. * For now we do not retransmit anything, while we have some new
  1637. * segments to send.
  1638. */
  1639. if (tcp_may_send_now(sk, tp))
  1640. return;
  1641. if (tp->forward_skb_hint) {
  1642. skb = tp->forward_skb_hint;
  1643. packet_cnt = tp->forward_cnt_hint;
  1644. } else{
  1645. skb = sk->sk_write_queue.next;
  1646. packet_cnt = 0;
  1647. }
  1648. sk_stream_for_retrans_queue_from(skb, sk) {
  1649. tp->forward_cnt_hint = packet_cnt;
  1650. tp->forward_skb_hint = skb;
  1651. /* Similar to the retransmit loop above we
  1652. * can pretend that the retransmitted SKB
  1653. * we send out here will be composed of one
  1654. * real MSS sized packet because tcp_retransmit_skb()
  1655. * will fragment it if necessary.
  1656. */
  1657. if (++packet_cnt > tp->fackets_out)
  1658. break;
  1659. if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
  1660. break;
  1661. if (TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS)
  1662. continue;
  1663. /* Ok, retransmit it. */
  1664. if (tcp_retransmit_skb(sk, skb)) {
  1665. tp->forward_skb_hint = NULL;
  1666. break;
  1667. }
  1668. if (skb == skb_peek(&sk->sk_write_queue))
  1669. inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
  1670. inet_csk(sk)->icsk_rto,
  1671. TCP_RTO_MAX);
  1672. NET_INC_STATS_BH(LINUX_MIB_TCPFORWARDRETRANS);
  1673. }
  1674. }
  1675. /* Send a fin. The caller locks the socket for us. This cannot be
  1676. * allowed to fail queueing a FIN frame under any circumstances.
  1677. */
  1678. void tcp_send_fin(struct sock *sk)
  1679. {
  1680. struct tcp_sock *tp = tcp_sk(sk);
  1681. struct sk_buff *skb = skb_peek_tail(&sk->sk_write_queue);
  1682. int mss_now;
  1683. /* Optimization, tack on the FIN if we have a queue of
  1684. * unsent frames. But be careful about outgoing SACKS
  1685. * and IP options.
  1686. */
  1687. mss_now = tcp_current_mss(sk, 1);
  1688. if (sk->sk_send_head != NULL) {
  1689. TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN;
  1690. TCP_SKB_CB(skb)->end_seq++;
  1691. tp->write_seq++;
  1692. } else {
  1693. /* Socket is locked, keep trying until memory is available. */
  1694. for (;;) {
  1695. skb = alloc_skb_fclone(MAX_TCP_HEADER, GFP_KERNEL);
  1696. if (skb)
  1697. break;
  1698. yield();
  1699. }
  1700. /* Reserve space for headers and prepare control bits. */
  1701. skb_reserve(skb, MAX_TCP_HEADER);
  1702. skb->csum = 0;
  1703. TCP_SKB_CB(skb)->flags = (TCPCB_FLAG_ACK | TCPCB_FLAG_FIN);
  1704. TCP_SKB_CB(skb)->sacked = 0;
  1705. skb_shinfo(skb)->gso_segs = 1;
  1706. skb_shinfo(skb)->gso_size = 0;
  1707. skb_shinfo(skb)->gso_type = 0;
  1708. /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
  1709. TCP_SKB_CB(skb)->seq = tp->write_seq;
  1710. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + 1;
  1711. tcp_queue_skb(sk, skb);
  1712. }
  1713. __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_OFF);
  1714. }
  1715. /* We get here when a process closes a file descriptor (either due to
  1716. * an explicit close() or as a byproduct of exit()'ing) and there
  1717. * was unread data in the receive queue. This behavior is recommended
  1718. * by draft-ietf-tcpimpl-prob-03.txt section 3.10. -DaveM
  1719. */
  1720. void tcp_send_active_reset(struct sock *sk, gfp_t priority)
  1721. {
  1722. struct tcp_sock *tp = tcp_sk(sk);
  1723. struct sk_buff *skb;
  1724. /* NOTE: No TCP options attached and we never retransmit this. */
  1725. skb = alloc_skb(MAX_TCP_HEADER, priority);
  1726. if (!skb) {
  1727. NET_INC_STATS(LINUX_MIB_TCPABORTFAILED);
  1728. return;
  1729. }
  1730. /* Reserve space for headers and prepare control bits. */
  1731. skb_reserve(skb, MAX_TCP_HEADER);
  1732. skb->csum = 0;
  1733. TCP_SKB_CB(skb)->flags = (TCPCB_FLAG_ACK | TCPCB_FLAG_RST);
  1734. TCP_SKB_CB(skb)->sacked = 0;
  1735. skb_shinfo(skb)->gso_segs = 1;
  1736. skb_shinfo(skb)->gso_size = 0;
  1737. skb_shinfo(skb)->gso_type = 0;
  1738. /* Send it off. */
  1739. TCP_SKB_CB(skb)->seq = tcp_acceptable_seq(sk, tp);
  1740. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq;
  1741. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  1742. if (tcp_transmit_skb(sk, skb, 0, priority))
  1743. NET_INC_STATS(LINUX_MIB_TCPABORTFAILED);
  1744. }
  1745. /* WARNING: This routine must only be called when we have already sent
  1746. * a SYN packet that crossed the incoming SYN that caused this routine
  1747. * to get called. If this assumption fails then the initial rcv_wnd
  1748. * and rcv_wscale values will not be correct.
  1749. */
  1750. int tcp_send_synack(struct sock *sk)
  1751. {
  1752. struct sk_buff* skb;
  1753. skb = skb_peek(&sk->sk_write_queue);
  1754. if (skb == NULL || !(TCP_SKB_CB(skb)->flags&TCPCB_FLAG_SYN)) {
  1755. printk(KERN_DEBUG "tcp_send_synack: wrong queue state\n");
  1756. return -EFAULT;
  1757. }
  1758. if (!(TCP_SKB_CB(skb)->flags&TCPCB_FLAG_ACK)) {
  1759. if (skb_cloned(skb)) {
  1760. struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
  1761. if (nskb == NULL)
  1762. return -ENOMEM;
  1763. __skb_unlink(skb, &sk->sk_write_queue);
  1764. skb_header_release(nskb);
  1765. __skb_queue_head(&sk->sk_write_queue, nskb);
  1766. sk_stream_free_skb(sk, skb);
  1767. sk_charge_skb(sk, nskb);
  1768. skb = nskb;
  1769. }
  1770. TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_ACK;
  1771. TCP_ECN_send_synack(tcp_sk(sk), skb);
  1772. }
  1773. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  1774. return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
  1775. }
  1776. /*
  1777. * Prepare a SYN-ACK.
  1778. */
  1779. struct sk_buff * tcp_make_synack(struct sock *sk, struct dst_entry *dst,
  1780. struct request_sock *req)
  1781. {
  1782. struct inet_request_sock *ireq = inet_rsk(req);
  1783. struct tcp_sock *tp = tcp_sk(sk);
  1784. struct tcphdr *th;
  1785. int tcp_header_size;
  1786. struct sk_buff *skb;
  1787. #ifdef CONFIG_TCP_MD5SIG
  1788. struct tcp_md5sig_key *md5;
  1789. __u8 *md5_hash_location;
  1790. #endif
  1791. skb = sock_wmalloc(sk, MAX_TCP_HEADER + 15, 1, GFP_ATOMIC);
  1792. if (skb == NULL)
  1793. return NULL;
  1794. /* Reserve space for headers. */
  1795. skb_reserve(skb, MAX_TCP_HEADER);
  1796. skb->dst = dst_clone(dst);
  1797. tcp_header_size = (sizeof(struct tcphdr) + TCPOLEN_MSS +
  1798. (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0) +
  1799. (ireq->wscale_ok ? TCPOLEN_WSCALE_ALIGNED : 0) +
  1800. /* SACK_PERM is in the place of NOP NOP of TS */
  1801. ((ireq->sack_ok && !ireq->tstamp_ok) ? TCPOLEN_SACKPERM_ALIGNED : 0));
  1802. #ifdef CONFIG_TCP_MD5SIG
  1803. /* Are we doing MD5 on this segment? If so - make room for it */
  1804. md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req);
  1805. if (md5)
  1806. tcp_header_size += TCPOLEN_MD5SIG_ALIGNED;
  1807. #endif
  1808. skb->h.th = th = (struct tcphdr *) skb_push(skb, tcp_header_size);
  1809. memset(th, 0, sizeof(struct tcphdr));
  1810. th->syn = 1;
  1811. th->ack = 1;
  1812. TCP_ECN_make_synack(req, th);
  1813. th->source = inet_sk(sk)->sport;
  1814. th->dest = ireq->rmt_port;
  1815. TCP_SKB_CB(skb)->seq = tcp_rsk(req)->snt_isn;
  1816. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + 1;
  1817. TCP_SKB_CB(skb)->sacked = 0;
  1818. skb_shinfo(skb)->gso_segs = 1;
  1819. skb_shinfo(skb)->gso_size = 0;
  1820. skb_shinfo(skb)->gso_type = 0;
  1821. th->seq = htonl(TCP_SKB_CB(skb)->seq);
  1822. th->ack_seq = htonl(tcp_rsk(req)->rcv_isn + 1);
  1823. if (req->rcv_wnd == 0) { /* ignored for retransmitted syns */
  1824. __u8 rcv_wscale;
  1825. /* Set this up on the first call only */
  1826. req->window_clamp = tp->window_clamp ? : dst_metric(dst, RTAX_WINDOW);
  1827. /* tcp_full_space because it is guaranteed to be the first packet */
  1828. tcp_select_initial_window(tcp_full_space(sk),
  1829. dst_metric(dst, RTAX_ADVMSS) - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
  1830. &req->rcv_wnd,
  1831. &req->window_clamp,
  1832. ireq->wscale_ok,
  1833. &rcv_wscale);
  1834. ireq->rcv_wscale = rcv_wscale;
  1835. }
  1836. /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
  1837. th->window = htons(min(req->rcv_wnd, 65535U));
  1838. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  1839. tcp_syn_build_options((__be32 *)(th + 1), dst_metric(dst, RTAX_ADVMSS), ireq->tstamp_ok,
  1840. ireq->sack_ok, ireq->wscale_ok, ireq->rcv_wscale,
  1841. TCP_SKB_CB(skb)->when,
  1842. req->ts_recent,
  1843. (
  1844. #ifdef CONFIG_TCP_MD5SIG
  1845. md5 ? &md5_hash_location :
  1846. #endif
  1847. NULL)
  1848. );
  1849. skb->csum = 0;
  1850. th->doff = (tcp_header_size >> 2);
  1851. TCP_INC_STATS(TCP_MIB_OUTSEGS);
  1852. #ifdef CONFIG_TCP_MD5SIG
  1853. /* Okay, we have all we need - do the md5 hash if needed */
  1854. if (md5) {
  1855. tp->af_specific->calc_md5_hash(md5_hash_location,
  1856. md5,
  1857. NULL, dst, req,
  1858. skb->h.th, sk->sk_protocol,
  1859. skb->len);
  1860. }
  1861. #endif
  1862. return skb;
  1863. }
  1864. /*
  1865. * Do all connect socket setups that can be done AF independent.
  1866. */
  1867. static void tcp_connect_init(struct sock *sk)
  1868. {
  1869. struct dst_entry *dst = __sk_dst_get(sk);
  1870. struct tcp_sock *tp = tcp_sk(sk);
  1871. __u8 rcv_wscale;
  1872. /* We'll fix this up when we get a response from the other end.
  1873. * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
  1874. */
  1875. tp->tcp_header_len = sizeof(struct tcphdr) +
  1876. (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0);
  1877. #ifdef CONFIG_TCP_MD5SIG
  1878. if (tp->af_specific->md5_lookup(sk, sk) != NULL)
  1879. tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
  1880. #endif
  1881. /* If user gave his TCP_MAXSEG, record it to clamp */
  1882. if (tp->rx_opt.user_mss)
  1883. tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
  1884. tp->max_window = 0;
  1885. tcp_mtup_init(sk);
  1886. tcp_sync_mss(sk, dst_mtu(dst));
  1887. if (!tp->window_clamp)
  1888. tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
  1889. tp->advmss = dst_metric(dst, RTAX_ADVMSS);
  1890. tcp_initialize_rcv_mss(sk);
  1891. tcp_select_initial_window(tcp_full_space(sk),
  1892. tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
  1893. &tp->rcv_wnd,
  1894. &tp->window_clamp,
  1895. sysctl_tcp_window_scaling,
  1896. &rcv_wscale);
  1897. tp->rx_opt.rcv_wscale = rcv_wscale;
  1898. tp->rcv_ssthresh = tp->rcv_wnd;
  1899. sk->sk_err = 0;
  1900. sock_reset_flag(sk, SOCK_DONE);
  1901. tp->snd_wnd = 0;
  1902. tcp_init_wl(tp, tp->write_seq, 0);
  1903. tp->snd_una = tp->write_seq;
  1904. tp->snd_sml = tp->write_seq;
  1905. tp->rcv_nxt = 0;
  1906. tp->rcv_wup = 0;
  1907. tp->copied_seq = 0;
  1908. inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT;
  1909. inet_csk(sk)->icsk_retransmits = 0;
  1910. tcp_clear_retrans(tp);
  1911. }
  1912. /*
  1913. * Build a SYN and send it off.
  1914. */
  1915. int tcp_connect(struct sock *sk)
  1916. {
  1917. struct tcp_sock *tp = tcp_sk(sk);
  1918. struct sk_buff *buff;
  1919. tcp_connect_init(sk);
  1920. buff = alloc_skb_fclone(MAX_TCP_HEADER + 15, sk->sk_allocation);
  1921. if (unlikely(buff == NULL))
  1922. return -ENOBUFS;
  1923. /* Reserve space for headers. */
  1924. skb_reserve(buff, MAX_TCP_HEADER);
  1925. TCP_SKB_CB(buff)->flags = TCPCB_FLAG_SYN;
  1926. TCP_ECN_send_syn(sk, tp, buff);
  1927. TCP_SKB_CB(buff)->sacked = 0;
  1928. skb_shinfo(buff)->gso_segs = 1;
  1929. skb_shinfo(buff)->gso_size = 0;
  1930. skb_shinfo(buff)->gso_type = 0;
  1931. buff->csum = 0;
  1932. tp->snd_nxt = tp->write_seq;
  1933. TCP_SKB_CB(buff)->seq = tp->write_seq++;
  1934. TCP_SKB_CB(buff)->end_seq = tp->write_seq;
  1935. /* Send it off. */
  1936. TCP_SKB_CB(buff)->when = tcp_time_stamp;
  1937. tp->retrans_stamp = TCP_SKB_CB(buff)->when;
  1938. skb_header_release(buff);
  1939. __skb_queue_tail(&sk->sk_write_queue, buff);
  1940. sk_charge_skb(sk, buff);
  1941. tp->packets_out += tcp_skb_pcount(buff);
  1942. tcp_transmit_skb(sk, buff, 1, GFP_KERNEL);
  1943. /* We change tp->snd_nxt after the tcp_transmit_skb() call
  1944. * in order to make this packet get counted in tcpOutSegs.
  1945. */
  1946. tp->snd_nxt = tp->write_seq;
  1947. tp->pushed_seq = tp->write_seq;
  1948. TCP_INC_STATS(TCP_MIB_ACTIVEOPENS);
  1949. /* Timer for repeating the SYN until an answer. */
  1950. inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
  1951. inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
  1952. return 0;
  1953. }
  1954. /* Send out a delayed ack, the caller does the policy checking
  1955. * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check()
  1956. * for details.
  1957. */
  1958. void tcp_send_delayed_ack(struct sock *sk)
  1959. {
  1960. struct inet_connection_sock *icsk = inet_csk(sk);
  1961. int ato = icsk->icsk_ack.ato;
  1962. unsigned long timeout;
  1963. if (ato > TCP_DELACK_MIN) {
  1964. const struct tcp_sock *tp = tcp_sk(sk);
  1965. int max_ato = HZ/2;
  1966. if (icsk->icsk_ack.pingpong || (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
  1967. max_ato = TCP_DELACK_MAX;
  1968. /* Slow path, intersegment interval is "high". */
  1969. /* If some rtt estimate is known, use it to bound delayed ack.
  1970. * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
  1971. * directly.
  1972. */
  1973. if (tp->srtt) {
  1974. int rtt = max(tp->srtt>>3, TCP_DELACK_MIN);
  1975. if (rtt < max_ato)
  1976. max_ato = rtt;
  1977. }
  1978. ato = min(ato, max_ato);
  1979. }
  1980. /* Stay within the limit we were given */
  1981. timeout = jiffies + ato;
  1982. /* Use new timeout only if there wasn't a older one earlier. */
  1983. if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
  1984. /* If delack timer was blocked or is about to expire,
  1985. * send ACK now.
  1986. */
  1987. if (icsk->icsk_ack.blocked ||
  1988. time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
  1989. tcp_send_ack(sk);
  1990. return;
  1991. }
  1992. if (!time_before(timeout, icsk->icsk_ack.timeout))
  1993. timeout = icsk->icsk_ack.timeout;
  1994. }
  1995. icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
  1996. icsk->icsk_ack.timeout = timeout;
  1997. sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
  1998. }
  1999. /* This routine sends an ack and also updates the window. */
  2000. void tcp_send_ack(struct sock *sk)
  2001. {
  2002. /* If we have been reset, we may not send again. */
  2003. if (sk->sk_state != TCP_CLOSE) {
  2004. struct tcp_sock *tp = tcp_sk(sk);
  2005. struct sk_buff *buff;
  2006. /* We are not putting this on the write queue, so
  2007. * tcp_transmit_skb() will set the ownership to this
  2008. * sock.
  2009. */
  2010. buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
  2011. if (buff == NULL) {
  2012. inet_csk_schedule_ack(sk);
  2013. inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
  2014. inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
  2015. TCP_DELACK_MAX, TCP_RTO_MAX);
  2016. return;
  2017. }
  2018. /* Reserve space for headers and prepare control bits. */
  2019. skb_reserve(buff, MAX_TCP_HEADER);
  2020. buff->csum = 0;
  2021. TCP_SKB_CB(buff)->flags = TCPCB_FLAG_ACK;
  2022. TCP_SKB_CB(buff)->sacked = 0;
  2023. skb_shinfo(buff)->gso_segs = 1;
  2024. skb_shinfo(buff)->gso_size = 0;
  2025. skb_shinfo(buff)->gso_type = 0;
  2026. /* Send it off, this clears delayed acks for us. */
  2027. TCP_SKB_CB(buff)->seq = TCP_SKB_CB(buff)->end_seq = tcp_acceptable_seq(sk, tp);
  2028. TCP_SKB_CB(buff)->when = tcp_time_stamp;
  2029. tcp_transmit_skb(sk, buff, 0, GFP_ATOMIC);
  2030. }
  2031. }
  2032. /* This routine sends a packet with an out of date sequence
  2033. * number. It assumes the other end will try to ack it.
  2034. *
  2035. * Question: what should we make while urgent mode?
  2036. * 4.4BSD forces sending single byte of data. We cannot send
  2037. * out of window data, because we have SND.NXT==SND.MAX...
  2038. *
  2039. * Current solution: to send TWO zero-length segments in urgent mode:
  2040. * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
  2041. * out-of-date with SND.UNA-1 to probe window.
  2042. */
  2043. static int tcp_xmit_probe_skb(struct sock *sk, int urgent)
  2044. {
  2045. struct tcp_sock *tp = tcp_sk(sk);
  2046. struct sk_buff *skb;
  2047. /* We don't queue it, tcp_transmit_skb() sets ownership. */
  2048. skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
  2049. if (skb == NULL)
  2050. return -1;
  2051. /* Reserve space for headers and set control bits. */
  2052. skb_reserve(skb, MAX_TCP_HEADER);
  2053. skb->csum = 0;
  2054. TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK;
  2055. TCP_SKB_CB(skb)->sacked = urgent;
  2056. skb_shinfo(skb)->gso_segs = 1;
  2057. skb_shinfo(skb)->gso_size = 0;
  2058. skb_shinfo(skb)->gso_type = 0;
  2059. /* Use a previous sequence. This should cause the other
  2060. * end to send an ack. Don't queue or clone SKB, just
  2061. * send it.
  2062. */
  2063. TCP_SKB_CB(skb)->seq = urgent ? tp->snd_una : tp->snd_una - 1;
  2064. TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq;
  2065. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  2066. return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC);
  2067. }
  2068. int tcp_write_wakeup(struct sock *sk)
  2069. {
  2070. if (sk->sk_state != TCP_CLOSE) {
  2071. struct tcp_sock *tp = tcp_sk(sk);
  2072. struct sk_buff *skb;
  2073. if ((skb = sk->sk_send_head) != NULL &&
  2074. before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) {
  2075. int err;
  2076. unsigned int mss = tcp_current_mss(sk, 0);
  2077. unsigned int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq;
  2078. if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
  2079. tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
  2080. /* We are probing the opening of a window
  2081. * but the window size is != 0
  2082. * must have been a result SWS avoidance ( sender )
  2083. */
  2084. if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
  2085. skb->len > mss) {
  2086. seg_size = min(seg_size, mss);
  2087. TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
  2088. if (tcp_fragment(sk, skb, seg_size, mss))
  2089. return -1;
  2090. } else if (!tcp_skb_pcount(skb))
  2091. tcp_set_skb_tso_segs(sk, skb, mss);
  2092. TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
  2093. TCP_SKB_CB(skb)->when = tcp_time_stamp;
  2094. err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
  2095. if (!err) {
  2096. update_send_head(sk, tp, skb);
  2097. }
  2098. return err;
  2099. } else {
  2100. if (tp->urg_mode &&
  2101. between(tp->snd_up, tp->snd_una+1, tp->snd_una+0xFFFF))
  2102. tcp_xmit_probe_skb(sk, TCPCB_URG);
  2103. return tcp_xmit_probe_skb(sk, 0);
  2104. }
  2105. }
  2106. return -1;
  2107. }
  2108. /* A window probe timeout has occurred. If window is not closed send
  2109. * a partial packet else a zero probe.
  2110. */
  2111. void tcp_send_probe0(struct sock *sk)
  2112. {
  2113. struct inet_connection_sock *icsk = inet_csk(sk);
  2114. struct tcp_sock *tp = tcp_sk(sk);
  2115. int err;
  2116. err = tcp_write_wakeup(sk);
  2117. if (tp->packets_out || !sk->sk_send_head) {
  2118. /* Cancel probe timer, if it is not required. */
  2119. icsk->icsk_probes_out = 0;
  2120. icsk->icsk_backoff = 0;
  2121. return;
  2122. }
  2123. if (err <= 0) {
  2124. if (icsk->icsk_backoff < sysctl_tcp_retries2)
  2125. icsk->icsk_backoff++;
  2126. icsk->icsk_probes_out++;
  2127. inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
  2128. min(icsk->icsk_rto << icsk->icsk_backoff, TCP_RTO_MAX),
  2129. TCP_RTO_MAX);
  2130. } else {
  2131. /* If packet was not sent due to local congestion,
  2132. * do not backoff and do not remember icsk_probes_out.
  2133. * Let local senders to fight for local resources.
  2134. *
  2135. * Use accumulated backoff yet.
  2136. */
  2137. if (!icsk->icsk_probes_out)
  2138. icsk->icsk_probes_out = 1;
  2139. inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
  2140. min(icsk->icsk_rto << icsk->icsk_backoff,
  2141. TCP_RESOURCE_PROBE_INTERVAL),
  2142. TCP_RTO_MAX);
  2143. }
  2144. }
  2145. EXPORT_SYMBOL(tcp_connect);
  2146. EXPORT_SYMBOL(tcp_make_synack);
  2147. EXPORT_SYMBOL(tcp_simple_retransmit);
  2148. EXPORT_SYMBOL(tcp_sync_mss);
  2149. EXPORT_SYMBOL(sysctl_tcp_tso_win_divisor);
  2150. EXPORT_SYMBOL(tcp_mtup_init);