input.c 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* RxRPC packet reception
  3. *
  4. * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells (dhowells@redhat.com)
  6. */
  7. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  8. #include <linux/module.h>
  9. #include <linux/net.h>
  10. #include <linux/skbuff.h>
  11. #include <linux/errqueue.h>
  12. #include <linux/udp.h>
  13. #include <linux/in.h>
  14. #include <linux/in6.h>
  15. #include <linux/icmp.h>
  16. #include <linux/gfp.h>
  17. #include <net/sock.h>
  18. #include <net/af_rxrpc.h>
  19. #include <net/ip.h>
  20. #include <net/udp.h>
  21. #include <net/net_namespace.h>
  22. #include "ar-internal.h"
  23. static void rxrpc_proto_abort(const char *why,
  24. struct rxrpc_call *call, rxrpc_seq_t seq)
  25. {
  26. if (rxrpc_abort_call(why, call, seq, RX_PROTOCOL_ERROR, -EBADMSG)) {
  27. set_bit(RXRPC_CALL_EV_ABORT, &call->events);
  28. rxrpc_queue_call(call);
  29. }
  30. }
  31. /*
  32. * Do TCP-style congestion management [RFC 5681].
  33. */
  34. static void rxrpc_congestion_management(struct rxrpc_call *call,
  35. struct sk_buff *skb,
  36. struct rxrpc_ack_summary *summary,
  37. rxrpc_serial_t acked_serial)
  38. {
  39. enum rxrpc_congest_change change = rxrpc_cong_no_change;
  40. unsigned int cumulative_acks = call->cong_cumul_acks;
  41. unsigned int cwnd = call->cong_cwnd;
  42. bool resend = false;
  43. summary->flight_size =
  44. (call->tx_top - call->tx_hard_ack) - summary->nr_acks;
  45. if (test_and_clear_bit(RXRPC_CALL_RETRANS_TIMEOUT, &call->flags)) {
  46. summary->retrans_timeo = true;
  47. call->cong_ssthresh = max_t(unsigned int,
  48. summary->flight_size / 2, 2);
  49. cwnd = 1;
  50. if (cwnd >= call->cong_ssthresh &&
  51. call->cong_mode == RXRPC_CALL_SLOW_START) {
  52. call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
  53. call->cong_tstamp = skb->tstamp;
  54. cumulative_acks = 0;
  55. }
  56. }
  57. cumulative_acks += summary->nr_new_acks;
  58. cumulative_acks += summary->nr_rot_new_acks;
  59. if (cumulative_acks > 255)
  60. cumulative_acks = 255;
  61. summary->mode = call->cong_mode;
  62. summary->cwnd = call->cong_cwnd;
  63. summary->ssthresh = call->cong_ssthresh;
  64. summary->cumulative_acks = cumulative_acks;
  65. summary->dup_acks = call->cong_dup_acks;
  66. switch (call->cong_mode) {
  67. case RXRPC_CALL_SLOW_START:
  68. if (summary->nr_nacks > 0)
  69. goto packet_loss_detected;
  70. if (summary->cumulative_acks > 0)
  71. cwnd += 1;
  72. if (cwnd >= call->cong_ssthresh) {
  73. call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
  74. call->cong_tstamp = skb->tstamp;
  75. }
  76. goto out;
  77. case RXRPC_CALL_CONGEST_AVOIDANCE:
  78. if (summary->nr_nacks > 0)
  79. goto packet_loss_detected;
  80. /* We analyse the number of packets that get ACK'd per RTT
  81. * period and increase the window if we managed to fill it.
  82. */
  83. if (call->peer->rtt_count == 0)
  84. goto out;
  85. if (ktime_before(skb->tstamp,
  86. ktime_add_us(call->cong_tstamp,
  87. call->peer->srtt_us >> 3)))
  88. goto out_no_clear_ca;
  89. change = rxrpc_cong_rtt_window_end;
  90. call->cong_tstamp = skb->tstamp;
  91. if (cumulative_acks >= cwnd)
  92. cwnd++;
  93. goto out;
  94. case RXRPC_CALL_PACKET_LOSS:
  95. if (summary->nr_nacks == 0)
  96. goto resume_normality;
  97. if (summary->new_low_nack) {
  98. change = rxrpc_cong_new_low_nack;
  99. call->cong_dup_acks = 1;
  100. if (call->cong_extra > 1)
  101. call->cong_extra = 1;
  102. goto send_extra_data;
  103. }
  104. call->cong_dup_acks++;
  105. if (call->cong_dup_acks < 3)
  106. goto send_extra_data;
  107. change = rxrpc_cong_begin_retransmission;
  108. call->cong_mode = RXRPC_CALL_FAST_RETRANSMIT;
  109. call->cong_ssthresh = max_t(unsigned int,
  110. summary->flight_size / 2, 2);
  111. cwnd = call->cong_ssthresh + 3;
  112. call->cong_extra = 0;
  113. call->cong_dup_acks = 0;
  114. resend = true;
  115. goto out;
  116. case RXRPC_CALL_FAST_RETRANSMIT:
  117. if (!summary->new_low_nack) {
  118. if (summary->nr_new_acks == 0)
  119. cwnd += 1;
  120. call->cong_dup_acks++;
  121. if (call->cong_dup_acks == 2) {
  122. change = rxrpc_cong_retransmit_again;
  123. call->cong_dup_acks = 0;
  124. resend = true;
  125. }
  126. } else {
  127. change = rxrpc_cong_progress;
  128. cwnd = call->cong_ssthresh;
  129. if (summary->nr_nacks == 0)
  130. goto resume_normality;
  131. }
  132. goto out;
  133. default:
  134. BUG();
  135. goto out;
  136. }
  137. resume_normality:
  138. change = rxrpc_cong_cleared_nacks;
  139. call->cong_dup_acks = 0;
  140. call->cong_extra = 0;
  141. call->cong_tstamp = skb->tstamp;
  142. if (cwnd < call->cong_ssthresh)
  143. call->cong_mode = RXRPC_CALL_SLOW_START;
  144. else
  145. call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
  146. out:
  147. cumulative_acks = 0;
  148. out_no_clear_ca:
  149. if (cwnd >= RXRPC_RXTX_BUFF_SIZE - 1)
  150. cwnd = RXRPC_RXTX_BUFF_SIZE - 1;
  151. call->cong_cwnd = cwnd;
  152. call->cong_cumul_acks = cumulative_acks;
  153. trace_rxrpc_congest(call, summary, acked_serial, change);
  154. if (resend && !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))
  155. rxrpc_queue_call(call);
  156. return;
  157. packet_loss_detected:
  158. change = rxrpc_cong_saw_nack;
  159. call->cong_mode = RXRPC_CALL_PACKET_LOSS;
  160. call->cong_dup_acks = 0;
  161. goto send_extra_data;
  162. send_extra_data:
  163. /* Send some previously unsent DATA if we have some to advance the ACK
  164. * state.
  165. */
  166. if (call->rxtx_annotations[call->tx_top & RXRPC_RXTX_BUFF_MASK] &
  167. RXRPC_TX_ANNO_LAST ||
  168. summary->nr_acks != call->tx_top - call->tx_hard_ack) {
  169. call->cong_extra++;
  170. wake_up(&call->waitq);
  171. }
  172. goto out_no_clear_ca;
  173. }
  174. /*
  175. * Apply a hard ACK by advancing the Tx window.
  176. */
  177. static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to,
  178. struct rxrpc_ack_summary *summary)
  179. {
  180. struct sk_buff *skb, *list = NULL;
  181. bool rot_last = false;
  182. int ix;
  183. u8 annotation;
  184. if (call->acks_lowest_nak == call->tx_hard_ack) {
  185. call->acks_lowest_nak = to;
  186. } else if (before_eq(call->acks_lowest_nak, to)) {
  187. summary->new_low_nack = true;
  188. call->acks_lowest_nak = to;
  189. }
  190. spin_lock(&call->lock);
  191. while (before(call->tx_hard_ack, to)) {
  192. call->tx_hard_ack++;
  193. ix = call->tx_hard_ack & RXRPC_RXTX_BUFF_MASK;
  194. skb = call->rxtx_buffer[ix];
  195. annotation = call->rxtx_annotations[ix];
  196. rxrpc_see_skb(skb, rxrpc_skb_rotated);
  197. call->rxtx_buffer[ix] = NULL;
  198. call->rxtx_annotations[ix] = 0;
  199. skb->next = list;
  200. list = skb;
  201. if (annotation & RXRPC_TX_ANNO_LAST) {
  202. set_bit(RXRPC_CALL_TX_LAST, &call->flags);
  203. rot_last = true;
  204. }
  205. if ((annotation & RXRPC_TX_ANNO_MASK) != RXRPC_TX_ANNO_ACK)
  206. summary->nr_rot_new_acks++;
  207. }
  208. spin_unlock(&call->lock);
  209. trace_rxrpc_transmit(call, (rot_last ?
  210. rxrpc_transmit_rotate_last :
  211. rxrpc_transmit_rotate));
  212. wake_up(&call->waitq);
  213. while (list) {
  214. skb = list;
  215. list = skb->next;
  216. skb_mark_not_on_list(skb);
  217. rxrpc_free_skb(skb, rxrpc_skb_freed);
  218. }
  219. return rot_last;
  220. }
  221. /*
  222. * End the transmission phase of a call.
  223. *
  224. * This occurs when we get an ACKALL packet, the first DATA packet of a reply,
  225. * or a final ACK packet.
  226. */
  227. static bool rxrpc_end_tx_phase(struct rxrpc_call *call, bool reply_begun,
  228. const char *abort_why)
  229. {
  230. unsigned int state;
  231. ASSERT(test_bit(RXRPC_CALL_TX_LAST, &call->flags));
  232. write_lock(&call->state_lock);
  233. state = call->state;
  234. switch (state) {
  235. case RXRPC_CALL_CLIENT_SEND_REQUEST:
  236. case RXRPC_CALL_CLIENT_AWAIT_REPLY:
  237. if (reply_begun)
  238. call->state = state = RXRPC_CALL_CLIENT_RECV_REPLY;
  239. else
  240. call->state = state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
  241. break;
  242. case RXRPC_CALL_SERVER_AWAIT_ACK:
  243. __rxrpc_call_completed(call);
  244. state = call->state;
  245. break;
  246. default:
  247. goto bad_state;
  248. }
  249. write_unlock(&call->state_lock);
  250. if (state == RXRPC_CALL_CLIENT_AWAIT_REPLY)
  251. trace_rxrpc_transmit(call, rxrpc_transmit_await_reply);
  252. else
  253. trace_rxrpc_transmit(call, rxrpc_transmit_end);
  254. _leave(" = ok");
  255. return true;
  256. bad_state:
  257. write_unlock(&call->state_lock);
  258. kdebug("end_tx %s", rxrpc_call_states[call->state]);
  259. rxrpc_proto_abort(abort_why, call, call->tx_top);
  260. return false;
  261. }
  262. /*
  263. * Begin the reply reception phase of a call.
  264. */
  265. static bool rxrpc_receiving_reply(struct rxrpc_call *call)
  266. {
  267. struct rxrpc_ack_summary summary = { 0 };
  268. unsigned long now, timo;
  269. rxrpc_seq_t top = READ_ONCE(call->tx_top);
  270. if (call->ackr_reason) {
  271. spin_lock_bh(&call->lock);
  272. call->ackr_reason = 0;
  273. spin_unlock_bh(&call->lock);
  274. now = jiffies;
  275. timo = now + MAX_JIFFY_OFFSET;
  276. WRITE_ONCE(call->resend_at, timo);
  277. WRITE_ONCE(call->ack_at, timo);
  278. trace_rxrpc_timer(call, rxrpc_timer_init_for_reply, now);
  279. }
  280. if (!test_bit(RXRPC_CALL_TX_LAST, &call->flags)) {
  281. if (!rxrpc_rotate_tx_window(call, top, &summary)) {
  282. rxrpc_proto_abort("TXL", call, top);
  283. return false;
  284. }
  285. }
  286. if (!rxrpc_end_tx_phase(call, true, "ETD"))
  287. return false;
  288. call->tx_phase = false;
  289. return true;
  290. }
  291. /*
  292. * Scan a data packet to validate its structure and to work out how many
  293. * subpackets it contains.
  294. *
  295. * A jumbo packet is a collection of consecutive packets glued together with
  296. * little headers between that indicate how to change the initial header for
  297. * each subpacket.
  298. *
  299. * RXRPC_JUMBO_PACKET must be set on all but the last subpacket - and all but
  300. * the last are RXRPC_JUMBO_DATALEN in size. The last subpacket may be of any
  301. * size.
  302. */
  303. static bool rxrpc_validate_data(struct sk_buff *skb)
  304. {
  305. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  306. unsigned int offset = sizeof(struct rxrpc_wire_header);
  307. unsigned int len = skb->len;
  308. u8 flags = sp->hdr.flags;
  309. for (;;) {
  310. if (flags & RXRPC_REQUEST_ACK)
  311. __set_bit(sp->nr_subpackets, sp->rx_req_ack);
  312. sp->nr_subpackets++;
  313. if (!(flags & RXRPC_JUMBO_PACKET))
  314. break;
  315. if (len - offset < RXRPC_JUMBO_SUBPKTLEN)
  316. goto protocol_error;
  317. if (flags & RXRPC_LAST_PACKET)
  318. goto protocol_error;
  319. offset += RXRPC_JUMBO_DATALEN;
  320. if (skb_copy_bits(skb, offset, &flags, 1) < 0)
  321. goto protocol_error;
  322. offset += sizeof(struct rxrpc_jumbo_header);
  323. }
  324. if (flags & RXRPC_LAST_PACKET)
  325. sp->rx_flags |= RXRPC_SKB_INCL_LAST;
  326. return true;
  327. protocol_error:
  328. return false;
  329. }
  330. /*
  331. * Handle reception of a duplicate packet.
  332. *
  333. * We have to take care to avoid an attack here whereby we're given a series of
  334. * jumbograms, each with a sequence number one before the preceding one and
  335. * filled up to maximum UDP size. If they never send us the first packet in
  336. * the sequence, they can cause us to have to hold on to around 2MiB of kernel
  337. * space until the call times out.
  338. *
  339. * We limit the space usage by only accepting three duplicate jumbo packets per
  340. * call. After that, we tell the other side we're no longer accepting jumbos
  341. * (that information is encoded in the ACK packet).
  342. */
  343. static void rxrpc_input_dup_data(struct rxrpc_call *call, rxrpc_seq_t seq,
  344. bool is_jumbo, bool *_jumbo_bad)
  345. {
  346. /* Discard normal packets that are duplicates. */
  347. if (is_jumbo)
  348. return;
  349. /* Skip jumbo subpackets that are duplicates. When we've had three or
  350. * more partially duplicate jumbo packets, we refuse to take any more
  351. * jumbos for this call.
  352. */
  353. if (!*_jumbo_bad) {
  354. call->nr_jumbo_bad++;
  355. *_jumbo_bad = true;
  356. }
  357. }
  358. /*
  359. * Process a DATA packet, adding the packet to the Rx ring. The caller's
  360. * packet ref must be passed on or discarded.
  361. */
  362. static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb)
  363. {
  364. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  365. enum rxrpc_call_state state;
  366. unsigned int j, nr_subpackets;
  367. rxrpc_serial_t serial = sp->hdr.serial, ack_serial = 0;
  368. rxrpc_seq_t seq0 = sp->hdr.seq, hard_ack;
  369. bool immediate_ack = false, jumbo_bad = false;
  370. u8 ack = 0;
  371. _enter("{%u,%u},{%u,%u}",
  372. call->rx_hard_ack, call->rx_top, skb->len, seq0);
  373. _proto("Rx DATA %%%u { #%u f=%02x n=%u }",
  374. sp->hdr.serial, seq0, sp->hdr.flags, sp->nr_subpackets);
  375. state = READ_ONCE(call->state);
  376. if (state >= RXRPC_CALL_COMPLETE) {
  377. rxrpc_free_skb(skb, rxrpc_skb_freed);
  378. return;
  379. }
  380. if (state == RXRPC_CALL_SERVER_RECV_REQUEST) {
  381. unsigned long timo = READ_ONCE(call->next_req_timo);
  382. unsigned long now, expect_req_by;
  383. if (timo) {
  384. now = jiffies;
  385. expect_req_by = now + timo;
  386. WRITE_ONCE(call->expect_req_by, expect_req_by);
  387. rxrpc_reduce_call_timer(call, expect_req_by, now,
  388. rxrpc_timer_set_for_idle);
  389. }
  390. }
  391. spin_lock(&call->input_lock);
  392. /* Received data implicitly ACKs all of the request packets we sent
  393. * when we're acting as a client.
  394. */
  395. if ((state == RXRPC_CALL_CLIENT_SEND_REQUEST ||
  396. state == RXRPC_CALL_CLIENT_AWAIT_REPLY) &&
  397. !rxrpc_receiving_reply(call))
  398. goto unlock;
  399. call->ackr_prev_seq = seq0;
  400. hard_ack = READ_ONCE(call->rx_hard_ack);
  401. nr_subpackets = sp->nr_subpackets;
  402. if (nr_subpackets > 1) {
  403. if (call->nr_jumbo_bad > 3) {
  404. ack = RXRPC_ACK_NOSPACE;
  405. ack_serial = serial;
  406. goto ack;
  407. }
  408. }
  409. for (j = 0; j < nr_subpackets; j++) {
  410. rxrpc_serial_t serial = sp->hdr.serial + j;
  411. rxrpc_seq_t seq = seq0 + j;
  412. unsigned int ix = seq & RXRPC_RXTX_BUFF_MASK;
  413. bool terminal = (j == nr_subpackets - 1);
  414. bool last = terminal && (sp->rx_flags & RXRPC_SKB_INCL_LAST);
  415. u8 flags, annotation = j;
  416. _proto("Rx DATA+%u %%%u { #%x t=%u l=%u }",
  417. j, serial, seq, terminal, last);
  418. if (last) {
  419. if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
  420. seq != call->rx_top) {
  421. rxrpc_proto_abort("LSN", call, seq);
  422. goto unlock;
  423. }
  424. } else {
  425. if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
  426. after_eq(seq, call->rx_top)) {
  427. rxrpc_proto_abort("LSA", call, seq);
  428. goto unlock;
  429. }
  430. }
  431. flags = 0;
  432. if (last)
  433. flags |= RXRPC_LAST_PACKET;
  434. if (!terminal)
  435. flags |= RXRPC_JUMBO_PACKET;
  436. if (test_bit(j, sp->rx_req_ack))
  437. flags |= RXRPC_REQUEST_ACK;
  438. trace_rxrpc_rx_data(call->debug_id, seq, serial, flags, annotation);
  439. if (before_eq(seq, hard_ack)) {
  440. ack = RXRPC_ACK_DUPLICATE;
  441. ack_serial = serial;
  442. continue;
  443. }
  444. if (call->rxtx_buffer[ix]) {
  445. rxrpc_input_dup_data(call, seq, nr_subpackets > 1,
  446. &jumbo_bad);
  447. if (ack != RXRPC_ACK_DUPLICATE) {
  448. ack = RXRPC_ACK_DUPLICATE;
  449. ack_serial = serial;
  450. }
  451. immediate_ack = true;
  452. continue;
  453. }
  454. if (after(seq, hard_ack + call->rx_winsize)) {
  455. ack = RXRPC_ACK_EXCEEDS_WINDOW;
  456. ack_serial = serial;
  457. if (flags & RXRPC_JUMBO_PACKET) {
  458. if (!jumbo_bad) {
  459. call->nr_jumbo_bad++;
  460. jumbo_bad = true;
  461. }
  462. }
  463. goto ack;
  464. }
  465. if (flags & RXRPC_REQUEST_ACK && !ack) {
  466. ack = RXRPC_ACK_REQUESTED;
  467. ack_serial = serial;
  468. }
  469. /* Queue the packet. We use a couple of memory barriers here as need
  470. * to make sure that rx_top is perceived to be set after the buffer
  471. * pointer and that the buffer pointer is set after the annotation and
  472. * the skb data.
  473. *
  474. * Barriers against rxrpc_recvmsg_data() and rxrpc_rotate_rx_window()
  475. * and also rxrpc_fill_out_ack().
  476. */
  477. if (!terminal)
  478. rxrpc_get_skb(skb, rxrpc_skb_got);
  479. call->rxtx_annotations[ix] = annotation;
  480. smp_wmb();
  481. call->rxtx_buffer[ix] = skb;
  482. if (after(seq, call->rx_top)) {
  483. smp_store_release(&call->rx_top, seq);
  484. } else if (before(seq, call->rx_top)) {
  485. /* Send an immediate ACK if we fill in a hole */
  486. if (!ack) {
  487. ack = RXRPC_ACK_DELAY;
  488. ack_serial = serial;
  489. }
  490. immediate_ack = true;
  491. }
  492. if (terminal) {
  493. /* From this point on, we're not allowed to touch the
  494. * packet any longer as its ref now belongs to the Rx
  495. * ring.
  496. */
  497. skb = NULL;
  498. sp = NULL;
  499. }
  500. if (last) {
  501. set_bit(RXRPC_CALL_RX_LAST, &call->flags);
  502. if (!ack) {
  503. ack = RXRPC_ACK_DELAY;
  504. ack_serial = serial;
  505. }
  506. trace_rxrpc_receive(call, rxrpc_receive_queue_last, serial, seq);
  507. } else {
  508. trace_rxrpc_receive(call, rxrpc_receive_queue, serial, seq);
  509. }
  510. if (after_eq(seq, call->rx_expect_next)) {
  511. if (after(seq, call->rx_expect_next)) {
  512. _net("OOS %u > %u", seq, call->rx_expect_next);
  513. ack = RXRPC_ACK_OUT_OF_SEQUENCE;
  514. ack_serial = serial;
  515. }
  516. call->rx_expect_next = seq + 1;
  517. }
  518. }
  519. ack:
  520. if (ack)
  521. rxrpc_propose_ACK(call, ack, ack_serial,
  522. immediate_ack, true,
  523. rxrpc_propose_ack_input_data);
  524. else
  525. rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial,
  526. false, true,
  527. rxrpc_propose_ack_input_data);
  528. trace_rxrpc_notify_socket(call->debug_id, serial);
  529. rxrpc_notify_socket(call);
  530. unlock:
  531. spin_unlock(&call->input_lock);
  532. rxrpc_free_skb(skb, rxrpc_skb_freed);
  533. _leave(" [queued]");
  534. }
  535. /*
  536. * See if there's a cached RTT probe to complete.
  537. */
  538. static void rxrpc_complete_rtt_probe(struct rxrpc_call *call,
  539. ktime_t resp_time,
  540. rxrpc_serial_t acked_serial,
  541. rxrpc_serial_t ack_serial,
  542. enum rxrpc_rtt_rx_trace type)
  543. {
  544. rxrpc_serial_t orig_serial;
  545. unsigned long avail;
  546. ktime_t sent_at;
  547. bool matched = false;
  548. int i;
  549. avail = READ_ONCE(call->rtt_avail);
  550. smp_rmb(); /* Read avail bits before accessing data. */
  551. for (i = 0; i < ARRAY_SIZE(call->rtt_serial); i++) {
  552. if (!test_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &avail))
  553. continue;
  554. sent_at = call->rtt_sent_at[i];
  555. orig_serial = call->rtt_serial[i];
  556. if (orig_serial == acked_serial) {
  557. clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
  558. smp_mb(); /* Read data before setting avail bit */
  559. set_bit(i, &call->rtt_avail);
  560. if (type != rxrpc_rtt_rx_cancel)
  561. rxrpc_peer_add_rtt(call, type, i, acked_serial, ack_serial,
  562. sent_at, resp_time);
  563. else
  564. trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_cancel, i,
  565. orig_serial, acked_serial, 0, 0);
  566. matched = true;
  567. }
  568. /* If a later serial is being acked, then mark this slot as
  569. * being available.
  570. */
  571. if (after(acked_serial, orig_serial)) {
  572. trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_obsolete, i,
  573. orig_serial, acked_serial, 0, 0);
  574. clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
  575. smp_wmb();
  576. set_bit(i, &call->rtt_avail);
  577. }
  578. }
  579. if (!matched)
  580. trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_lost, 9, 0, acked_serial, 0, 0);
  581. }
  582. /*
  583. * Process the response to a ping that we sent to find out if we lost an ACK.
  584. *
  585. * If we got back a ping response that indicates a lower tx_top than what we
  586. * had at the time of the ping transmission, we adjudge all the DATA packets
  587. * sent between the response tx_top and the ping-time tx_top to have been lost.
  588. */
  589. static void rxrpc_input_check_for_lost_ack(struct rxrpc_call *call)
  590. {
  591. rxrpc_seq_t top, bottom, seq;
  592. bool resend = false;
  593. spin_lock_bh(&call->lock);
  594. bottom = call->tx_hard_ack + 1;
  595. top = call->acks_lost_top;
  596. if (before(bottom, top)) {
  597. for (seq = bottom; before_eq(seq, top); seq++) {
  598. int ix = seq & RXRPC_RXTX_BUFF_MASK;
  599. u8 annotation = call->rxtx_annotations[ix];
  600. u8 anno_type = annotation & RXRPC_TX_ANNO_MASK;
  601. if (anno_type != RXRPC_TX_ANNO_UNACK)
  602. continue;
  603. annotation &= ~RXRPC_TX_ANNO_MASK;
  604. annotation |= RXRPC_TX_ANNO_RETRANS;
  605. call->rxtx_annotations[ix] = annotation;
  606. resend = true;
  607. }
  608. }
  609. spin_unlock_bh(&call->lock);
  610. if (resend && !test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))
  611. rxrpc_queue_call(call);
  612. }
  613. /*
  614. * Process a ping response.
  615. */
  616. static void rxrpc_input_ping_response(struct rxrpc_call *call,
  617. ktime_t resp_time,
  618. rxrpc_serial_t acked_serial,
  619. rxrpc_serial_t ack_serial)
  620. {
  621. if (acked_serial == call->acks_lost_ping)
  622. rxrpc_input_check_for_lost_ack(call);
  623. }
  624. /*
  625. * Process the extra information that may be appended to an ACK packet
  626. */
  627. static void rxrpc_input_ackinfo(struct rxrpc_call *call, struct sk_buff *skb,
  628. struct rxrpc_ackinfo *ackinfo)
  629. {
  630. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  631. struct rxrpc_peer *peer;
  632. unsigned int mtu;
  633. bool wake = false;
  634. u32 rwind = ntohl(ackinfo->rwind);
  635. _proto("Rx ACK %%%u Info { rx=%u max=%u rwin=%u jm=%u }",
  636. sp->hdr.serial,
  637. ntohl(ackinfo->rxMTU), ntohl(ackinfo->maxMTU),
  638. rwind, ntohl(ackinfo->jumbo_max));
  639. if (rwind > RXRPC_RXTX_BUFF_SIZE - 1)
  640. rwind = RXRPC_RXTX_BUFF_SIZE - 1;
  641. if (call->tx_winsize != rwind) {
  642. if (rwind > call->tx_winsize)
  643. wake = true;
  644. trace_rxrpc_rx_rwind_change(call, sp->hdr.serial, rwind, wake);
  645. call->tx_winsize = rwind;
  646. }
  647. if (call->cong_ssthresh > rwind)
  648. call->cong_ssthresh = rwind;
  649. mtu = min(ntohl(ackinfo->rxMTU), ntohl(ackinfo->maxMTU));
  650. peer = call->peer;
  651. if (mtu < peer->maxdata) {
  652. spin_lock_bh(&peer->lock);
  653. peer->maxdata = mtu;
  654. peer->mtu = mtu + peer->hdrsize;
  655. spin_unlock_bh(&peer->lock);
  656. _net("Net MTU %u (maxdata %u)", peer->mtu, peer->maxdata);
  657. }
  658. if (wake)
  659. wake_up(&call->waitq);
  660. }
  661. /*
  662. * Process individual soft ACKs.
  663. *
  664. * Each ACK in the array corresponds to one packet and can be either an ACK or
  665. * a NAK. If we get find an explicitly NAK'd packet we resend immediately;
  666. * packets that lie beyond the end of the ACK list are scheduled for resend by
  667. * the timer on the basis that the peer might just not have processed them at
  668. * the time the ACK was sent.
  669. */
  670. static void rxrpc_input_soft_acks(struct rxrpc_call *call, u8 *acks,
  671. rxrpc_seq_t seq, int nr_acks,
  672. struct rxrpc_ack_summary *summary)
  673. {
  674. int ix;
  675. u8 annotation, anno_type;
  676. for (; nr_acks > 0; nr_acks--, seq++) {
  677. ix = seq & RXRPC_RXTX_BUFF_MASK;
  678. annotation = call->rxtx_annotations[ix];
  679. anno_type = annotation & RXRPC_TX_ANNO_MASK;
  680. annotation &= ~RXRPC_TX_ANNO_MASK;
  681. switch (*acks++) {
  682. case RXRPC_ACK_TYPE_ACK:
  683. summary->nr_acks++;
  684. if (anno_type == RXRPC_TX_ANNO_ACK)
  685. continue;
  686. summary->nr_new_acks++;
  687. call->rxtx_annotations[ix] =
  688. RXRPC_TX_ANNO_ACK | annotation;
  689. break;
  690. case RXRPC_ACK_TYPE_NACK:
  691. if (!summary->nr_nacks &&
  692. call->acks_lowest_nak != seq) {
  693. call->acks_lowest_nak = seq;
  694. summary->new_low_nack = true;
  695. }
  696. summary->nr_nacks++;
  697. if (anno_type == RXRPC_TX_ANNO_NAK)
  698. continue;
  699. summary->nr_new_nacks++;
  700. if (anno_type == RXRPC_TX_ANNO_RETRANS)
  701. continue;
  702. call->rxtx_annotations[ix] =
  703. RXRPC_TX_ANNO_NAK | annotation;
  704. break;
  705. default:
  706. return rxrpc_proto_abort("SFT", call, 0);
  707. }
  708. }
  709. }
  710. /*
  711. * Return true if the ACK is valid - ie. it doesn't appear to have regressed
  712. * with respect to the ack state conveyed by preceding ACKs.
  713. */
  714. static bool rxrpc_is_ack_valid(struct rxrpc_call *call,
  715. rxrpc_seq_t first_pkt, rxrpc_seq_t prev_pkt)
  716. {
  717. rxrpc_seq_t base = READ_ONCE(call->ackr_first_seq);
  718. if (after(first_pkt, base))
  719. return true; /* The window advanced */
  720. if (before(first_pkt, base))
  721. return false; /* firstPacket regressed */
  722. if (after_eq(prev_pkt, call->ackr_prev_seq))
  723. return true; /* previousPacket hasn't regressed. */
  724. /* Some rx implementations put a serial number in previousPacket. */
  725. if (after_eq(prev_pkt, base + call->tx_winsize))
  726. return false;
  727. return true;
  728. }
  729. /*
  730. * Process an ACK packet.
  731. *
  732. * ack.firstPacket is the sequence number of the first soft-ACK'd/NAK'd packet
  733. * in the ACK array. Anything before that is hard-ACK'd and may be discarded.
  734. *
  735. * A hard-ACK means that a packet has been processed and may be discarded; a
  736. * soft-ACK means that the packet may be discarded and retransmission
  737. * requested. A phase is complete when all packets are hard-ACK'd.
  738. */
  739. static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
  740. {
  741. struct rxrpc_ack_summary summary = { 0 };
  742. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  743. union {
  744. struct rxrpc_ackpacket ack;
  745. struct rxrpc_ackinfo info;
  746. u8 acks[RXRPC_MAXACKS];
  747. } buf;
  748. rxrpc_serial_t ack_serial, acked_serial;
  749. rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt;
  750. int nr_acks, offset, ioffset;
  751. _enter("");
  752. offset = sizeof(struct rxrpc_wire_header);
  753. if (skb_copy_bits(skb, offset, &buf.ack, sizeof(buf.ack)) < 0) {
  754. _debug("extraction failure");
  755. return rxrpc_proto_abort("XAK", call, 0);
  756. }
  757. offset += sizeof(buf.ack);
  758. ack_serial = sp->hdr.serial;
  759. acked_serial = ntohl(buf.ack.serial);
  760. first_soft_ack = ntohl(buf.ack.firstPacket);
  761. prev_pkt = ntohl(buf.ack.previousPacket);
  762. hard_ack = first_soft_ack - 1;
  763. nr_acks = buf.ack.nAcks;
  764. summary.ack_reason = (buf.ack.reason < RXRPC_ACK__INVALID ?
  765. buf.ack.reason : RXRPC_ACK__INVALID);
  766. trace_rxrpc_rx_ack(call, ack_serial, acked_serial,
  767. first_soft_ack, prev_pkt,
  768. summary.ack_reason, nr_acks);
  769. switch (buf.ack.reason) {
  770. case RXRPC_ACK_PING_RESPONSE:
  771. rxrpc_input_ping_response(call, skb->tstamp, acked_serial,
  772. ack_serial);
  773. rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
  774. rxrpc_rtt_rx_ping_response);
  775. break;
  776. case RXRPC_ACK_REQUESTED:
  777. rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
  778. rxrpc_rtt_rx_requested_ack);
  779. break;
  780. default:
  781. if (acked_serial != 0)
  782. rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
  783. rxrpc_rtt_rx_cancel);
  784. break;
  785. }
  786. if (buf.ack.reason == RXRPC_ACK_PING) {
  787. _proto("Rx ACK %%%u PING Request", ack_serial);
  788. rxrpc_propose_ACK(call, RXRPC_ACK_PING_RESPONSE,
  789. ack_serial, true, true,
  790. rxrpc_propose_ack_respond_to_ping);
  791. } else if (sp->hdr.flags & RXRPC_REQUEST_ACK) {
  792. rxrpc_propose_ACK(call, RXRPC_ACK_REQUESTED,
  793. ack_serial, true, true,
  794. rxrpc_propose_ack_respond_to_ack);
  795. }
  796. /* Discard any out-of-order or duplicate ACKs (outside lock). */
  797. if (!rxrpc_is_ack_valid(call, first_soft_ack, prev_pkt)) {
  798. trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial,
  799. first_soft_ack, call->ackr_first_seq,
  800. prev_pkt, call->ackr_prev_seq);
  801. return;
  802. }
  803. buf.info.rxMTU = 0;
  804. ioffset = offset + nr_acks + 3;
  805. if (skb->len >= ioffset + sizeof(buf.info) &&
  806. skb_copy_bits(skb, ioffset, &buf.info, sizeof(buf.info)) < 0)
  807. return rxrpc_proto_abort("XAI", call, 0);
  808. spin_lock(&call->input_lock);
  809. /* Discard any out-of-order or duplicate ACKs (inside lock). */
  810. if (!rxrpc_is_ack_valid(call, first_soft_ack, prev_pkt)) {
  811. trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial,
  812. first_soft_ack, call->ackr_first_seq,
  813. prev_pkt, call->ackr_prev_seq);
  814. goto out;
  815. }
  816. call->acks_latest_ts = skb->tstamp;
  817. call->ackr_first_seq = first_soft_ack;
  818. call->ackr_prev_seq = prev_pkt;
  819. /* Parse rwind and mtu sizes if provided. */
  820. if (buf.info.rxMTU)
  821. rxrpc_input_ackinfo(call, skb, &buf.info);
  822. if (first_soft_ack == 0) {
  823. rxrpc_proto_abort("AK0", call, 0);
  824. goto out;
  825. }
  826. /* Ignore ACKs unless we are or have just been transmitting. */
  827. switch (READ_ONCE(call->state)) {
  828. case RXRPC_CALL_CLIENT_SEND_REQUEST:
  829. case RXRPC_CALL_CLIENT_AWAIT_REPLY:
  830. case RXRPC_CALL_SERVER_SEND_REPLY:
  831. case RXRPC_CALL_SERVER_AWAIT_ACK:
  832. break;
  833. default:
  834. goto out;
  835. }
  836. if (before(hard_ack, call->tx_hard_ack) ||
  837. after(hard_ack, call->tx_top)) {
  838. rxrpc_proto_abort("AKW", call, 0);
  839. goto out;
  840. }
  841. if (nr_acks > call->tx_top - hard_ack) {
  842. rxrpc_proto_abort("AKN", call, 0);
  843. goto out;
  844. }
  845. if (after(hard_ack, call->tx_hard_ack)) {
  846. if (rxrpc_rotate_tx_window(call, hard_ack, &summary)) {
  847. rxrpc_end_tx_phase(call, false, "ETA");
  848. goto out;
  849. }
  850. }
  851. if (nr_acks > 0) {
  852. if (skb_copy_bits(skb, offset, buf.acks, nr_acks) < 0) {
  853. rxrpc_proto_abort("XSA", call, 0);
  854. goto out;
  855. }
  856. rxrpc_input_soft_acks(call, buf.acks, first_soft_ack, nr_acks,
  857. &summary);
  858. }
  859. if (call->rxtx_annotations[call->tx_top & RXRPC_RXTX_BUFF_MASK] &
  860. RXRPC_TX_ANNO_LAST &&
  861. summary.nr_acks == call->tx_top - hard_ack &&
  862. rxrpc_is_client_call(call))
  863. rxrpc_propose_ACK(call, RXRPC_ACK_PING, ack_serial,
  864. false, true,
  865. rxrpc_propose_ack_ping_for_lost_reply);
  866. rxrpc_congestion_management(call, skb, &summary, acked_serial);
  867. out:
  868. spin_unlock(&call->input_lock);
  869. }
  870. /*
  871. * Process an ACKALL packet.
  872. */
  873. static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb)
  874. {
  875. struct rxrpc_ack_summary summary = { 0 };
  876. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  877. _proto("Rx ACKALL %%%u", sp->hdr.serial);
  878. spin_lock(&call->input_lock);
  879. if (rxrpc_rotate_tx_window(call, call->tx_top, &summary))
  880. rxrpc_end_tx_phase(call, false, "ETL");
  881. spin_unlock(&call->input_lock);
  882. }
  883. /*
  884. * Process an ABORT packet directed at a call.
  885. */
  886. static void rxrpc_input_abort(struct rxrpc_call *call, struct sk_buff *skb)
  887. {
  888. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  889. __be32 wtmp;
  890. u32 abort_code = RX_CALL_DEAD;
  891. _enter("");
  892. if (skb->len >= 4 &&
  893. skb_copy_bits(skb, sizeof(struct rxrpc_wire_header),
  894. &wtmp, sizeof(wtmp)) >= 0)
  895. abort_code = ntohl(wtmp);
  896. trace_rxrpc_rx_abort(call, sp->hdr.serial, abort_code);
  897. _proto("Rx ABORT %%%u { %x }", sp->hdr.serial, abort_code);
  898. rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
  899. abort_code, -ECONNABORTED);
  900. }
  901. /*
  902. * Process an incoming call packet.
  903. */
  904. static void rxrpc_input_call_packet(struct rxrpc_call *call,
  905. struct sk_buff *skb)
  906. {
  907. struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
  908. unsigned long timo;
  909. _enter("%p,%p", call, skb);
  910. timo = READ_ONCE(call->next_rx_timo);
  911. if (timo) {
  912. unsigned long now = jiffies, expect_rx_by;
  913. expect_rx_by = now + timo;
  914. WRITE_ONCE(call->expect_rx_by, expect_rx_by);
  915. rxrpc_reduce_call_timer(call, expect_rx_by, now,
  916. rxrpc_timer_set_for_normal);
  917. }
  918. switch (sp->hdr.type) {
  919. case RXRPC_PACKET_TYPE_DATA:
  920. rxrpc_input_data(call, skb);
  921. goto no_free;
  922. case RXRPC_PACKET_TYPE_ACK:
  923. rxrpc_input_ack(call, skb);
  924. break;
  925. case RXRPC_PACKET_TYPE_BUSY:
  926. _proto("Rx BUSY %%%u", sp->hdr.serial);
  927. /* Just ignore BUSY packets from the server; the retry and
  928. * lifespan timers will take care of business. BUSY packets
  929. * from the client don't make sense.
  930. */
  931. break;
  932. case RXRPC_PACKET_TYPE_ABORT:
  933. rxrpc_input_abort(call, skb);
  934. break;
  935. case RXRPC_PACKET_TYPE_ACKALL:
  936. rxrpc_input_ackall(call, skb);
  937. break;
  938. default:
  939. break;
  940. }
  941. rxrpc_free_skb(skb, rxrpc_skb_freed);
  942. no_free:
  943. _leave("");
  944. }
  945. /*
  946. * Handle a new service call on a channel implicitly completing the preceding
  947. * call on that channel. This does not apply to client conns.
  948. *
  949. * TODO: If callNumber > call_id + 1, renegotiate security.
  950. */
  951. static void rxrpc_input_implicit_end_call(struct rxrpc_sock *rx,
  952. struct rxrpc_connection *conn,
  953. struct rxrpc_call *call)
  954. {
  955. switch (READ_ONCE(call->state)) {
  956. case RXRPC_CALL_SERVER_AWAIT_ACK:
  957. rxrpc_call_completed(call);
  958. fallthrough;
  959. case RXRPC_CALL_COMPLETE:
  960. break;
  961. default:
  962. if (rxrpc_abort_call("IMP", call, 0, RX_CALL_DEAD, -ESHUTDOWN)) {
  963. set_bit(RXRPC_CALL_EV_ABORT, &call->events);
  964. rxrpc_queue_call(call);
  965. }
  966. trace_rxrpc_improper_term(call);
  967. break;
  968. }
  969. spin_lock(&rx->incoming_lock);
  970. __rxrpc_disconnect_call(conn, call);
  971. spin_unlock(&rx->incoming_lock);
  972. }
  973. /*
  974. * post connection-level events to the connection
  975. * - this includes challenges, responses, some aborts and call terminal packet
  976. * retransmission.
  977. */
  978. static void rxrpc_post_packet_to_conn(struct rxrpc_connection *conn,
  979. struct sk_buff *skb)
  980. {
  981. _enter("%p,%p", conn, skb);
  982. skb_queue_tail(&conn->rx_queue, skb);
  983. rxrpc_queue_conn(conn);
  984. }
  985. /*
  986. * post endpoint-level events to the local endpoint
  987. * - this includes debug and version messages
  988. */
  989. static void rxrpc_post_packet_to_local(struct rxrpc_local *local,
  990. struct sk_buff *skb)
  991. {
  992. _enter("%p,%p", local, skb);
  993. if (rxrpc_get_local_maybe(local)) {
  994. skb_queue_tail(&local->event_queue, skb);
  995. rxrpc_queue_local(local);
  996. } else {
  997. rxrpc_free_skb(skb, rxrpc_skb_freed);
  998. }
  999. }
  1000. /*
  1001. * put a packet up for transport-level abort
  1002. */
  1003. static void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb)
  1004. {
  1005. CHECK_SLAB_OKAY(&local->usage);
  1006. if (rxrpc_get_local_maybe(local)) {
  1007. skb_queue_tail(&local->reject_queue, skb);
  1008. rxrpc_queue_local(local);
  1009. } else {
  1010. rxrpc_free_skb(skb, rxrpc_skb_freed);
  1011. }
  1012. }
  1013. /*
  1014. * Extract the wire header from a packet and translate the byte order.
  1015. */
  1016. static noinline
  1017. int rxrpc_extract_header(struct rxrpc_skb_priv *sp, struct sk_buff *skb)
  1018. {
  1019. struct rxrpc_wire_header whdr;
  1020. /* dig out the RxRPC connection details */
  1021. if (skb_copy_bits(skb, 0, &whdr, sizeof(whdr)) < 0) {
  1022. trace_rxrpc_rx_eproto(NULL, sp->hdr.serial,
  1023. tracepoint_string("bad_hdr"));
  1024. return -EBADMSG;
  1025. }
  1026. memset(sp, 0, sizeof(*sp));
  1027. sp->hdr.epoch = ntohl(whdr.epoch);
  1028. sp->hdr.cid = ntohl(whdr.cid);
  1029. sp->hdr.callNumber = ntohl(whdr.callNumber);
  1030. sp->hdr.seq = ntohl(whdr.seq);
  1031. sp->hdr.serial = ntohl(whdr.serial);
  1032. sp->hdr.flags = whdr.flags;
  1033. sp->hdr.type = whdr.type;
  1034. sp->hdr.userStatus = whdr.userStatus;
  1035. sp->hdr.securityIndex = whdr.securityIndex;
  1036. sp->hdr._rsvd = ntohs(whdr._rsvd);
  1037. sp->hdr.serviceId = ntohs(whdr.serviceId);
  1038. return 0;
  1039. }
  1040. /*
  1041. * handle data received on the local endpoint
  1042. * - may be called in interrupt context
  1043. *
  1044. * [!] Note that as this is called from the encap_rcv hook, the socket is not
  1045. * held locked by the caller and nothing prevents sk_user_data on the UDP from
  1046. * being cleared in the middle of processing this function.
  1047. *
  1048. * Called with the RCU read lock held from the IP layer via UDP.
  1049. */
  1050. int rxrpc_input_packet(struct sock *udp_sk, struct sk_buff *skb)
  1051. {
  1052. struct rxrpc_local *local = rcu_dereference_sk_user_data(udp_sk);
  1053. struct rxrpc_connection *conn;
  1054. struct rxrpc_channel *chan;
  1055. struct rxrpc_call *call = NULL;
  1056. struct rxrpc_skb_priv *sp;
  1057. struct rxrpc_peer *peer = NULL;
  1058. struct rxrpc_sock *rx = NULL;
  1059. unsigned int channel;
  1060. _enter("%p", udp_sk);
  1061. if (unlikely(!local)) {
  1062. kfree_skb(skb);
  1063. return 0;
  1064. }
  1065. if (skb->tstamp == 0)
  1066. skb->tstamp = ktime_get_real();
  1067. rxrpc_new_skb(skb, rxrpc_skb_received);
  1068. skb_pull(skb, sizeof(struct udphdr));
  1069. /* The UDP protocol already released all skb resources;
  1070. * we are free to add our own data there.
  1071. */
  1072. sp = rxrpc_skb(skb);
  1073. /* dig out the RxRPC connection details */
  1074. if (rxrpc_extract_header(sp, skb) < 0)
  1075. goto bad_message;
  1076. if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS)) {
  1077. static int lose;
  1078. if ((lose++ & 7) == 7) {
  1079. trace_rxrpc_rx_lose(sp);
  1080. rxrpc_free_skb(skb, rxrpc_skb_lost);
  1081. return 0;
  1082. }
  1083. }
  1084. if (skb->tstamp == 0)
  1085. skb->tstamp = ktime_get_real();
  1086. trace_rxrpc_rx_packet(sp);
  1087. switch (sp->hdr.type) {
  1088. case RXRPC_PACKET_TYPE_VERSION:
  1089. if (rxrpc_to_client(sp))
  1090. goto discard;
  1091. rxrpc_post_packet_to_local(local, skb);
  1092. goto out;
  1093. case RXRPC_PACKET_TYPE_BUSY:
  1094. if (rxrpc_to_server(sp))
  1095. goto discard;
  1096. fallthrough;
  1097. case RXRPC_PACKET_TYPE_ACK:
  1098. case RXRPC_PACKET_TYPE_ACKALL:
  1099. if (sp->hdr.callNumber == 0)
  1100. goto bad_message;
  1101. fallthrough;
  1102. case RXRPC_PACKET_TYPE_ABORT:
  1103. break;
  1104. case RXRPC_PACKET_TYPE_DATA:
  1105. if (sp->hdr.callNumber == 0 ||
  1106. sp->hdr.seq == 0)
  1107. goto bad_message;
  1108. if (!rxrpc_validate_data(skb))
  1109. goto bad_message;
  1110. /* Unshare the packet so that it can be modified for in-place
  1111. * decryption.
  1112. */
  1113. if (sp->hdr.securityIndex != 0) {
  1114. struct sk_buff *nskb = skb_unshare(skb, GFP_ATOMIC);
  1115. if (!nskb) {
  1116. rxrpc_eaten_skb(skb, rxrpc_skb_unshared_nomem);
  1117. goto out;
  1118. }
  1119. if (nskb != skb) {
  1120. rxrpc_eaten_skb(skb, rxrpc_skb_received);
  1121. skb = nskb;
  1122. rxrpc_new_skb(skb, rxrpc_skb_unshared);
  1123. sp = rxrpc_skb(skb);
  1124. }
  1125. }
  1126. break;
  1127. case RXRPC_PACKET_TYPE_CHALLENGE:
  1128. if (rxrpc_to_server(sp))
  1129. goto discard;
  1130. break;
  1131. case RXRPC_PACKET_TYPE_RESPONSE:
  1132. if (rxrpc_to_client(sp))
  1133. goto discard;
  1134. break;
  1135. /* Packet types 9-11 should just be ignored. */
  1136. case RXRPC_PACKET_TYPE_PARAMS:
  1137. case RXRPC_PACKET_TYPE_10:
  1138. case RXRPC_PACKET_TYPE_11:
  1139. goto discard;
  1140. default:
  1141. _proto("Rx Bad Packet Type %u", sp->hdr.type);
  1142. goto bad_message;
  1143. }
  1144. if (sp->hdr.serviceId == 0)
  1145. goto bad_message;
  1146. if (rxrpc_to_server(sp)) {
  1147. /* Weed out packets to services we're not offering. Packets
  1148. * that would begin a call are explicitly rejected and the rest
  1149. * are just discarded.
  1150. */
  1151. rx = rcu_dereference(local->service);
  1152. if (!rx || (sp->hdr.serviceId != rx->srx.srx_service &&
  1153. sp->hdr.serviceId != rx->second_service)) {
  1154. if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
  1155. sp->hdr.seq == 1)
  1156. goto unsupported_service;
  1157. goto discard;
  1158. }
  1159. }
  1160. conn = rxrpc_find_connection_rcu(local, skb, &peer);
  1161. if (conn) {
  1162. if (sp->hdr.securityIndex != conn->security_ix)
  1163. goto wrong_security;
  1164. if (sp->hdr.serviceId != conn->service_id) {
  1165. int old_id;
  1166. if (!test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags))
  1167. goto reupgrade;
  1168. old_id = cmpxchg(&conn->service_id, conn->params.service_id,
  1169. sp->hdr.serviceId);
  1170. if (old_id != conn->params.service_id &&
  1171. old_id != sp->hdr.serviceId)
  1172. goto reupgrade;
  1173. }
  1174. if (sp->hdr.callNumber == 0) {
  1175. /* Connection-level packet */
  1176. _debug("CONN %p {%d}", conn, conn->debug_id);
  1177. rxrpc_post_packet_to_conn(conn, skb);
  1178. goto out;
  1179. }
  1180. if ((int)sp->hdr.serial - (int)conn->hi_serial > 0)
  1181. conn->hi_serial = sp->hdr.serial;
  1182. /* Call-bound packets are routed by connection channel. */
  1183. channel = sp->hdr.cid & RXRPC_CHANNELMASK;
  1184. chan = &conn->channels[channel];
  1185. /* Ignore really old calls */
  1186. if (sp->hdr.callNumber < chan->last_call)
  1187. goto discard;
  1188. if (sp->hdr.callNumber == chan->last_call) {
  1189. if (chan->call ||
  1190. sp->hdr.type == RXRPC_PACKET_TYPE_ABORT)
  1191. goto discard;
  1192. /* For the previous service call, if completed
  1193. * successfully, we discard all further packets.
  1194. */
  1195. if (rxrpc_conn_is_service(conn) &&
  1196. chan->last_type == RXRPC_PACKET_TYPE_ACK)
  1197. goto discard;
  1198. /* But otherwise we need to retransmit the final packet
  1199. * from data cached in the connection record.
  1200. */
  1201. if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA)
  1202. trace_rxrpc_rx_data(chan->call_debug_id,
  1203. sp->hdr.seq,
  1204. sp->hdr.serial,
  1205. sp->hdr.flags, 0);
  1206. rxrpc_post_packet_to_conn(conn, skb);
  1207. goto out;
  1208. }
  1209. call = rcu_dereference(chan->call);
  1210. if (sp->hdr.callNumber > chan->call_id) {
  1211. if (rxrpc_to_client(sp))
  1212. goto reject_packet;
  1213. if (call)
  1214. rxrpc_input_implicit_end_call(rx, conn, call);
  1215. call = NULL;
  1216. }
  1217. if (call) {
  1218. if (sp->hdr.serviceId != call->service_id)
  1219. call->service_id = sp->hdr.serviceId;
  1220. if ((int)sp->hdr.serial - (int)call->rx_serial > 0)
  1221. call->rx_serial = sp->hdr.serial;
  1222. if (!test_bit(RXRPC_CALL_RX_HEARD, &call->flags))
  1223. set_bit(RXRPC_CALL_RX_HEARD, &call->flags);
  1224. }
  1225. }
  1226. if (!call || atomic_read(&call->usage) == 0) {
  1227. if (rxrpc_to_client(sp) ||
  1228. sp->hdr.type != RXRPC_PACKET_TYPE_DATA)
  1229. goto bad_message;
  1230. if (sp->hdr.seq != 1)
  1231. goto discard;
  1232. call = rxrpc_new_incoming_call(local, rx, skb);
  1233. if (!call)
  1234. goto reject_packet;
  1235. }
  1236. /* Process a call packet; this either discards or passes on the ref
  1237. * elsewhere.
  1238. */
  1239. rxrpc_input_call_packet(call, skb);
  1240. goto out;
  1241. discard:
  1242. rxrpc_free_skb(skb, rxrpc_skb_freed);
  1243. out:
  1244. trace_rxrpc_rx_done(0, 0);
  1245. return 0;
  1246. wrong_security:
  1247. trace_rxrpc_abort(0, "SEC", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
  1248. RXKADINCONSISTENCY, EBADMSG);
  1249. skb->priority = RXKADINCONSISTENCY;
  1250. goto post_abort;
  1251. unsupported_service:
  1252. trace_rxrpc_abort(0, "INV", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
  1253. RX_INVALID_OPERATION, EOPNOTSUPP);
  1254. skb->priority = RX_INVALID_OPERATION;
  1255. goto post_abort;
  1256. reupgrade:
  1257. trace_rxrpc_abort(0, "UPG", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
  1258. RX_PROTOCOL_ERROR, EBADMSG);
  1259. goto protocol_error;
  1260. bad_message:
  1261. trace_rxrpc_abort(0, "BAD", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
  1262. RX_PROTOCOL_ERROR, EBADMSG);
  1263. protocol_error:
  1264. skb->priority = RX_PROTOCOL_ERROR;
  1265. post_abort:
  1266. skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
  1267. reject_packet:
  1268. trace_rxrpc_rx_done(skb->mark, skb->priority);
  1269. rxrpc_reject_packet(local, skb);
  1270. _leave(" [badmsg]");
  1271. return 0;
  1272. }