conn_client.c 29 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* Client connection-specific management code.
  3. *
  4. * Copyright (C) 2016, 2020 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells (dhowells@redhat.com)
  6. *
  7. * Client connections need to be cached for a little while after they've made a
  8. * call so as to handle retransmitted DATA packets in case the server didn't
  9. * receive the final ACK or terminating ABORT we sent it.
  10. *
  11. * There are flags of relevance to the cache:
  12. *
  13. * (2) DONT_REUSE - The connection should be discarded as soon as possible and
  14. * should not be reused. This is set when an exclusive connection is used
  15. * or a call ID counter overflows.
  16. *
  17. * The caching state may only be changed if the cache lock is held.
  18. *
  19. * There are two idle client connection expiry durations. If the total number
  20. * of connections is below the reap threshold, we use the normal duration; if
  21. * it's above, we use the fast duration.
  22. */
  23. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24. #include <linux/slab.h>
  25. #include <linux/idr.h>
  26. #include <linux/timer.h>
  27. #include <linux/sched/signal.h>
  28. #include "ar-internal.h"
  29. __read_mostly unsigned int rxrpc_reap_client_connections = 900;
  30. __read_mostly unsigned long rxrpc_conn_idle_client_expiry = 2 * 60 * HZ;
  31. __read_mostly unsigned long rxrpc_conn_idle_client_fast_expiry = 2 * HZ;
  32. /*
  33. * We use machine-unique IDs for our client connections.
  34. */
  35. DEFINE_IDR(rxrpc_client_conn_ids);
  36. static DEFINE_SPINLOCK(rxrpc_conn_id_lock);
  37. /*
  38. * Get a connection ID and epoch for a client connection from the global pool.
  39. * The connection struct pointer is then recorded in the idr radix tree. The
  40. * epoch doesn't change until the client is rebooted (or, at least, unless the
  41. * module is unloaded).
  42. */
  43. static int rxrpc_get_client_connection_id(struct rxrpc_connection *conn,
  44. gfp_t gfp)
  45. {
  46. struct rxrpc_net *rxnet = conn->params.local->rxnet;
  47. int id;
  48. _enter("");
  49. idr_preload(gfp);
  50. spin_lock(&rxrpc_conn_id_lock);
  51. id = idr_alloc_cyclic(&rxrpc_client_conn_ids, conn,
  52. 1, 0x40000000, GFP_NOWAIT);
  53. if (id < 0)
  54. goto error;
  55. spin_unlock(&rxrpc_conn_id_lock);
  56. idr_preload_end();
  57. conn->proto.epoch = rxnet->epoch;
  58. conn->proto.cid = id << RXRPC_CIDSHIFT;
  59. set_bit(RXRPC_CONN_HAS_IDR, &conn->flags);
  60. _leave(" [CID %x]", conn->proto.cid);
  61. return 0;
  62. error:
  63. spin_unlock(&rxrpc_conn_id_lock);
  64. idr_preload_end();
  65. _leave(" = %d", id);
  66. return id;
  67. }
  68. /*
  69. * Release a connection ID for a client connection from the global pool.
  70. */
  71. static void rxrpc_put_client_connection_id(struct rxrpc_connection *conn)
  72. {
  73. if (test_bit(RXRPC_CONN_HAS_IDR, &conn->flags)) {
  74. spin_lock(&rxrpc_conn_id_lock);
  75. idr_remove(&rxrpc_client_conn_ids,
  76. conn->proto.cid >> RXRPC_CIDSHIFT);
  77. spin_unlock(&rxrpc_conn_id_lock);
  78. }
  79. }
  80. /*
  81. * Destroy the client connection ID tree.
  82. */
  83. void rxrpc_destroy_client_conn_ids(void)
  84. {
  85. struct rxrpc_connection *conn;
  86. int id;
  87. if (!idr_is_empty(&rxrpc_client_conn_ids)) {
  88. idr_for_each_entry(&rxrpc_client_conn_ids, conn, id) {
  89. pr_err("AF_RXRPC: Leaked client conn %p {%d}\n",
  90. conn, atomic_read(&conn->usage));
  91. }
  92. BUG();
  93. }
  94. idr_destroy(&rxrpc_client_conn_ids);
  95. }
  96. /*
  97. * Allocate a connection bundle.
  98. */
  99. static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_conn_parameters *cp,
  100. gfp_t gfp)
  101. {
  102. struct rxrpc_bundle *bundle;
  103. bundle = kzalloc(sizeof(*bundle), gfp);
  104. if (bundle) {
  105. bundle->params = *cp;
  106. rxrpc_get_peer(bundle->params.peer);
  107. atomic_set(&bundle->usage, 1);
  108. spin_lock_init(&bundle->channel_lock);
  109. INIT_LIST_HEAD(&bundle->waiting_calls);
  110. }
  111. return bundle;
  112. }
  113. struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle)
  114. {
  115. atomic_inc(&bundle->usage);
  116. return bundle;
  117. }
  118. static void rxrpc_free_bundle(struct rxrpc_bundle *bundle)
  119. {
  120. rxrpc_put_peer(bundle->params.peer);
  121. kfree(bundle);
  122. }
  123. void rxrpc_put_bundle(struct rxrpc_bundle *bundle)
  124. {
  125. unsigned int d = bundle->debug_id;
  126. unsigned int u = atomic_dec_return(&bundle->usage);
  127. _debug("PUT B=%x %u", d, u);
  128. if (u == 0)
  129. rxrpc_free_bundle(bundle);
  130. }
  131. /*
  132. * Allocate a client connection.
  133. */
  134. static struct rxrpc_connection *
  135. rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle, gfp_t gfp)
  136. {
  137. struct rxrpc_connection *conn;
  138. struct rxrpc_net *rxnet = bundle->params.local->rxnet;
  139. int ret;
  140. _enter("");
  141. conn = rxrpc_alloc_connection(gfp);
  142. if (!conn) {
  143. _leave(" = -ENOMEM");
  144. return ERR_PTR(-ENOMEM);
  145. }
  146. atomic_set(&conn->usage, 1);
  147. conn->bundle = bundle;
  148. conn->params = bundle->params;
  149. conn->out_clientflag = RXRPC_CLIENT_INITIATED;
  150. conn->state = RXRPC_CONN_CLIENT;
  151. conn->service_id = conn->params.service_id;
  152. ret = rxrpc_get_client_connection_id(conn, gfp);
  153. if (ret < 0)
  154. goto error_0;
  155. ret = rxrpc_init_client_conn_security(conn);
  156. if (ret < 0)
  157. goto error_1;
  158. ret = conn->security->prime_packet_security(conn);
  159. if (ret < 0)
  160. goto error_2;
  161. atomic_inc(&rxnet->nr_conns);
  162. write_lock(&rxnet->conn_lock);
  163. list_add_tail(&conn->proc_link, &rxnet->conn_proc_list);
  164. write_unlock(&rxnet->conn_lock);
  165. rxrpc_get_bundle(bundle);
  166. rxrpc_get_peer(conn->params.peer);
  167. rxrpc_get_local(conn->params.local);
  168. key_get(conn->params.key);
  169. trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_client,
  170. atomic_read(&conn->usage),
  171. __builtin_return_address(0));
  172. atomic_inc(&rxnet->nr_client_conns);
  173. trace_rxrpc_client(conn, -1, rxrpc_client_alloc);
  174. _leave(" = %p", conn);
  175. return conn;
  176. error_2:
  177. conn->security->clear(conn);
  178. error_1:
  179. rxrpc_put_client_connection_id(conn);
  180. error_0:
  181. kfree(conn);
  182. _leave(" = %d", ret);
  183. return ERR_PTR(ret);
  184. }
  185. /*
  186. * Determine if a connection may be reused.
  187. */
  188. static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
  189. {
  190. struct rxrpc_net *rxnet;
  191. int id_cursor, id, distance, limit;
  192. if (!conn)
  193. goto dont_reuse;
  194. rxnet = conn->params.local->rxnet;
  195. if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags))
  196. goto dont_reuse;
  197. if (conn->state != RXRPC_CONN_CLIENT ||
  198. conn->proto.epoch != rxnet->epoch)
  199. goto mark_dont_reuse;
  200. /* The IDR tree gets very expensive on memory if the connection IDs are
  201. * widely scattered throughout the number space, so we shall want to
  202. * kill off connections that, say, have an ID more than about four
  203. * times the maximum number of client conns away from the current
  204. * allocation point to try and keep the IDs concentrated.
  205. */
  206. id_cursor = idr_get_cursor(&rxrpc_client_conn_ids);
  207. id = conn->proto.cid >> RXRPC_CIDSHIFT;
  208. distance = id - id_cursor;
  209. if (distance < 0)
  210. distance = -distance;
  211. limit = max_t(unsigned long, atomic_read(&rxnet->nr_conns) * 4, 1024);
  212. if (distance > limit)
  213. goto mark_dont_reuse;
  214. return true;
  215. mark_dont_reuse:
  216. set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
  217. dont_reuse:
  218. return false;
  219. }
  220. /*
  221. * Look up the conn bundle that matches the connection parameters, adding it if
  222. * it doesn't yet exist.
  223. */
  224. static struct rxrpc_bundle *rxrpc_look_up_bundle(struct rxrpc_conn_parameters *cp,
  225. gfp_t gfp)
  226. {
  227. static atomic_t rxrpc_bundle_id;
  228. struct rxrpc_bundle *bundle, *candidate;
  229. struct rxrpc_local *local = cp->local;
  230. struct rb_node *p, **pp, *parent;
  231. long diff;
  232. _enter("{%px,%x,%u,%u}",
  233. cp->peer, key_serial(cp->key), cp->security_level, cp->upgrade);
  234. if (cp->exclusive)
  235. return rxrpc_alloc_bundle(cp, gfp);
  236. /* First, see if the bundle is already there. */
  237. _debug("search 1");
  238. spin_lock(&local->client_bundles_lock);
  239. p = local->client_bundles.rb_node;
  240. while (p) {
  241. bundle = rb_entry(p, struct rxrpc_bundle, local_node);
  242. #define cmp(X) ((long)bundle->params.X - (long)cp->X)
  243. diff = (cmp(peer) ?:
  244. cmp(key) ?:
  245. cmp(security_level) ?:
  246. cmp(upgrade));
  247. #undef cmp
  248. if (diff < 0)
  249. p = p->rb_left;
  250. else if (diff > 0)
  251. p = p->rb_right;
  252. else
  253. goto found_bundle;
  254. }
  255. spin_unlock(&local->client_bundles_lock);
  256. _debug("not found");
  257. /* It wasn't. We need to add one. */
  258. candidate = rxrpc_alloc_bundle(cp, gfp);
  259. if (!candidate)
  260. return NULL;
  261. _debug("search 2");
  262. spin_lock(&local->client_bundles_lock);
  263. pp = &local->client_bundles.rb_node;
  264. parent = NULL;
  265. while (*pp) {
  266. parent = *pp;
  267. bundle = rb_entry(parent, struct rxrpc_bundle, local_node);
  268. #define cmp(X) ((long)bundle->params.X - (long)cp->X)
  269. diff = (cmp(peer) ?:
  270. cmp(key) ?:
  271. cmp(security_level) ?:
  272. cmp(upgrade));
  273. #undef cmp
  274. if (diff < 0)
  275. pp = &(*pp)->rb_left;
  276. else if (diff > 0)
  277. pp = &(*pp)->rb_right;
  278. else
  279. goto found_bundle_free;
  280. }
  281. _debug("new bundle");
  282. candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id);
  283. rb_link_node(&candidate->local_node, parent, pp);
  284. rb_insert_color(&candidate->local_node, &local->client_bundles);
  285. rxrpc_get_bundle(candidate);
  286. spin_unlock(&local->client_bundles_lock);
  287. _leave(" = %u [new]", candidate->debug_id);
  288. return candidate;
  289. found_bundle_free:
  290. rxrpc_free_bundle(candidate);
  291. found_bundle:
  292. rxrpc_get_bundle(bundle);
  293. spin_unlock(&local->client_bundles_lock);
  294. _leave(" = %u [found]", bundle->debug_id);
  295. return bundle;
  296. }
  297. /*
  298. * Create or find a client bundle to use for a call.
  299. *
  300. * If we return with a connection, the call will be on its waiting list. It's
  301. * left to the caller to assign a channel and wake up the call.
  302. */
  303. static struct rxrpc_bundle *rxrpc_prep_call(struct rxrpc_sock *rx,
  304. struct rxrpc_call *call,
  305. struct rxrpc_conn_parameters *cp,
  306. struct sockaddr_rxrpc *srx,
  307. gfp_t gfp)
  308. {
  309. struct rxrpc_bundle *bundle;
  310. _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
  311. cp->peer = rxrpc_lookup_peer(rx, cp->local, srx, gfp);
  312. if (!cp->peer)
  313. goto error;
  314. call->cong_cwnd = cp->peer->cong_cwnd;
  315. if (call->cong_cwnd >= call->cong_ssthresh)
  316. call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE;
  317. else
  318. call->cong_mode = RXRPC_CALL_SLOW_START;
  319. if (cp->upgrade)
  320. __set_bit(RXRPC_CALL_UPGRADE, &call->flags);
  321. /* Find the client connection bundle. */
  322. bundle = rxrpc_look_up_bundle(cp, gfp);
  323. if (!bundle)
  324. goto error;
  325. /* Get this call queued. Someone else may activate it whilst we're
  326. * lining up a new connection, but that's fine.
  327. */
  328. spin_lock(&bundle->channel_lock);
  329. list_add_tail(&call->chan_wait_link, &bundle->waiting_calls);
  330. spin_unlock(&bundle->channel_lock);
  331. _leave(" = [B=%x]", bundle->debug_id);
  332. return bundle;
  333. error:
  334. _leave(" = -ENOMEM");
  335. return ERR_PTR(-ENOMEM);
  336. }
  337. /*
  338. * Allocate a new connection and add it into a bundle.
  339. */
  340. static void rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, gfp_t gfp)
  341. __releases(bundle->channel_lock)
  342. {
  343. struct rxrpc_connection *candidate = NULL, *old = NULL;
  344. bool conflict;
  345. int i;
  346. _enter("");
  347. conflict = bundle->alloc_conn;
  348. if (!conflict)
  349. bundle->alloc_conn = true;
  350. spin_unlock(&bundle->channel_lock);
  351. if (conflict) {
  352. _leave(" [conf]");
  353. return;
  354. }
  355. candidate = rxrpc_alloc_client_connection(bundle, gfp);
  356. spin_lock(&bundle->channel_lock);
  357. bundle->alloc_conn = false;
  358. if (IS_ERR(candidate)) {
  359. bundle->alloc_error = PTR_ERR(candidate);
  360. spin_unlock(&bundle->channel_lock);
  361. _leave(" [err %ld]", PTR_ERR(candidate));
  362. return;
  363. }
  364. bundle->alloc_error = 0;
  365. for (i = 0; i < ARRAY_SIZE(bundle->conns); i++) {
  366. unsigned int shift = i * RXRPC_MAXCALLS;
  367. int j;
  368. old = bundle->conns[i];
  369. if (!rxrpc_may_reuse_conn(old)) {
  370. if (old)
  371. trace_rxrpc_client(old, -1, rxrpc_client_replace);
  372. candidate->bundle_shift = shift;
  373. bundle->conns[i] = candidate;
  374. for (j = 0; j < RXRPC_MAXCALLS; j++)
  375. set_bit(shift + j, &bundle->avail_chans);
  376. candidate = NULL;
  377. break;
  378. }
  379. old = NULL;
  380. }
  381. spin_unlock(&bundle->channel_lock);
  382. if (candidate) {
  383. _debug("discard C=%x", candidate->debug_id);
  384. trace_rxrpc_client(candidate, -1, rxrpc_client_duplicate);
  385. rxrpc_put_connection(candidate);
  386. }
  387. rxrpc_put_connection(old);
  388. _leave("");
  389. }
  390. /*
  391. * Add a connection to a bundle if there are no usable connections or we have
  392. * connections waiting for extra capacity.
  393. */
  394. static void rxrpc_maybe_add_conn(struct rxrpc_bundle *bundle, gfp_t gfp)
  395. {
  396. struct rxrpc_call *call;
  397. int i, usable;
  398. _enter("");
  399. spin_lock(&bundle->channel_lock);
  400. /* See if there are any usable connections. */
  401. usable = 0;
  402. for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
  403. if (rxrpc_may_reuse_conn(bundle->conns[i]))
  404. usable++;
  405. if (!usable && !list_empty(&bundle->waiting_calls)) {
  406. call = list_first_entry(&bundle->waiting_calls,
  407. struct rxrpc_call, chan_wait_link);
  408. if (test_bit(RXRPC_CALL_UPGRADE, &call->flags))
  409. bundle->try_upgrade = true;
  410. }
  411. if (!usable)
  412. goto alloc_conn;
  413. if (!bundle->avail_chans &&
  414. !bundle->try_upgrade &&
  415. !list_empty(&bundle->waiting_calls) &&
  416. usable < ARRAY_SIZE(bundle->conns))
  417. goto alloc_conn;
  418. spin_unlock(&bundle->channel_lock);
  419. _leave("");
  420. return;
  421. alloc_conn:
  422. return rxrpc_add_conn_to_bundle(bundle, gfp);
  423. }
  424. /*
  425. * Assign a channel to the call at the front of the queue and wake the call up.
  426. * We don't increment the callNumber counter until this number has been exposed
  427. * to the world.
  428. */
  429. static void rxrpc_activate_one_channel(struct rxrpc_connection *conn,
  430. unsigned int channel)
  431. {
  432. struct rxrpc_channel *chan = &conn->channels[channel];
  433. struct rxrpc_bundle *bundle = conn->bundle;
  434. struct rxrpc_call *call = list_entry(bundle->waiting_calls.next,
  435. struct rxrpc_call, chan_wait_link);
  436. u32 call_id = chan->call_counter + 1;
  437. _enter("C=%x,%u", conn->debug_id, channel);
  438. trace_rxrpc_client(conn, channel, rxrpc_client_chan_activate);
  439. /* Cancel the final ACK on the previous call if it hasn't been sent yet
  440. * as the DATA packet will implicitly ACK it.
  441. */
  442. clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
  443. clear_bit(conn->bundle_shift + channel, &bundle->avail_chans);
  444. rxrpc_see_call(call);
  445. list_del_init(&call->chan_wait_link);
  446. call->peer = rxrpc_get_peer(conn->params.peer);
  447. call->conn = rxrpc_get_connection(conn);
  448. call->cid = conn->proto.cid | channel;
  449. call->call_id = call_id;
  450. call->security = conn->security;
  451. call->security_ix = conn->security_ix;
  452. call->service_id = conn->service_id;
  453. trace_rxrpc_connect_call(call);
  454. _net("CONNECT call %08x:%08x as call %d on conn %d",
  455. call->cid, call->call_id, call->debug_id, conn->debug_id);
  456. write_lock_bh(&call->state_lock);
  457. call->state = RXRPC_CALL_CLIENT_SEND_REQUEST;
  458. write_unlock_bh(&call->state_lock);
  459. /* Paired with the read barrier in rxrpc_connect_call(). This orders
  460. * cid and epoch in the connection wrt to call_id without the need to
  461. * take the channel_lock.
  462. *
  463. * We provisionally assign a callNumber at this point, but we don't
  464. * confirm it until the call is about to be exposed.
  465. *
  466. * TODO: Pair with a barrier in the data_ready handler when that looks
  467. * at the call ID through a connection channel.
  468. */
  469. smp_wmb();
  470. chan->call_id = call_id;
  471. chan->call_debug_id = call->debug_id;
  472. rcu_assign_pointer(chan->call, call);
  473. wake_up(&call->waitq);
  474. }
  475. /*
  476. * Remove a connection from the idle list if it's on it.
  477. */
  478. static void rxrpc_unidle_conn(struct rxrpc_bundle *bundle, struct rxrpc_connection *conn)
  479. {
  480. struct rxrpc_net *rxnet = bundle->params.local->rxnet;
  481. bool drop_ref;
  482. if (!list_empty(&conn->cache_link)) {
  483. drop_ref = false;
  484. spin_lock(&rxnet->client_conn_cache_lock);
  485. if (!list_empty(&conn->cache_link)) {
  486. list_del_init(&conn->cache_link);
  487. drop_ref = true;
  488. }
  489. spin_unlock(&rxnet->client_conn_cache_lock);
  490. if (drop_ref)
  491. rxrpc_put_connection(conn);
  492. }
  493. }
  494. /*
  495. * Assign channels and callNumbers to waiting calls with channel_lock
  496. * held by caller.
  497. */
  498. static void rxrpc_activate_channels_locked(struct rxrpc_bundle *bundle)
  499. {
  500. struct rxrpc_connection *conn;
  501. unsigned long avail, mask;
  502. unsigned int channel, slot;
  503. if (bundle->try_upgrade)
  504. mask = 1;
  505. else
  506. mask = ULONG_MAX;
  507. while (!list_empty(&bundle->waiting_calls)) {
  508. avail = bundle->avail_chans & mask;
  509. if (!avail)
  510. break;
  511. channel = __ffs(avail);
  512. clear_bit(channel, &bundle->avail_chans);
  513. slot = channel / RXRPC_MAXCALLS;
  514. conn = bundle->conns[slot];
  515. if (!conn)
  516. break;
  517. if (bundle->try_upgrade)
  518. set_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags);
  519. rxrpc_unidle_conn(bundle, conn);
  520. channel &= (RXRPC_MAXCALLS - 1);
  521. conn->act_chans |= 1 << channel;
  522. rxrpc_activate_one_channel(conn, channel);
  523. }
  524. }
  525. /*
  526. * Assign channels and callNumbers to waiting calls.
  527. */
  528. static void rxrpc_activate_channels(struct rxrpc_bundle *bundle)
  529. {
  530. _enter("B=%x", bundle->debug_id);
  531. trace_rxrpc_client(NULL, -1, rxrpc_client_activate_chans);
  532. if (!bundle->avail_chans)
  533. return;
  534. spin_lock(&bundle->channel_lock);
  535. rxrpc_activate_channels_locked(bundle);
  536. spin_unlock(&bundle->channel_lock);
  537. _leave("");
  538. }
  539. /*
  540. * Wait for a callNumber and a channel to be granted to a call.
  541. */
  542. static int rxrpc_wait_for_channel(struct rxrpc_bundle *bundle,
  543. struct rxrpc_call *call, gfp_t gfp)
  544. {
  545. DECLARE_WAITQUEUE(myself, current);
  546. int ret = 0;
  547. _enter("%d", call->debug_id);
  548. if (!gfpflags_allow_blocking(gfp)) {
  549. rxrpc_maybe_add_conn(bundle, gfp);
  550. rxrpc_activate_channels(bundle);
  551. ret = bundle->alloc_error ?: -EAGAIN;
  552. goto out;
  553. }
  554. add_wait_queue_exclusive(&call->waitq, &myself);
  555. for (;;) {
  556. rxrpc_maybe_add_conn(bundle, gfp);
  557. rxrpc_activate_channels(bundle);
  558. ret = bundle->alloc_error;
  559. if (ret < 0)
  560. break;
  561. switch (call->interruptibility) {
  562. case RXRPC_INTERRUPTIBLE:
  563. case RXRPC_PREINTERRUPTIBLE:
  564. set_current_state(TASK_INTERRUPTIBLE);
  565. break;
  566. case RXRPC_UNINTERRUPTIBLE:
  567. default:
  568. set_current_state(TASK_UNINTERRUPTIBLE);
  569. break;
  570. }
  571. if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_AWAIT_CONN)
  572. break;
  573. if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
  574. call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
  575. signal_pending(current)) {
  576. ret = -ERESTARTSYS;
  577. break;
  578. }
  579. schedule();
  580. }
  581. remove_wait_queue(&call->waitq, &myself);
  582. __set_current_state(TASK_RUNNING);
  583. out:
  584. _leave(" = %d", ret);
  585. return ret;
  586. }
  587. /*
  588. * find a connection for a call
  589. * - called in process context with IRQs enabled
  590. */
  591. int rxrpc_connect_call(struct rxrpc_sock *rx,
  592. struct rxrpc_call *call,
  593. struct rxrpc_conn_parameters *cp,
  594. struct sockaddr_rxrpc *srx,
  595. gfp_t gfp)
  596. {
  597. struct rxrpc_bundle *bundle;
  598. struct rxrpc_net *rxnet = cp->local->rxnet;
  599. int ret = 0;
  600. _enter("{%d,%lx},", call->debug_id, call->user_call_ID);
  601. rxrpc_discard_expired_client_conns(&rxnet->client_conn_reaper);
  602. bundle = rxrpc_prep_call(rx, call, cp, srx, gfp);
  603. if (IS_ERR(bundle)) {
  604. ret = PTR_ERR(bundle);
  605. goto out;
  606. }
  607. if (call->state == RXRPC_CALL_CLIENT_AWAIT_CONN) {
  608. ret = rxrpc_wait_for_channel(bundle, call, gfp);
  609. if (ret < 0)
  610. goto wait_failed;
  611. }
  612. granted_channel:
  613. /* Paired with the write barrier in rxrpc_activate_one_channel(). */
  614. smp_rmb();
  615. out_put_bundle:
  616. rxrpc_put_bundle(bundle);
  617. out:
  618. _leave(" = %d", ret);
  619. return ret;
  620. wait_failed:
  621. spin_lock(&bundle->channel_lock);
  622. list_del_init(&call->chan_wait_link);
  623. spin_unlock(&bundle->channel_lock);
  624. if (call->state != RXRPC_CALL_CLIENT_AWAIT_CONN) {
  625. ret = 0;
  626. goto granted_channel;
  627. }
  628. trace_rxrpc_client(call->conn, ret, rxrpc_client_chan_wait_failed);
  629. rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR, 0, ret);
  630. rxrpc_disconnect_client_call(bundle, call);
  631. goto out_put_bundle;
  632. }
  633. /*
  634. * Note that a call, and thus a connection, is about to be exposed to the
  635. * world.
  636. */
  637. void rxrpc_expose_client_call(struct rxrpc_call *call)
  638. {
  639. unsigned int channel = call->cid & RXRPC_CHANNELMASK;
  640. struct rxrpc_connection *conn = call->conn;
  641. struct rxrpc_channel *chan = &conn->channels[channel];
  642. if (!test_and_set_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
  643. /* Mark the call ID as being used. If the callNumber counter
  644. * exceeds ~2 billion, we kill the connection after its
  645. * outstanding calls have finished so that the counter doesn't
  646. * wrap.
  647. */
  648. chan->call_counter++;
  649. if (chan->call_counter >= INT_MAX)
  650. set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags);
  651. trace_rxrpc_client(conn, channel, rxrpc_client_exposed);
  652. }
  653. }
  654. /*
  655. * Set the reap timer.
  656. */
  657. static void rxrpc_set_client_reap_timer(struct rxrpc_net *rxnet)
  658. {
  659. if (!rxnet->kill_all_client_conns) {
  660. unsigned long now = jiffies;
  661. unsigned long reap_at = now + rxrpc_conn_idle_client_expiry;
  662. if (rxnet->live)
  663. timer_reduce(&rxnet->client_conn_reap_timer, reap_at);
  664. }
  665. }
  666. /*
  667. * Disconnect a client call.
  668. */
  669. void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call *call)
  670. {
  671. struct rxrpc_connection *conn;
  672. struct rxrpc_channel *chan = NULL;
  673. struct rxrpc_net *rxnet = bundle->params.local->rxnet;
  674. unsigned int channel;
  675. bool may_reuse;
  676. u32 cid;
  677. _enter("c=%x", call->debug_id);
  678. spin_lock(&bundle->channel_lock);
  679. set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
  680. /* Calls that have never actually been assigned a channel can simply be
  681. * discarded.
  682. */
  683. conn = call->conn;
  684. if (!conn) {
  685. _debug("call is waiting");
  686. ASSERTCMP(call->call_id, ==, 0);
  687. ASSERT(!test_bit(RXRPC_CALL_EXPOSED, &call->flags));
  688. list_del_init(&call->chan_wait_link);
  689. goto out;
  690. }
  691. cid = call->cid;
  692. channel = cid & RXRPC_CHANNELMASK;
  693. chan = &conn->channels[channel];
  694. trace_rxrpc_client(conn, channel, rxrpc_client_chan_disconnect);
  695. if (rcu_access_pointer(chan->call) != call) {
  696. spin_unlock(&bundle->channel_lock);
  697. BUG();
  698. }
  699. may_reuse = rxrpc_may_reuse_conn(conn);
  700. /* If a client call was exposed to the world, we save the result for
  701. * retransmission.
  702. *
  703. * We use a barrier here so that the call number and abort code can be
  704. * read without needing to take a lock.
  705. *
  706. * TODO: Make the incoming packet handler check this and handle
  707. * terminal retransmission without requiring access to the call.
  708. */
  709. if (test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
  710. _debug("exposed %u,%u", call->call_id, call->abort_code);
  711. __rxrpc_disconnect_call(conn, call);
  712. if (test_and_clear_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) {
  713. trace_rxrpc_client(conn, channel, rxrpc_client_to_active);
  714. bundle->try_upgrade = false;
  715. if (may_reuse)
  716. rxrpc_activate_channels_locked(bundle);
  717. }
  718. }
  719. /* See if we can pass the channel directly to another call. */
  720. if (may_reuse && !list_empty(&bundle->waiting_calls)) {
  721. trace_rxrpc_client(conn, channel, rxrpc_client_chan_pass);
  722. rxrpc_activate_one_channel(conn, channel);
  723. goto out;
  724. }
  725. /* Schedule the final ACK to be transmitted in a short while so that it
  726. * can be skipped if we find a follow-on call. The first DATA packet
  727. * of the follow on call will implicitly ACK this call.
  728. */
  729. if (call->completion == RXRPC_CALL_SUCCEEDED &&
  730. test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
  731. unsigned long final_ack_at = jiffies + 2;
  732. WRITE_ONCE(chan->final_ack_at, final_ack_at);
  733. smp_wmb(); /* vs rxrpc_process_delayed_final_acks() */
  734. set_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags);
  735. rxrpc_reduce_conn_timer(conn, final_ack_at);
  736. }
  737. /* Deactivate the channel. */
  738. rcu_assign_pointer(chan->call, NULL);
  739. set_bit(conn->bundle_shift + channel, &conn->bundle->avail_chans);
  740. conn->act_chans &= ~(1 << channel);
  741. /* If no channels remain active, then put the connection on the idle
  742. * list for a short while. Give it a ref to stop it going away if it
  743. * becomes unbundled.
  744. */
  745. if (!conn->act_chans) {
  746. trace_rxrpc_client(conn, channel, rxrpc_client_to_idle);
  747. conn->idle_timestamp = jiffies;
  748. rxrpc_get_connection(conn);
  749. spin_lock(&rxnet->client_conn_cache_lock);
  750. list_move_tail(&conn->cache_link, &rxnet->idle_client_conns);
  751. spin_unlock(&rxnet->client_conn_cache_lock);
  752. rxrpc_set_client_reap_timer(rxnet);
  753. }
  754. out:
  755. spin_unlock(&bundle->channel_lock);
  756. _leave("");
  757. return;
  758. }
  759. /*
  760. * Remove a connection from a bundle.
  761. */
  762. static void rxrpc_unbundle_conn(struct rxrpc_connection *conn)
  763. {
  764. struct rxrpc_bundle *bundle = conn->bundle;
  765. struct rxrpc_local *local = bundle->params.local;
  766. unsigned int bindex;
  767. bool need_drop = false, need_put = false;
  768. int i;
  769. _enter("C=%x", conn->debug_id);
  770. if (conn->flags & RXRPC_CONN_FINAL_ACK_MASK)
  771. rxrpc_process_delayed_final_acks(conn, true);
  772. spin_lock(&bundle->channel_lock);
  773. bindex = conn->bundle_shift / RXRPC_MAXCALLS;
  774. if (bundle->conns[bindex] == conn) {
  775. _debug("clear slot %u", bindex);
  776. bundle->conns[bindex] = NULL;
  777. for (i = 0; i < RXRPC_MAXCALLS; i++)
  778. clear_bit(conn->bundle_shift + i, &bundle->avail_chans);
  779. need_drop = true;
  780. }
  781. spin_unlock(&bundle->channel_lock);
  782. /* If there are no more connections, remove the bundle */
  783. if (!bundle->avail_chans) {
  784. _debug("maybe unbundle");
  785. spin_lock(&local->client_bundles_lock);
  786. for (i = 0; i < ARRAY_SIZE(bundle->conns); i++)
  787. if (bundle->conns[i])
  788. break;
  789. if (i == ARRAY_SIZE(bundle->conns) && !bundle->params.exclusive) {
  790. _debug("erase bundle");
  791. rb_erase(&bundle->local_node, &local->client_bundles);
  792. need_put = true;
  793. }
  794. spin_unlock(&local->client_bundles_lock);
  795. if (need_put)
  796. rxrpc_put_bundle(bundle);
  797. }
  798. if (need_drop)
  799. rxrpc_put_connection(conn);
  800. _leave("");
  801. }
  802. /*
  803. * Clean up a dead client connection.
  804. */
  805. static void rxrpc_kill_client_conn(struct rxrpc_connection *conn)
  806. {
  807. struct rxrpc_local *local = conn->params.local;
  808. struct rxrpc_net *rxnet = local->rxnet;
  809. _enter("C=%x", conn->debug_id);
  810. trace_rxrpc_client(conn, -1, rxrpc_client_cleanup);
  811. atomic_dec(&rxnet->nr_client_conns);
  812. rxrpc_put_client_connection_id(conn);
  813. rxrpc_kill_connection(conn);
  814. }
  815. /*
  816. * Clean up a dead client connections.
  817. */
  818. void rxrpc_put_client_conn(struct rxrpc_connection *conn)
  819. {
  820. const void *here = __builtin_return_address(0);
  821. unsigned int debug_id = conn->debug_id;
  822. int n;
  823. n = atomic_dec_return(&conn->usage);
  824. trace_rxrpc_conn(debug_id, rxrpc_conn_put_client, n, here);
  825. if (n <= 0) {
  826. ASSERTCMP(n, >=, 0);
  827. rxrpc_kill_client_conn(conn);
  828. }
  829. }
  830. /*
  831. * Discard expired client connections from the idle list. Each conn in the
  832. * idle list has been exposed and holds an extra ref because of that.
  833. *
  834. * This may be called from conn setup or from a work item so cannot be
  835. * considered non-reentrant.
  836. */
  837. void rxrpc_discard_expired_client_conns(struct work_struct *work)
  838. {
  839. struct rxrpc_connection *conn;
  840. struct rxrpc_net *rxnet =
  841. container_of(work, struct rxrpc_net, client_conn_reaper);
  842. unsigned long expiry, conn_expires_at, now;
  843. unsigned int nr_conns;
  844. _enter("");
  845. if (list_empty(&rxnet->idle_client_conns)) {
  846. _leave(" [empty]");
  847. return;
  848. }
  849. /* Don't double up on the discarding */
  850. if (!spin_trylock(&rxnet->client_conn_discard_lock)) {
  851. _leave(" [already]");
  852. return;
  853. }
  854. /* We keep an estimate of what the number of conns ought to be after
  855. * we've discarded some so that we don't overdo the discarding.
  856. */
  857. nr_conns = atomic_read(&rxnet->nr_client_conns);
  858. next:
  859. spin_lock(&rxnet->client_conn_cache_lock);
  860. if (list_empty(&rxnet->idle_client_conns))
  861. goto out;
  862. conn = list_entry(rxnet->idle_client_conns.next,
  863. struct rxrpc_connection, cache_link);
  864. if (!rxnet->kill_all_client_conns) {
  865. /* If the number of connections is over the reap limit, we
  866. * expedite discard by reducing the expiry timeout. We must,
  867. * however, have at least a short grace period to be able to do
  868. * final-ACK or ABORT retransmission.
  869. */
  870. expiry = rxrpc_conn_idle_client_expiry;
  871. if (nr_conns > rxrpc_reap_client_connections)
  872. expiry = rxrpc_conn_idle_client_fast_expiry;
  873. if (conn->params.local->service_closed)
  874. expiry = rxrpc_closed_conn_expiry * HZ;
  875. conn_expires_at = conn->idle_timestamp + expiry;
  876. now = READ_ONCE(jiffies);
  877. if (time_after(conn_expires_at, now))
  878. goto not_yet_expired;
  879. }
  880. trace_rxrpc_client(conn, -1, rxrpc_client_discard);
  881. list_del_init(&conn->cache_link);
  882. spin_unlock(&rxnet->client_conn_cache_lock);
  883. rxrpc_unbundle_conn(conn);
  884. rxrpc_put_connection(conn); /* Drop the ->cache_link ref */
  885. nr_conns--;
  886. goto next;
  887. not_yet_expired:
  888. /* The connection at the front of the queue hasn't yet expired, so
  889. * schedule the work item for that point if we discarded something.
  890. *
  891. * We don't worry if the work item is already scheduled - it can look
  892. * after rescheduling itself at a later time. We could cancel it, but
  893. * then things get messier.
  894. */
  895. _debug("not yet");
  896. if (!rxnet->kill_all_client_conns)
  897. timer_reduce(&rxnet->client_conn_reap_timer, conn_expires_at);
  898. out:
  899. spin_unlock(&rxnet->client_conn_cache_lock);
  900. spin_unlock(&rxnet->client_conn_discard_lock);
  901. _leave("");
  902. }
  903. /*
  904. * Preemptively destroy all the client connection records rather than waiting
  905. * for them to time out
  906. */
  907. void rxrpc_destroy_all_client_connections(struct rxrpc_net *rxnet)
  908. {
  909. _enter("");
  910. spin_lock(&rxnet->client_conn_cache_lock);
  911. rxnet->kill_all_client_conns = true;
  912. spin_unlock(&rxnet->client_conn_cache_lock);
  913. del_timer_sync(&rxnet->client_conn_reap_timer);
  914. if (!rxrpc_queue_work(&rxnet->client_conn_reaper))
  915. _debug("destroy: queue failed");
  916. _leave("");
  917. }
  918. /*
  919. * Clean up the client connections on a local endpoint.
  920. */
  921. void rxrpc_clean_up_local_conns(struct rxrpc_local *local)
  922. {
  923. struct rxrpc_connection *conn, *tmp;
  924. struct rxrpc_net *rxnet = local->rxnet;
  925. LIST_HEAD(graveyard);
  926. _enter("");
  927. spin_lock(&rxnet->client_conn_cache_lock);
  928. list_for_each_entry_safe(conn, tmp, &rxnet->idle_client_conns,
  929. cache_link) {
  930. if (conn->params.local == local) {
  931. trace_rxrpc_client(conn, -1, rxrpc_client_discard);
  932. list_move(&conn->cache_link, &graveyard);
  933. }
  934. }
  935. spin_unlock(&rxnet->client_conn_cache_lock);
  936. while (!list_empty(&graveyard)) {
  937. conn = list_entry(graveyard.next,
  938. struct rxrpc_connection, cache_link);
  939. list_del_init(&conn->cache_link);
  940. rxrpc_unbundle_conn(conn);
  941. rxrpc_put_connection(conn);
  942. }
  943. _leave(" [culled]");
  944. }