kcmsock.c 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Kernel Connection Multiplexor
  4. *
  5. * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
  6. */
  7. #include <linux/bpf.h>
  8. #include <linux/errno.h>
  9. #include <linux/errqueue.h>
  10. #include <linux/file.h>
  11. #include <linux/in.h>
  12. #include <linux/kernel.h>
  13. #include <linux/module.h>
  14. #include <linux/net.h>
  15. #include <linux/netdevice.h>
  16. #include <linux/poll.h>
  17. #include <linux/rculist.h>
  18. #include <linux/skbuff.h>
  19. #include <linux/socket.h>
  20. #include <linux/uaccess.h>
  21. #include <linux/workqueue.h>
  22. #include <linux/syscalls.h>
  23. #include <linux/sched/signal.h>
  24. #include <net/kcm.h>
  25. #include <net/netns/generic.h>
  26. #include <net/sock.h>
  27. #include <uapi/linux/kcm.h>
  28. unsigned int kcm_net_id;
  29. static struct kmem_cache *kcm_psockp __read_mostly;
  30. static struct kmem_cache *kcm_muxp __read_mostly;
  31. static struct workqueue_struct *kcm_wq;
  32. static inline struct kcm_sock *kcm_sk(const struct sock *sk)
  33. {
  34. return (struct kcm_sock *)sk;
  35. }
  36. static inline struct kcm_tx_msg *kcm_tx_msg(struct sk_buff *skb)
  37. {
  38. return (struct kcm_tx_msg *)skb->cb;
  39. }
  40. static void report_csk_error(struct sock *csk, int err)
  41. {
  42. csk->sk_err = EPIPE;
  43. csk->sk_error_report(csk);
  44. }
  45. static void kcm_abort_tx_psock(struct kcm_psock *psock, int err,
  46. bool wakeup_kcm)
  47. {
  48. struct sock *csk = psock->sk;
  49. struct kcm_mux *mux = psock->mux;
  50. /* Unrecoverable error in transmit */
  51. spin_lock_bh(&mux->lock);
  52. if (psock->tx_stopped) {
  53. spin_unlock_bh(&mux->lock);
  54. return;
  55. }
  56. psock->tx_stopped = 1;
  57. KCM_STATS_INCR(psock->stats.tx_aborts);
  58. if (!psock->tx_kcm) {
  59. /* Take off psocks_avail list */
  60. list_del(&psock->psock_avail_list);
  61. } else if (wakeup_kcm) {
  62. /* In this case psock is being aborted while outside of
  63. * write_msgs and psock is reserved. Schedule tx_work
  64. * to handle the failure there. Need to commit tx_stopped
  65. * before queuing work.
  66. */
  67. smp_mb();
  68. queue_work(kcm_wq, &psock->tx_kcm->tx_work);
  69. }
  70. spin_unlock_bh(&mux->lock);
  71. /* Report error on lower socket */
  72. report_csk_error(csk, err);
  73. }
  74. /* RX mux lock held. */
  75. static void kcm_update_rx_mux_stats(struct kcm_mux *mux,
  76. struct kcm_psock *psock)
  77. {
  78. STRP_STATS_ADD(mux->stats.rx_bytes,
  79. psock->strp.stats.bytes -
  80. psock->saved_rx_bytes);
  81. mux->stats.rx_msgs +=
  82. psock->strp.stats.msgs - psock->saved_rx_msgs;
  83. psock->saved_rx_msgs = psock->strp.stats.msgs;
  84. psock->saved_rx_bytes = psock->strp.stats.bytes;
  85. }
  86. static void kcm_update_tx_mux_stats(struct kcm_mux *mux,
  87. struct kcm_psock *psock)
  88. {
  89. KCM_STATS_ADD(mux->stats.tx_bytes,
  90. psock->stats.tx_bytes - psock->saved_tx_bytes);
  91. mux->stats.tx_msgs +=
  92. psock->stats.tx_msgs - psock->saved_tx_msgs;
  93. psock->saved_tx_msgs = psock->stats.tx_msgs;
  94. psock->saved_tx_bytes = psock->stats.tx_bytes;
  95. }
  96. static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
  97. /* KCM is ready to receive messages on its queue-- either the KCM is new or
  98. * has become unblocked after being blocked on full socket buffer. Queue any
  99. * pending ready messages on a psock. RX mux lock held.
  100. */
  101. static void kcm_rcv_ready(struct kcm_sock *kcm)
  102. {
  103. struct kcm_mux *mux = kcm->mux;
  104. struct kcm_psock *psock;
  105. struct sk_buff *skb;
  106. if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))
  107. return;
  108. while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {
  109. if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
  110. /* Assuming buffer limit has been reached */
  111. skb_queue_head(&mux->rx_hold_queue, skb);
  112. WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
  113. return;
  114. }
  115. }
  116. while (!list_empty(&mux->psocks_ready)) {
  117. psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,
  118. psock_ready_list);
  119. if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
  120. /* Assuming buffer limit has been reached */
  121. WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
  122. return;
  123. }
  124. /* Consumed the ready message on the psock. Schedule rx_work to
  125. * get more messages.
  126. */
  127. list_del(&psock->psock_ready_list);
  128. psock->ready_rx_msg = NULL;
  129. /* Commit clearing of ready_rx_msg for queuing work */
  130. smp_mb();
  131. strp_unpause(&psock->strp);
  132. strp_check_rcv(&psock->strp);
  133. }
  134. /* Buffer limit is okay now, add to ready list */
  135. list_add_tail(&kcm->wait_rx_list,
  136. &kcm->mux->kcm_rx_waiters);
  137. kcm->rx_wait = true;
  138. }
  139. static void kcm_rfree(struct sk_buff *skb)
  140. {
  141. struct sock *sk = skb->sk;
  142. struct kcm_sock *kcm = kcm_sk(sk);
  143. struct kcm_mux *mux = kcm->mux;
  144. unsigned int len = skb->truesize;
  145. sk_mem_uncharge(sk, len);
  146. atomic_sub(len, &sk->sk_rmem_alloc);
  147. /* For reading rx_wait and rx_psock without holding lock */
  148. smp_mb__after_atomic();
  149. if (!kcm->rx_wait && !kcm->rx_psock &&
  150. sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) {
  151. spin_lock_bh(&mux->rx_lock);
  152. kcm_rcv_ready(kcm);
  153. spin_unlock_bh(&mux->rx_lock);
  154. }
  155. }
  156. static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
  157. {
  158. struct sk_buff_head *list = &sk->sk_receive_queue;
  159. if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
  160. return -ENOMEM;
  161. if (!sk_rmem_schedule(sk, skb, skb->truesize))
  162. return -ENOBUFS;
  163. skb->dev = NULL;
  164. skb_orphan(skb);
  165. skb->sk = sk;
  166. skb->destructor = kcm_rfree;
  167. atomic_add(skb->truesize, &sk->sk_rmem_alloc);
  168. sk_mem_charge(sk, skb->truesize);
  169. skb_queue_tail(list, skb);
  170. if (!sock_flag(sk, SOCK_DEAD))
  171. sk->sk_data_ready(sk);
  172. return 0;
  173. }
  174. /* Requeue received messages for a kcm socket to other kcm sockets. This is
  175. * called with a kcm socket is receive disabled.
  176. * RX mux lock held.
  177. */
  178. static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)
  179. {
  180. struct sk_buff *skb;
  181. struct kcm_sock *kcm;
  182. while ((skb = __skb_dequeue(head))) {
  183. /* Reset destructor to avoid calling kcm_rcv_ready */
  184. skb->destructor = sock_rfree;
  185. skb_orphan(skb);
  186. try_again:
  187. if (list_empty(&mux->kcm_rx_waiters)) {
  188. skb_queue_tail(&mux->rx_hold_queue, skb);
  189. continue;
  190. }
  191. kcm = list_first_entry(&mux->kcm_rx_waiters,
  192. struct kcm_sock, wait_rx_list);
  193. if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
  194. /* Should mean socket buffer full */
  195. list_del(&kcm->wait_rx_list);
  196. kcm->rx_wait = false;
  197. /* Commit rx_wait to read in kcm_free */
  198. smp_wmb();
  199. goto try_again;
  200. }
  201. }
  202. }
  203. /* Lower sock lock held */
  204. static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock,
  205. struct sk_buff *head)
  206. {
  207. struct kcm_mux *mux = psock->mux;
  208. struct kcm_sock *kcm;
  209. WARN_ON(psock->ready_rx_msg);
  210. if (psock->rx_kcm)
  211. return psock->rx_kcm;
  212. spin_lock_bh(&mux->rx_lock);
  213. if (psock->rx_kcm) {
  214. spin_unlock_bh(&mux->rx_lock);
  215. return psock->rx_kcm;
  216. }
  217. kcm_update_rx_mux_stats(mux, psock);
  218. if (list_empty(&mux->kcm_rx_waiters)) {
  219. psock->ready_rx_msg = head;
  220. strp_pause(&psock->strp);
  221. list_add_tail(&psock->psock_ready_list,
  222. &mux->psocks_ready);
  223. spin_unlock_bh(&mux->rx_lock);
  224. return NULL;
  225. }
  226. kcm = list_first_entry(&mux->kcm_rx_waiters,
  227. struct kcm_sock, wait_rx_list);
  228. list_del(&kcm->wait_rx_list);
  229. kcm->rx_wait = false;
  230. psock->rx_kcm = kcm;
  231. kcm->rx_psock = psock;
  232. spin_unlock_bh(&mux->rx_lock);
  233. return kcm;
  234. }
  235. static void kcm_done(struct kcm_sock *kcm);
  236. static void kcm_done_work(struct work_struct *w)
  237. {
  238. kcm_done(container_of(w, struct kcm_sock, done_work));
  239. }
  240. /* Lower sock held */
  241. static void unreserve_rx_kcm(struct kcm_psock *psock,
  242. bool rcv_ready)
  243. {
  244. struct kcm_sock *kcm = psock->rx_kcm;
  245. struct kcm_mux *mux = psock->mux;
  246. if (!kcm)
  247. return;
  248. spin_lock_bh(&mux->rx_lock);
  249. psock->rx_kcm = NULL;
  250. kcm->rx_psock = NULL;
  251. /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with
  252. * kcm_rfree
  253. */
  254. smp_mb();
  255. if (unlikely(kcm->done)) {
  256. spin_unlock_bh(&mux->rx_lock);
  257. /* Need to run kcm_done in a task since we need to qcquire
  258. * callback locks which may already be held here.
  259. */
  260. INIT_WORK(&kcm->done_work, kcm_done_work);
  261. schedule_work(&kcm->done_work);
  262. return;
  263. }
  264. if (unlikely(kcm->rx_disabled)) {
  265. requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
  266. } else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {
  267. /* Check for degenerative race with rx_wait that all
  268. * data was dequeued (accounted for in kcm_rfree).
  269. */
  270. kcm_rcv_ready(kcm);
  271. }
  272. spin_unlock_bh(&mux->rx_lock);
  273. }
  274. /* Lower sock lock held */
  275. static void psock_data_ready(struct sock *sk)
  276. {
  277. struct kcm_psock *psock;
  278. read_lock_bh(&sk->sk_callback_lock);
  279. psock = (struct kcm_psock *)sk->sk_user_data;
  280. if (likely(psock))
  281. strp_data_ready(&psock->strp);
  282. read_unlock_bh(&sk->sk_callback_lock);
  283. }
  284. /* Called with lower sock held */
  285. static void kcm_rcv_strparser(struct strparser *strp, struct sk_buff *skb)
  286. {
  287. struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
  288. struct kcm_sock *kcm;
  289. try_queue:
  290. kcm = reserve_rx_kcm(psock, skb);
  291. if (!kcm) {
  292. /* Unable to reserve a KCM, message is held in psock and strp
  293. * is paused.
  294. */
  295. return;
  296. }
  297. if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
  298. /* Should mean socket buffer full */
  299. unreserve_rx_kcm(psock, false);
  300. goto try_queue;
  301. }
  302. }
  303. static int kcm_parse_func_strparser(struct strparser *strp, struct sk_buff *skb)
  304. {
  305. struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
  306. struct bpf_prog *prog = psock->bpf_prog;
  307. int res;
  308. res = bpf_prog_run_pin_on_cpu(prog, skb);
  309. return res;
  310. }
  311. static int kcm_read_sock_done(struct strparser *strp, int err)
  312. {
  313. struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
  314. unreserve_rx_kcm(psock, true);
  315. return err;
  316. }
  317. static void psock_state_change(struct sock *sk)
  318. {
  319. /* TCP only does a EPOLLIN for a half close. Do a EPOLLHUP here
  320. * since application will normally not poll with EPOLLIN
  321. * on the TCP sockets.
  322. */
  323. report_csk_error(sk, EPIPE);
  324. }
  325. static void psock_write_space(struct sock *sk)
  326. {
  327. struct kcm_psock *psock;
  328. struct kcm_mux *mux;
  329. struct kcm_sock *kcm;
  330. read_lock_bh(&sk->sk_callback_lock);
  331. psock = (struct kcm_psock *)sk->sk_user_data;
  332. if (unlikely(!psock))
  333. goto out;
  334. mux = psock->mux;
  335. spin_lock_bh(&mux->lock);
  336. /* Check if the socket is reserved so someone is waiting for sending. */
  337. kcm = psock->tx_kcm;
  338. if (kcm && !unlikely(kcm->tx_stopped))
  339. queue_work(kcm_wq, &kcm->tx_work);
  340. spin_unlock_bh(&mux->lock);
  341. out:
  342. read_unlock_bh(&sk->sk_callback_lock);
  343. }
  344. static void unreserve_psock(struct kcm_sock *kcm);
  345. /* kcm sock is locked. */
  346. static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)
  347. {
  348. struct kcm_mux *mux = kcm->mux;
  349. struct kcm_psock *psock;
  350. psock = kcm->tx_psock;
  351. smp_rmb(); /* Must read tx_psock before tx_wait */
  352. if (psock) {
  353. WARN_ON(kcm->tx_wait);
  354. if (unlikely(psock->tx_stopped))
  355. unreserve_psock(kcm);
  356. else
  357. return kcm->tx_psock;
  358. }
  359. spin_lock_bh(&mux->lock);
  360. /* Check again under lock to see if psock was reserved for this
  361. * psock via psock_unreserve.
  362. */
  363. psock = kcm->tx_psock;
  364. if (unlikely(psock)) {
  365. WARN_ON(kcm->tx_wait);
  366. spin_unlock_bh(&mux->lock);
  367. return kcm->tx_psock;
  368. }
  369. if (!list_empty(&mux->psocks_avail)) {
  370. psock = list_first_entry(&mux->psocks_avail,
  371. struct kcm_psock,
  372. psock_avail_list);
  373. list_del(&psock->psock_avail_list);
  374. if (kcm->tx_wait) {
  375. list_del(&kcm->wait_psock_list);
  376. kcm->tx_wait = false;
  377. }
  378. kcm->tx_psock = psock;
  379. psock->tx_kcm = kcm;
  380. KCM_STATS_INCR(psock->stats.reserved);
  381. } else if (!kcm->tx_wait) {
  382. list_add_tail(&kcm->wait_psock_list,
  383. &mux->kcm_tx_waiters);
  384. kcm->tx_wait = true;
  385. }
  386. spin_unlock_bh(&mux->lock);
  387. return psock;
  388. }
  389. /* mux lock held */
  390. static void psock_now_avail(struct kcm_psock *psock)
  391. {
  392. struct kcm_mux *mux = psock->mux;
  393. struct kcm_sock *kcm;
  394. if (list_empty(&mux->kcm_tx_waiters)) {
  395. list_add_tail(&psock->psock_avail_list,
  396. &mux->psocks_avail);
  397. } else {
  398. kcm = list_first_entry(&mux->kcm_tx_waiters,
  399. struct kcm_sock,
  400. wait_psock_list);
  401. list_del(&kcm->wait_psock_list);
  402. kcm->tx_wait = false;
  403. psock->tx_kcm = kcm;
  404. /* Commit before changing tx_psock since that is read in
  405. * reserve_psock before queuing work.
  406. */
  407. smp_mb();
  408. kcm->tx_psock = psock;
  409. KCM_STATS_INCR(psock->stats.reserved);
  410. queue_work(kcm_wq, &kcm->tx_work);
  411. }
  412. }
  413. /* kcm sock is locked. */
  414. static void unreserve_psock(struct kcm_sock *kcm)
  415. {
  416. struct kcm_psock *psock;
  417. struct kcm_mux *mux = kcm->mux;
  418. spin_lock_bh(&mux->lock);
  419. psock = kcm->tx_psock;
  420. if (WARN_ON(!psock)) {
  421. spin_unlock_bh(&mux->lock);
  422. return;
  423. }
  424. smp_rmb(); /* Read tx_psock before tx_wait */
  425. kcm_update_tx_mux_stats(mux, psock);
  426. WARN_ON(kcm->tx_wait);
  427. kcm->tx_psock = NULL;
  428. psock->tx_kcm = NULL;
  429. KCM_STATS_INCR(psock->stats.unreserved);
  430. if (unlikely(psock->tx_stopped)) {
  431. if (psock->done) {
  432. /* Deferred free */
  433. list_del(&psock->psock_list);
  434. mux->psocks_cnt--;
  435. sock_put(psock->sk);
  436. fput(psock->sk->sk_socket->file);
  437. kmem_cache_free(kcm_psockp, psock);
  438. }
  439. /* Don't put back on available list */
  440. spin_unlock_bh(&mux->lock);
  441. return;
  442. }
  443. psock_now_avail(psock);
  444. spin_unlock_bh(&mux->lock);
  445. }
  446. static void kcm_report_tx_retry(struct kcm_sock *kcm)
  447. {
  448. struct kcm_mux *mux = kcm->mux;
  449. spin_lock_bh(&mux->lock);
  450. KCM_STATS_INCR(mux->stats.tx_retries);
  451. spin_unlock_bh(&mux->lock);
  452. }
  453. /* Write any messages ready on the kcm socket. Called with kcm sock lock
  454. * held. Return bytes actually sent or error.
  455. */
  456. static int kcm_write_msgs(struct kcm_sock *kcm)
  457. {
  458. struct sock *sk = &kcm->sk;
  459. struct kcm_psock *psock;
  460. struct sk_buff *skb, *head;
  461. struct kcm_tx_msg *txm;
  462. unsigned short fragidx, frag_offset;
  463. unsigned int sent, total_sent = 0;
  464. int ret = 0;
  465. kcm->tx_wait_more = false;
  466. psock = kcm->tx_psock;
  467. if (unlikely(psock && psock->tx_stopped)) {
  468. /* A reserved psock was aborted asynchronously. Unreserve
  469. * it and we'll retry the message.
  470. */
  471. unreserve_psock(kcm);
  472. kcm_report_tx_retry(kcm);
  473. if (skb_queue_empty(&sk->sk_write_queue))
  474. return 0;
  475. kcm_tx_msg(skb_peek(&sk->sk_write_queue))->sent = 0;
  476. } else if (skb_queue_empty(&sk->sk_write_queue)) {
  477. return 0;
  478. }
  479. head = skb_peek(&sk->sk_write_queue);
  480. txm = kcm_tx_msg(head);
  481. if (txm->sent) {
  482. /* Send of first skbuff in queue already in progress */
  483. if (WARN_ON(!psock)) {
  484. ret = -EINVAL;
  485. goto out;
  486. }
  487. sent = txm->sent;
  488. frag_offset = txm->frag_offset;
  489. fragidx = txm->fragidx;
  490. skb = txm->frag_skb;
  491. goto do_frag;
  492. }
  493. try_again:
  494. psock = reserve_psock(kcm);
  495. if (!psock)
  496. goto out;
  497. do {
  498. skb = head;
  499. txm = kcm_tx_msg(head);
  500. sent = 0;
  501. do_frag_list:
  502. if (WARN_ON(!skb_shinfo(skb)->nr_frags)) {
  503. ret = -EINVAL;
  504. goto out;
  505. }
  506. for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags;
  507. fragidx++) {
  508. skb_frag_t *frag;
  509. frag_offset = 0;
  510. do_frag:
  511. frag = &skb_shinfo(skb)->frags[fragidx];
  512. if (WARN_ON(!skb_frag_size(frag))) {
  513. ret = -EINVAL;
  514. goto out;
  515. }
  516. ret = kernel_sendpage(psock->sk->sk_socket,
  517. skb_frag_page(frag),
  518. skb_frag_off(frag) + frag_offset,
  519. skb_frag_size(frag) - frag_offset,
  520. MSG_DONTWAIT);
  521. if (ret <= 0) {
  522. if (ret == -EAGAIN) {
  523. /* Save state to try again when there's
  524. * write space on the socket
  525. */
  526. txm->sent = sent;
  527. txm->frag_offset = frag_offset;
  528. txm->fragidx = fragidx;
  529. txm->frag_skb = skb;
  530. ret = 0;
  531. goto out;
  532. }
  533. /* Hard failure in sending message, abort this
  534. * psock since it has lost framing
  535. * synchonization and retry sending the
  536. * message from the beginning.
  537. */
  538. kcm_abort_tx_psock(psock, ret ? -ret : EPIPE,
  539. true);
  540. unreserve_psock(kcm);
  541. txm->sent = 0;
  542. kcm_report_tx_retry(kcm);
  543. ret = 0;
  544. goto try_again;
  545. }
  546. sent += ret;
  547. frag_offset += ret;
  548. KCM_STATS_ADD(psock->stats.tx_bytes, ret);
  549. if (frag_offset < skb_frag_size(frag)) {
  550. /* Not finished with this frag */
  551. goto do_frag;
  552. }
  553. }
  554. if (skb == head) {
  555. if (skb_has_frag_list(skb)) {
  556. skb = skb_shinfo(skb)->frag_list;
  557. goto do_frag_list;
  558. }
  559. } else if (skb->next) {
  560. skb = skb->next;
  561. goto do_frag_list;
  562. }
  563. /* Successfully sent the whole packet, account for it. */
  564. skb_dequeue(&sk->sk_write_queue);
  565. kfree_skb(head);
  566. sk->sk_wmem_queued -= sent;
  567. total_sent += sent;
  568. KCM_STATS_INCR(psock->stats.tx_msgs);
  569. } while ((head = skb_peek(&sk->sk_write_queue)));
  570. out:
  571. if (!head) {
  572. /* Done with all queued messages. */
  573. WARN_ON(!skb_queue_empty(&sk->sk_write_queue));
  574. unreserve_psock(kcm);
  575. }
  576. /* Check if write space is available */
  577. sk->sk_write_space(sk);
  578. return total_sent ? : ret;
  579. }
  580. static void kcm_tx_work(struct work_struct *w)
  581. {
  582. struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);
  583. struct sock *sk = &kcm->sk;
  584. int err;
  585. lock_sock(sk);
  586. /* Primarily for SOCK_DGRAM sockets, also handle asynchronous tx
  587. * aborts
  588. */
  589. err = kcm_write_msgs(kcm);
  590. if (err < 0) {
  591. /* Hard failure in write, report error on KCM socket */
  592. pr_warn("KCM: Hard failure on kcm_write_msgs %d\n", err);
  593. report_csk_error(&kcm->sk, -err);
  594. goto out;
  595. }
  596. /* Primarily for SOCK_SEQPACKET sockets */
  597. if (likely(sk->sk_socket) &&
  598. test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
  599. clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  600. sk->sk_write_space(sk);
  601. }
  602. out:
  603. release_sock(sk);
  604. }
  605. static void kcm_push(struct kcm_sock *kcm)
  606. {
  607. if (kcm->tx_wait_more)
  608. kcm_write_msgs(kcm);
  609. }
  610. static ssize_t kcm_sendpage(struct socket *sock, struct page *page,
  611. int offset, size_t size, int flags)
  612. {
  613. struct sock *sk = sock->sk;
  614. struct kcm_sock *kcm = kcm_sk(sk);
  615. struct sk_buff *skb = NULL, *head = NULL;
  616. long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
  617. bool eor;
  618. int err = 0;
  619. int i;
  620. if (flags & MSG_SENDPAGE_NOTLAST)
  621. flags |= MSG_MORE;
  622. /* No MSG_EOR from splice, only look at MSG_MORE */
  623. eor = !(flags & MSG_MORE);
  624. lock_sock(sk);
  625. sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
  626. err = -EPIPE;
  627. if (sk->sk_err)
  628. goto out_error;
  629. if (kcm->seq_skb) {
  630. /* Previously opened message */
  631. head = kcm->seq_skb;
  632. skb = kcm_tx_msg(head)->last_skb;
  633. i = skb_shinfo(skb)->nr_frags;
  634. if (skb_can_coalesce(skb, i, page, offset)) {
  635. skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
  636. skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
  637. goto coalesced;
  638. }
  639. if (i >= MAX_SKB_FRAGS) {
  640. struct sk_buff *tskb;
  641. tskb = alloc_skb(0, sk->sk_allocation);
  642. while (!tskb) {
  643. kcm_push(kcm);
  644. err = sk_stream_wait_memory(sk, &timeo);
  645. if (err)
  646. goto out_error;
  647. }
  648. if (head == skb)
  649. skb_shinfo(head)->frag_list = tskb;
  650. else
  651. skb->next = tskb;
  652. skb = tskb;
  653. skb->ip_summed = CHECKSUM_UNNECESSARY;
  654. i = 0;
  655. }
  656. } else {
  657. /* Call the sk_stream functions to manage the sndbuf mem. */
  658. if (!sk_stream_memory_free(sk)) {
  659. kcm_push(kcm);
  660. set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  661. err = sk_stream_wait_memory(sk, &timeo);
  662. if (err)
  663. goto out_error;
  664. }
  665. head = alloc_skb(0, sk->sk_allocation);
  666. while (!head) {
  667. kcm_push(kcm);
  668. err = sk_stream_wait_memory(sk, &timeo);
  669. if (err)
  670. goto out_error;
  671. }
  672. skb = head;
  673. i = 0;
  674. }
  675. get_page(page);
  676. skb_fill_page_desc(skb, i, page, offset, size);
  677. skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
  678. coalesced:
  679. skb->len += size;
  680. skb->data_len += size;
  681. skb->truesize += size;
  682. sk->sk_wmem_queued += size;
  683. sk_mem_charge(sk, size);
  684. if (head != skb) {
  685. head->len += size;
  686. head->data_len += size;
  687. head->truesize += size;
  688. }
  689. if (eor) {
  690. bool not_busy = skb_queue_empty(&sk->sk_write_queue);
  691. /* Message complete, queue it on send buffer */
  692. __skb_queue_tail(&sk->sk_write_queue, head);
  693. kcm->seq_skb = NULL;
  694. KCM_STATS_INCR(kcm->stats.tx_msgs);
  695. if (flags & MSG_BATCH) {
  696. kcm->tx_wait_more = true;
  697. } else if (kcm->tx_wait_more || not_busy) {
  698. err = kcm_write_msgs(kcm);
  699. if (err < 0) {
  700. /* We got a hard error in write_msgs but have
  701. * already queued this message. Report an error
  702. * in the socket, but don't affect return value
  703. * from sendmsg
  704. */
  705. pr_warn("KCM: Hard failure on kcm_write_msgs\n");
  706. report_csk_error(&kcm->sk, -err);
  707. }
  708. }
  709. } else {
  710. /* Message not complete, save state */
  711. kcm->seq_skb = head;
  712. kcm_tx_msg(head)->last_skb = skb;
  713. }
  714. KCM_STATS_ADD(kcm->stats.tx_bytes, size);
  715. release_sock(sk);
  716. return size;
  717. out_error:
  718. kcm_push(kcm);
  719. err = sk_stream_error(sk, flags, err);
  720. /* make sure we wake any epoll edge trigger waiter */
  721. if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
  722. sk->sk_write_space(sk);
  723. release_sock(sk);
  724. return err;
  725. }
  726. static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
  727. {
  728. struct sock *sk = sock->sk;
  729. struct kcm_sock *kcm = kcm_sk(sk);
  730. struct sk_buff *skb = NULL, *head = NULL;
  731. size_t copy, copied = 0;
  732. long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
  733. int eor = (sock->type == SOCK_DGRAM) ?
  734. !(msg->msg_flags & MSG_MORE) : !!(msg->msg_flags & MSG_EOR);
  735. int err = -EPIPE;
  736. lock_sock(sk);
  737. /* Per tcp_sendmsg this should be in poll */
  738. sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
  739. if (sk->sk_err)
  740. goto out_error;
  741. if (kcm->seq_skb) {
  742. /* Previously opened message */
  743. head = kcm->seq_skb;
  744. skb = kcm_tx_msg(head)->last_skb;
  745. goto start;
  746. }
  747. /* Call the sk_stream functions to manage the sndbuf mem. */
  748. if (!sk_stream_memory_free(sk)) {
  749. kcm_push(kcm);
  750. set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  751. err = sk_stream_wait_memory(sk, &timeo);
  752. if (err)
  753. goto out_error;
  754. }
  755. if (msg_data_left(msg)) {
  756. /* New message, alloc head skb */
  757. head = alloc_skb(0, sk->sk_allocation);
  758. while (!head) {
  759. kcm_push(kcm);
  760. err = sk_stream_wait_memory(sk, &timeo);
  761. if (err)
  762. goto out_error;
  763. head = alloc_skb(0, sk->sk_allocation);
  764. }
  765. skb = head;
  766. /* Set ip_summed to CHECKSUM_UNNECESSARY to avoid calling
  767. * csum_and_copy_from_iter from skb_do_copy_data_nocache.
  768. */
  769. skb->ip_summed = CHECKSUM_UNNECESSARY;
  770. }
  771. start:
  772. while (msg_data_left(msg)) {
  773. bool merge = true;
  774. int i = skb_shinfo(skb)->nr_frags;
  775. struct page_frag *pfrag = sk_page_frag(sk);
  776. if (!sk_page_frag_refill(sk, pfrag))
  777. goto wait_for_memory;
  778. if (!skb_can_coalesce(skb, i, pfrag->page,
  779. pfrag->offset)) {
  780. if (i == MAX_SKB_FRAGS) {
  781. struct sk_buff *tskb;
  782. tskb = alloc_skb(0, sk->sk_allocation);
  783. if (!tskb)
  784. goto wait_for_memory;
  785. if (head == skb)
  786. skb_shinfo(head)->frag_list = tskb;
  787. else
  788. skb->next = tskb;
  789. skb = tskb;
  790. skb->ip_summed = CHECKSUM_UNNECESSARY;
  791. continue;
  792. }
  793. merge = false;
  794. }
  795. copy = min_t(int, msg_data_left(msg),
  796. pfrag->size - pfrag->offset);
  797. if (!sk_wmem_schedule(sk, copy))
  798. goto wait_for_memory;
  799. err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
  800. pfrag->page,
  801. pfrag->offset,
  802. copy);
  803. if (err)
  804. goto out_error;
  805. /* Update the skb. */
  806. if (merge) {
  807. skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
  808. } else {
  809. skb_fill_page_desc(skb, i, pfrag->page,
  810. pfrag->offset, copy);
  811. get_page(pfrag->page);
  812. }
  813. pfrag->offset += copy;
  814. copied += copy;
  815. if (head != skb) {
  816. head->len += copy;
  817. head->data_len += copy;
  818. }
  819. continue;
  820. wait_for_memory:
  821. kcm_push(kcm);
  822. err = sk_stream_wait_memory(sk, &timeo);
  823. if (err)
  824. goto out_error;
  825. }
  826. if (eor) {
  827. bool not_busy = skb_queue_empty(&sk->sk_write_queue);
  828. if (head) {
  829. /* Message complete, queue it on send buffer */
  830. __skb_queue_tail(&sk->sk_write_queue, head);
  831. kcm->seq_skb = NULL;
  832. KCM_STATS_INCR(kcm->stats.tx_msgs);
  833. }
  834. if (msg->msg_flags & MSG_BATCH) {
  835. kcm->tx_wait_more = true;
  836. } else if (kcm->tx_wait_more || not_busy) {
  837. err = kcm_write_msgs(kcm);
  838. if (err < 0) {
  839. /* We got a hard error in write_msgs but have
  840. * already queued this message. Report an error
  841. * in the socket, but don't affect return value
  842. * from sendmsg
  843. */
  844. pr_warn("KCM: Hard failure on kcm_write_msgs\n");
  845. report_csk_error(&kcm->sk, -err);
  846. }
  847. }
  848. } else {
  849. /* Message not complete, save state */
  850. partial_message:
  851. if (head) {
  852. kcm->seq_skb = head;
  853. kcm_tx_msg(head)->last_skb = skb;
  854. }
  855. }
  856. KCM_STATS_ADD(kcm->stats.tx_bytes, copied);
  857. release_sock(sk);
  858. return copied;
  859. out_error:
  860. kcm_push(kcm);
  861. if (copied && sock->type == SOCK_SEQPACKET) {
  862. /* Wrote some bytes before encountering an
  863. * error, return partial success.
  864. */
  865. goto partial_message;
  866. }
  867. if (head != kcm->seq_skb)
  868. kfree_skb(head);
  869. err = sk_stream_error(sk, msg->msg_flags, err);
  870. /* make sure we wake any epoll edge trigger waiter */
  871. if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
  872. sk->sk_write_space(sk);
  873. release_sock(sk);
  874. return err;
  875. }
  876. static struct sk_buff *kcm_wait_data(struct sock *sk, int flags,
  877. long timeo, int *err)
  878. {
  879. struct sk_buff *skb;
  880. while (!(skb = skb_peek(&sk->sk_receive_queue))) {
  881. if (sk->sk_err) {
  882. *err = sock_error(sk);
  883. return NULL;
  884. }
  885. if (sock_flag(sk, SOCK_DONE))
  886. return NULL;
  887. if ((flags & MSG_DONTWAIT) || !timeo) {
  888. *err = -EAGAIN;
  889. return NULL;
  890. }
  891. sk_wait_data(sk, &timeo, NULL);
  892. /* Handle signals */
  893. if (signal_pending(current)) {
  894. *err = sock_intr_errno(timeo);
  895. return NULL;
  896. }
  897. }
  898. return skb;
  899. }
  900. static int kcm_recvmsg(struct socket *sock, struct msghdr *msg,
  901. size_t len, int flags)
  902. {
  903. struct sock *sk = sock->sk;
  904. struct kcm_sock *kcm = kcm_sk(sk);
  905. int err = 0;
  906. long timeo;
  907. struct strp_msg *stm;
  908. int copied = 0;
  909. struct sk_buff *skb;
  910. timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
  911. lock_sock(sk);
  912. skb = kcm_wait_data(sk, flags, timeo, &err);
  913. if (!skb)
  914. goto out;
  915. /* Okay, have a message on the receive queue */
  916. stm = strp_msg(skb);
  917. if (len > stm->full_len)
  918. len = stm->full_len;
  919. err = skb_copy_datagram_msg(skb, stm->offset, msg, len);
  920. if (err < 0)
  921. goto out;
  922. copied = len;
  923. if (likely(!(flags & MSG_PEEK))) {
  924. KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
  925. if (copied < stm->full_len) {
  926. if (sock->type == SOCK_DGRAM) {
  927. /* Truncated message */
  928. msg->msg_flags |= MSG_TRUNC;
  929. goto msg_finished;
  930. }
  931. stm->offset += copied;
  932. stm->full_len -= copied;
  933. } else {
  934. msg_finished:
  935. /* Finished with message */
  936. msg->msg_flags |= MSG_EOR;
  937. KCM_STATS_INCR(kcm->stats.rx_msgs);
  938. skb_unlink(skb, &sk->sk_receive_queue);
  939. kfree_skb(skb);
  940. }
  941. }
  942. out:
  943. release_sock(sk);
  944. return copied ? : err;
  945. }
  946. static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
  947. struct pipe_inode_info *pipe, size_t len,
  948. unsigned int flags)
  949. {
  950. struct sock *sk = sock->sk;
  951. struct kcm_sock *kcm = kcm_sk(sk);
  952. long timeo;
  953. struct strp_msg *stm;
  954. int err = 0;
  955. ssize_t copied;
  956. struct sk_buff *skb;
  957. /* Only support splice for SOCKSEQPACKET */
  958. timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
  959. lock_sock(sk);
  960. skb = kcm_wait_data(sk, flags, timeo, &err);
  961. if (!skb)
  962. goto err_out;
  963. /* Okay, have a message on the receive queue */
  964. stm = strp_msg(skb);
  965. if (len > stm->full_len)
  966. len = stm->full_len;
  967. copied = skb_splice_bits(skb, sk, stm->offset, pipe, len, flags);
  968. if (copied < 0) {
  969. err = copied;
  970. goto err_out;
  971. }
  972. KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
  973. stm->offset += copied;
  974. stm->full_len -= copied;
  975. /* We have no way to return MSG_EOR. If all the bytes have been
  976. * read we still leave the message in the receive socket buffer.
  977. * A subsequent recvmsg needs to be done to return MSG_EOR and
  978. * finish reading the message.
  979. */
  980. release_sock(sk);
  981. return copied;
  982. err_out:
  983. release_sock(sk);
  984. return err;
  985. }
  986. /* kcm sock lock held */
  987. static void kcm_recv_disable(struct kcm_sock *kcm)
  988. {
  989. struct kcm_mux *mux = kcm->mux;
  990. if (kcm->rx_disabled)
  991. return;
  992. spin_lock_bh(&mux->rx_lock);
  993. kcm->rx_disabled = 1;
  994. /* If a psock is reserved we'll do cleanup in unreserve */
  995. if (!kcm->rx_psock) {
  996. if (kcm->rx_wait) {
  997. list_del(&kcm->wait_rx_list);
  998. kcm->rx_wait = false;
  999. }
  1000. requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
  1001. }
  1002. spin_unlock_bh(&mux->rx_lock);
  1003. }
  1004. /* kcm sock lock held */
  1005. static void kcm_recv_enable(struct kcm_sock *kcm)
  1006. {
  1007. struct kcm_mux *mux = kcm->mux;
  1008. if (!kcm->rx_disabled)
  1009. return;
  1010. spin_lock_bh(&mux->rx_lock);
  1011. kcm->rx_disabled = 0;
  1012. kcm_rcv_ready(kcm);
  1013. spin_unlock_bh(&mux->rx_lock);
  1014. }
  1015. static int kcm_setsockopt(struct socket *sock, int level, int optname,
  1016. sockptr_t optval, unsigned int optlen)
  1017. {
  1018. struct kcm_sock *kcm = kcm_sk(sock->sk);
  1019. int val, valbool;
  1020. int err = 0;
  1021. if (level != SOL_KCM)
  1022. return -ENOPROTOOPT;
  1023. if (optlen < sizeof(int))
  1024. return -EINVAL;
  1025. if (copy_from_sockptr(&val, optval, sizeof(int)))
  1026. return -EFAULT;
  1027. valbool = val ? 1 : 0;
  1028. switch (optname) {
  1029. case KCM_RECV_DISABLE:
  1030. lock_sock(&kcm->sk);
  1031. if (valbool)
  1032. kcm_recv_disable(kcm);
  1033. else
  1034. kcm_recv_enable(kcm);
  1035. release_sock(&kcm->sk);
  1036. break;
  1037. default:
  1038. err = -ENOPROTOOPT;
  1039. }
  1040. return err;
  1041. }
  1042. static int kcm_getsockopt(struct socket *sock, int level, int optname,
  1043. char __user *optval, int __user *optlen)
  1044. {
  1045. struct kcm_sock *kcm = kcm_sk(sock->sk);
  1046. int val, len;
  1047. if (level != SOL_KCM)
  1048. return -ENOPROTOOPT;
  1049. if (get_user(len, optlen))
  1050. return -EFAULT;
  1051. len = min_t(unsigned int, len, sizeof(int));
  1052. if (len < 0)
  1053. return -EINVAL;
  1054. switch (optname) {
  1055. case KCM_RECV_DISABLE:
  1056. val = kcm->rx_disabled;
  1057. break;
  1058. default:
  1059. return -ENOPROTOOPT;
  1060. }
  1061. if (put_user(len, optlen))
  1062. return -EFAULT;
  1063. if (copy_to_user(optval, &val, len))
  1064. return -EFAULT;
  1065. return 0;
  1066. }
  1067. static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)
  1068. {
  1069. struct kcm_sock *tkcm;
  1070. struct list_head *head;
  1071. int index = 0;
  1072. /* For SOCK_SEQPACKET sock type, datagram_poll checks the sk_state, so
  1073. * we set sk_state, otherwise epoll_wait always returns right away with
  1074. * EPOLLHUP
  1075. */
  1076. kcm->sk.sk_state = TCP_ESTABLISHED;
  1077. /* Add to mux's kcm sockets list */
  1078. kcm->mux = mux;
  1079. spin_lock_bh(&mux->lock);
  1080. head = &mux->kcm_socks;
  1081. list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {
  1082. if (tkcm->index != index)
  1083. break;
  1084. head = &tkcm->kcm_sock_list;
  1085. index++;
  1086. }
  1087. list_add(&kcm->kcm_sock_list, head);
  1088. kcm->index = index;
  1089. mux->kcm_socks_cnt++;
  1090. spin_unlock_bh(&mux->lock);
  1091. INIT_WORK(&kcm->tx_work, kcm_tx_work);
  1092. spin_lock_bh(&mux->rx_lock);
  1093. kcm_rcv_ready(kcm);
  1094. spin_unlock_bh(&mux->rx_lock);
  1095. }
  1096. static int kcm_attach(struct socket *sock, struct socket *csock,
  1097. struct bpf_prog *prog)
  1098. {
  1099. struct kcm_sock *kcm = kcm_sk(sock->sk);
  1100. struct kcm_mux *mux = kcm->mux;
  1101. struct sock *csk;
  1102. struct kcm_psock *psock = NULL, *tpsock;
  1103. struct list_head *head;
  1104. int index = 0;
  1105. static const struct strp_callbacks cb = {
  1106. .rcv_msg = kcm_rcv_strparser,
  1107. .parse_msg = kcm_parse_func_strparser,
  1108. .read_sock_done = kcm_read_sock_done,
  1109. };
  1110. int err = 0;
  1111. csk = csock->sk;
  1112. if (!csk)
  1113. return -EINVAL;
  1114. lock_sock(csk);
  1115. /* Only allow TCP sockets to be attached for now */
  1116. if ((csk->sk_family != AF_INET && csk->sk_family != AF_INET6) ||
  1117. csk->sk_protocol != IPPROTO_TCP) {
  1118. err = -EOPNOTSUPP;
  1119. goto out;
  1120. }
  1121. /* Don't allow listeners or closed sockets */
  1122. if (csk->sk_state == TCP_LISTEN || csk->sk_state == TCP_CLOSE) {
  1123. err = -EOPNOTSUPP;
  1124. goto out;
  1125. }
  1126. psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL);
  1127. if (!psock) {
  1128. err = -ENOMEM;
  1129. goto out;
  1130. }
  1131. psock->mux = mux;
  1132. psock->sk = csk;
  1133. psock->bpf_prog = prog;
  1134. err = strp_init(&psock->strp, csk, &cb);
  1135. if (err) {
  1136. kmem_cache_free(kcm_psockp, psock);
  1137. goto out;
  1138. }
  1139. write_lock_bh(&csk->sk_callback_lock);
  1140. /* Check if sk_user_data is aready by KCM or someone else.
  1141. * Must be done under lock to prevent race conditions.
  1142. */
  1143. if (csk->sk_user_data) {
  1144. write_unlock_bh(&csk->sk_callback_lock);
  1145. strp_stop(&psock->strp);
  1146. strp_done(&psock->strp);
  1147. kmem_cache_free(kcm_psockp, psock);
  1148. err = -EALREADY;
  1149. goto out;
  1150. }
  1151. psock->save_data_ready = csk->sk_data_ready;
  1152. psock->save_write_space = csk->sk_write_space;
  1153. psock->save_state_change = csk->sk_state_change;
  1154. csk->sk_user_data = psock;
  1155. csk->sk_data_ready = psock_data_ready;
  1156. csk->sk_write_space = psock_write_space;
  1157. csk->sk_state_change = psock_state_change;
  1158. write_unlock_bh(&csk->sk_callback_lock);
  1159. sock_hold(csk);
  1160. /* Finished initialization, now add the psock to the MUX. */
  1161. spin_lock_bh(&mux->lock);
  1162. head = &mux->psocks;
  1163. list_for_each_entry(tpsock, &mux->psocks, psock_list) {
  1164. if (tpsock->index != index)
  1165. break;
  1166. head = &tpsock->psock_list;
  1167. index++;
  1168. }
  1169. list_add(&psock->psock_list, head);
  1170. psock->index = index;
  1171. KCM_STATS_INCR(mux->stats.psock_attach);
  1172. mux->psocks_cnt++;
  1173. psock_now_avail(psock);
  1174. spin_unlock_bh(&mux->lock);
  1175. /* Schedule RX work in case there are already bytes queued */
  1176. strp_check_rcv(&psock->strp);
  1177. out:
  1178. release_sock(csk);
  1179. return err;
  1180. }
  1181. static int kcm_attach_ioctl(struct socket *sock, struct kcm_attach *info)
  1182. {
  1183. struct socket *csock;
  1184. struct bpf_prog *prog;
  1185. int err;
  1186. csock = sockfd_lookup(info->fd, &err);
  1187. if (!csock)
  1188. return -ENOENT;
  1189. prog = bpf_prog_get_type(info->bpf_fd, BPF_PROG_TYPE_SOCKET_FILTER);
  1190. if (IS_ERR(prog)) {
  1191. err = PTR_ERR(prog);
  1192. goto out;
  1193. }
  1194. err = kcm_attach(sock, csock, prog);
  1195. if (err) {
  1196. bpf_prog_put(prog);
  1197. goto out;
  1198. }
  1199. /* Keep reference on file also */
  1200. return 0;
  1201. out:
  1202. fput(csock->file);
  1203. return err;
  1204. }
  1205. static void kcm_unattach(struct kcm_psock *psock)
  1206. {
  1207. struct sock *csk = psock->sk;
  1208. struct kcm_mux *mux = psock->mux;
  1209. lock_sock(csk);
  1210. /* Stop getting callbacks from TCP socket. After this there should
  1211. * be no way to reserve a kcm for this psock.
  1212. */
  1213. write_lock_bh(&csk->sk_callback_lock);
  1214. csk->sk_user_data = NULL;
  1215. csk->sk_data_ready = psock->save_data_ready;
  1216. csk->sk_write_space = psock->save_write_space;
  1217. csk->sk_state_change = psock->save_state_change;
  1218. strp_stop(&psock->strp);
  1219. if (WARN_ON(psock->rx_kcm)) {
  1220. write_unlock_bh(&csk->sk_callback_lock);
  1221. release_sock(csk);
  1222. return;
  1223. }
  1224. spin_lock_bh(&mux->rx_lock);
  1225. /* Stop receiver activities. After this point psock should not be
  1226. * able to get onto ready list either through callbacks or work.
  1227. */
  1228. if (psock->ready_rx_msg) {
  1229. list_del(&psock->psock_ready_list);
  1230. kfree_skb(psock->ready_rx_msg);
  1231. psock->ready_rx_msg = NULL;
  1232. KCM_STATS_INCR(mux->stats.rx_ready_drops);
  1233. }
  1234. spin_unlock_bh(&mux->rx_lock);
  1235. write_unlock_bh(&csk->sk_callback_lock);
  1236. /* Call strp_done without sock lock */
  1237. release_sock(csk);
  1238. strp_done(&psock->strp);
  1239. lock_sock(csk);
  1240. bpf_prog_put(psock->bpf_prog);
  1241. spin_lock_bh(&mux->lock);
  1242. aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);
  1243. save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);
  1244. KCM_STATS_INCR(mux->stats.psock_unattach);
  1245. if (psock->tx_kcm) {
  1246. /* psock was reserved. Just mark it finished and we will clean
  1247. * up in the kcm paths, we need kcm lock which can not be
  1248. * acquired here.
  1249. */
  1250. KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);
  1251. spin_unlock_bh(&mux->lock);
  1252. /* We are unattaching a socket that is reserved. Abort the
  1253. * socket since we may be out of sync in sending on it. We need
  1254. * to do this without the mux lock.
  1255. */
  1256. kcm_abort_tx_psock(psock, EPIPE, false);
  1257. spin_lock_bh(&mux->lock);
  1258. if (!psock->tx_kcm) {
  1259. /* psock now unreserved in window mux was unlocked */
  1260. goto no_reserved;
  1261. }
  1262. psock->done = 1;
  1263. /* Commit done before queuing work to process it */
  1264. smp_mb();
  1265. /* Queue tx work to make sure psock->done is handled */
  1266. queue_work(kcm_wq, &psock->tx_kcm->tx_work);
  1267. spin_unlock_bh(&mux->lock);
  1268. } else {
  1269. no_reserved:
  1270. if (!psock->tx_stopped)
  1271. list_del(&psock->psock_avail_list);
  1272. list_del(&psock->psock_list);
  1273. mux->psocks_cnt--;
  1274. spin_unlock_bh(&mux->lock);
  1275. sock_put(csk);
  1276. fput(csk->sk_socket->file);
  1277. kmem_cache_free(kcm_psockp, psock);
  1278. }
  1279. release_sock(csk);
  1280. }
  1281. static int kcm_unattach_ioctl(struct socket *sock, struct kcm_unattach *info)
  1282. {
  1283. struct kcm_sock *kcm = kcm_sk(sock->sk);
  1284. struct kcm_mux *mux = kcm->mux;
  1285. struct kcm_psock *psock;
  1286. struct socket *csock;
  1287. struct sock *csk;
  1288. int err;
  1289. csock = sockfd_lookup(info->fd, &err);
  1290. if (!csock)
  1291. return -ENOENT;
  1292. csk = csock->sk;
  1293. if (!csk) {
  1294. err = -EINVAL;
  1295. goto out;
  1296. }
  1297. err = -ENOENT;
  1298. spin_lock_bh(&mux->lock);
  1299. list_for_each_entry(psock, &mux->psocks, psock_list) {
  1300. if (psock->sk != csk)
  1301. continue;
  1302. /* Found the matching psock */
  1303. if (psock->unattaching || WARN_ON(psock->done)) {
  1304. err = -EALREADY;
  1305. break;
  1306. }
  1307. psock->unattaching = 1;
  1308. spin_unlock_bh(&mux->lock);
  1309. /* Lower socket lock should already be held */
  1310. kcm_unattach(psock);
  1311. err = 0;
  1312. goto out;
  1313. }
  1314. spin_unlock_bh(&mux->lock);
  1315. out:
  1316. fput(csock->file);
  1317. return err;
  1318. }
  1319. static struct proto kcm_proto = {
  1320. .name = "KCM",
  1321. .owner = THIS_MODULE,
  1322. .obj_size = sizeof(struct kcm_sock),
  1323. };
  1324. /* Clone a kcm socket. */
  1325. static struct file *kcm_clone(struct socket *osock)
  1326. {
  1327. struct socket *newsock;
  1328. struct sock *newsk;
  1329. newsock = sock_alloc();
  1330. if (!newsock)
  1331. return ERR_PTR(-ENFILE);
  1332. newsock->type = osock->type;
  1333. newsock->ops = osock->ops;
  1334. __module_get(newsock->ops->owner);
  1335. newsk = sk_alloc(sock_net(osock->sk), PF_KCM, GFP_KERNEL,
  1336. &kcm_proto, false);
  1337. if (!newsk) {
  1338. sock_release(newsock);
  1339. return ERR_PTR(-ENOMEM);
  1340. }
  1341. sock_init_data(newsock, newsk);
  1342. init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);
  1343. return sock_alloc_file(newsock, 0, osock->sk->sk_prot_creator->name);
  1344. }
  1345. static int kcm_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
  1346. {
  1347. int err;
  1348. switch (cmd) {
  1349. case SIOCKCMATTACH: {
  1350. struct kcm_attach info;
  1351. if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
  1352. return -EFAULT;
  1353. err = kcm_attach_ioctl(sock, &info);
  1354. break;
  1355. }
  1356. case SIOCKCMUNATTACH: {
  1357. struct kcm_unattach info;
  1358. if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
  1359. return -EFAULT;
  1360. err = kcm_unattach_ioctl(sock, &info);
  1361. break;
  1362. }
  1363. case SIOCKCMCLONE: {
  1364. struct kcm_clone info;
  1365. struct file *file;
  1366. info.fd = get_unused_fd_flags(0);
  1367. if (unlikely(info.fd < 0))
  1368. return info.fd;
  1369. file = kcm_clone(sock);
  1370. if (IS_ERR(file)) {
  1371. put_unused_fd(info.fd);
  1372. return PTR_ERR(file);
  1373. }
  1374. if (copy_to_user((void __user *)arg, &info,
  1375. sizeof(info))) {
  1376. put_unused_fd(info.fd);
  1377. fput(file);
  1378. return -EFAULT;
  1379. }
  1380. fd_install(info.fd, file);
  1381. err = 0;
  1382. break;
  1383. }
  1384. default:
  1385. err = -ENOIOCTLCMD;
  1386. break;
  1387. }
  1388. return err;
  1389. }
  1390. static void free_mux(struct rcu_head *rcu)
  1391. {
  1392. struct kcm_mux *mux = container_of(rcu,
  1393. struct kcm_mux, rcu);
  1394. kmem_cache_free(kcm_muxp, mux);
  1395. }
  1396. static void release_mux(struct kcm_mux *mux)
  1397. {
  1398. struct kcm_net *knet = mux->knet;
  1399. struct kcm_psock *psock, *tmp_psock;
  1400. /* Release psocks */
  1401. list_for_each_entry_safe(psock, tmp_psock,
  1402. &mux->psocks, psock_list) {
  1403. if (!WARN_ON(psock->unattaching))
  1404. kcm_unattach(psock);
  1405. }
  1406. if (WARN_ON(mux->psocks_cnt))
  1407. return;
  1408. __skb_queue_purge(&mux->rx_hold_queue);
  1409. mutex_lock(&knet->mutex);
  1410. aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);
  1411. aggregate_psock_stats(&mux->aggregate_psock_stats,
  1412. &knet->aggregate_psock_stats);
  1413. aggregate_strp_stats(&mux->aggregate_strp_stats,
  1414. &knet->aggregate_strp_stats);
  1415. list_del_rcu(&mux->kcm_mux_list);
  1416. knet->count--;
  1417. mutex_unlock(&knet->mutex);
  1418. call_rcu(&mux->rcu, free_mux);
  1419. }
  1420. static void kcm_done(struct kcm_sock *kcm)
  1421. {
  1422. struct kcm_mux *mux = kcm->mux;
  1423. struct sock *sk = &kcm->sk;
  1424. int socks_cnt;
  1425. spin_lock_bh(&mux->rx_lock);
  1426. if (kcm->rx_psock) {
  1427. /* Cleanup in unreserve_rx_kcm */
  1428. WARN_ON(kcm->done);
  1429. kcm->rx_disabled = 1;
  1430. kcm->done = 1;
  1431. spin_unlock_bh(&mux->rx_lock);
  1432. return;
  1433. }
  1434. if (kcm->rx_wait) {
  1435. list_del(&kcm->wait_rx_list);
  1436. kcm->rx_wait = false;
  1437. }
  1438. /* Move any pending receive messages to other kcm sockets */
  1439. requeue_rx_msgs(mux, &sk->sk_receive_queue);
  1440. spin_unlock_bh(&mux->rx_lock);
  1441. if (WARN_ON(sk_rmem_alloc_get(sk)))
  1442. return;
  1443. /* Detach from MUX */
  1444. spin_lock_bh(&mux->lock);
  1445. list_del(&kcm->kcm_sock_list);
  1446. mux->kcm_socks_cnt--;
  1447. socks_cnt = mux->kcm_socks_cnt;
  1448. spin_unlock_bh(&mux->lock);
  1449. if (!socks_cnt) {
  1450. /* We are done with the mux now. */
  1451. release_mux(mux);
  1452. }
  1453. WARN_ON(kcm->rx_wait);
  1454. sock_put(&kcm->sk);
  1455. }
  1456. /* Called by kcm_release to close a KCM socket.
  1457. * If this is the last KCM socket on the MUX, destroy the MUX.
  1458. */
  1459. static int kcm_release(struct socket *sock)
  1460. {
  1461. struct sock *sk = sock->sk;
  1462. struct kcm_sock *kcm;
  1463. struct kcm_mux *mux;
  1464. struct kcm_psock *psock;
  1465. if (!sk)
  1466. return 0;
  1467. kcm = kcm_sk(sk);
  1468. mux = kcm->mux;
  1469. sock_orphan(sk);
  1470. kfree_skb(kcm->seq_skb);
  1471. lock_sock(sk);
  1472. /* Purge queue under lock to avoid race condition with tx_work trying
  1473. * to act when queue is nonempty. If tx_work runs after this point
  1474. * it will just return.
  1475. */
  1476. __skb_queue_purge(&sk->sk_write_queue);
  1477. /* Set tx_stopped. This is checked when psock is bound to a kcm and we
  1478. * get a writespace callback. This prevents further work being queued
  1479. * from the callback (unbinding the psock occurs after canceling work.
  1480. */
  1481. kcm->tx_stopped = 1;
  1482. release_sock(sk);
  1483. spin_lock_bh(&mux->lock);
  1484. if (kcm->tx_wait) {
  1485. /* Take of tx_wait list, after this point there should be no way
  1486. * that a psock will be assigned to this kcm.
  1487. */
  1488. list_del(&kcm->wait_psock_list);
  1489. kcm->tx_wait = false;
  1490. }
  1491. spin_unlock_bh(&mux->lock);
  1492. /* Cancel work. After this point there should be no outside references
  1493. * to the kcm socket.
  1494. */
  1495. cancel_work_sync(&kcm->tx_work);
  1496. lock_sock(sk);
  1497. psock = kcm->tx_psock;
  1498. if (psock) {
  1499. /* A psock was reserved, so we need to kill it since it
  1500. * may already have some bytes queued from a message. We
  1501. * need to do this after removing kcm from tx_wait list.
  1502. */
  1503. kcm_abort_tx_psock(psock, EPIPE, false);
  1504. unreserve_psock(kcm);
  1505. }
  1506. release_sock(sk);
  1507. WARN_ON(kcm->tx_wait);
  1508. WARN_ON(kcm->tx_psock);
  1509. sock->sk = NULL;
  1510. kcm_done(kcm);
  1511. return 0;
  1512. }
  1513. static const struct proto_ops kcm_dgram_ops = {
  1514. .family = PF_KCM,
  1515. .owner = THIS_MODULE,
  1516. .release = kcm_release,
  1517. .bind = sock_no_bind,
  1518. .connect = sock_no_connect,
  1519. .socketpair = sock_no_socketpair,
  1520. .accept = sock_no_accept,
  1521. .getname = sock_no_getname,
  1522. .poll = datagram_poll,
  1523. .ioctl = kcm_ioctl,
  1524. .listen = sock_no_listen,
  1525. .shutdown = sock_no_shutdown,
  1526. .setsockopt = kcm_setsockopt,
  1527. .getsockopt = kcm_getsockopt,
  1528. .sendmsg = kcm_sendmsg,
  1529. .recvmsg = kcm_recvmsg,
  1530. .mmap = sock_no_mmap,
  1531. .sendpage = kcm_sendpage,
  1532. };
  1533. static const struct proto_ops kcm_seqpacket_ops = {
  1534. .family = PF_KCM,
  1535. .owner = THIS_MODULE,
  1536. .release = kcm_release,
  1537. .bind = sock_no_bind,
  1538. .connect = sock_no_connect,
  1539. .socketpair = sock_no_socketpair,
  1540. .accept = sock_no_accept,
  1541. .getname = sock_no_getname,
  1542. .poll = datagram_poll,
  1543. .ioctl = kcm_ioctl,
  1544. .listen = sock_no_listen,
  1545. .shutdown = sock_no_shutdown,
  1546. .setsockopt = kcm_setsockopt,
  1547. .getsockopt = kcm_getsockopt,
  1548. .sendmsg = kcm_sendmsg,
  1549. .recvmsg = kcm_recvmsg,
  1550. .mmap = sock_no_mmap,
  1551. .sendpage = kcm_sendpage,
  1552. .splice_read = kcm_splice_read,
  1553. };
  1554. /* Create proto operation for kcm sockets */
  1555. static int kcm_create(struct net *net, struct socket *sock,
  1556. int protocol, int kern)
  1557. {
  1558. struct kcm_net *knet = net_generic(net, kcm_net_id);
  1559. struct sock *sk;
  1560. struct kcm_mux *mux;
  1561. switch (sock->type) {
  1562. case SOCK_DGRAM:
  1563. sock->ops = &kcm_dgram_ops;
  1564. break;
  1565. case SOCK_SEQPACKET:
  1566. sock->ops = &kcm_seqpacket_ops;
  1567. break;
  1568. default:
  1569. return -ESOCKTNOSUPPORT;
  1570. }
  1571. if (protocol != KCMPROTO_CONNECTED)
  1572. return -EPROTONOSUPPORT;
  1573. sk = sk_alloc(net, PF_KCM, GFP_KERNEL, &kcm_proto, kern);
  1574. if (!sk)
  1575. return -ENOMEM;
  1576. /* Allocate a kcm mux, shared between KCM sockets */
  1577. mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);
  1578. if (!mux) {
  1579. sk_free(sk);
  1580. return -ENOMEM;
  1581. }
  1582. spin_lock_init(&mux->lock);
  1583. spin_lock_init(&mux->rx_lock);
  1584. INIT_LIST_HEAD(&mux->kcm_socks);
  1585. INIT_LIST_HEAD(&mux->kcm_rx_waiters);
  1586. INIT_LIST_HEAD(&mux->kcm_tx_waiters);
  1587. INIT_LIST_HEAD(&mux->psocks);
  1588. INIT_LIST_HEAD(&mux->psocks_ready);
  1589. INIT_LIST_HEAD(&mux->psocks_avail);
  1590. mux->knet = knet;
  1591. /* Add new MUX to list */
  1592. mutex_lock(&knet->mutex);
  1593. list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);
  1594. knet->count++;
  1595. mutex_unlock(&knet->mutex);
  1596. skb_queue_head_init(&mux->rx_hold_queue);
  1597. /* Init KCM socket */
  1598. sock_init_data(sock, sk);
  1599. init_kcm_sock(kcm_sk(sk), mux);
  1600. return 0;
  1601. }
  1602. static const struct net_proto_family kcm_family_ops = {
  1603. .family = PF_KCM,
  1604. .create = kcm_create,
  1605. .owner = THIS_MODULE,
  1606. };
  1607. static __net_init int kcm_init_net(struct net *net)
  1608. {
  1609. struct kcm_net *knet = net_generic(net, kcm_net_id);
  1610. INIT_LIST_HEAD_RCU(&knet->mux_list);
  1611. mutex_init(&knet->mutex);
  1612. return 0;
  1613. }
  1614. static __net_exit void kcm_exit_net(struct net *net)
  1615. {
  1616. struct kcm_net *knet = net_generic(net, kcm_net_id);
  1617. /* All KCM sockets should be closed at this point, which should mean
  1618. * that all multiplexors and psocks have been destroyed.
  1619. */
  1620. WARN_ON(!list_empty(&knet->mux_list));
  1621. }
  1622. static struct pernet_operations kcm_net_ops = {
  1623. .init = kcm_init_net,
  1624. .exit = kcm_exit_net,
  1625. .id = &kcm_net_id,
  1626. .size = sizeof(struct kcm_net),
  1627. };
  1628. static int __init kcm_init(void)
  1629. {
  1630. int err = -ENOMEM;
  1631. kcm_muxp = kmem_cache_create("kcm_mux_cache",
  1632. sizeof(struct kcm_mux), 0,
  1633. SLAB_HWCACHE_ALIGN, NULL);
  1634. if (!kcm_muxp)
  1635. goto fail;
  1636. kcm_psockp = kmem_cache_create("kcm_psock_cache",
  1637. sizeof(struct kcm_psock), 0,
  1638. SLAB_HWCACHE_ALIGN, NULL);
  1639. if (!kcm_psockp)
  1640. goto fail;
  1641. kcm_wq = create_singlethread_workqueue("kkcmd");
  1642. if (!kcm_wq)
  1643. goto fail;
  1644. err = proto_register(&kcm_proto, 1);
  1645. if (err)
  1646. goto fail;
  1647. err = register_pernet_device(&kcm_net_ops);
  1648. if (err)
  1649. goto net_ops_fail;
  1650. err = sock_register(&kcm_family_ops);
  1651. if (err)
  1652. goto sock_register_fail;
  1653. err = kcm_proc_init();
  1654. if (err)
  1655. goto proc_init_fail;
  1656. return 0;
  1657. proc_init_fail:
  1658. sock_unregister(PF_KCM);
  1659. sock_register_fail:
  1660. unregister_pernet_device(&kcm_net_ops);
  1661. net_ops_fail:
  1662. proto_unregister(&kcm_proto);
  1663. fail:
  1664. kmem_cache_destroy(kcm_muxp);
  1665. kmem_cache_destroy(kcm_psockp);
  1666. if (kcm_wq)
  1667. destroy_workqueue(kcm_wq);
  1668. return err;
  1669. }
  1670. static void __exit kcm_exit(void)
  1671. {
  1672. kcm_proc_exit();
  1673. sock_unregister(PF_KCM);
  1674. unregister_pernet_device(&kcm_net_ops);
  1675. proto_unregister(&kcm_proto);
  1676. destroy_workqueue(kcm_wq);
  1677. kmem_cache_destroy(kcm_muxp);
  1678. kmem_cache_destroy(kcm_psockp);
  1679. }
  1680. module_init(kcm_init);
  1681. module_exit(kcm_exit);
  1682. MODULE_LICENSE("GPL");
  1683. MODULE_ALIAS_NETPROTO(PF_KCM);