svc.c 24 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045
  1. /*
  2. * linux/net/sunrpc/svc.c
  3. *
  4. * High-level RPC service routines
  5. *
  6. * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
  7. *
  8. * Multiple threads pools and NUMAisation
  9. * Copyright (c) 2006 Silicon Graphics, Inc.
  10. * by Greg Banks <gnb@melbourne.sgi.com>
  11. */
  12. #include <linux/linkage.h>
  13. #include <linux/sched.h>
  14. #include <linux/errno.h>
  15. #include <linux/net.h>
  16. #include <linux/in.h>
  17. #include <linux/mm.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/module.h>
  20. #include <linux/sunrpc/types.h>
  21. #include <linux/sunrpc/xdr.h>
  22. #include <linux/sunrpc/stats.h>
  23. #include <linux/sunrpc/svcsock.h>
  24. #include <linux/sunrpc/clnt.h>
  25. #define RPCDBG_FACILITY RPCDBG_SVCDSP
  26. #define svc_serv_is_pooled(serv) ((serv)->sv_function)
  27. /*
  28. * Mode for mapping cpus to pools.
  29. */
  30. enum {
  31. SVC_POOL_AUTO = -1, /* choose one of the others */
  32. SVC_POOL_GLOBAL, /* no mapping, just a single global pool
  33. * (legacy & UP mode) */
  34. SVC_POOL_PERCPU, /* one pool per cpu */
  35. SVC_POOL_PERNODE /* one pool per numa node */
  36. };
  37. #define SVC_POOL_DEFAULT SVC_POOL_GLOBAL
  38. /*
  39. * Structure for mapping cpus to pools and vice versa.
  40. * Setup once during sunrpc initialisation.
  41. */
  42. static struct svc_pool_map {
  43. int count; /* How many svc_servs use us */
  44. int mode; /* Note: int not enum to avoid
  45. * warnings about "enumeration value
  46. * not handled in switch" */
  47. unsigned int npools;
  48. unsigned int *pool_to; /* maps pool id to cpu or node */
  49. unsigned int *to_pool; /* maps cpu or node to pool id */
  50. } svc_pool_map = {
  51. .count = 0,
  52. .mode = SVC_POOL_DEFAULT
  53. };
  54. static DEFINE_MUTEX(svc_pool_map_mutex);/* protects svc_pool_map.count only */
  55. static int
  56. param_set_pool_mode(const char *val, struct kernel_param *kp)
  57. {
  58. int *ip = (int *)kp->arg;
  59. struct svc_pool_map *m = &svc_pool_map;
  60. int err;
  61. mutex_lock(&svc_pool_map_mutex);
  62. err = -EBUSY;
  63. if (m->count)
  64. goto out;
  65. err = 0;
  66. if (!strncmp(val, "auto", 4))
  67. *ip = SVC_POOL_AUTO;
  68. else if (!strncmp(val, "global", 6))
  69. *ip = SVC_POOL_GLOBAL;
  70. else if (!strncmp(val, "percpu", 6))
  71. *ip = SVC_POOL_PERCPU;
  72. else if (!strncmp(val, "pernode", 7))
  73. *ip = SVC_POOL_PERNODE;
  74. else
  75. err = -EINVAL;
  76. out:
  77. mutex_unlock(&svc_pool_map_mutex);
  78. return err;
  79. }
  80. static int
  81. param_get_pool_mode(char *buf, struct kernel_param *kp)
  82. {
  83. int *ip = (int *)kp->arg;
  84. switch (*ip)
  85. {
  86. case SVC_POOL_AUTO:
  87. return strlcpy(buf, "auto", 20);
  88. case SVC_POOL_GLOBAL:
  89. return strlcpy(buf, "global", 20);
  90. case SVC_POOL_PERCPU:
  91. return strlcpy(buf, "percpu", 20);
  92. case SVC_POOL_PERNODE:
  93. return strlcpy(buf, "pernode", 20);
  94. default:
  95. return sprintf(buf, "%d", *ip);
  96. }
  97. }
  98. module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
  99. &svc_pool_map.mode, 0644);
  100. /*
  101. * Detect best pool mapping mode heuristically,
  102. * according to the machine's topology.
  103. */
  104. static int
  105. svc_pool_map_choose_mode(void)
  106. {
  107. unsigned int node;
  108. if (num_online_nodes() > 1) {
  109. /*
  110. * Actually have multiple NUMA nodes,
  111. * so split pools on NUMA node boundaries
  112. */
  113. return SVC_POOL_PERNODE;
  114. }
  115. node = any_online_node(node_online_map);
  116. if (nr_cpus_node(node) > 2) {
  117. /*
  118. * Non-trivial SMP, or CONFIG_NUMA on
  119. * non-NUMA hardware, e.g. with a generic
  120. * x86_64 kernel on Xeons. In this case we
  121. * want to divide the pools on cpu boundaries.
  122. */
  123. return SVC_POOL_PERCPU;
  124. }
  125. /* default: one global pool */
  126. return SVC_POOL_GLOBAL;
  127. }
  128. /*
  129. * Allocate the to_pool[] and pool_to[] arrays.
  130. * Returns 0 on success or an errno.
  131. */
  132. static int
  133. svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
  134. {
  135. m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  136. if (!m->to_pool)
  137. goto fail;
  138. m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
  139. if (!m->pool_to)
  140. goto fail_free;
  141. return 0;
  142. fail_free:
  143. kfree(m->to_pool);
  144. fail:
  145. return -ENOMEM;
  146. }
  147. /*
  148. * Initialise the pool map for SVC_POOL_PERCPU mode.
  149. * Returns number of pools or <0 on error.
  150. */
  151. static int
  152. svc_pool_map_init_percpu(struct svc_pool_map *m)
  153. {
  154. unsigned int maxpools = nr_cpu_ids;
  155. unsigned int pidx = 0;
  156. unsigned int cpu;
  157. int err;
  158. err = svc_pool_map_alloc_arrays(m, maxpools);
  159. if (err)
  160. return err;
  161. for_each_online_cpu(cpu) {
  162. BUG_ON(pidx > maxpools);
  163. m->to_pool[cpu] = pidx;
  164. m->pool_to[pidx] = cpu;
  165. pidx++;
  166. }
  167. /* cpus brought online later all get mapped to pool0, sorry */
  168. return pidx;
  169. };
  170. /*
  171. * Initialise the pool map for SVC_POOL_PERNODE mode.
  172. * Returns number of pools or <0 on error.
  173. */
  174. static int
  175. svc_pool_map_init_pernode(struct svc_pool_map *m)
  176. {
  177. unsigned int maxpools = nr_node_ids;
  178. unsigned int pidx = 0;
  179. unsigned int node;
  180. int err;
  181. err = svc_pool_map_alloc_arrays(m, maxpools);
  182. if (err)
  183. return err;
  184. for_each_node_with_cpus(node) {
  185. /* some architectures (e.g. SN2) have cpuless nodes */
  186. BUG_ON(pidx > maxpools);
  187. m->to_pool[node] = pidx;
  188. m->pool_to[pidx] = node;
  189. pidx++;
  190. }
  191. /* nodes brought online later all get mapped to pool0, sorry */
  192. return pidx;
  193. }
  194. /*
  195. * Add a reference to the global map of cpus to pools (and
  196. * vice versa). Initialise the map if we're the first user.
  197. * Returns the number of pools.
  198. */
  199. static unsigned int
  200. svc_pool_map_get(void)
  201. {
  202. struct svc_pool_map *m = &svc_pool_map;
  203. int npools = -1;
  204. mutex_lock(&svc_pool_map_mutex);
  205. if (m->count++) {
  206. mutex_unlock(&svc_pool_map_mutex);
  207. return m->npools;
  208. }
  209. if (m->mode == SVC_POOL_AUTO)
  210. m->mode = svc_pool_map_choose_mode();
  211. switch (m->mode) {
  212. case SVC_POOL_PERCPU:
  213. npools = svc_pool_map_init_percpu(m);
  214. break;
  215. case SVC_POOL_PERNODE:
  216. npools = svc_pool_map_init_pernode(m);
  217. break;
  218. }
  219. if (npools < 0) {
  220. /* default, or memory allocation failure */
  221. npools = 1;
  222. m->mode = SVC_POOL_GLOBAL;
  223. }
  224. m->npools = npools;
  225. mutex_unlock(&svc_pool_map_mutex);
  226. return m->npools;
  227. }
  228. /*
  229. * Drop a reference to the global map of cpus to pools.
  230. * When the last reference is dropped, the map data is
  231. * freed; this allows the sysadmin to change the pool
  232. * mode using the pool_mode module option without
  233. * rebooting or re-loading sunrpc.ko.
  234. */
  235. static void
  236. svc_pool_map_put(void)
  237. {
  238. struct svc_pool_map *m = &svc_pool_map;
  239. mutex_lock(&svc_pool_map_mutex);
  240. if (!--m->count) {
  241. m->mode = SVC_POOL_DEFAULT;
  242. kfree(m->to_pool);
  243. kfree(m->pool_to);
  244. m->npools = 0;
  245. }
  246. mutex_unlock(&svc_pool_map_mutex);
  247. }
  248. /*
  249. * Set the current thread's cpus_allowed mask so that it
  250. * will only run on cpus in the given pool.
  251. *
  252. * Returns 1 and fills in oldmask iff a cpumask was applied.
  253. */
  254. static inline int
  255. svc_pool_map_set_cpumask(unsigned int pidx, cpumask_t *oldmask)
  256. {
  257. struct svc_pool_map *m = &svc_pool_map;
  258. unsigned int node; /* or cpu */
  259. /*
  260. * The caller checks for sv_nrpools > 1, which
  261. * implies that we've been initialized.
  262. */
  263. BUG_ON(m->count == 0);
  264. switch (m->mode)
  265. {
  266. default:
  267. return 0;
  268. case SVC_POOL_PERCPU:
  269. node = m->pool_to[pidx];
  270. *oldmask = current->cpus_allowed;
  271. set_cpus_allowed(current, cpumask_of_cpu(node));
  272. return 1;
  273. case SVC_POOL_PERNODE:
  274. node = m->pool_to[pidx];
  275. *oldmask = current->cpus_allowed;
  276. set_cpus_allowed(current, node_to_cpumask(node));
  277. return 1;
  278. }
  279. }
  280. /*
  281. * Use the mapping mode to choose a pool for a given CPU.
  282. * Used when enqueueing an incoming RPC. Always returns
  283. * a non-NULL pool pointer.
  284. */
  285. struct svc_pool *
  286. svc_pool_for_cpu(struct svc_serv *serv, int cpu)
  287. {
  288. struct svc_pool_map *m = &svc_pool_map;
  289. unsigned int pidx = 0;
  290. /*
  291. * An uninitialised map happens in a pure client when
  292. * lockd is brought up, so silently treat it the
  293. * same as SVC_POOL_GLOBAL.
  294. */
  295. if (svc_serv_is_pooled(serv)) {
  296. switch (m->mode) {
  297. case SVC_POOL_PERCPU:
  298. pidx = m->to_pool[cpu];
  299. break;
  300. case SVC_POOL_PERNODE:
  301. pidx = m->to_pool[cpu_to_node(cpu)];
  302. break;
  303. }
  304. }
  305. return &serv->sv_pools[pidx % serv->sv_nrpools];
  306. }
  307. /*
  308. * Create an RPC service
  309. */
  310. static struct svc_serv *
  311. __svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
  312. void (*shutdown)(struct svc_serv *serv))
  313. {
  314. struct svc_serv *serv;
  315. int vers;
  316. unsigned int xdrsize;
  317. unsigned int i;
  318. if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
  319. return NULL;
  320. serv->sv_name = prog->pg_name;
  321. serv->sv_program = prog;
  322. serv->sv_nrthreads = 1;
  323. serv->sv_stats = prog->pg_stats;
  324. if (bufsize > RPCSVC_MAXPAYLOAD)
  325. bufsize = RPCSVC_MAXPAYLOAD;
  326. serv->sv_max_payload = bufsize? bufsize : 4096;
  327. serv->sv_max_mesg = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
  328. serv->sv_shutdown = shutdown;
  329. xdrsize = 0;
  330. while (prog) {
  331. prog->pg_lovers = prog->pg_nvers-1;
  332. for (vers=0; vers<prog->pg_nvers ; vers++)
  333. if (prog->pg_vers[vers]) {
  334. prog->pg_hivers = vers;
  335. if (prog->pg_lovers > vers)
  336. prog->pg_lovers = vers;
  337. if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
  338. xdrsize = prog->pg_vers[vers]->vs_xdrsize;
  339. }
  340. prog = prog->pg_next;
  341. }
  342. serv->sv_xdrsize = xdrsize;
  343. INIT_LIST_HEAD(&serv->sv_tempsocks);
  344. INIT_LIST_HEAD(&serv->sv_permsocks);
  345. init_timer(&serv->sv_temptimer);
  346. spin_lock_init(&serv->sv_lock);
  347. serv->sv_nrpools = npools;
  348. serv->sv_pools =
  349. kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
  350. GFP_KERNEL);
  351. if (!serv->sv_pools) {
  352. kfree(serv);
  353. return NULL;
  354. }
  355. for (i = 0; i < serv->sv_nrpools; i++) {
  356. struct svc_pool *pool = &serv->sv_pools[i];
  357. dprintk("svc: initialising pool %u for %s\n",
  358. i, serv->sv_name);
  359. pool->sp_id = i;
  360. INIT_LIST_HEAD(&pool->sp_threads);
  361. INIT_LIST_HEAD(&pool->sp_sockets);
  362. INIT_LIST_HEAD(&pool->sp_all_threads);
  363. spin_lock_init(&pool->sp_lock);
  364. }
  365. /* Remove any stale portmap registrations */
  366. svc_register(serv, 0, 0);
  367. return serv;
  368. }
  369. struct svc_serv *
  370. svc_create(struct svc_program *prog, unsigned int bufsize,
  371. void (*shutdown)(struct svc_serv *serv))
  372. {
  373. return __svc_create(prog, bufsize, /*npools*/1, shutdown);
  374. }
  375. struct svc_serv *
  376. svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
  377. void (*shutdown)(struct svc_serv *serv),
  378. svc_thread_fn func, int sig, struct module *mod)
  379. {
  380. struct svc_serv *serv;
  381. unsigned int npools = svc_pool_map_get();
  382. serv = __svc_create(prog, bufsize, npools, shutdown);
  383. if (serv != NULL) {
  384. serv->sv_function = func;
  385. serv->sv_kill_signal = sig;
  386. serv->sv_module = mod;
  387. }
  388. return serv;
  389. }
  390. /*
  391. * Destroy an RPC service. Should be called with the BKL held
  392. */
  393. void
  394. svc_destroy(struct svc_serv *serv)
  395. {
  396. struct svc_sock *svsk;
  397. struct svc_sock *tmp;
  398. dprintk("svc: svc_destroy(%s, %d)\n",
  399. serv->sv_program->pg_name,
  400. serv->sv_nrthreads);
  401. if (serv->sv_nrthreads) {
  402. if (--(serv->sv_nrthreads) != 0) {
  403. svc_sock_update_bufs(serv);
  404. return;
  405. }
  406. } else
  407. printk("svc_destroy: no threads for serv=%p!\n", serv);
  408. del_timer_sync(&serv->sv_temptimer);
  409. list_for_each_entry_safe(svsk, tmp, &serv->sv_tempsocks, sk_list)
  410. svc_force_close_socket(svsk);
  411. if (serv->sv_shutdown)
  412. serv->sv_shutdown(serv);
  413. list_for_each_entry_safe(svsk, tmp, &serv->sv_permsocks, sk_list)
  414. svc_force_close_socket(svsk);
  415. BUG_ON(!list_empty(&serv->sv_permsocks));
  416. BUG_ON(!list_empty(&serv->sv_tempsocks));
  417. cache_clean_deferred(serv);
  418. if (svc_serv_is_pooled(serv))
  419. svc_pool_map_put();
  420. /* Unregister service with the portmapper */
  421. svc_register(serv, 0, 0);
  422. kfree(serv->sv_pools);
  423. kfree(serv);
  424. }
  425. /*
  426. * Allocate an RPC server's buffer space.
  427. * We allocate pages and place them in rq_argpages.
  428. */
  429. static int
  430. svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
  431. {
  432. int pages;
  433. int arghi;
  434. pages = size / PAGE_SIZE + 1; /* extra page as we hold both request and reply.
  435. * We assume one is at most one page
  436. */
  437. arghi = 0;
  438. BUG_ON(pages > RPCSVC_MAXPAGES);
  439. while (pages) {
  440. struct page *p = alloc_page(GFP_KERNEL);
  441. if (!p)
  442. break;
  443. rqstp->rq_pages[arghi++] = p;
  444. pages--;
  445. }
  446. return ! pages;
  447. }
  448. /*
  449. * Release an RPC server buffer
  450. */
  451. static void
  452. svc_release_buffer(struct svc_rqst *rqstp)
  453. {
  454. int i;
  455. for (i=0; i<ARRAY_SIZE(rqstp->rq_pages); i++)
  456. if (rqstp->rq_pages[i])
  457. put_page(rqstp->rq_pages[i]);
  458. }
  459. /*
  460. * Create a thread in the given pool. Caller must hold BKL.
  461. * On a NUMA or SMP machine, with a multi-pool serv, the thread
  462. * will be restricted to run on the cpus belonging to the pool.
  463. */
  464. static int
  465. __svc_create_thread(svc_thread_fn func, struct svc_serv *serv,
  466. struct svc_pool *pool)
  467. {
  468. struct svc_rqst *rqstp;
  469. int error = -ENOMEM;
  470. int have_oldmask = 0;
  471. cpumask_t oldmask;
  472. rqstp = kzalloc(sizeof(*rqstp), GFP_KERNEL);
  473. if (!rqstp)
  474. goto out;
  475. init_waitqueue_head(&rqstp->rq_wait);
  476. if (!(rqstp->rq_argp = kmalloc(serv->sv_xdrsize, GFP_KERNEL))
  477. || !(rqstp->rq_resp = kmalloc(serv->sv_xdrsize, GFP_KERNEL))
  478. || !svc_init_buffer(rqstp, serv->sv_max_mesg))
  479. goto out_thread;
  480. serv->sv_nrthreads++;
  481. spin_lock_bh(&pool->sp_lock);
  482. pool->sp_nrthreads++;
  483. list_add(&rqstp->rq_all, &pool->sp_all_threads);
  484. spin_unlock_bh(&pool->sp_lock);
  485. rqstp->rq_server = serv;
  486. rqstp->rq_pool = pool;
  487. if (serv->sv_nrpools > 1)
  488. have_oldmask = svc_pool_map_set_cpumask(pool->sp_id, &oldmask);
  489. error = kernel_thread((int (*)(void *)) func, rqstp, 0);
  490. if (have_oldmask)
  491. set_cpus_allowed(current, oldmask);
  492. if (error < 0)
  493. goto out_thread;
  494. svc_sock_update_bufs(serv);
  495. error = 0;
  496. out:
  497. return error;
  498. out_thread:
  499. svc_exit_thread(rqstp);
  500. goto out;
  501. }
  502. /*
  503. * Create a thread in the default pool. Caller must hold BKL.
  504. */
  505. int
  506. svc_create_thread(svc_thread_fn func, struct svc_serv *serv)
  507. {
  508. return __svc_create_thread(func, serv, &serv->sv_pools[0]);
  509. }
  510. /*
  511. * Choose a pool in which to create a new thread, for svc_set_num_threads
  512. */
  513. static inline struct svc_pool *
  514. choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  515. {
  516. if (pool != NULL)
  517. return pool;
  518. return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
  519. }
  520. /*
  521. * Choose a thread to kill, for svc_set_num_threads
  522. */
  523. static inline struct task_struct *
  524. choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
  525. {
  526. unsigned int i;
  527. struct task_struct *task = NULL;
  528. if (pool != NULL) {
  529. spin_lock_bh(&pool->sp_lock);
  530. } else {
  531. /* choose a pool in round-robin fashion */
  532. for (i = 0; i < serv->sv_nrpools; i++) {
  533. pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
  534. spin_lock_bh(&pool->sp_lock);
  535. if (!list_empty(&pool->sp_all_threads))
  536. goto found_pool;
  537. spin_unlock_bh(&pool->sp_lock);
  538. }
  539. return NULL;
  540. }
  541. found_pool:
  542. if (!list_empty(&pool->sp_all_threads)) {
  543. struct svc_rqst *rqstp;
  544. /*
  545. * Remove from the pool->sp_all_threads list
  546. * so we don't try to kill it again.
  547. */
  548. rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
  549. list_del_init(&rqstp->rq_all);
  550. task = rqstp->rq_task;
  551. }
  552. spin_unlock_bh(&pool->sp_lock);
  553. return task;
  554. }
  555. /*
  556. * Create or destroy enough new threads to make the number
  557. * of threads the given number. If `pool' is non-NULL, applies
  558. * only to threads in that pool, otherwise round-robins between
  559. * all pools. Must be called with a svc_get() reference and
  560. * the BKL held.
  561. *
  562. * Destroying threads relies on the service threads filling in
  563. * rqstp->rq_task, which only the nfs ones do. Assumes the serv
  564. * has been created using svc_create_pooled().
  565. *
  566. * Based on code that used to be in nfsd_svc() but tweaked
  567. * to be pool-aware.
  568. */
  569. int
  570. svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  571. {
  572. struct task_struct *victim;
  573. int error = 0;
  574. unsigned int state = serv->sv_nrthreads-1;
  575. if (pool == NULL) {
  576. /* The -1 assumes caller has done a svc_get() */
  577. nrservs -= (serv->sv_nrthreads-1);
  578. } else {
  579. spin_lock_bh(&pool->sp_lock);
  580. nrservs -= pool->sp_nrthreads;
  581. spin_unlock_bh(&pool->sp_lock);
  582. }
  583. /* create new threads */
  584. while (nrservs > 0) {
  585. nrservs--;
  586. __module_get(serv->sv_module);
  587. error = __svc_create_thread(serv->sv_function, serv,
  588. choose_pool(serv, pool, &state));
  589. if (error < 0) {
  590. module_put(serv->sv_module);
  591. break;
  592. }
  593. }
  594. /* destroy old threads */
  595. while (nrservs < 0 &&
  596. (victim = choose_victim(serv, pool, &state)) != NULL) {
  597. send_sig(serv->sv_kill_signal, victim, 1);
  598. nrservs++;
  599. }
  600. return error;
  601. }
  602. /*
  603. * Called from a server thread as it's exiting. Caller must hold BKL.
  604. */
  605. void
  606. svc_exit_thread(struct svc_rqst *rqstp)
  607. {
  608. struct svc_serv *serv = rqstp->rq_server;
  609. struct svc_pool *pool = rqstp->rq_pool;
  610. svc_release_buffer(rqstp);
  611. kfree(rqstp->rq_resp);
  612. kfree(rqstp->rq_argp);
  613. kfree(rqstp->rq_auth_data);
  614. spin_lock_bh(&pool->sp_lock);
  615. pool->sp_nrthreads--;
  616. list_del(&rqstp->rq_all);
  617. spin_unlock_bh(&pool->sp_lock);
  618. kfree(rqstp);
  619. /* Release the server */
  620. if (serv)
  621. svc_destroy(serv);
  622. }
  623. /*
  624. * Register an RPC service with the local portmapper.
  625. * To unregister a service, call this routine with
  626. * proto and port == 0.
  627. */
  628. int
  629. svc_register(struct svc_serv *serv, int proto, unsigned short port)
  630. {
  631. struct svc_program *progp;
  632. unsigned long flags;
  633. int i, error = 0, dummy;
  634. if (!port)
  635. clear_thread_flag(TIF_SIGPENDING);
  636. for (progp = serv->sv_program; progp; progp = progp->pg_next) {
  637. for (i = 0; i < progp->pg_nvers; i++) {
  638. if (progp->pg_vers[i] == NULL)
  639. continue;
  640. dprintk("svc: svc_register(%s, %s, %d, %d)%s\n",
  641. progp->pg_name,
  642. proto == IPPROTO_UDP? "udp" : "tcp",
  643. port,
  644. i,
  645. progp->pg_vers[i]->vs_hidden?
  646. " (but not telling portmap)" : "");
  647. if (progp->pg_vers[i]->vs_hidden)
  648. continue;
  649. error = rpc_register(progp->pg_prog, i, proto, port, &dummy);
  650. if (error < 0)
  651. break;
  652. if (port && !dummy) {
  653. error = -EACCES;
  654. break;
  655. }
  656. }
  657. }
  658. if (!port) {
  659. spin_lock_irqsave(&current->sighand->siglock, flags);
  660. recalc_sigpending();
  661. spin_unlock_irqrestore(&current->sighand->siglock, flags);
  662. }
  663. return error;
  664. }
  665. /*
  666. * Process the RPC request.
  667. */
  668. int
  669. svc_process(struct svc_rqst *rqstp)
  670. {
  671. struct svc_program *progp;
  672. struct svc_version *versp = NULL; /* compiler food */
  673. struct svc_procedure *procp = NULL;
  674. struct kvec * argv = &rqstp->rq_arg.head[0];
  675. struct kvec * resv = &rqstp->rq_res.head[0];
  676. struct svc_serv *serv = rqstp->rq_server;
  677. kxdrproc_t xdr;
  678. __be32 *statp;
  679. u32 dir, prog, vers, proc;
  680. __be32 auth_stat, rpc_stat;
  681. int auth_res;
  682. __be32 *reply_statp;
  683. rpc_stat = rpc_success;
  684. if (argv->iov_len < 6*4)
  685. goto err_short_len;
  686. /* setup response xdr_buf.
  687. * Initially it has just one page
  688. */
  689. rqstp->rq_resused = 1;
  690. resv->iov_base = page_address(rqstp->rq_respages[0]);
  691. resv->iov_len = 0;
  692. rqstp->rq_res.pages = rqstp->rq_respages + 1;
  693. rqstp->rq_res.len = 0;
  694. rqstp->rq_res.page_base = 0;
  695. rqstp->rq_res.page_len = 0;
  696. rqstp->rq_res.buflen = PAGE_SIZE;
  697. rqstp->rq_res.tail[0].iov_base = NULL;
  698. rqstp->rq_res.tail[0].iov_len = 0;
  699. /* Will be turned off only in gss privacy case: */
  700. rqstp->rq_sendfile_ok = 1;
  701. /* tcp needs a space for the record length... */
  702. if (rqstp->rq_prot == IPPROTO_TCP)
  703. svc_putnl(resv, 0);
  704. rqstp->rq_xid = svc_getu32(argv);
  705. svc_putu32(resv, rqstp->rq_xid);
  706. dir = svc_getnl(argv);
  707. vers = svc_getnl(argv);
  708. /* First words of reply: */
  709. svc_putnl(resv, 1); /* REPLY */
  710. if (dir != 0) /* direction != CALL */
  711. goto err_bad_dir;
  712. if (vers != 2) /* RPC version number */
  713. goto err_bad_rpc;
  714. /* Save position in case we later decide to reject: */
  715. reply_statp = resv->iov_base + resv->iov_len;
  716. svc_putnl(resv, 0); /* ACCEPT */
  717. rqstp->rq_prog = prog = svc_getnl(argv); /* program number */
  718. rqstp->rq_vers = vers = svc_getnl(argv); /* version number */
  719. rqstp->rq_proc = proc = svc_getnl(argv); /* procedure number */
  720. progp = serv->sv_program;
  721. for (progp = serv->sv_program; progp; progp = progp->pg_next)
  722. if (prog == progp->pg_prog)
  723. break;
  724. /*
  725. * Decode auth data, and add verifier to reply buffer.
  726. * We do this before anything else in order to get a decent
  727. * auth verifier.
  728. */
  729. auth_res = svc_authenticate(rqstp, &auth_stat);
  730. /* Also give the program a chance to reject this call: */
  731. if (auth_res == SVC_OK && progp) {
  732. auth_stat = rpc_autherr_badcred;
  733. auth_res = progp->pg_authenticate(rqstp);
  734. }
  735. switch (auth_res) {
  736. case SVC_OK:
  737. break;
  738. case SVC_GARBAGE:
  739. rpc_stat = rpc_garbage_args;
  740. goto err_bad;
  741. case SVC_SYSERR:
  742. rpc_stat = rpc_system_err;
  743. goto err_bad;
  744. case SVC_DENIED:
  745. goto err_bad_auth;
  746. case SVC_DROP:
  747. goto dropit;
  748. case SVC_COMPLETE:
  749. goto sendit;
  750. }
  751. if (progp == NULL)
  752. goto err_bad_prog;
  753. if (vers >= progp->pg_nvers ||
  754. !(versp = progp->pg_vers[vers]))
  755. goto err_bad_vers;
  756. procp = versp->vs_proc + proc;
  757. if (proc >= versp->vs_nproc || !procp->pc_func)
  758. goto err_bad_proc;
  759. rqstp->rq_server = serv;
  760. rqstp->rq_procinfo = procp;
  761. /* Syntactic check complete */
  762. serv->sv_stats->rpccnt++;
  763. /* Build the reply header. */
  764. statp = resv->iov_base +resv->iov_len;
  765. svc_putnl(resv, RPC_SUCCESS);
  766. /* Bump per-procedure stats counter */
  767. procp->pc_count++;
  768. /* Initialize storage for argp and resp */
  769. memset(rqstp->rq_argp, 0, procp->pc_argsize);
  770. memset(rqstp->rq_resp, 0, procp->pc_ressize);
  771. /* un-reserve some of the out-queue now that we have a
  772. * better idea of reply size
  773. */
  774. if (procp->pc_xdrressize)
  775. svc_reserve(rqstp, procp->pc_xdrressize<<2);
  776. /* Call the function that processes the request. */
  777. if (!versp->vs_dispatch) {
  778. /* Decode arguments */
  779. xdr = procp->pc_decode;
  780. if (xdr && !xdr(rqstp, argv->iov_base, rqstp->rq_argp))
  781. goto err_garbage;
  782. *statp = procp->pc_func(rqstp, rqstp->rq_argp, rqstp->rq_resp);
  783. /* Encode reply */
  784. if (*statp == rpc_drop_reply) {
  785. if (procp->pc_release)
  786. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  787. goto dropit;
  788. }
  789. if (*statp == rpc_success && (xdr = procp->pc_encode)
  790. && !xdr(rqstp, resv->iov_base+resv->iov_len, rqstp->rq_resp)) {
  791. dprintk("svc: failed to encode reply\n");
  792. /* serv->sv_stats->rpcsystemerr++; */
  793. *statp = rpc_system_err;
  794. }
  795. } else {
  796. dprintk("svc: calling dispatcher\n");
  797. if (!versp->vs_dispatch(rqstp, statp)) {
  798. /* Release reply info */
  799. if (procp->pc_release)
  800. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  801. goto dropit;
  802. }
  803. }
  804. /* Check RPC status result */
  805. if (*statp != rpc_success)
  806. resv->iov_len = ((void*)statp) - resv->iov_base + 4;
  807. /* Release reply info */
  808. if (procp->pc_release)
  809. procp->pc_release(rqstp, NULL, rqstp->rq_resp);
  810. if (procp->pc_encode == NULL)
  811. goto dropit;
  812. sendit:
  813. if (svc_authorise(rqstp))
  814. goto dropit;
  815. return svc_send(rqstp);
  816. dropit:
  817. svc_authorise(rqstp); /* doesn't hurt to call this twice */
  818. dprintk("svc: svc_process dropit\n");
  819. svc_drop(rqstp);
  820. return 0;
  821. err_short_len:
  822. if (net_ratelimit())
  823. printk("svc: short len %Zd, dropping request\n", argv->iov_len);
  824. goto dropit; /* drop request */
  825. err_bad_dir:
  826. if (net_ratelimit())
  827. printk("svc: bad direction %d, dropping request\n", dir);
  828. serv->sv_stats->rpcbadfmt++;
  829. goto dropit; /* drop request */
  830. err_bad_rpc:
  831. serv->sv_stats->rpcbadfmt++;
  832. svc_putnl(resv, 1); /* REJECT */
  833. svc_putnl(resv, 0); /* RPC_MISMATCH */
  834. svc_putnl(resv, 2); /* Only RPCv2 supported */
  835. svc_putnl(resv, 2);
  836. goto sendit;
  837. err_bad_auth:
  838. dprintk("svc: authentication failed (%d)\n", ntohl(auth_stat));
  839. serv->sv_stats->rpcbadauth++;
  840. /* Restore write pointer to location of accept status: */
  841. xdr_ressize_check(rqstp, reply_statp);
  842. svc_putnl(resv, 1); /* REJECT */
  843. svc_putnl(resv, 1); /* AUTH_ERROR */
  844. svc_putnl(resv, ntohl(auth_stat)); /* status */
  845. goto sendit;
  846. err_bad_prog:
  847. dprintk("svc: unknown program %d\n", prog);
  848. serv->sv_stats->rpcbadfmt++;
  849. svc_putnl(resv, RPC_PROG_UNAVAIL);
  850. goto sendit;
  851. err_bad_vers:
  852. if (net_ratelimit())
  853. printk("svc: unknown version (%d for prog %d, %s)\n",
  854. vers, prog, progp->pg_name);
  855. serv->sv_stats->rpcbadfmt++;
  856. svc_putnl(resv, RPC_PROG_MISMATCH);
  857. svc_putnl(resv, progp->pg_lovers);
  858. svc_putnl(resv, progp->pg_hivers);
  859. goto sendit;
  860. err_bad_proc:
  861. if (net_ratelimit())
  862. printk("svc: unknown procedure (%d)\n", proc);
  863. serv->sv_stats->rpcbadfmt++;
  864. svc_putnl(resv, RPC_PROC_UNAVAIL);
  865. goto sendit;
  866. err_garbage:
  867. if (net_ratelimit())
  868. printk("svc: failed to decode args\n");
  869. rpc_stat = rpc_garbage_args;
  870. err_bad:
  871. serv->sv_stats->rpcbadfmt++;
  872. svc_putnl(resv, ntohl(rpc_stat));
  873. goto sendit;
  874. }
  875. /*
  876. * Return (transport-specific) limit on the rpc payload.
  877. */
  878. u32 svc_max_payload(const struct svc_rqst *rqstp)
  879. {
  880. int max = RPCSVC_MAXPAYLOAD_TCP;
  881. if (rqstp->rq_sock->sk_sock->type == SOCK_DGRAM)
  882. max = RPCSVC_MAXPAYLOAD_UDP;
  883. if (rqstp->rq_server->sv_max_payload < max)
  884. max = rqstp->rq_server->sv_max_payload;
  885. return max;
  886. }
  887. EXPORT_SYMBOL_GPL(svc_max_payload);