rdma.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * RDMA resource limiting controller for cgroups.
  4. *
  5. * Used to allow a cgroup hierarchy to stop processes from consuming
  6. * additional RDMA resources after a certain limit is reached.
  7. *
  8. * Copyright (C) 2016 Parav Pandit <pandit.parav@gmail.com>
  9. */
  10. #include <linux/bitops.h>
  11. #include <linux/slab.h>
  12. #include <linux/seq_file.h>
  13. #include <linux/cgroup.h>
  14. #include <linux/parser.h>
  15. #include <linux/cgroup_rdma.h>
  16. #define RDMACG_MAX_STR "max"
  17. /*
  18. * Protects list of resource pools maintained on per cgroup basis
  19. * and rdma device list.
  20. */
  21. static DEFINE_MUTEX(rdmacg_mutex);
  22. static LIST_HEAD(rdmacg_devices);
  23. enum rdmacg_file_type {
  24. RDMACG_RESOURCE_TYPE_MAX,
  25. RDMACG_RESOURCE_TYPE_STAT,
  26. };
  27. /*
  28. * resource table definition as to be seen by the user.
  29. * Need to add entries to it when more resources are
  30. * added/defined at IB verb/core layer.
  31. */
  32. static char const *rdmacg_resource_names[] = {
  33. [RDMACG_RESOURCE_HCA_HANDLE] = "hca_handle",
  34. [RDMACG_RESOURCE_HCA_OBJECT] = "hca_object",
  35. };
  36. /* resource tracker for each resource of rdma cgroup */
  37. struct rdmacg_resource {
  38. int max;
  39. int usage;
  40. };
  41. /*
  42. * resource pool object which represents per cgroup, per device
  43. * resources. There are multiple instances of this object per cgroup,
  44. * therefore it cannot be embedded within rdma_cgroup structure. It
  45. * is maintained as list.
  46. */
  47. struct rdmacg_resource_pool {
  48. struct rdmacg_device *device;
  49. struct rdmacg_resource resources[RDMACG_RESOURCE_MAX];
  50. struct list_head cg_node;
  51. struct list_head dev_node;
  52. /* count active user tasks of this pool */
  53. u64 usage_sum;
  54. /* total number counts which are set to max */
  55. int num_max_cnt;
  56. };
  57. static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css)
  58. {
  59. return container_of(css, struct rdma_cgroup, css);
  60. }
  61. static struct rdma_cgroup *parent_rdmacg(struct rdma_cgroup *cg)
  62. {
  63. return css_rdmacg(cg->css.parent);
  64. }
  65. static inline struct rdma_cgroup *get_current_rdmacg(void)
  66. {
  67. return css_rdmacg(task_get_css(current, rdma_cgrp_id));
  68. }
  69. static void set_resource_limit(struct rdmacg_resource_pool *rpool,
  70. int index, int new_max)
  71. {
  72. if (new_max == S32_MAX) {
  73. if (rpool->resources[index].max != S32_MAX)
  74. rpool->num_max_cnt++;
  75. } else {
  76. if (rpool->resources[index].max == S32_MAX)
  77. rpool->num_max_cnt--;
  78. }
  79. rpool->resources[index].max = new_max;
  80. }
  81. static void set_all_resource_max_limit(struct rdmacg_resource_pool *rpool)
  82. {
  83. int i;
  84. for (i = 0; i < RDMACG_RESOURCE_MAX; i++)
  85. set_resource_limit(rpool, i, S32_MAX);
  86. }
  87. static void free_cg_rpool_locked(struct rdmacg_resource_pool *rpool)
  88. {
  89. lockdep_assert_held(&rdmacg_mutex);
  90. list_del(&rpool->cg_node);
  91. list_del(&rpool->dev_node);
  92. kfree(rpool);
  93. }
  94. static struct rdmacg_resource_pool *
  95. find_cg_rpool_locked(struct rdma_cgroup *cg,
  96. struct rdmacg_device *device)
  97. {
  98. struct rdmacg_resource_pool *pool;
  99. lockdep_assert_held(&rdmacg_mutex);
  100. list_for_each_entry(pool, &cg->rpools, cg_node)
  101. if (pool->device == device)
  102. return pool;
  103. return NULL;
  104. }
  105. static struct rdmacg_resource_pool *
  106. get_cg_rpool_locked(struct rdma_cgroup *cg, struct rdmacg_device *device)
  107. {
  108. struct rdmacg_resource_pool *rpool;
  109. rpool = find_cg_rpool_locked(cg, device);
  110. if (rpool)
  111. return rpool;
  112. rpool = kzalloc(sizeof(*rpool), GFP_KERNEL);
  113. if (!rpool)
  114. return ERR_PTR(-ENOMEM);
  115. rpool->device = device;
  116. set_all_resource_max_limit(rpool);
  117. INIT_LIST_HEAD(&rpool->cg_node);
  118. INIT_LIST_HEAD(&rpool->dev_node);
  119. list_add_tail(&rpool->cg_node, &cg->rpools);
  120. list_add_tail(&rpool->dev_node, &device->rpools);
  121. return rpool;
  122. }
  123. /**
  124. * uncharge_cg_locked - uncharge resource for rdma cgroup
  125. * @cg: pointer to cg to uncharge and all parents in hierarchy
  126. * @device: pointer to rdmacg device
  127. * @index: index of the resource to uncharge in cg (resource pool)
  128. *
  129. * It also frees the resource pool which was created as part of
  130. * charging operation when there are no resources attached to
  131. * resource pool.
  132. */
  133. static void
  134. uncharge_cg_locked(struct rdma_cgroup *cg,
  135. struct rdmacg_device *device,
  136. enum rdmacg_resource_type index)
  137. {
  138. struct rdmacg_resource_pool *rpool;
  139. rpool = find_cg_rpool_locked(cg, device);
  140. /*
  141. * rpool cannot be null at this stage. Let kernel operate in case
  142. * if there a bug in IB stack or rdma controller, instead of crashing
  143. * the system.
  144. */
  145. if (unlikely(!rpool)) {
  146. pr_warn("Invalid device %p or rdma cgroup %p\n", cg, device);
  147. return;
  148. }
  149. rpool->resources[index].usage--;
  150. /*
  151. * A negative count (or overflow) is invalid,
  152. * it indicates a bug in the rdma controller.
  153. */
  154. WARN_ON_ONCE(rpool->resources[index].usage < 0);
  155. rpool->usage_sum--;
  156. if (rpool->usage_sum == 0 &&
  157. rpool->num_max_cnt == RDMACG_RESOURCE_MAX) {
  158. /*
  159. * No user of the rpool and all entries are set to max, so
  160. * safe to delete this rpool.
  161. */
  162. free_cg_rpool_locked(rpool);
  163. }
  164. }
  165. /**
  166. * rdmacg_uncharge_hierarchy - hierarchically uncharge rdma resource count
  167. * @device: pointer to rdmacg device
  168. * @stop_cg: while traversing hirerchy, when meet with stop_cg cgroup
  169. * stop uncharging
  170. * @index: index of the resource to uncharge in cg in given resource pool
  171. */
  172. static void rdmacg_uncharge_hierarchy(struct rdma_cgroup *cg,
  173. struct rdmacg_device *device,
  174. struct rdma_cgroup *stop_cg,
  175. enum rdmacg_resource_type index)
  176. {
  177. struct rdma_cgroup *p;
  178. mutex_lock(&rdmacg_mutex);
  179. for (p = cg; p != stop_cg; p = parent_rdmacg(p))
  180. uncharge_cg_locked(p, device, index);
  181. mutex_unlock(&rdmacg_mutex);
  182. css_put(&cg->css);
  183. }
  184. /**
  185. * rdmacg_uncharge - hierarchically uncharge rdma resource count
  186. * @device: pointer to rdmacg device
  187. * @index: index of the resource to uncharge in cgroup in given resource pool
  188. */
  189. void rdmacg_uncharge(struct rdma_cgroup *cg,
  190. struct rdmacg_device *device,
  191. enum rdmacg_resource_type index)
  192. {
  193. if (index >= RDMACG_RESOURCE_MAX)
  194. return;
  195. rdmacg_uncharge_hierarchy(cg, device, NULL, index);
  196. }
  197. EXPORT_SYMBOL(rdmacg_uncharge);
  198. /**
  199. * rdmacg_try_charge - hierarchically try to charge the rdma resource
  200. * @rdmacg: pointer to rdma cgroup which will own this resource
  201. * @device: pointer to rdmacg device
  202. * @index: index of the resource to charge in cgroup (resource pool)
  203. *
  204. * This function follows charging resource in hierarchical way.
  205. * It will fail if the charge would cause the new value to exceed the
  206. * hierarchical limit.
  207. * Returns 0 if the charge succeded, otherwise -EAGAIN, -ENOMEM or -EINVAL.
  208. * Returns pointer to rdmacg for this resource when charging is successful.
  209. *
  210. * Charger needs to account resources on two criteria.
  211. * (a) per cgroup & (b) per device resource usage.
  212. * Per cgroup resource usage ensures that tasks of cgroup doesn't cross
  213. * the configured limits. Per device provides granular configuration
  214. * in multi device usage. It allocates resource pool in the hierarchy
  215. * for each parent it come across for first resource. Later on resource
  216. * pool will be available. Therefore it will be much faster thereon
  217. * to charge/uncharge.
  218. */
  219. int rdmacg_try_charge(struct rdma_cgroup **rdmacg,
  220. struct rdmacg_device *device,
  221. enum rdmacg_resource_type index)
  222. {
  223. struct rdma_cgroup *cg, *p;
  224. struct rdmacg_resource_pool *rpool;
  225. s64 new;
  226. int ret = 0;
  227. if (index >= RDMACG_RESOURCE_MAX)
  228. return -EINVAL;
  229. /*
  230. * hold on to css, as cgroup can be removed but resource
  231. * accounting happens on css.
  232. */
  233. cg = get_current_rdmacg();
  234. mutex_lock(&rdmacg_mutex);
  235. for (p = cg; p; p = parent_rdmacg(p)) {
  236. rpool = get_cg_rpool_locked(p, device);
  237. if (IS_ERR(rpool)) {
  238. ret = PTR_ERR(rpool);
  239. goto err;
  240. } else {
  241. new = rpool->resources[index].usage + 1;
  242. if (new > rpool->resources[index].max) {
  243. ret = -EAGAIN;
  244. goto err;
  245. } else {
  246. rpool->resources[index].usage = new;
  247. rpool->usage_sum++;
  248. }
  249. }
  250. }
  251. mutex_unlock(&rdmacg_mutex);
  252. *rdmacg = cg;
  253. return 0;
  254. err:
  255. mutex_unlock(&rdmacg_mutex);
  256. rdmacg_uncharge_hierarchy(cg, device, p, index);
  257. return ret;
  258. }
  259. EXPORT_SYMBOL(rdmacg_try_charge);
  260. /**
  261. * rdmacg_register_device - register rdmacg device to rdma controller.
  262. * @device: pointer to rdmacg device whose resources need to be accounted.
  263. *
  264. * If IB stack wish a device to participate in rdma cgroup resource
  265. * tracking, it must invoke this API to register with rdma cgroup before
  266. * any user space application can start using the RDMA resources.
  267. */
  268. void rdmacg_register_device(struct rdmacg_device *device)
  269. {
  270. INIT_LIST_HEAD(&device->dev_node);
  271. INIT_LIST_HEAD(&device->rpools);
  272. mutex_lock(&rdmacg_mutex);
  273. list_add_tail(&device->dev_node, &rdmacg_devices);
  274. mutex_unlock(&rdmacg_mutex);
  275. }
  276. EXPORT_SYMBOL(rdmacg_register_device);
  277. /**
  278. * rdmacg_unregister_device - unregister rdmacg device from rdma controller.
  279. * @device: pointer to rdmacg device which was previously registered with rdma
  280. * controller using rdmacg_register_device().
  281. *
  282. * IB stack must invoke this after all the resources of the IB device
  283. * are destroyed and after ensuring that no more resources will be created
  284. * when this API is invoked.
  285. */
  286. void rdmacg_unregister_device(struct rdmacg_device *device)
  287. {
  288. struct rdmacg_resource_pool *rpool, *tmp;
  289. /*
  290. * Synchronize with any active resource settings,
  291. * usage query happening via configfs.
  292. */
  293. mutex_lock(&rdmacg_mutex);
  294. list_del_init(&device->dev_node);
  295. /*
  296. * Now that this device is off the cgroup list, its safe to free
  297. * all the rpool resources.
  298. */
  299. list_for_each_entry_safe(rpool, tmp, &device->rpools, dev_node)
  300. free_cg_rpool_locked(rpool);
  301. mutex_unlock(&rdmacg_mutex);
  302. }
  303. EXPORT_SYMBOL(rdmacg_unregister_device);
  304. static int parse_resource(char *c, int *intval)
  305. {
  306. substring_t argstr;
  307. char *name, *value = c;
  308. size_t len;
  309. int ret, i;
  310. name = strsep(&value, "=");
  311. if (!name || !value)
  312. return -EINVAL;
  313. i = match_string(rdmacg_resource_names, RDMACG_RESOURCE_MAX, name);
  314. if (i < 0)
  315. return i;
  316. len = strlen(value);
  317. argstr.from = value;
  318. argstr.to = value + len;
  319. ret = match_int(&argstr, intval);
  320. if (ret >= 0) {
  321. if (*intval < 0)
  322. return -EINVAL;
  323. return i;
  324. }
  325. if (strncmp(value, RDMACG_MAX_STR, len) == 0) {
  326. *intval = S32_MAX;
  327. return i;
  328. }
  329. return -EINVAL;
  330. }
  331. static int rdmacg_parse_limits(char *options,
  332. int *new_limits, unsigned long *enables)
  333. {
  334. char *c;
  335. int err = -EINVAL;
  336. /* parse resource options */
  337. while ((c = strsep(&options, " ")) != NULL) {
  338. int index, intval;
  339. index = parse_resource(c, &intval);
  340. if (index < 0)
  341. goto err;
  342. new_limits[index] = intval;
  343. *enables |= BIT(index);
  344. }
  345. return 0;
  346. err:
  347. return err;
  348. }
  349. static struct rdmacg_device *rdmacg_get_device_locked(const char *name)
  350. {
  351. struct rdmacg_device *device;
  352. lockdep_assert_held(&rdmacg_mutex);
  353. list_for_each_entry(device, &rdmacg_devices, dev_node)
  354. if (!strcmp(name, device->name))
  355. return device;
  356. return NULL;
  357. }
  358. static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of,
  359. char *buf, size_t nbytes, loff_t off)
  360. {
  361. struct rdma_cgroup *cg = css_rdmacg(of_css(of));
  362. const char *dev_name;
  363. struct rdmacg_resource_pool *rpool;
  364. struct rdmacg_device *device;
  365. char *options = strstrip(buf);
  366. int *new_limits;
  367. unsigned long enables = 0;
  368. int i = 0, ret = 0;
  369. /* extract the device name first */
  370. dev_name = strsep(&options, " ");
  371. if (!dev_name) {
  372. ret = -EINVAL;
  373. goto err;
  374. }
  375. new_limits = kcalloc(RDMACG_RESOURCE_MAX, sizeof(int), GFP_KERNEL);
  376. if (!new_limits) {
  377. ret = -ENOMEM;
  378. goto err;
  379. }
  380. ret = rdmacg_parse_limits(options, new_limits, &enables);
  381. if (ret)
  382. goto parse_err;
  383. /* acquire lock to synchronize with hot plug devices */
  384. mutex_lock(&rdmacg_mutex);
  385. device = rdmacg_get_device_locked(dev_name);
  386. if (!device) {
  387. ret = -ENODEV;
  388. goto dev_err;
  389. }
  390. rpool = get_cg_rpool_locked(cg, device);
  391. if (IS_ERR(rpool)) {
  392. ret = PTR_ERR(rpool);
  393. goto dev_err;
  394. }
  395. /* now set the new limits of the rpool */
  396. for_each_set_bit(i, &enables, RDMACG_RESOURCE_MAX)
  397. set_resource_limit(rpool, i, new_limits[i]);
  398. if (rpool->usage_sum == 0 &&
  399. rpool->num_max_cnt == RDMACG_RESOURCE_MAX) {
  400. /*
  401. * No user of the rpool and all entries are set to max, so
  402. * safe to delete this rpool.
  403. */
  404. free_cg_rpool_locked(rpool);
  405. }
  406. dev_err:
  407. mutex_unlock(&rdmacg_mutex);
  408. parse_err:
  409. kfree(new_limits);
  410. err:
  411. return ret ?: nbytes;
  412. }
  413. static void print_rpool_values(struct seq_file *sf,
  414. struct rdmacg_resource_pool *rpool)
  415. {
  416. enum rdmacg_file_type sf_type;
  417. int i;
  418. u32 value;
  419. sf_type = seq_cft(sf)->private;
  420. for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
  421. seq_puts(sf, rdmacg_resource_names[i]);
  422. seq_putc(sf, '=');
  423. if (sf_type == RDMACG_RESOURCE_TYPE_MAX) {
  424. if (rpool)
  425. value = rpool->resources[i].max;
  426. else
  427. value = S32_MAX;
  428. } else {
  429. if (rpool)
  430. value = rpool->resources[i].usage;
  431. else
  432. value = 0;
  433. }
  434. if (value == S32_MAX)
  435. seq_puts(sf, RDMACG_MAX_STR);
  436. else
  437. seq_printf(sf, "%d", value);
  438. seq_putc(sf, ' ');
  439. }
  440. }
  441. static int rdmacg_resource_read(struct seq_file *sf, void *v)
  442. {
  443. struct rdmacg_device *device;
  444. struct rdmacg_resource_pool *rpool;
  445. struct rdma_cgroup *cg = css_rdmacg(seq_css(sf));
  446. mutex_lock(&rdmacg_mutex);
  447. list_for_each_entry(device, &rdmacg_devices, dev_node) {
  448. seq_printf(sf, "%s ", device->name);
  449. rpool = find_cg_rpool_locked(cg, device);
  450. print_rpool_values(sf, rpool);
  451. seq_putc(sf, '\n');
  452. }
  453. mutex_unlock(&rdmacg_mutex);
  454. return 0;
  455. }
  456. static struct cftype rdmacg_files[] = {
  457. {
  458. .name = "max",
  459. .write = rdmacg_resource_set_max,
  460. .seq_show = rdmacg_resource_read,
  461. .private = RDMACG_RESOURCE_TYPE_MAX,
  462. .flags = CFTYPE_NOT_ON_ROOT,
  463. },
  464. {
  465. .name = "current",
  466. .seq_show = rdmacg_resource_read,
  467. .private = RDMACG_RESOURCE_TYPE_STAT,
  468. .flags = CFTYPE_NOT_ON_ROOT,
  469. },
  470. { } /* terminate */
  471. };
  472. static struct cgroup_subsys_state *
  473. rdmacg_css_alloc(struct cgroup_subsys_state *parent)
  474. {
  475. struct rdma_cgroup *cg;
  476. cg = kzalloc(sizeof(*cg), GFP_KERNEL);
  477. if (!cg)
  478. return ERR_PTR(-ENOMEM);
  479. INIT_LIST_HEAD(&cg->rpools);
  480. return &cg->css;
  481. }
  482. static void rdmacg_css_free(struct cgroup_subsys_state *css)
  483. {
  484. struct rdma_cgroup *cg = css_rdmacg(css);
  485. kfree(cg);
  486. }
  487. /**
  488. * rdmacg_css_offline - cgroup css_offline callback
  489. * @css: css of interest
  490. *
  491. * This function is called when @css is about to go away and responsible
  492. * for shooting down all rdmacg associated with @css. As part of that it
  493. * marks all the resource pool entries to max value, so that when resources are
  494. * uncharged, associated resource pool can be freed as well.
  495. */
  496. static void rdmacg_css_offline(struct cgroup_subsys_state *css)
  497. {
  498. struct rdma_cgroup *cg = css_rdmacg(css);
  499. struct rdmacg_resource_pool *rpool;
  500. mutex_lock(&rdmacg_mutex);
  501. list_for_each_entry(rpool, &cg->rpools, cg_node)
  502. set_all_resource_max_limit(rpool);
  503. mutex_unlock(&rdmacg_mutex);
  504. }
  505. struct cgroup_subsys rdma_cgrp_subsys = {
  506. .css_alloc = rdmacg_css_alloc,
  507. .css_free = rdmacg_css_free,
  508. .css_offline = rdmacg_css_offline,
  509. .legacy_cftypes = rdmacg_files,
  510. .dfl_cftypes = rdmacg_files,
  511. };