bpf_iter.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright (c) 2020 Facebook */
  3. #include <linux/fs.h>
  4. #include <linux/anon_inodes.h>
  5. #include <linux/filter.h>
  6. #include <linux/bpf.h>
  7. struct bpf_iter_target_info {
  8. struct list_head list;
  9. const struct bpf_iter_reg *reg_info;
  10. u32 btf_id; /* cached value */
  11. };
  12. struct bpf_iter_link {
  13. struct bpf_link link;
  14. struct bpf_iter_aux_info aux;
  15. struct bpf_iter_target_info *tinfo;
  16. };
  17. struct bpf_iter_priv_data {
  18. struct bpf_iter_target_info *tinfo;
  19. const struct bpf_iter_seq_info *seq_info;
  20. struct bpf_prog *prog;
  21. u64 session_id;
  22. u64 seq_num;
  23. bool done_stop;
  24. u8 target_private[] __aligned(8);
  25. };
  26. static struct list_head targets = LIST_HEAD_INIT(targets);
  27. static DEFINE_MUTEX(targets_mutex);
  28. /* protect bpf_iter_link changes */
  29. static DEFINE_MUTEX(link_mutex);
  30. /* incremented on every opened seq_file */
  31. static atomic64_t session_id;
  32. static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
  33. const struct bpf_iter_seq_info *seq_info);
  34. static void bpf_iter_inc_seq_num(struct seq_file *seq)
  35. {
  36. struct bpf_iter_priv_data *iter_priv;
  37. iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  38. target_private);
  39. iter_priv->seq_num++;
  40. }
  41. static void bpf_iter_dec_seq_num(struct seq_file *seq)
  42. {
  43. struct bpf_iter_priv_data *iter_priv;
  44. iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  45. target_private);
  46. iter_priv->seq_num--;
  47. }
  48. static void bpf_iter_done_stop(struct seq_file *seq)
  49. {
  50. struct bpf_iter_priv_data *iter_priv;
  51. iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  52. target_private);
  53. iter_priv->done_stop = true;
  54. }
  55. /* maximum visited objects before bailing out */
  56. #define MAX_ITER_OBJECTS 1000000
  57. /* bpf_seq_read, a customized and simpler version for bpf iterator.
  58. * no_llseek is assumed for this file.
  59. * The following are differences from seq_read():
  60. * . fixed buffer size (PAGE_SIZE)
  61. * . assuming no_llseek
  62. * . stop() may call bpf program, handling potential overflow there
  63. */
  64. static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
  65. loff_t *ppos)
  66. {
  67. struct seq_file *seq = file->private_data;
  68. size_t n, offs, copied = 0;
  69. int err = 0, num_objs = 0;
  70. void *p;
  71. mutex_lock(&seq->lock);
  72. if (!seq->buf) {
  73. seq->size = PAGE_SIZE << 3;
  74. seq->buf = kvmalloc(seq->size, GFP_KERNEL);
  75. if (!seq->buf) {
  76. err = -ENOMEM;
  77. goto done;
  78. }
  79. }
  80. if (seq->count) {
  81. n = min(seq->count, size);
  82. err = copy_to_user(buf, seq->buf + seq->from, n);
  83. if (err) {
  84. err = -EFAULT;
  85. goto done;
  86. }
  87. seq->count -= n;
  88. seq->from += n;
  89. copied = n;
  90. goto done;
  91. }
  92. seq->from = 0;
  93. p = seq->op->start(seq, &seq->index);
  94. if (!p)
  95. goto stop;
  96. if (IS_ERR(p)) {
  97. err = PTR_ERR(p);
  98. seq->op->stop(seq, p);
  99. seq->count = 0;
  100. goto done;
  101. }
  102. err = seq->op->show(seq, p);
  103. if (err > 0) {
  104. /* object is skipped, decrease seq_num, so next
  105. * valid object can reuse the same seq_num.
  106. */
  107. bpf_iter_dec_seq_num(seq);
  108. seq->count = 0;
  109. } else if (err < 0 || seq_has_overflowed(seq)) {
  110. if (!err)
  111. err = -E2BIG;
  112. seq->op->stop(seq, p);
  113. seq->count = 0;
  114. goto done;
  115. }
  116. while (1) {
  117. loff_t pos = seq->index;
  118. num_objs++;
  119. offs = seq->count;
  120. p = seq->op->next(seq, p, &seq->index);
  121. if (pos == seq->index) {
  122. pr_info_ratelimited("buggy seq_file .next function %ps "
  123. "did not updated position index\n",
  124. seq->op->next);
  125. seq->index++;
  126. }
  127. if (IS_ERR_OR_NULL(p))
  128. break;
  129. /* got a valid next object, increase seq_num */
  130. bpf_iter_inc_seq_num(seq);
  131. if (seq->count >= size)
  132. break;
  133. if (num_objs >= MAX_ITER_OBJECTS) {
  134. if (offs == 0) {
  135. err = -EAGAIN;
  136. seq->op->stop(seq, p);
  137. goto done;
  138. }
  139. break;
  140. }
  141. err = seq->op->show(seq, p);
  142. if (err > 0) {
  143. bpf_iter_dec_seq_num(seq);
  144. seq->count = offs;
  145. } else if (err < 0 || seq_has_overflowed(seq)) {
  146. seq->count = offs;
  147. if (offs == 0) {
  148. if (!err)
  149. err = -E2BIG;
  150. seq->op->stop(seq, p);
  151. goto done;
  152. }
  153. break;
  154. }
  155. }
  156. stop:
  157. offs = seq->count;
  158. /* bpf program called if !p */
  159. seq->op->stop(seq, p);
  160. if (!p) {
  161. if (!seq_has_overflowed(seq)) {
  162. bpf_iter_done_stop(seq);
  163. } else {
  164. seq->count = offs;
  165. if (offs == 0) {
  166. err = -E2BIG;
  167. goto done;
  168. }
  169. }
  170. }
  171. n = min(seq->count, size);
  172. err = copy_to_user(buf, seq->buf, n);
  173. if (err) {
  174. err = -EFAULT;
  175. goto done;
  176. }
  177. copied = n;
  178. seq->count -= n;
  179. seq->from = n;
  180. done:
  181. if (!copied)
  182. copied = err;
  183. else
  184. *ppos += copied;
  185. mutex_unlock(&seq->lock);
  186. return copied;
  187. }
  188. static const struct bpf_iter_seq_info *
  189. __get_seq_info(struct bpf_iter_link *link)
  190. {
  191. const struct bpf_iter_seq_info *seq_info;
  192. if (link->aux.map) {
  193. seq_info = link->aux.map->ops->iter_seq_info;
  194. if (seq_info)
  195. return seq_info;
  196. }
  197. return link->tinfo->reg_info->seq_info;
  198. }
  199. static int iter_open(struct inode *inode, struct file *file)
  200. {
  201. struct bpf_iter_link *link = inode->i_private;
  202. return prepare_seq_file(file, link, __get_seq_info(link));
  203. }
  204. static int iter_release(struct inode *inode, struct file *file)
  205. {
  206. struct bpf_iter_priv_data *iter_priv;
  207. struct seq_file *seq;
  208. seq = file->private_data;
  209. if (!seq)
  210. return 0;
  211. iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  212. target_private);
  213. if (iter_priv->seq_info->fini_seq_private)
  214. iter_priv->seq_info->fini_seq_private(seq->private);
  215. bpf_prog_put(iter_priv->prog);
  216. seq->private = iter_priv;
  217. return seq_release_private(inode, file);
  218. }
  219. const struct file_operations bpf_iter_fops = {
  220. .open = iter_open,
  221. .llseek = no_llseek,
  222. .read = bpf_seq_read,
  223. .release = iter_release,
  224. };
  225. /* The argument reg_info will be cached in bpf_iter_target_info.
  226. * The common practice is to declare target reg_info as
  227. * a const static variable and passed as an argument to
  228. * bpf_iter_reg_target().
  229. */
  230. int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
  231. {
  232. struct bpf_iter_target_info *tinfo;
  233. tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
  234. if (!tinfo)
  235. return -ENOMEM;
  236. tinfo->reg_info = reg_info;
  237. INIT_LIST_HEAD(&tinfo->list);
  238. mutex_lock(&targets_mutex);
  239. list_add(&tinfo->list, &targets);
  240. mutex_unlock(&targets_mutex);
  241. return 0;
  242. }
  243. void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
  244. {
  245. struct bpf_iter_target_info *tinfo;
  246. bool found = false;
  247. mutex_lock(&targets_mutex);
  248. list_for_each_entry(tinfo, &targets, list) {
  249. if (reg_info == tinfo->reg_info) {
  250. list_del(&tinfo->list);
  251. kfree(tinfo);
  252. found = true;
  253. break;
  254. }
  255. }
  256. mutex_unlock(&targets_mutex);
  257. WARN_ON(found == false);
  258. }
  259. static void cache_btf_id(struct bpf_iter_target_info *tinfo,
  260. struct bpf_prog *prog)
  261. {
  262. tinfo->btf_id = prog->aux->attach_btf_id;
  263. }
  264. bool bpf_iter_prog_supported(struct bpf_prog *prog)
  265. {
  266. const char *attach_fname = prog->aux->attach_func_name;
  267. u32 prog_btf_id = prog->aux->attach_btf_id;
  268. const char *prefix = BPF_ITER_FUNC_PREFIX;
  269. struct bpf_iter_target_info *tinfo;
  270. int prefix_len = strlen(prefix);
  271. bool supported = false;
  272. if (strncmp(attach_fname, prefix, prefix_len))
  273. return false;
  274. mutex_lock(&targets_mutex);
  275. list_for_each_entry(tinfo, &targets, list) {
  276. if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
  277. supported = true;
  278. break;
  279. }
  280. if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
  281. cache_btf_id(tinfo, prog);
  282. supported = true;
  283. break;
  284. }
  285. }
  286. mutex_unlock(&targets_mutex);
  287. if (supported) {
  288. prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
  289. prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
  290. }
  291. return supported;
  292. }
  293. static void bpf_iter_link_release(struct bpf_link *link)
  294. {
  295. struct bpf_iter_link *iter_link =
  296. container_of(link, struct bpf_iter_link, link);
  297. if (iter_link->tinfo->reg_info->detach_target)
  298. iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
  299. }
  300. static void bpf_iter_link_dealloc(struct bpf_link *link)
  301. {
  302. struct bpf_iter_link *iter_link =
  303. container_of(link, struct bpf_iter_link, link);
  304. kfree(iter_link);
  305. }
  306. static int bpf_iter_link_replace(struct bpf_link *link,
  307. struct bpf_prog *new_prog,
  308. struct bpf_prog *old_prog)
  309. {
  310. int ret = 0;
  311. mutex_lock(&link_mutex);
  312. if (old_prog && link->prog != old_prog) {
  313. ret = -EPERM;
  314. goto out_unlock;
  315. }
  316. if (link->prog->type != new_prog->type ||
  317. link->prog->expected_attach_type != new_prog->expected_attach_type ||
  318. link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
  319. ret = -EINVAL;
  320. goto out_unlock;
  321. }
  322. old_prog = xchg(&link->prog, new_prog);
  323. bpf_prog_put(old_prog);
  324. out_unlock:
  325. mutex_unlock(&link_mutex);
  326. return ret;
  327. }
  328. static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
  329. struct seq_file *seq)
  330. {
  331. struct bpf_iter_link *iter_link =
  332. container_of(link, struct bpf_iter_link, link);
  333. bpf_iter_show_fdinfo_t show_fdinfo;
  334. seq_printf(seq,
  335. "target_name:\t%s\n",
  336. iter_link->tinfo->reg_info->target);
  337. show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
  338. if (show_fdinfo)
  339. show_fdinfo(&iter_link->aux, seq);
  340. }
  341. static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
  342. struct bpf_link_info *info)
  343. {
  344. struct bpf_iter_link *iter_link =
  345. container_of(link, struct bpf_iter_link, link);
  346. char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
  347. bpf_iter_fill_link_info_t fill_link_info;
  348. u32 ulen = info->iter.target_name_len;
  349. const char *target_name;
  350. u32 target_len;
  351. if (!ulen ^ !ubuf)
  352. return -EINVAL;
  353. target_name = iter_link->tinfo->reg_info->target;
  354. target_len = strlen(target_name);
  355. info->iter.target_name_len = target_len + 1;
  356. if (ubuf) {
  357. if (ulen >= target_len + 1) {
  358. if (copy_to_user(ubuf, target_name, target_len + 1))
  359. return -EFAULT;
  360. } else {
  361. char zero = '\0';
  362. if (copy_to_user(ubuf, target_name, ulen - 1))
  363. return -EFAULT;
  364. if (put_user(zero, ubuf + ulen - 1))
  365. return -EFAULT;
  366. return -ENOSPC;
  367. }
  368. }
  369. fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
  370. if (fill_link_info)
  371. return fill_link_info(&iter_link->aux, info);
  372. return 0;
  373. }
  374. static const struct bpf_link_ops bpf_iter_link_lops = {
  375. .release = bpf_iter_link_release,
  376. .dealloc = bpf_iter_link_dealloc,
  377. .update_prog = bpf_iter_link_replace,
  378. .show_fdinfo = bpf_iter_link_show_fdinfo,
  379. .fill_link_info = bpf_iter_link_fill_link_info,
  380. };
  381. bool bpf_link_is_iter(struct bpf_link *link)
  382. {
  383. return link->ops == &bpf_iter_link_lops;
  384. }
  385. int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
  386. {
  387. union bpf_iter_link_info __user *ulinfo;
  388. struct bpf_link_primer link_primer;
  389. struct bpf_iter_target_info *tinfo;
  390. union bpf_iter_link_info linfo;
  391. struct bpf_iter_link *link;
  392. u32 prog_btf_id, linfo_len;
  393. bool existed = false;
  394. int err;
  395. if (attr->link_create.target_fd || attr->link_create.flags)
  396. return -EINVAL;
  397. memset(&linfo, 0, sizeof(union bpf_iter_link_info));
  398. ulinfo = u64_to_user_ptr(attr->link_create.iter_info);
  399. linfo_len = attr->link_create.iter_info_len;
  400. if (!ulinfo ^ !linfo_len)
  401. return -EINVAL;
  402. if (ulinfo) {
  403. err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
  404. linfo_len);
  405. if (err)
  406. return err;
  407. linfo_len = min_t(u32, linfo_len, sizeof(linfo));
  408. if (copy_from_user(&linfo, ulinfo, linfo_len))
  409. return -EFAULT;
  410. }
  411. prog_btf_id = prog->aux->attach_btf_id;
  412. mutex_lock(&targets_mutex);
  413. list_for_each_entry(tinfo, &targets, list) {
  414. if (tinfo->btf_id == prog_btf_id) {
  415. existed = true;
  416. break;
  417. }
  418. }
  419. mutex_unlock(&targets_mutex);
  420. if (!existed)
  421. return -ENOENT;
  422. link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
  423. if (!link)
  424. return -ENOMEM;
  425. bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
  426. link->tinfo = tinfo;
  427. err = bpf_link_prime(&link->link, &link_primer);
  428. if (err) {
  429. kfree(link);
  430. return err;
  431. }
  432. if (tinfo->reg_info->attach_target) {
  433. err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
  434. if (err) {
  435. bpf_link_cleanup(&link_primer);
  436. return err;
  437. }
  438. }
  439. return bpf_link_settle(&link_primer);
  440. }
  441. static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
  442. struct bpf_iter_target_info *tinfo,
  443. const struct bpf_iter_seq_info *seq_info,
  444. struct bpf_prog *prog)
  445. {
  446. priv_data->tinfo = tinfo;
  447. priv_data->seq_info = seq_info;
  448. priv_data->prog = prog;
  449. priv_data->session_id = atomic64_inc_return(&session_id);
  450. priv_data->seq_num = 0;
  451. priv_data->done_stop = false;
  452. }
  453. static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
  454. const struct bpf_iter_seq_info *seq_info)
  455. {
  456. struct bpf_iter_priv_data *priv_data;
  457. struct bpf_iter_target_info *tinfo;
  458. struct bpf_prog *prog;
  459. u32 total_priv_dsize;
  460. struct seq_file *seq;
  461. int err = 0;
  462. mutex_lock(&link_mutex);
  463. prog = link->link.prog;
  464. bpf_prog_inc(prog);
  465. mutex_unlock(&link_mutex);
  466. tinfo = link->tinfo;
  467. total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
  468. seq_info->seq_priv_size;
  469. priv_data = __seq_open_private(file, seq_info->seq_ops,
  470. total_priv_dsize);
  471. if (!priv_data) {
  472. err = -ENOMEM;
  473. goto release_prog;
  474. }
  475. if (seq_info->init_seq_private) {
  476. err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
  477. if (err)
  478. goto release_seq_file;
  479. }
  480. init_seq_meta(priv_data, tinfo, seq_info, prog);
  481. seq = file->private_data;
  482. seq->private = priv_data->target_private;
  483. return 0;
  484. release_seq_file:
  485. seq_release_private(file->f_inode, file);
  486. file->private_data = NULL;
  487. release_prog:
  488. bpf_prog_put(prog);
  489. return err;
  490. }
  491. int bpf_iter_new_fd(struct bpf_link *link)
  492. {
  493. struct bpf_iter_link *iter_link;
  494. struct file *file;
  495. unsigned int flags;
  496. int err, fd;
  497. if (link->ops != &bpf_iter_link_lops)
  498. return -EINVAL;
  499. flags = O_RDONLY | O_CLOEXEC;
  500. fd = get_unused_fd_flags(flags);
  501. if (fd < 0)
  502. return fd;
  503. file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
  504. if (IS_ERR(file)) {
  505. err = PTR_ERR(file);
  506. goto free_fd;
  507. }
  508. iter_link = container_of(link, struct bpf_iter_link, link);
  509. err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
  510. if (err)
  511. goto free_file;
  512. fd_install(fd, file);
  513. return fd;
  514. free_file:
  515. fput(file);
  516. free_fd:
  517. put_unused_fd(fd);
  518. return err;
  519. }
  520. struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
  521. {
  522. struct bpf_iter_priv_data *iter_priv;
  523. struct seq_file *seq;
  524. void *seq_priv;
  525. seq = meta->seq;
  526. if (seq->file->f_op != &bpf_iter_fops)
  527. return NULL;
  528. seq_priv = seq->private;
  529. iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
  530. target_private);
  531. if (in_stop && iter_priv->done_stop)
  532. return NULL;
  533. meta->session_id = iter_priv->session_id;
  534. meta->seq_num = iter_priv->seq_num;
  535. return iter_priv->prog;
  536. }
  537. int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
  538. {
  539. int ret;
  540. rcu_read_lock();
  541. migrate_disable();
  542. ret = BPF_PROG_RUN(prog, ctx);
  543. migrate_enable();
  544. rcu_read_unlock();
  545. /* bpf program can only return 0 or 1:
  546. * 0 : okay
  547. * 1 : retry the same object
  548. * The bpf_iter_run_prog() return value
  549. * will be seq_ops->show() return value.
  550. */
  551. return ret == 0 ? 0 : -EAGAIN;
  552. }