ring_buffer.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Performance events ring-buffer code:
  4. *
  5. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  6. * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  7. * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  8. * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  9. */
  10. #include <linux/perf_event.h>
  11. #include <linux/vmalloc.h>
  12. #include <linux/slab.h>
  13. #include <linux/circ_buf.h>
  14. #include <linux/poll.h>
  15. #include <linux/nospec.h>
  16. #include "internal.h"
  17. static void perf_output_wakeup(struct perf_output_handle *handle)
  18. {
  19. atomic_set(&handle->rb->poll, EPOLLIN);
  20. handle->event->pending_wakeup = 1;
  21. irq_work_queue(&handle->event->pending);
  22. }
  23. /*
  24. * We need to ensure a later event_id doesn't publish a head when a former
  25. * event isn't done writing. However since we need to deal with NMIs we
  26. * cannot fully serialize things.
  27. *
  28. * We only publish the head (and generate a wakeup) when the outer-most
  29. * event completes.
  30. */
  31. static void perf_output_get_handle(struct perf_output_handle *handle)
  32. {
  33. struct perf_buffer *rb = handle->rb;
  34. preempt_disable();
  35. /*
  36. * Avoid an explicit LOAD/STORE such that architectures with memops
  37. * can use them.
  38. */
  39. (*(volatile unsigned int *)&rb->nest)++;
  40. handle->wakeup = local_read(&rb->wakeup);
  41. }
  42. static void perf_output_put_handle(struct perf_output_handle *handle)
  43. {
  44. struct perf_buffer *rb = handle->rb;
  45. unsigned long head;
  46. unsigned int nest;
  47. /*
  48. * If this isn't the outermost nesting, we don't have to update
  49. * @rb->user_page->data_head.
  50. */
  51. nest = READ_ONCE(rb->nest);
  52. if (nest > 1) {
  53. WRITE_ONCE(rb->nest, nest - 1);
  54. goto out;
  55. }
  56. again:
  57. /*
  58. * In order to avoid publishing a head value that goes backwards,
  59. * we must ensure the load of @rb->head happens after we've
  60. * incremented @rb->nest.
  61. *
  62. * Otherwise we can observe a @rb->head value before one published
  63. * by an IRQ/NMI happening between the load and the increment.
  64. */
  65. barrier();
  66. head = local_read(&rb->head);
  67. /*
  68. * IRQ/NMI can happen here and advance @rb->head, causing our
  69. * load above to be stale.
  70. */
  71. /*
  72. * Since the mmap() consumer (userspace) can run on a different CPU:
  73. *
  74. * kernel user
  75. *
  76. * if (LOAD ->data_tail) { LOAD ->data_head
  77. * (A) smp_rmb() (C)
  78. * STORE $data LOAD $data
  79. * smp_wmb() (B) smp_mb() (D)
  80. * STORE ->data_head STORE ->data_tail
  81. * }
  82. *
  83. * Where A pairs with D, and B pairs with C.
  84. *
  85. * In our case (A) is a control dependency that separates the load of
  86. * the ->data_tail and the stores of $data. In case ->data_tail
  87. * indicates there is no room in the buffer to store $data we do not.
  88. *
  89. * D needs to be a full barrier since it separates the data READ
  90. * from the tail WRITE.
  91. *
  92. * For B a WMB is sufficient since it separates two WRITEs, and for C
  93. * an RMB is sufficient since it separates two READs.
  94. *
  95. * See perf_output_begin().
  96. */
  97. smp_wmb(); /* B, matches C */
  98. WRITE_ONCE(rb->user_page->data_head, head);
  99. /*
  100. * We must publish the head before decrementing the nest count,
  101. * otherwise an IRQ/NMI can publish a more recent head value and our
  102. * write will (temporarily) publish a stale value.
  103. */
  104. barrier();
  105. WRITE_ONCE(rb->nest, 0);
  106. /*
  107. * Ensure we decrement @rb->nest before we validate the @rb->head.
  108. * Otherwise we cannot be sure we caught the 'last' nested update.
  109. */
  110. barrier();
  111. if (unlikely(head != local_read(&rb->head))) {
  112. WRITE_ONCE(rb->nest, 1);
  113. goto again;
  114. }
  115. if (handle->wakeup != local_read(&rb->wakeup))
  116. perf_output_wakeup(handle);
  117. out:
  118. preempt_enable();
  119. }
  120. static __always_inline bool
  121. ring_buffer_has_space(unsigned long head, unsigned long tail,
  122. unsigned long data_size, unsigned int size,
  123. bool backward)
  124. {
  125. if (!backward)
  126. return CIRC_SPACE(head, tail, data_size) >= size;
  127. else
  128. return CIRC_SPACE(tail, head, data_size) >= size;
  129. }
  130. static __always_inline int
  131. __perf_output_begin(struct perf_output_handle *handle,
  132. struct perf_sample_data *data,
  133. struct perf_event *event, unsigned int size,
  134. bool backward)
  135. {
  136. struct perf_buffer *rb;
  137. unsigned long tail, offset, head;
  138. int have_lost, page_shift;
  139. struct {
  140. struct perf_event_header header;
  141. u64 id;
  142. u64 lost;
  143. } lost_event;
  144. rcu_read_lock();
  145. /*
  146. * For inherited events we send all the output towards the parent.
  147. */
  148. if (event->parent)
  149. event = event->parent;
  150. rb = rcu_dereference(event->rb);
  151. if (unlikely(!rb))
  152. goto out;
  153. if (unlikely(rb->paused)) {
  154. if (rb->nr_pages)
  155. local_inc(&rb->lost);
  156. goto out;
  157. }
  158. handle->rb = rb;
  159. handle->event = event;
  160. have_lost = local_read(&rb->lost);
  161. if (unlikely(have_lost)) {
  162. size += sizeof(lost_event);
  163. if (event->attr.sample_id_all)
  164. size += event->id_header_size;
  165. }
  166. perf_output_get_handle(handle);
  167. do {
  168. tail = READ_ONCE(rb->user_page->data_tail);
  169. offset = head = local_read(&rb->head);
  170. if (!rb->overwrite) {
  171. if (unlikely(!ring_buffer_has_space(head, tail,
  172. perf_data_size(rb),
  173. size, backward)))
  174. goto fail;
  175. }
  176. /*
  177. * The above forms a control dependency barrier separating the
  178. * @tail load above from the data stores below. Since the @tail
  179. * load is required to compute the branch to fail below.
  180. *
  181. * A, matches D; the full memory barrier userspace SHOULD issue
  182. * after reading the data and before storing the new tail
  183. * position.
  184. *
  185. * See perf_output_put_handle().
  186. */
  187. if (!backward)
  188. head += size;
  189. else
  190. head -= size;
  191. } while (local_cmpxchg(&rb->head, offset, head) != offset);
  192. if (backward) {
  193. offset = head;
  194. head = (u64)(-head);
  195. }
  196. /*
  197. * We rely on the implied barrier() by local_cmpxchg() to ensure
  198. * none of the data stores below can be lifted up by the compiler.
  199. */
  200. if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))
  201. local_add(rb->watermark, &rb->wakeup);
  202. page_shift = PAGE_SHIFT + page_order(rb);
  203. handle->page = (offset >> page_shift) & (rb->nr_pages - 1);
  204. offset &= (1UL << page_shift) - 1;
  205. handle->addr = rb->data_pages[handle->page] + offset;
  206. handle->size = (1UL << page_shift) - offset;
  207. if (unlikely(have_lost)) {
  208. lost_event.header.size = sizeof(lost_event);
  209. lost_event.header.type = PERF_RECORD_LOST;
  210. lost_event.header.misc = 0;
  211. lost_event.id = event->id;
  212. lost_event.lost = local_xchg(&rb->lost, 0);
  213. /* XXX mostly redundant; @data is already fully initializes */
  214. perf_event_header__init_id(&lost_event.header, data, event);
  215. perf_output_put(handle, lost_event);
  216. perf_event__output_id_sample(event, handle, data);
  217. }
  218. return 0;
  219. fail:
  220. local_inc(&rb->lost);
  221. perf_output_put_handle(handle);
  222. out:
  223. rcu_read_unlock();
  224. return -ENOSPC;
  225. }
  226. int perf_output_begin_forward(struct perf_output_handle *handle,
  227. struct perf_sample_data *data,
  228. struct perf_event *event, unsigned int size)
  229. {
  230. return __perf_output_begin(handle, data, event, size, false);
  231. }
  232. int perf_output_begin_backward(struct perf_output_handle *handle,
  233. struct perf_sample_data *data,
  234. struct perf_event *event, unsigned int size)
  235. {
  236. return __perf_output_begin(handle, data, event, size, true);
  237. }
  238. int perf_output_begin(struct perf_output_handle *handle,
  239. struct perf_sample_data *data,
  240. struct perf_event *event, unsigned int size)
  241. {
  242. return __perf_output_begin(handle, data, event, size,
  243. unlikely(is_write_backward(event)));
  244. }
  245. unsigned int perf_output_copy(struct perf_output_handle *handle,
  246. const void *buf, unsigned int len)
  247. {
  248. return __output_copy(handle, buf, len);
  249. }
  250. unsigned int perf_output_skip(struct perf_output_handle *handle,
  251. unsigned int len)
  252. {
  253. return __output_skip(handle, NULL, len);
  254. }
  255. void perf_output_end(struct perf_output_handle *handle)
  256. {
  257. perf_output_put_handle(handle);
  258. rcu_read_unlock();
  259. }
  260. static void
  261. ring_buffer_init(struct perf_buffer *rb, long watermark, int flags)
  262. {
  263. long max_size = perf_data_size(rb);
  264. if (watermark)
  265. rb->watermark = min(max_size, watermark);
  266. if (!rb->watermark)
  267. rb->watermark = max_size / 2;
  268. if (flags & RING_BUFFER_WRITABLE)
  269. rb->overwrite = 0;
  270. else
  271. rb->overwrite = 1;
  272. refcount_set(&rb->refcount, 1);
  273. INIT_LIST_HEAD(&rb->event_list);
  274. spin_lock_init(&rb->event_lock);
  275. /*
  276. * perf_output_begin() only checks rb->paused, therefore
  277. * rb->paused must be true if we have no pages for output.
  278. */
  279. if (!rb->nr_pages)
  280. rb->paused = 1;
  281. }
  282. void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)
  283. {
  284. /*
  285. * OVERWRITE is determined by perf_aux_output_end() and can't
  286. * be passed in directly.
  287. */
  288. if (WARN_ON_ONCE(flags & PERF_AUX_FLAG_OVERWRITE))
  289. return;
  290. handle->aux_flags |= flags;
  291. }
  292. EXPORT_SYMBOL_GPL(perf_aux_output_flag);
  293. /*
  294. * This is called before hardware starts writing to the AUX area to
  295. * obtain an output handle and make sure there's room in the buffer.
  296. * When the capture completes, call perf_aux_output_end() to commit
  297. * the recorded data to the buffer.
  298. *
  299. * The ordering is similar to that of perf_output_{begin,end}, with
  300. * the exception of (B), which should be taken care of by the pmu
  301. * driver, since ordering rules will differ depending on hardware.
  302. *
  303. * Call this from pmu::start(); see the comment in perf_aux_output_end()
  304. * about its use in pmu callbacks. Both can also be called from the PMI
  305. * handler if needed.
  306. */
  307. void *perf_aux_output_begin(struct perf_output_handle *handle,
  308. struct perf_event *event)
  309. {
  310. struct perf_event *output_event = event;
  311. unsigned long aux_head, aux_tail;
  312. struct perf_buffer *rb;
  313. unsigned int nest;
  314. if (output_event->parent)
  315. output_event = output_event->parent;
  316. /*
  317. * Since this will typically be open across pmu::add/pmu::del, we
  318. * grab ring_buffer's refcount instead of holding rcu read lock
  319. * to make sure it doesn't disappear under us.
  320. */
  321. rb = ring_buffer_get(output_event);
  322. if (!rb)
  323. return NULL;
  324. if (!rb_has_aux(rb))
  325. goto err;
  326. /*
  327. * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
  328. * about to get freed, so we leave immediately.
  329. *
  330. * Checking rb::aux_mmap_count and rb::refcount has to be done in
  331. * the same order, see perf_mmap_close. Otherwise we end up freeing
  332. * aux pages in this path, which is a bug, because in_atomic().
  333. */
  334. if (!atomic_read(&rb->aux_mmap_count))
  335. goto err;
  336. if (!refcount_inc_not_zero(&rb->aux_refcount))
  337. goto err;
  338. nest = READ_ONCE(rb->aux_nest);
  339. /*
  340. * Nesting is not supported for AUX area, make sure nested
  341. * writers are caught early
  342. */
  343. if (WARN_ON_ONCE(nest))
  344. goto err_put;
  345. WRITE_ONCE(rb->aux_nest, nest + 1);
  346. aux_head = rb->aux_head;
  347. handle->rb = rb;
  348. handle->event = event;
  349. handle->head = aux_head;
  350. handle->size = 0;
  351. handle->aux_flags = 0;
  352. /*
  353. * In overwrite mode, AUX data stores do not depend on aux_tail,
  354. * therefore (A) control dependency barrier does not exist. The
  355. * (B) <-> (C) ordering is still observed by the pmu driver.
  356. */
  357. if (!rb->aux_overwrite) {
  358. aux_tail = READ_ONCE(rb->user_page->aux_tail);
  359. handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
  360. if (aux_head - aux_tail < perf_aux_size(rb))
  361. handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
  362. /*
  363. * handle->size computation depends on aux_tail load; this forms a
  364. * control dependency barrier separating aux_tail load from aux data
  365. * store that will be enabled on successful return
  366. */
  367. if (!handle->size) { /* A, matches D */
  368. event->pending_disable = smp_processor_id();
  369. perf_output_wakeup(handle);
  370. WRITE_ONCE(rb->aux_nest, 0);
  371. goto err_put;
  372. }
  373. }
  374. return handle->rb->aux_priv;
  375. err_put:
  376. /* can't be last */
  377. rb_free_aux(rb);
  378. err:
  379. ring_buffer_put(rb);
  380. handle->event = NULL;
  381. return NULL;
  382. }
  383. EXPORT_SYMBOL_GPL(perf_aux_output_begin);
  384. static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb)
  385. {
  386. if (rb->aux_overwrite)
  387. return false;
  388. if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) {
  389. rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark);
  390. return true;
  391. }
  392. return false;
  393. }
  394. /*
  395. * Commit the data written by hardware into the ring buffer by adjusting
  396. * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
  397. * pmu driver's responsibility to observe ordering rules of the hardware,
  398. * so that all the data is externally visible before this is called.
  399. *
  400. * Note: this has to be called from pmu::stop() callback, as the assumption
  401. * of the AUX buffer management code is that after pmu::stop(), the AUX
  402. * transaction must be stopped and therefore drop the AUX reference count.
  403. */
  404. void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
  405. {
  406. bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);
  407. struct perf_buffer *rb = handle->rb;
  408. unsigned long aux_head;
  409. /* in overwrite mode, driver provides aux_head via handle */
  410. if (rb->aux_overwrite) {
  411. handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;
  412. aux_head = handle->head;
  413. rb->aux_head = aux_head;
  414. } else {
  415. handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;
  416. aux_head = rb->aux_head;
  417. rb->aux_head += size;
  418. }
  419. /*
  420. * Only send RECORD_AUX if we have something useful to communicate
  421. *
  422. * Note: the OVERWRITE records by themselves are not considered
  423. * useful, as they don't communicate any *new* information,
  424. * aside from the short-lived offset, that becomes history at
  425. * the next event sched-in and therefore isn't useful.
  426. * The userspace that needs to copy out AUX data in overwrite
  427. * mode should know to use user_page::aux_head for the actual
  428. * offset. So, from now on we don't output AUX records that
  429. * have *only* OVERWRITE flag set.
  430. */
  431. if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))
  432. perf_event_aux_event(handle->event, aux_head, size,
  433. handle->aux_flags);
  434. WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
  435. if (rb_need_aux_wakeup(rb))
  436. wakeup = true;
  437. if (wakeup) {
  438. if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)
  439. handle->event->pending_disable = smp_processor_id();
  440. perf_output_wakeup(handle);
  441. }
  442. handle->event = NULL;
  443. WRITE_ONCE(rb->aux_nest, 0);
  444. /* can't be last */
  445. rb_free_aux(rb);
  446. ring_buffer_put(rb);
  447. }
  448. EXPORT_SYMBOL_GPL(perf_aux_output_end);
  449. /*
  450. * Skip over a given number of bytes in the AUX buffer, due to, for example,
  451. * hardware's alignment constraints.
  452. */
  453. int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)
  454. {
  455. struct perf_buffer *rb = handle->rb;
  456. if (size > handle->size)
  457. return -ENOSPC;
  458. rb->aux_head += size;
  459. WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
  460. if (rb_need_aux_wakeup(rb)) {
  461. perf_output_wakeup(handle);
  462. handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
  463. }
  464. handle->head = rb->aux_head;
  465. handle->size -= size;
  466. return 0;
  467. }
  468. EXPORT_SYMBOL_GPL(perf_aux_output_skip);
  469. void *perf_get_aux(struct perf_output_handle *handle)
  470. {
  471. /* this is only valid between perf_aux_output_begin and *_end */
  472. if (!handle->event)
  473. return NULL;
  474. return handle->rb->aux_priv;
  475. }
  476. EXPORT_SYMBOL_GPL(perf_get_aux);
  477. /*
  478. * Copy out AUX data from an AUX handle.
  479. */
  480. long perf_output_copy_aux(struct perf_output_handle *aux_handle,
  481. struct perf_output_handle *handle,
  482. unsigned long from, unsigned long to)
  483. {
  484. struct perf_buffer *rb = aux_handle->rb;
  485. unsigned long tocopy, remainder, len = 0;
  486. void *addr;
  487. from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
  488. to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
  489. do {
  490. tocopy = PAGE_SIZE - offset_in_page(from);
  491. if (to > from)
  492. tocopy = min(tocopy, to - from);
  493. if (!tocopy)
  494. break;
  495. addr = rb->aux_pages[from >> PAGE_SHIFT];
  496. addr += offset_in_page(from);
  497. remainder = perf_output_copy(handle, addr, tocopy);
  498. if (remainder)
  499. return -EFAULT;
  500. len += tocopy;
  501. from += tocopy;
  502. from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
  503. } while (to != from);
  504. return len;
  505. }
  506. #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
  507. static struct page *rb_alloc_aux_page(int node, int order)
  508. {
  509. struct page *page;
  510. if (order > MAX_ORDER)
  511. order = MAX_ORDER;
  512. do {
  513. page = alloc_pages_node(node, PERF_AUX_GFP, order);
  514. } while (!page && order--);
  515. if (page && order) {
  516. /*
  517. * Communicate the allocation size to the driver:
  518. * if we managed to secure a high-order allocation,
  519. * set its first page's private to this order;
  520. * !PagePrivate(page) means it's just a normal page.
  521. */
  522. split_page(page, order);
  523. SetPagePrivate(page);
  524. set_page_private(page, order);
  525. }
  526. return page;
  527. }
  528. static void rb_free_aux_page(struct perf_buffer *rb, int idx)
  529. {
  530. struct page *page = virt_to_page(rb->aux_pages[idx]);
  531. ClearPagePrivate(page);
  532. page->mapping = NULL;
  533. __free_page(page);
  534. }
  535. static void __rb_free_aux(struct perf_buffer *rb)
  536. {
  537. int pg;
  538. /*
  539. * Should never happen, the last reference should be dropped from
  540. * perf_mmap_close() path, which first stops aux transactions (which
  541. * in turn are the atomic holders of aux_refcount) and then does the
  542. * last rb_free_aux().
  543. */
  544. WARN_ON_ONCE(in_atomic());
  545. if (rb->aux_priv) {
  546. rb->free_aux(rb->aux_priv);
  547. rb->free_aux = NULL;
  548. rb->aux_priv = NULL;
  549. }
  550. if (rb->aux_nr_pages) {
  551. for (pg = 0; pg < rb->aux_nr_pages; pg++)
  552. rb_free_aux_page(rb, pg);
  553. kfree(rb->aux_pages);
  554. rb->aux_nr_pages = 0;
  555. }
  556. }
  557. int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
  558. pgoff_t pgoff, int nr_pages, long watermark, int flags)
  559. {
  560. bool overwrite = !(flags & RING_BUFFER_WRITABLE);
  561. int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
  562. int ret = -ENOMEM, max_order;
  563. if (!has_aux(event))
  564. return -EOPNOTSUPP;
  565. /*
  566. * We need to start with the max_order that fits in nr_pages,
  567. * not the other way around, hence ilog2() and not get_order.
  568. */
  569. max_order = ilog2(nr_pages);
  570. /*
  571. * PMU requests more than one contiguous chunks of memory
  572. * for SW double buffering
  573. */
  574. if (!overwrite) {
  575. if (!max_order)
  576. return -EINVAL;
  577. max_order--;
  578. }
  579. rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL,
  580. node);
  581. if (!rb->aux_pages)
  582. return -ENOMEM;
  583. rb->free_aux = event->pmu->free_aux;
  584. for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) {
  585. struct page *page;
  586. int last, order;
  587. order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages));
  588. page = rb_alloc_aux_page(node, order);
  589. if (!page)
  590. goto out;
  591. for (last = rb->aux_nr_pages + (1 << page_private(page));
  592. last > rb->aux_nr_pages; rb->aux_nr_pages++)
  593. rb->aux_pages[rb->aux_nr_pages] = page_address(page++);
  594. }
  595. /*
  596. * In overwrite mode, PMUs that don't support SG may not handle more
  597. * than one contiguous allocation, since they rely on PMI to do double
  598. * buffering. In this case, the entire buffer has to be one contiguous
  599. * chunk.
  600. */
  601. if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) &&
  602. overwrite) {
  603. struct page *page = virt_to_page(rb->aux_pages[0]);
  604. if (page_private(page) != max_order)
  605. goto out;
  606. }
  607. rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages,
  608. overwrite);
  609. if (!rb->aux_priv)
  610. goto out;
  611. ret = 0;
  612. /*
  613. * aux_pages (and pmu driver's private data, aux_priv) will be
  614. * referenced in both producer's and consumer's contexts, thus
  615. * we keep a refcount here to make sure either of the two can
  616. * reference them safely.
  617. */
  618. refcount_set(&rb->aux_refcount, 1);
  619. rb->aux_overwrite = overwrite;
  620. rb->aux_watermark = watermark;
  621. if (!rb->aux_watermark && !rb->aux_overwrite)
  622. rb->aux_watermark = nr_pages << (PAGE_SHIFT - 1);
  623. out:
  624. if (!ret)
  625. rb->aux_pgoff = pgoff;
  626. else
  627. __rb_free_aux(rb);
  628. return ret;
  629. }
  630. void rb_free_aux(struct perf_buffer *rb)
  631. {
  632. if (refcount_dec_and_test(&rb->aux_refcount))
  633. __rb_free_aux(rb);
  634. }
  635. #ifndef CONFIG_PERF_USE_VMALLOC
  636. /*
  637. * Back perf_mmap() with regular GFP_KERNEL-0 pages.
  638. */
  639. static struct page *
  640. __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
  641. {
  642. if (pgoff > rb->nr_pages)
  643. return NULL;
  644. if (pgoff == 0)
  645. return virt_to_page(rb->user_page);
  646. return virt_to_page(rb->data_pages[pgoff - 1]);
  647. }
  648. static void *perf_mmap_alloc_page(int cpu)
  649. {
  650. struct page *page;
  651. int node;
  652. node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  653. page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
  654. if (!page)
  655. return NULL;
  656. return page_address(page);
  657. }
  658. static void perf_mmap_free_page(void *addr)
  659. {
  660. struct page *page = virt_to_page(addr);
  661. page->mapping = NULL;
  662. __free_page(page);
  663. }
  664. struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
  665. {
  666. struct perf_buffer *rb;
  667. unsigned long size;
  668. int i;
  669. size = sizeof(struct perf_buffer);
  670. size += nr_pages * sizeof(void *);
  671. if (order_base_2(size) >= PAGE_SHIFT+MAX_ORDER)
  672. goto fail;
  673. rb = kzalloc(size, GFP_KERNEL);
  674. if (!rb)
  675. goto fail;
  676. rb->user_page = perf_mmap_alloc_page(cpu);
  677. if (!rb->user_page)
  678. goto fail_user_page;
  679. for (i = 0; i < nr_pages; i++) {
  680. rb->data_pages[i] = perf_mmap_alloc_page(cpu);
  681. if (!rb->data_pages[i])
  682. goto fail_data_pages;
  683. }
  684. rb->nr_pages = nr_pages;
  685. ring_buffer_init(rb, watermark, flags);
  686. return rb;
  687. fail_data_pages:
  688. for (i--; i >= 0; i--)
  689. perf_mmap_free_page(rb->data_pages[i]);
  690. perf_mmap_free_page(rb->user_page);
  691. fail_user_page:
  692. kfree(rb);
  693. fail:
  694. return NULL;
  695. }
  696. void rb_free(struct perf_buffer *rb)
  697. {
  698. int i;
  699. perf_mmap_free_page(rb->user_page);
  700. for (i = 0; i < rb->nr_pages; i++)
  701. perf_mmap_free_page(rb->data_pages[i]);
  702. kfree(rb);
  703. }
  704. #else
  705. static struct page *
  706. __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
  707. {
  708. /* The '>' counts in the user page. */
  709. if (pgoff > data_page_nr(rb))
  710. return NULL;
  711. return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
  712. }
  713. static void perf_mmap_unmark_page(void *addr)
  714. {
  715. struct page *page = vmalloc_to_page(addr);
  716. page->mapping = NULL;
  717. }
  718. static void rb_free_work(struct work_struct *work)
  719. {
  720. struct perf_buffer *rb;
  721. void *base;
  722. int i, nr;
  723. rb = container_of(work, struct perf_buffer, work);
  724. nr = data_page_nr(rb);
  725. base = rb->user_page;
  726. /* The '<=' counts in the user page. */
  727. for (i = 0; i <= nr; i++)
  728. perf_mmap_unmark_page(base + (i * PAGE_SIZE));
  729. vfree(base);
  730. kfree(rb);
  731. }
  732. void rb_free(struct perf_buffer *rb)
  733. {
  734. schedule_work(&rb->work);
  735. }
  736. struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
  737. {
  738. struct perf_buffer *rb;
  739. unsigned long size;
  740. void *all_buf;
  741. size = sizeof(struct perf_buffer);
  742. size += sizeof(void *);
  743. rb = kzalloc(size, GFP_KERNEL);
  744. if (!rb)
  745. goto fail;
  746. INIT_WORK(&rb->work, rb_free_work);
  747. all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
  748. if (!all_buf)
  749. goto fail_all_buf;
  750. rb->user_page = all_buf;
  751. rb->data_pages[0] = all_buf + PAGE_SIZE;
  752. if (nr_pages) {
  753. rb->nr_pages = 1;
  754. rb->page_order = ilog2(nr_pages);
  755. }
  756. ring_buffer_init(rb, watermark, flags);
  757. return rb;
  758. fail_all_buf:
  759. kfree(rb);
  760. fail:
  761. return NULL;
  762. }
  763. #endif
  764. struct page *
  765. perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
  766. {
  767. if (rb->aux_nr_pages) {
  768. /* above AUX space */
  769. if (pgoff > rb->aux_pgoff + rb->aux_nr_pages)
  770. return NULL;
  771. /* AUX space */
  772. if (pgoff >= rb->aux_pgoff) {
  773. int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages);
  774. return virt_to_page(rb->aux_pages[aux_pgoff]);
  775. }
  776. }
  777. return __perf_mmap_to_page(rb, pgoff);
  778. }