relay.c 26 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097
  1. /*
  2. * Public API and common code for kernel->userspace relay file support.
  3. *
  4. * See Documentation/filesystems/relayfs.txt for an overview of relayfs.
  5. *
  6. * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
  7. * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
  8. *
  9. * Moved to kernel/relay.c by Paul Mundt, 2006.
  10. * November 2006 - CPU hotplug support by Mathieu Desnoyers
  11. * (mathieu.desnoyers@polymtl.ca)
  12. *
  13. * This file is released under the GPL.
  14. */
  15. #include <linux/errno.h>
  16. #include <linux/stddef.h>
  17. #include <linux/slab.h>
  18. #include <linux/module.h>
  19. #include <linux/string.h>
  20. #include <linux/relay.h>
  21. #include <linux/vmalloc.h>
  22. #include <linux/mm.h>
  23. #include <linux/cpu.h>
  24. /* list of open channels, for cpu hotplug */
  25. static DEFINE_MUTEX(relay_channels_mutex);
  26. static LIST_HEAD(relay_channels);
  27. /*
  28. * close() vm_op implementation for relay file mapping.
  29. */
  30. static void relay_file_mmap_close(struct vm_area_struct *vma)
  31. {
  32. struct rchan_buf *buf = vma->vm_private_data;
  33. buf->chan->cb->buf_unmapped(buf, vma->vm_file);
  34. }
  35. /*
  36. * nopage() vm_op implementation for relay file mapping.
  37. */
  38. static struct page *relay_buf_nopage(struct vm_area_struct *vma,
  39. unsigned long address,
  40. int *type)
  41. {
  42. struct page *page;
  43. struct rchan_buf *buf = vma->vm_private_data;
  44. unsigned long offset = address - vma->vm_start;
  45. if (address > vma->vm_end)
  46. return NOPAGE_SIGBUS; /* Disallow mremap */
  47. if (!buf)
  48. return NOPAGE_OOM;
  49. page = vmalloc_to_page(buf->start + offset);
  50. if (!page)
  51. return NOPAGE_OOM;
  52. get_page(page);
  53. if (type)
  54. *type = VM_FAULT_MINOR;
  55. return page;
  56. }
  57. /*
  58. * vm_ops for relay file mappings.
  59. */
  60. static struct vm_operations_struct relay_file_mmap_ops = {
  61. .nopage = relay_buf_nopage,
  62. .close = relay_file_mmap_close,
  63. };
  64. /**
  65. * relay_mmap_buf: - mmap channel buffer to process address space
  66. * @buf: relay channel buffer
  67. * @vma: vm_area_struct describing memory to be mapped
  68. *
  69. * Returns 0 if ok, negative on error
  70. *
  71. * Caller should already have grabbed mmap_sem.
  72. */
  73. int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)
  74. {
  75. unsigned long length = vma->vm_end - vma->vm_start;
  76. struct file *filp = vma->vm_file;
  77. if (!buf)
  78. return -EBADF;
  79. if (length != (unsigned long)buf->chan->alloc_size)
  80. return -EINVAL;
  81. vma->vm_ops = &relay_file_mmap_ops;
  82. vma->vm_private_data = buf;
  83. buf->chan->cb->buf_mapped(buf, filp);
  84. return 0;
  85. }
  86. /**
  87. * relay_alloc_buf - allocate a channel buffer
  88. * @buf: the buffer struct
  89. * @size: total size of the buffer
  90. *
  91. * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
  92. * passed in size will get page aligned, if it isn't already.
  93. */
  94. static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)
  95. {
  96. void *mem;
  97. unsigned int i, j, n_pages;
  98. *size = PAGE_ALIGN(*size);
  99. n_pages = *size >> PAGE_SHIFT;
  100. buf->page_array = kcalloc(n_pages, sizeof(struct page *), GFP_KERNEL);
  101. if (!buf->page_array)
  102. return NULL;
  103. for (i = 0; i < n_pages; i++) {
  104. buf->page_array[i] = alloc_page(GFP_KERNEL);
  105. if (unlikely(!buf->page_array[i]))
  106. goto depopulate;
  107. }
  108. mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);
  109. if (!mem)
  110. goto depopulate;
  111. memset(mem, 0, *size);
  112. buf->page_count = n_pages;
  113. return mem;
  114. depopulate:
  115. for (j = 0; j < i; j++)
  116. __free_page(buf->page_array[j]);
  117. kfree(buf->page_array);
  118. return NULL;
  119. }
  120. /**
  121. * relay_create_buf - allocate and initialize a channel buffer
  122. * @chan: the relay channel
  123. *
  124. * Returns channel buffer if successful, %NULL otherwise.
  125. */
  126. struct rchan_buf *relay_create_buf(struct rchan *chan)
  127. {
  128. struct rchan_buf *buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);
  129. if (!buf)
  130. return NULL;
  131. buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL);
  132. if (!buf->padding)
  133. goto free_buf;
  134. buf->start = relay_alloc_buf(buf, &chan->alloc_size);
  135. if (!buf->start)
  136. goto free_buf;
  137. buf->chan = chan;
  138. kref_get(&buf->chan->kref);
  139. return buf;
  140. free_buf:
  141. kfree(buf->padding);
  142. kfree(buf);
  143. return NULL;
  144. }
  145. /**
  146. * relay_destroy_channel - free the channel struct
  147. * @kref: target kernel reference that contains the relay channel
  148. *
  149. * Should only be called from kref_put().
  150. */
  151. void relay_destroy_channel(struct kref *kref)
  152. {
  153. struct rchan *chan = container_of(kref, struct rchan, kref);
  154. kfree(chan);
  155. }
  156. /**
  157. * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
  158. * @buf: the buffer struct
  159. */
  160. void relay_destroy_buf(struct rchan_buf *buf)
  161. {
  162. struct rchan *chan = buf->chan;
  163. unsigned int i;
  164. if (likely(buf->start)) {
  165. vunmap(buf->start);
  166. for (i = 0; i < buf->page_count; i++)
  167. __free_page(buf->page_array[i]);
  168. kfree(buf->page_array);
  169. }
  170. chan->buf[buf->cpu] = NULL;
  171. kfree(buf->padding);
  172. kfree(buf);
  173. kref_put(&chan->kref, relay_destroy_channel);
  174. }
  175. /**
  176. * relay_remove_buf - remove a channel buffer
  177. * @kref: target kernel reference that contains the relay buffer
  178. *
  179. * Removes the file from the fileystem, which also frees the
  180. * rchan_buf_struct and the channel buffer. Should only be called from
  181. * kref_put().
  182. */
  183. void relay_remove_buf(struct kref *kref)
  184. {
  185. struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);
  186. buf->chan->cb->remove_buf_file(buf->dentry);
  187. relay_destroy_buf(buf);
  188. }
  189. /**
  190. * relay_buf_empty - boolean, is the channel buffer empty?
  191. * @buf: channel buffer
  192. *
  193. * Returns 1 if the buffer is empty, 0 otherwise.
  194. */
  195. int relay_buf_empty(struct rchan_buf *buf)
  196. {
  197. return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;
  198. }
  199. EXPORT_SYMBOL_GPL(relay_buf_empty);
  200. /**
  201. * relay_buf_full - boolean, is the channel buffer full?
  202. * @buf: channel buffer
  203. *
  204. * Returns 1 if the buffer is full, 0 otherwise.
  205. */
  206. int relay_buf_full(struct rchan_buf *buf)
  207. {
  208. size_t ready = buf->subbufs_produced - buf->subbufs_consumed;
  209. return (ready >= buf->chan->n_subbufs) ? 1 : 0;
  210. }
  211. EXPORT_SYMBOL_GPL(relay_buf_full);
  212. /*
  213. * High-level relay kernel API and associated functions.
  214. */
  215. /*
  216. * rchan_callback implementations defining default channel behavior. Used
  217. * in place of corresponding NULL values in client callback struct.
  218. */
  219. /*
  220. * subbuf_start() default callback. Does nothing.
  221. */
  222. static int subbuf_start_default_callback (struct rchan_buf *buf,
  223. void *subbuf,
  224. void *prev_subbuf,
  225. size_t prev_padding)
  226. {
  227. if (relay_buf_full(buf))
  228. return 0;
  229. return 1;
  230. }
  231. /*
  232. * buf_mapped() default callback. Does nothing.
  233. */
  234. static void buf_mapped_default_callback(struct rchan_buf *buf,
  235. struct file *filp)
  236. {
  237. }
  238. /*
  239. * buf_unmapped() default callback. Does nothing.
  240. */
  241. static void buf_unmapped_default_callback(struct rchan_buf *buf,
  242. struct file *filp)
  243. {
  244. }
  245. /*
  246. * create_buf_file_create() default callback. Does nothing.
  247. */
  248. static struct dentry *create_buf_file_default_callback(const char *filename,
  249. struct dentry *parent,
  250. int mode,
  251. struct rchan_buf *buf,
  252. int *is_global)
  253. {
  254. return NULL;
  255. }
  256. /*
  257. * remove_buf_file() default callback. Does nothing.
  258. */
  259. static int remove_buf_file_default_callback(struct dentry *dentry)
  260. {
  261. return -EINVAL;
  262. }
  263. /* relay channel default callbacks */
  264. static struct rchan_callbacks default_channel_callbacks = {
  265. .subbuf_start = subbuf_start_default_callback,
  266. .buf_mapped = buf_mapped_default_callback,
  267. .buf_unmapped = buf_unmapped_default_callback,
  268. .create_buf_file = create_buf_file_default_callback,
  269. .remove_buf_file = remove_buf_file_default_callback,
  270. };
  271. /**
  272. * wakeup_readers - wake up readers waiting on a channel
  273. * @work: work struct that contains the the channel buffer
  274. *
  275. * This is the work function used to defer reader waking. The
  276. * reason waking is deferred is that calling directly from write
  277. * causes problems if you're writing from say the scheduler.
  278. */
  279. static void wakeup_readers(struct work_struct *work)
  280. {
  281. struct rchan_buf *buf =
  282. container_of(work, struct rchan_buf, wake_readers.work);
  283. wake_up_interruptible(&buf->read_wait);
  284. }
  285. /**
  286. * __relay_reset - reset a channel buffer
  287. * @buf: the channel buffer
  288. * @init: 1 if this is a first-time initialization
  289. *
  290. * See relay_reset() for description of effect.
  291. */
  292. static void __relay_reset(struct rchan_buf *buf, unsigned int init)
  293. {
  294. size_t i;
  295. if (init) {
  296. init_waitqueue_head(&buf->read_wait);
  297. kref_init(&buf->kref);
  298. INIT_DELAYED_WORK(&buf->wake_readers, NULL);
  299. } else {
  300. cancel_delayed_work(&buf->wake_readers);
  301. flush_scheduled_work();
  302. }
  303. buf->subbufs_produced = 0;
  304. buf->subbufs_consumed = 0;
  305. buf->bytes_consumed = 0;
  306. buf->finalized = 0;
  307. buf->data = buf->start;
  308. buf->offset = 0;
  309. for (i = 0; i < buf->chan->n_subbufs; i++)
  310. buf->padding[i] = 0;
  311. buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);
  312. }
  313. /**
  314. * relay_reset - reset the channel
  315. * @chan: the channel
  316. *
  317. * This has the effect of erasing all data from all channel buffers
  318. * and restarting the channel in its initial state. The buffers
  319. * are not freed, so any mappings are still in effect.
  320. *
  321. * NOTE. Care should be taken that the channel isn't actually
  322. * being used by anything when this call is made.
  323. */
  324. void relay_reset(struct rchan *chan)
  325. {
  326. unsigned int i;
  327. if (!chan)
  328. return;
  329. if (chan->is_global && chan->buf[0]) {
  330. __relay_reset(chan->buf[0], 0);
  331. return;
  332. }
  333. mutex_lock(&relay_channels_mutex);
  334. for_each_online_cpu(i)
  335. if (chan->buf[i])
  336. __relay_reset(chan->buf[i], 0);
  337. mutex_unlock(&relay_channels_mutex);
  338. }
  339. EXPORT_SYMBOL_GPL(relay_reset);
  340. /*
  341. * relay_open_buf - create a new relay channel buffer
  342. *
  343. * used by relay_open() and CPU hotplug.
  344. */
  345. static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)
  346. {
  347. struct rchan_buf *buf = NULL;
  348. struct dentry *dentry;
  349. char *tmpname;
  350. if (chan->is_global)
  351. return chan->buf[0];
  352. tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL);
  353. if (!tmpname)
  354. goto end;
  355. snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);
  356. buf = relay_create_buf(chan);
  357. if (!buf)
  358. goto free_name;
  359. buf->cpu = cpu;
  360. __relay_reset(buf, 1);
  361. /* Create file in fs */
  362. dentry = chan->cb->create_buf_file(tmpname, chan->parent, S_IRUSR,
  363. buf, &chan->is_global);
  364. if (!dentry)
  365. goto free_buf;
  366. buf->dentry = dentry;
  367. if(chan->is_global) {
  368. chan->buf[0] = buf;
  369. buf->cpu = 0;
  370. }
  371. goto free_name;
  372. free_buf:
  373. relay_destroy_buf(buf);
  374. free_name:
  375. kfree(tmpname);
  376. end:
  377. return buf;
  378. }
  379. /**
  380. * relay_close_buf - close a channel buffer
  381. * @buf: channel buffer
  382. *
  383. * Marks the buffer finalized and restores the default callbacks.
  384. * The channel buffer and channel buffer data structure are then freed
  385. * automatically when the last reference is given up.
  386. */
  387. static void relay_close_buf(struct rchan_buf *buf)
  388. {
  389. buf->finalized = 1;
  390. cancel_delayed_work(&buf->wake_readers);
  391. flush_scheduled_work();
  392. kref_put(&buf->kref, relay_remove_buf);
  393. }
  394. static void setup_callbacks(struct rchan *chan,
  395. struct rchan_callbacks *cb)
  396. {
  397. if (!cb) {
  398. chan->cb = &default_channel_callbacks;
  399. return;
  400. }
  401. if (!cb->subbuf_start)
  402. cb->subbuf_start = subbuf_start_default_callback;
  403. if (!cb->buf_mapped)
  404. cb->buf_mapped = buf_mapped_default_callback;
  405. if (!cb->buf_unmapped)
  406. cb->buf_unmapped = buf_unmapped_default_callback;
  407. if (!cb->create_buf_file)
  408. cb->create_buf_file = create_buf_file_default_callback;
  409. if (!cb->remove_buf_file)
  410. cb->remove_buf_file = remove_buf_file_default_callback;
  411. chan->cb = cb;
  412. }
  413. /**
  414. * relay_hotcpu_callback - CPU hotplug callback
  415. * @nb: notifier block
  416. * @action: hotplug action to take
  417. * @hcpu: CPU number
  418. *
  419. * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
  420. */
  421. static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb,
  422. unsigned long action,
  423. void *hcpu)
  424. {
  425. unsigned int hotcpu = (unsigned long)hcpu;
  426. struct rchan *chan;
  427. switch(action) {
  428. case CPU_UP_PREPARE:
  429. mutex_lock(&relay_channels_mutex);
  430. list_for_each_entry(chan, &relay_channels, list) {
  431. if (chan->buf[hotcpu])
  432. continue;
  433. chan->buf[hotcpu] = relay_open_buf(chan, hotcpu);
  434. if(!chan->buf[hotcpu]) {
  435. printk(KERN_ERR
  436. "relay_hotcpu_callback: cpu %d buffer "
  437. "creation failed\n", hotcpu);
  438. mutex_unlock(&relay_channels_mutex);
  439. return NOTIFY_BAD;
  440. }
  441. }
  442. mutex_unlock(&relay_channels_mutex);
  443. break;
  444. case CPU_DEAD:
  445. /* No need to flush the cpu : will be flushed upon
  446. * final relay_flush() call. */
  447. break;
  448. }
  449. return NOTIFY_OK;
  450. }
  451. /**
  452. * relay_open - create a new relay channel
  453. * @base_filename: base name of files to create
  454. * @parent: dentry of parent directory, %NULL for root directory
  455. * @subbuf_size: size of sub-buffers
  456. * @n_subbufs: number of sub-buffers
  457. * @cb: client callback functions
  458. * @private_data: user-defined data
  459. *
  460. * Returns channel pointer if successful, %NULL otherwise.
  461. *
  462. * Creates a channel buffer for each cpu using the sizes and
  463. * attributes specified. The created channel buffer files
  464. * will be named base_filename0...base_filenameN-1. File
  465. * permissions will be %S_IRUSR.
  466. */
  467. struct rchan *relay_open(const char *base_filename,
  468. struct dentry *parent,
  469. size_t subbuf_size,
  470. size_t n_subbufs,
  471. struct rchan_callbacks *cb,
  472. void *private_data)
  473. {
  474. unsigned int i;
  475. struct rchan *chan;
  476. if (!base_filename)
  477. return NULL;
  478. if (!(subbuf_size && n_subbufs))
  479. return NULL;
  480. chan = kzalloc(sizeof(struct rchan), GFP_KERNEL);
  481. if (!chan)
  482. return NULL;
  483. chan->version = RELAYFS_CHANNEL_VERSION;
  484. chan->n_subbufs = n_subbufs;
  485. chan->subbuf_size = subbuf_size;
  486. chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs);
  487. chan->parent = parent;
  488. chan->private_data = private_data;
  489. strlcpy(chan->base_filename, base_filename, NAME_MAX);
  490. setup_callbacks(chan, cb);
  491. kref_init(&chan->kref);
  492. mutex_lock(&relay_channels_mutex);
  493. for_each_online_cpu(i) {
  494. chan->buf[i] = relay_open_buf(chan, i);
  495. if (!chan->buf[i])
  496. goto free_bufs;
  497. }
  498. list_add(&chan->list, &relay_channels);
  499. mutex_unlock(&relay_channels_mutex);
  500. return chan;
  501. free_bufs:
  502. for_each_online_cpu(i) {
  503. if (!chan->buf[i])
  504. break;
  505. relay_close_buf(chan->buf[i]);
  506. }
  507. kref_put(&chan->kref, relay_destroy_channel);
  508. mutex_unlock(&relay_channels_mutex);
  509. return NULL;
  510. }
  511. EXPORT_SYMBOL_GPL(relay_open);
  512. /**
  513. * relay_switch_subbuf - switch to a new sub-buffer
  514. * @buf: channel buffer
  515. * @length: size of current event
  516. *
  517. * Returns either the length passed in or 0 if full.
  518. *
  519. * Performs sub-buffer-switch tasks such as invoking callbacks,
  520. * updating padding counts, waking up readers, etc.
  521. */
  522. size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)
  523. {
  524. void *old, *new;
  525. size_t old_subbuf, new_subbuf;
  526. if (unlikely(length > buf->chan->subbuf_size))
  527. goto toobig;
  528. if (buf->offset != buf->chan->subbuf_size + 1) {
  529. buf->prev_padding = buf->chan->subbuf_size - buf->offset;
  530. old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
  531. buf->padding[old_subbuf] = buf->prev_padding;
  532. buf->subbufs_produced++;
  533. buf->dentry->d_inode->i_size += buf->chan->subbuf_size -
  534. buf->padding[old_subbuf];
  535. smp_mb();
  536. if (waitqueue_active(&buf->read_wait)) {
  537. PREPARE_DELAYED_WORK(&buf->wake_readers,
  538. wakeup_readers);
  539. schedule_delayed_work(&buf->wake_readers, 1);
  540. }
  541. }
  542. old = buf->data;
  543. new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
  544. new = buf->start + new_subbuf * buf->chan->subbuf_size;
  545. buf->offset = 0;
  546. if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {
  547. buf->offset = buf->chan->subbuf_size + 1;
  548. return 0;
  549. }
  550. buf->data = new;
  551. buf->padding[new_subbuf] = 0;
  552. if (unlikely(length + buf->offset > buf->chan->subbuf_size))
  553. goto toobig;
  554. return length;
  555. toobig:
  556. buf->chan->last_toobig = length;
  557. return 0;
  558. }
  559. EXPORT_SYMBOL_GPL(relay_switch_subbuf);
  560. /**
  561. * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
  562. * @chan: the channel
  563. * @cpu: the cpu associated with the channel buffer to update
  564. * @subbufs_consumed: number of sub-buffers to add to current buf's count
  565. *
  566. * Adds to the channel buffer's consumed sub-buffer count.
  567. * subbufs_consumed should be the number of sub-buffers newly consumed,
  568. * not the total consumed.
  569. *
  570. * NOTE. Kernel clients don't need to call this function if the channel
  571. * mode is 'overwrite'.
  572. */
  573. void relay_subbufs_consumed(struct rchan *chan,
  574. unsigned int cpu,
  575. size_t subbufs_consumed)
  576. {
  577. struct rchan_buf *buf;
  578. if (!chan)
  579. return;
  580. if (cpu >= NR_CPUS || !chan->buf[cpu])
  581. return;
  582. buf = chan->buf[cpu];
  583. buf->subbufs_consumed += subbufs_consumed;
  584. if (buf->subbufs_consumed > buf->subbufs_produced)
  585. buf->subbufs_consumed = buf->subbufs_produced;
  586. }
  587. EXPORT_SYMBOL_GPL(relay_subbufs_consumed);
  588. /**
  589. * relay_close - close the channel
  590. * @chan: the channel
  591. *
  592. * Closes all channel buffers and frees the channel.
  593. */
  594. void relay_close(struct rchan *chan)
  595. {
  596. unsigned int i;
  597. if (!chan)
  598. return;
  599. mutex_lock(&relay_channels_mutex);
  600. if (chan->is_global && chan->buf[0])
  601. relay_close_buf(chan->buf[0]);
  602. else
  603. for_each_possible_cpu(i)
  604. if (chan->buf[i])
  605. relay_close_buf(chan->buf[i]);
  606. if (chan->last_toobig)
  607. printk(KERN_WARNING "relay: one or more items not logged "
  608. "[item size (%Zd) > sub-buffer size (%Zd)]\n",
  609. chan->last_toobig, chan->subbuf_size);
  610. list_del(&chan->list);
  611. kref_put(&chan->kref, relay_destroy_channel);
  612. mutex_unlock(&relay_channels_mutex);
  613. }
  614. EXPORT_SYMBOL_GPL(relay_close);
  615. /**
  616. * relay_flush - close the channel
  617. * @chan: the channel
  618. *
  619. * Flushes all channel buffers, i.e. forces buffer switch.
  620. */
  621. void relay_flush(struct rchan *chan)
  622. {
  623. unsigned int i;
  624. if (!chan)
  625. return;
  626. if (chan->is_global && chan->buf[0]) {
  627. relay_switch_subbuf(chan->buf[0], 0);
  628. return;
  629. }
  630. mutex_lock(&relay_channels_mutex);
  631. for_each_possible_cpu(i)
  632. if (chan->buf[i])
  633. relay_switch_subbuf(chan->buf[i], 0);
  634. mutex_unlock(&relay_channels_mutex);
  635. }
  636. EXPORT_SYMBOL_GPL(relay_flush);
  637. /**
  638. * relay_file_open - open file op for relay files
  639. * @inode: the inode
  640. * @filp: the file
  641. *
  642. * Increments the channel buffer refcount.
  643. */
  644. static int relay_file_open(struct inode *inode, struct file *filp)
  645. {
  646. struct rchan_buf *buf = inode->i_private;
  647. kref_get(&buf->kref);
  648. filp->private_data = buf;
  649. return 0;
  650. }
  651. /**
  652. * relay_file_mmap - mmap file op for relay files
  653. * @filp: the file
  654. * @vma: the vma describing what to map
  655. *
  656. * Calls upon relay_mmap_buf() to map the file into user space.
  657. */
  658. static int relay_file_mmap(struct file *filp, struct vm_area_struct *vma)
  659. {
  660. struct rchan_buf *buf = filp->private_data;
  661. return relay_mmap_buf(buf, vma);
  662. }
  663. /**
  664. * relay_file_poll - poll file op for relay files
  665. * @filp: the file
  666. * @wait: poll table
  667. *
  668. * Poll implemention.
  669. */
  670. static unsigned int relay_file_poll(struct file *filp, poll_table *wait)
  671. {
  672. unsigned int mask = 0;
  673. struct rchan_buf *buf = filp->private_data;
  674. if (buf->finalized)
  675. return POLLERR;
  676. if (filp->f_mode & FMODE_READ) {
  677. poll_wait(filp, &buf->read_wait, wait);
  678. if (!relay_buf_empty(buf))
  679. mask |= POLLIN | POLLRDNORM;
  680. }
  681. return mask;
  682. }
  683. /**
  684. * relay_file_release - release file op for relay files
  685. * @inode: the inode
  686. * @filp: the file
  687. *
  688. * Decrements the channel refcount, as the filesystem is
  689. * no longer using it.
  690. */
  691. static int relay_file_release(struct inode *inode, struct file *filp)
  692. {
  693. struct rchan_buf *buf = filp->private_data;
  694. kref_put(&buf->kref, relay_remove_buf);
  695. return 0;
  696. }
  697. /*
  698. * relay_file_read_consume - update the consumed count for the buffer
  699. */
  700. static void relay_file_read_consume(struct rchan_buf *buf,
  701. size_t read_pos,
  702. size_t bytes_consumed)
  703. {
  704. size_t subbuf_size = buf->chan->subbuf_size;
  705. size_t n_subbufs = buf->chan->n_subbufs;
  706. size_t read_subbuf;
  707. if (buf->bytes_consumed + bytes_consumed > subbuf_size) {
  708. relay_subbufs_consumed(buf->chan, buf->cpu, 1);
  709. buf->bytes_consumed = 0;
  710. }
  711. buf->bytes_consumed += bytes_consumed;
  712. read_subbuf = read_pos / buf->chan->subbuf_size;
  713. if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {
  714. if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&
  715. (buf->offset == subbuf_size))
  716. return;
  717. relay_subbufs_consumed(buf->chan, buf->cpu, 1);
  718. buf->bytes_consumed = 0;
  719. }
  720. }
  721. /*
  722. * relay_file_read_avail - boolean, are there unconsumed bytes available?
  723. */
  724. static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos)
  725. {
  726. size_t subbuf_size = buf->chan->subbuf_size;
  727. size_t n_subbufs = buf->chan->n_subbufs;
  728. size_t produced = buf->subbufs_produced;
  729. size_t consumed = buf->subbufs_consumed;
  730. relay_file_read_consume(buf, read_pos, 0);
  731. if (unlikely(buf->offset > subbuf_size)) {
  732. if (produced == consumed)
  733. return 0;
  734. return 1;
  735. }
  736. if (unlikely(produced - consumed >= n_subbufs)) {
  737. consumed = (produced / n_subbufs) * n_subbufs;
  738. buf->subbufs_consumed = consumed;
  739. }
  740. produced = (produced % n_subbufs) * subbuf_size + buf->offset;
  741. consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;
  742. if (consumed > produced)
  743. produced += n_subbufs * subbuf_size;
  744. if (consumed == produced)
  745. return 0;
  746. return 1;
  747. }
  748. /**
  749. * relay_file_read_subbuf_avail - return bytes available in sub-buffer
  750. * @read_pos: file read position
  751. * @buf: relay channel buffer
  752. */
  753. static size_t relay_file_read_subbuf_avail(size_t read_pos,
  754. struct rchan_buf *buf)
  755. {
  756. size_t padding, avail = 0;
  757. size_t read_subbuf, read_offset, write_subbuf, write_offset;
  758. size_t subbuf_size = buf->chan->subbuf_size;
  759. write_subbuf = (buf->data - buf->start) / subbuf_size;
  760. write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;
  761. read_subbuf = read_pos / subbuf_size;
  762. read_offset = read_pos % subbuf_size;
  763. padding = buf->padding[read_subbuf];
  764. if (read_subbuf == write_subbuf) {
  765. if (read_offset + padding < write_offset)
  766. avail = write_offset - (read_offset + padding);
  767. } else
  768. avail = (subbuf_size - padding) - read_offset;
  769. return avail;
  770. }
  771. /**
  772. * relay_file_read_start_pos - find the first available byte to read
  773. * @read_pos: file read position
  774. * @buf: relay channel buffer
  775. *
  776. * If the @read_pos is in the middle of padding, return the
  777. * position of the first actually available byte, otherwise
  778. * return the original value.
  779. */
  780. static size_t relay_file_read_start_pos(size_t read_pos,
  781. struct rchan_buf *buf)
  782. {
  783. size_t read_subbuf, padding, padding_start, padding_end;
  784. size_t subbuf_size = buf->chan->subbuf_size;
  785. size_t n_subbufs = buf->chan->n_subbufs;
  786. read_subbuf = read_pos / subbuf_size;
  787. padding = buf->padding[read_subbuf];
  788. padding_start = (read_subbuf + 1) * subbuf_size - padding;
  789. padding_end = (read_subbuf + 1) * subbuf_size;
  790. if (read_pos >= padding_start && read_pos < padding_end) {
  791. read_subbuf = (read_subbuf + 1) % n_subbufs;
  792. read_pos = read_subbuf * subbuf_size;
  793. }
  794. return read_pos;
  795. }
  796. /**
  797. * relay_file_read_end_pos - return the new read position
  798. * @read_pos: file read position
  799. * @buf: relay channel buffer
  800. * @count: number of bytes to be read
  801. */
  802. static size_t relay_file_read_end_pos(struct rchan_buf *buf,
  803. size_t read_pos,
  804. size_t count)
  805. {
  806. size_t read_subbuf, padding, end_pos;
  807. size_t subbuf_size = buf->chan->subbuf_size;
  808. size_t n_subbufs = buf->chan->n_subbufs;
  809. read_subbuf = read_pos / subbuf_size;
  810. padding = buf->padding[read_subbuf];
  811. if (read_pos % subbuf_size + count + padding == subbuf_size)
  812. end_pos = (read_subbuf + 1) * subbuf_size;
  813. else
  814. end_pos = read_pos + count;
  815. if (end_pos >= subbuf_size * n_subbufs)
  816. end_pos = 0;
  817. return end_pos;
  818. }
  819. /*
  820. * subbuf_read_actor - read up to one subbuf's worth of data
  821. */
  822. static int subbuf_read_actor(size_t read_start,
  823. struct rchan_buf *buf,
  824. size_t avail,
  825. read_descriptor_t *desc,
  826. read_actor_t actor)
  827. {
  828. void *from;
  829. int ret = 0;
  830. from = buf->start + read_start;
  831. ret = avail;
  832. if (copy_to_user(desc->arg.buf, from, avail)) {
  833. desc->error = -EFAULT;
  834. ret = 0;
  835. }
  836. desc->arg.data += ret;
  837. desc->written += ret;
  838. desc->count -= ret;
  839. return ret;
  840. }
  841. /*
  842. * subbuf_send_actor - send up to one subbuf's worth of data
  843. */
  844. static int subbuf_send_actor(size_t read_start,
  845. struct rchan_buf *buf,
  846. size_t avail,
  847. read_descriptor_t *desc,
  848. read_actor_t actor)
  849. {
  850. unsigned long pidx, poff;
  851. unsigned int subbuf_pages;
  852. int ret = 0;
  853. subbuf_pages = buf->chan->alloc_size >> PAGE_SHIFT;
  854. pidx = (read_start / PAGE_SIZE) % subbuf_pages;
  855. poff = read_start & ~PAGE_MASK;
  856. while (avail) {
  857. struct page *p = buf->page_array[pidx];
  858. unsigned int len;
  859. len = PAGE_SIZE - poff;
  860. if (len > avail)
  861. len = avail;
  862. len = actor(desc, p, poff, len);
  863. if (desc->error)
  864. break;
  865. avail -= len;
  866. ret += len;
  867. poff = 0;
  868. pidx = (pidx + 1) % subbuf_pages;
  869. }
  870. return ret;
  871. }
  872. typedef int (*subbuf_actor_t) (size_t read_start,
  873. struct rchan_buf *buf,
  874. size_t avail,
  875. read_descriptor_t *desc,
  876. read_actor_t actor);
  877. /*
  878. * relay_file_read_subbufs - read count bytes, bridging subbuf boundaries
  879. */
  880. static ssize_t relay_file_read_subbufs(struct file *filp, loff_t *ppos,
  881. subbuf_actor_t subbuf_actor,
  882. read_actor_t actor,
  883. read_descriptor_t *desc)
  884. {
  885. struct rchan_buf *buf = filp->private_data;
  886. size_t read_start, avail;
  887. int ret;
  888. if (!desc->count)
  889. return 0;
  890. mutex_lock(&filp->f_path.dentry->d_inode->i_mutex);
  891. do {
  892. if (!relay_file_read_avail(buf, *ppos))
  893. break;
  894. read_start = relay_file_read_start_pos(*ppos, buf);
  895. avail = relay_file_read_subbuf_avail(read_start, buf);
  896. if (!avail)
  897. break;
  898. avail = min(desc->count, avail);
  899. ret = subbuf_actor(read_start, buf, avail, desc, actor);
  900. if (desc->error < 0)
  901. break;
  902. if (ret) {
  903. relay_file_read_consume(buf, read_start, ret);
  904. *ppos = relay_file_read_end_pos(buf, read_start, ret);
  905. }
  906. } while (desc->count && ret);
  907. mutex_unlock(&filp->f_path.dentry->d_inode->i_mutex);
  908. return desc->written;
  909. }
  910. static ssize_t relay_file_read(struct file *filp,
  911. char __user *buffer,
  912. size_t count,
  913. loff_t *ppos)
  914. {
  915. read_descriptor_t desc;
  916. desc.written = 0;
  917. desc.count = count;
  918. desc.arg.buf = buffer;
  919. desc.error = 0;
  920. return relay_file_read_subbufs(filp, ppos, subbuf_read_actor,
  921. NULL, &desc);
  922. }
  923. static ssize_t relay_file_sendfile(struct file *filp,
  924. loff_t *ppos,
  925. size_t count,
  926. read_actor_t actor,
  927. void *target)
  928. {
  929. read_descriptor_t desc;
  930. desc.written = 0;
  931. desc.count = count;
  932. desc.arg.data = target;
  933. desc.error = 0;
  934. return relay_file_read_subbufs(filp, ppos, subbuf_send_actor,
  935. actor, &desc);
  936. }
  937. const struct file_operations relay_file_operations = {
  938. .open = relay_file_open,
  939. .poll = relay_file_poll,
  940. .mmap = relay_file_mmap,
  941. .read = relay_file_read,
  942. .llseek = no_llseek,
  943. .release = relay_file_release,
  944. .sendfile = relay_file_sendfile,
  945. };
  946. EXPORT_SYMBOL_GPL(relay_file_operations);
  947. static __init int relay_init(void)
  948. {
  949. hotcpu_notifier(relay_hotcpu_callback, 0);
  950. return 0;
  951. }
  952. module_init(relay_init);