tty_buffer.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Tty buffer allocation management
  4. */
  5. #include <linux/types.h>
  6. #include <linux/errno.h>
  7. #include <linux/tty.h>
  8. #include <linux/tty_driver.h>
  9. #include <linux/tty_flip.h>
  10. #include <linux/timer.h>
  11. #include <linux/string.h>
  12. #include <linux/slab.h>
  13. #include <linux/sched.h>
  14. #include <linux/wait.h>
  15. #include <linux/bitops.h>
  16. #include <linux/delay.h>
  17. #include <linux/module.h>
  18. #include <linux/ratelimit.h>
  19. #define MIN_TTYB_SIZE 256
  20. #define TTYB_ALIGN_MASK 255
  21. /*
  22. * Byte threshold to limit memory consumption for flip buffers.
  23. * The actual memory limit is > 2x this amount.
  24. */
  25. #define TTYB_DEFAULT_MEM_LIMIT (640 * 1024UL)
  26. /*
  27. * We default to dicing tty buffer allocations to this many characters
  28. * in order to avoid multiple page allocations. We know the size of
  29. * tty_buffer itself but it must also be taken into account that the
  30. * the buffer is 256 byte aligned. See tty_buffer_find for the allocation
  31. * logic this must match
  32. */
  33. #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF)
  34. /**
  35. * tty_buffer_lock_exclusive - gain exclusive access to buffer
  36. * tty_buffer_unlock_exclusive - release exclusive access
  37. *
  38. * @port: tty port owning the flip buffer
  39. *
  40. * Guarantees safe use of the line discipline's receive_buf() method by
  41. * excluding the buffer work and any pending flush from using the flip
  42. * buffer. Data can continue to be added concurrently to the flip buffer
  43. * from the driver side.
  44. *
  45. * On release, the buffer work is restarted if there is data in the
  46. * flip buffer
  47. */
  48. void tty_buffer_lock_exclusive(struct tty_port *port)
  49. {
  50. struct tty_bufhead *buf = &port->buf;
  51. atomic_inc(&buf->priority);
  52. mutex_lock(&buf->lock);
  53. }
  54. EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive);
  55. void tty_buffer_unlock_exclusive(struct tty_port *port)
  56. {
  57. struct tty_bufhead *buf = &port->buf;
  58. int restart;
  59. restart = buf->head->commit != buf->head->read;
  60. atomic_dec(&buf->priority);
  61. mutex_unlock(&buf->lock);
  62. if (restart)
  63. queue_work(system_unbound_wq, &buf->work);
  64. }
  65. EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive);
  66. /**
  67. * tty_buffer_space_avail - return unused buffer space
  68. * @port: tty port owning the flip buffer
  69. *
  70. * Returns the # of bytes which can be written by the driver without
  71. * reaching the buffer limit.
  72. *
  73. * Note: this does not guarantee that memory is available to write
  74. * the returned # of bytes (use tty_prepare_flip_string_xxx() to
  75. * pre-allocate if memory guarantee is required).
  76. */
  77. int tty_buffer_space_avail(struct tty_port *port)
  78. {
  79. int space = port->buf.mem_limit - atomic_read(&port->buf.mem_used);
  80. return max(space, 0);
  81. }
  82. EXPORT_SYMBOL_GPL(tty_buffer_space_avail);
  83. static void tty_buffer_reset(struct tty_buffer *p, size_t size)
  84. {
  85. p->used = 0;
  86. p->size = size;
  87. p->next = NULL;
  88. p->commit = 0;
  89. p->read = 0;
  90. p->flags = 0;
  91. }
  92. /**
  93. * tty_buffer_free_all - free buffers used by a tty
  94. * @port: tty port to free from
  95. *
  96. * Remove all the buffers pending on a tty whether queued with data
  97. * or in the free ring. Must be called when the tty is no longer in use
  98. */
  99. void tty_buffer_free_all(struct tty_port *port)
  100. {
  101. struct tty_bufhead *buf = &port->buf;
  102. struct tty_buffer *p, *next;
  103. struct llist_node *llist;
  104. unsigned int freed = 0;
  105. int still_used;
  106. while ((p = buf->head) != NULL) {
  107. buf->head = p->next;
  108. freed += p->size;
  109. if (p->size > 0)
  110. kfree(p);
  111. }
  112. llist = llist_del_all(&buf->free);
  113. llist_for_each_entry_safe(p, next, llist, free)
  114. kfree(p);
  115. tty_buffer_reset(&buf->sentinel, 0);
  116. buf->head = &buf->sentinel;
  117. buf->tail = &buf->sentinel;
  118. still_used = atomic_xchg(&buf->mem_used, 0);
  119. WARN(still_used != freed, "we still have not freed %d bytes!",
  120. still_used - freed);
  121. }
  122. /**
  123. * tty_buffer_alloc - allocate a tty buffer
  124. * @port: tty port
  125. * @size: desired size (characters)
  126. *
  127. * Allocate a new tty buffer to hold the desired number of characters.
  128. * We round our buffers off in 256 character chunks to get better
  129. * allocation behaviour.
  130. * Return NULL if out of memory or the allocation would exceed the
  131. * per device queue
  132. */
  133. static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
  134. {
  135. struct llist_node *free;
  136. struct tty_buffer *p;
  137. /* Round the buffer size out */
  138. size = __ALIGN_MASK(size, TTYB_ALIGN_MASK);
  139. if (size <= MIN_TTYB_SIZE) {
  140. free = llist_del_first(&port->buf.free);
  141. if (free) {
  142. p = llist_entry(free, struct tty_buffer, free);
  143. goto found;
  144. }
  145. }
  146. /* Should possibly check if this fails for the largest buffer we
  147. have queued and recycle that ? */
  148. if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit)
  149. return NULL;
  150. p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC);
  151. if (p == NULL)
  152. return NULL;
  153. found:
  154. tty_buffer_reset(p, size);
  155. atomic_add(size, &port->buf.mem_used);
  156. return p;
  157. }
  158. /**
  159. * tty_buffer_free - free a tty buffer
  160. * @port: tty port owning the buffer
  161. * @b: the buffer to free
  162. *
  163. * Free a tty buffer, or add it to the free list according to our
  164. * internal strategy
  165. */
  166. static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
  167. {
  168. struct tty_bufhead *buf = &port->buf;
  169. /* Dumb strategy for now - should keep some stats */
  170. WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0);
  171. if (b->size > MIN_TTYB_SIZE)
  172. kfree(b);
  173. else if (b->size > 0)
  174. llist_add(&b->free, &buf->free);
  175. }
  176. /**
  177. * tty_buffer_flush - flush full tty buffers
  178. * @tty: tty to flush
  179. * @ld: optional ldisc ptr (must be referenced)
  180. *
  181. * flush all the buffers containing receive data. If ld != NULL,
  182. * flush the ldisc input buffer.
  183. *
  184. * Locking: takes buffer lock to ensure single-threaded flip buffer
  185. * 'consumer'
  186. */
  187. void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld)
  188. {
  189. struct tty_port *port = tty->port;
  190. struct tty_bufhead *buf = &port->buf;
  191. struct tty_buffer *next;
  192. atomic_inc(&buf->priority);
  193. mutex_lock(&buf->lock);
  194. /* paired w/ release in __tty_buffer_request_room; ensures there are
  195. * no pending memory accesses to the freed buffer
  196. */
  197. while ((next = smp_load_acquire(&buf->head->next)) != NULL) {
  198. tty_buffer_free(port, buf->head);
  199. buf->head = next;
  200. }
  201. buf->head->read = buf->head->commit;
  202. if (ld && ld->ops->flush_buffer)
  203. ld->ops->flush_buffer(tty);
  204. atomic_dec(&buf->priority);
  205. mutex_unlock(&buf->lock);
  206. }
  207. /**
  208. * tty_buffer_request_room - grow tty buffer if needed
  209. * @port: tty port
  210. * @size: size desired
  211. * @flags: buffer flags if new buffer allocated (default = 0)
  212. *
  213. * Make at least size bytes of linear space available for the tty
  214. * buffer. If we fail return the size we managed to find.
  215. *
  216. * Will change over to a new buffer if the current buffer is encoded as
  217. * TTY_NORMAL (so has no flags buffer) and the new buffer requires
  218. * a flags buffer.
  219. */
  220. static int __tty_buffer_request_room(struct tty_port *port, size_t size,
  221. int flags)
  222. {
  223. struct tty_bufhead *buf = &port->buf;
  224. struct tty_buffer *b, *n;
  225. int left, change;
  226. b = buf->tail;
  227. if (b->flags & TTYB_NORMAL)
  228. left = 2 * b->size - b->used;
  229. else
  230. left = b->size - b->used;
  231. change = (b->flags & TTYB_NORMAL) && (~flags & TTYB_NORMAL);
  232. if (change || left < size) {
  233. /* This is the slow path - looking for new buffers to use */
  234. n = tty_buffer_alloc(port, size);
  235. if (n != NULL) {
  236. n->flags = flags;
  237. buf->tail = n;
  238. /* paired w/ acquire in flush_to_ldisc(); ensures
  239. * flush_to_ldisc() sees buffer data.
  240. */
  241. smp_store_release(&b->commit, b->used);
  242. /* paired w/ acquire in flush_to_ldisc(); ensures the
  243. * latest commit value can be read before the head is
  244. * advanced to the next buffer
  245. */
  246. smp_store_release(&b->next, n);
  247. } else if (change)
  248. size = 0;
  249. else
  250. size = left;
  251. }
  252. return size;
  253. }
  254. int tty_buffer_request_room(struct tty_port *port, size_t size)
  255. {
  256. return __tty_buffer_request_room(port, size, 0);
  257. }
  258. EXPORT_SYMBOL_GPL(tty_buffer_request_room);
  259. /**
  260. * tty_insert_flip_string_fixed_flag - Add characters to the tty buffer
  261. * @port: tty port
  262. * @chars: characters
  263. * @flag: flag value for each character
  264. * @size: size
  265. *
  266. * Queue a series of bytes to the tty buffering. All the characters
  267. * passed are marked with the supplied flag. Returns the number added.
  268. */
  269. int tty_insert_flip_string_fixed_flag(struct tty_port *port,
  270. const unsigned char *chars, char flag, size_t size)
  271. {
  272. int copied = 0;
  273. do {
  274. int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
  275. int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0;
  276. int space = __tty_buffer_request_room(port, goal, flags);
  277. struct tty_buffer *tb = port->buf.tail;
  278. if (unlikely(space == 0))
  279. break;
  280. memcpy(char_buf_ptr(tb, tb->used), chars, space);
  281. if (~tb->flags & TTYB_NORMAL)
  282. memset(flag_buf_ptr(tb, tb->used), flag, space);
  283. tb->used += space;
  284. copied += space;
  285. chars += space;
  286. /* There is a small chance that we need to split the data over
  287. several buffers. If this is the case we must loop */
  288. } while (unlikely(size > copied));
  289. return copied;
  290. }
  291. EXPORT_SYMBOL(tty_insert_flip_string_fixed_flag);
  292. /**
  293. * tty_insert_flip_string_flags - Add characters to the tty buffer
  294. * @port: tty port
  295. * @chars: characters
  296. * @flags: flag bytes
  297. * @size: size
  298. *
  299. * Queue a series of bytes to the tty buffering. For each character
  300. * the flags array indicates the status of the character. Returns the
  301. * number added.
  302. */
  303. int tty_insert_flip_string_flags(struct tty_port *port,
  304. const unsigned char *chars, const char *flags, size_t size)
  305. {
  306. int copied = 0;
  307. do {
  308. int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
  309. int space = tty_buffer_request_room(port, goal);
  310. struct tty_buffer *tb = port->buf.tail;
  311. if (unlikely(space == 0))
  312. break;
  313. memcpy(char_buf_ptr(tb, tb->used), chars, space);
  314. memcpy(flag_buf_ptr(tb, tb->used), flags, space);
  315. tb->used += space;
  316. copied += space;
  317. chars += space;
  318. flags += space;
  319. /* There is a small chance that we need to split the data over
  320. several buffers. If this is the case we must loop */
  321. } while (unlikely(size > copied));
  322. return copied;
  323. }
  324. EXPORT_SYMBOL(tty_insert_flip_string_flags);
  325. /**
  326. * __tty_insert_flip_char - Add one character to the tty buffer
  327. * @port: tty port
  328. * @ch: character
  329. * @flag: flag byte
  330. *
  331. * Queue a single byte to the tty buffering, with an optional flag.
  332. * This is the slow path of tty_insert_flip_char.
  333. */
  334. int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag)
  335. {
  336. struct tty_buffer *tb;
  337. int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0;
  338. if (!__tty_buffer_request_room(port, 1, flags))
  339. return 0;
  340. tb = port->buf.tail;
  341. if (~tb->flags & TTYB_NORMAL)
  342. *flag_buf_ptr(tb, tb->used) = flag;
  343. *char_buf_ptr(tb, tb->used++) = ch;
  344. return 1;
  345. }
  346. EXPORT_SYMBOL(__tty_insert_flip_char);
  347. /**
  348. * tty_schedule_flip - push characters to ldisc
  349. * @port: tty port to push from
  350. *
  351. * Takes any pending buffers and transfers their ownership to the
  352. * ldisc side of the queue. It then schedules those characters for
  353. * processing by the line discipline.
  354. */
  355. void tty_schedule_flip(struct tty_port *port)
  356. {
  357. struct tty_bufhead *buf = &port->buf;
  358. /* paired w/ acquire in flush_to_ldisc(); ensures
  359. * flush_to_ldisc() sees buffer data.
  360. */
  361. smp_store_release(&buf->tail->commit, buf->tail->used);
  362. queue_work(system_unbound_wq, &buf->work);
  363. }
  364. EXPORT_SYMBOL(tty_schedule_flip);
  365. /**
  366. * tty_prepare_flip_string - make room for characters
  367. * @port: tty port
  368. * @chars: return pointer for character write area
  369. * @size: desired size
  370. *
  371. * Prepare a block of space in the buffer for data. Returns the length
  372. * available and buffer pointer to the space which is now allocated and
  373. * accounted for as ready for normal characters. This is used for drivers
  374. * that need their own block copy routines into the buffer. There is no
  375. * guarantee the buffer is a DMA target!
  376. */
  377. int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
  378. size_t size)
  379. {
  380. int space = __tty_buffer_request_room(port, size, TTYB_NORMAL);
  381. if (likely(space)) {
  382. struct tty_buffer *tb = port->buf.tail;
  383. *chars = char_buf_ptr(tb, tb->used);
  384. if (~tb->flags & TTYB_NORMAL)
  385. memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space);
  386. tb->used += space;
  387. }
  388. return space;
  389. }
  390. EXPORT_SYMBOL_GPL(tty_prepare_flip_string);
  391. /**
  392. * tty_ldisc_receive_buf - forward data to line discipline
  393. * @ld: line discipline to process input
  394. * @p: char buffer
  395. * @f: TTY_* flags buffer
  396. * @count: number of bytes to process
  397. *
  398. * Callers other than flush_to_ldisc() need to exclude the kworker
  399. * from concurrent use of the line discipline, see paste_selection().
  400. *
  401. * Returns the number of bytes processed
  402. */
  403. int tty_ldisc_receive_buf(struct tty_ldisc *ld, const unsigned char *p,
  404. char *f, int count)
  405. {
  406. if (ld->ops->receive_buf2)
  407. count = ld->ops->receive_buf2(ld->tty, p, f, count);
  408. else {
  409. count = min_t(int, count, ld->tty->receive_room);
  410. if (count && ld->ops->receive_buf)
  411. ld->ops->receive_buf(ld->tty, p, f, count);
  412. }
  413. return count;
  414. }
  415. EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf);
  416. static int
  417. receive_buf(struct tty_port *port, struct tty_buffer *head, int count)
  418. {
  419. unsigned char *p = char_buf_ptr(head, head->read);
  420. char *f = NULL;
  421. int n;
  422. if (~head->flags & TTYB_NORMAL)
  423. f = flag_buf_ptr(head, head->read);
  424. n = port->client_ops->receive_buf(port, p, f, count);
  425. if (n > 0)
  426. memset(p, 0, n);
  427. return n;
  428. }
  429. /**
  430. * flush_to_ldisc
  431. * @work: tty structure passed from work queue.
  432. *
  433. * This routine is called out of the software interrupt to flush data
  434. * from the buffer chain to the line discipline.
  435. *
  436. * The receive_buf method is single threaded for each tty instance.
  437. *
  438. * Locking: takes buffer lock to ensure single-threaded flip buffer
  439. * 'consumer'
  440. */
  441. static void flush_to_ldisc(struct work_struct *work)
  442. {
  443. struct tty_port *port = container_of(work, struct tty_port, buf.work);
  444. struct tty_bufhead *buf = &port->buf;
  445. mutex_lock(&buf->lock);
  446. while (1) {
  447. struct tty_buffer *head = buf->head;
  448. struct tty_buffer *next;
  449. int count;
  450. /* Ldisc or user is trying to gain exclusive access */
  451. if (atomic_read(&buf->priority))
  452. break;
  453. /* paired w/ release in __tty_buffer_request_room();
  454. * ensures commit value read is not stale if the head
  455. * is advancing to the next buffer
  456. */
  457. next = smp_load_acquire(&head->next);
  458. /* paired w/ release in __tty_buffer_request_room() or in
  459. * tty_buffer_flush(); ensures we see the committed buffer data
  460. */
  461. count = smp_load_acquire(&head->commit) - head->read;
  462. if (!count) {
  463. if (next == NULL)
  464. break;
  465. buf->head = next;
  466. tty_buffer_free(port, head);
  467. continue;
  468. }
  469. count = receive_buf(port, head, count);
  470. if (!count)
  471. break;
  472. head->read += count;
  473. if (need_resched())
  474. cond_resched();
  475. }
  476. mutex_unlock(&buf->lock);
  477. }
  478. /**
  479. * tty_flip_buffer_push - terminal
  480. * @port: tty port to push
  481. *
  482. * Queue a push of the terminal flip buffers to the line discipline.
  483. * Can be called from IRQ/atomic context.
  484. *
  485. * In the event of the queue being busy for flipping the work will be
  486. * held off and retried later.
  487. */
  488. void tty_flip_buffer_push(struct tty_port *port)
  489. {
  490. tty_schedule_flip(port);
  491. }
  492. EXPORT_SYMBOL(tty_flip_buffer_push);
  493. /**
  494. * tty_buffer_init - prepare a tty buffer structure
  495. * @port: tty port to initialise
  496. *
  497. * Set up the initial state of the buffer management for a tty device.
  498. * Must be called before the other tty buffer functions are used.
  499. */
  500. void tty_buffer_init(struct tty_port *port)
  501. {
  502. struct tty_bufhead *buf = &port->buf;
  503. mutex_init(&buf->lock);
  504. tty_buffer_reset(&buf->sentinel, 0);
  505. buf->head = &buf->sentinel;
  506. buf->tail = &buf->sentinel;
  507. init_llist_head(&buf->free);
  508. atomic_set(&buf->mem_used, 0);
  509. atomic_set(&buf->priority, 0);
  510. INIT_WORK(&buf->work, flush_to_ldisc);
  511. buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT;
  512. }
  513. /**
  514. * tty_buffer_set_limit - change the tty buffer memory limit
  515. * @port: tty port to change
  516. *
  517. * Change the tty buffer memory limit.
  518. * Must be called before the other tty buffer functions are used.
  519. */
  520. int tty_buffer_set_limit(struct tty_port *port, int limit)
  521. {
  522. if (limit < MIN_TTYB_SIZE)
  523. return -EINVAL;
  524. port->buf.mem_limit = limit;
  525. return 0;
  526. }
  527. EXPORT_SYMBOL_GPL(tty_buffer_set_limit);
  528. /* slave ptys can claim nested buffer lock when handling BRK and INTR */
  529. void tty_buffer_set_lock_subclass(struct tty_port *port)
  530. {
  531. lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE);
  532. }
  533. bool tty_buffer_restart_work(struct tty_port *port)
  534. {
  535. return queue_work(system_unbound_wq, &port->buf.work);
  536. }
  537. bool tty_buffer_cancel_work(struct tty_port *port)
  538. {
  539. return cancel_work_sync(&port->buf.work);
  540. }
  541. void tty_buffer_flush_work(struct tty_port *port)
  542. {
  543. flush_work(&port->buf.work);
  544. }