Commit | Line | Data |
---|---|---|
e3b3d0f5 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e0495736 AC |
2 | /* |
3 | * Tty buffer allocation management | |
4 | */ | |
5 | ||
6 | #include <linux/types.h> | |
7 | #include <linux/errno.h> | |
6bb6fa69 | 8 | #include <linux/minmax.h> |
e0495736 AC |
9 | #include <linux/tty.h> |
10 | #include <linux/tty_driver.h> | |
11 | #include <linux/tty_flip.h> | |
12 | #include <linux/timer.h> | |
13 | #include <linux/string.h> | |
14 | #include <linux/slab.h> | |
15 | #include <linux/sched.h> | |
e0495736 AC |
16 | #include <linux/wait.h> |
17 | #include <linux/bitops.h> | |
18 | #include <linux/delay.h> | |
19 | #include <linux/module.h> | |
593fb1ae | 20 | #include <linux/ratelimit.h> |
6c80c0b9 | 21 | #include "tty.h" |
1cef50e3 PH |
22 | |
23 | #define MIN_TTYB_SIZE 256 | |
15730dc4 | 24 | #define TTYB_ALIGN_MASK 0xff |
1cef50e3 | 25 | |
7bfe0b71 PH |
26 | /* |
27 | * Byte threshold to limit memory consumption for flip buffers. | |
28 | * The actual memory limit is > 2x this amount. | |
29 | */ | |
7ab57b76 | 30 | #define TTYB_DEFAULT_MEM_LIMIT (640 * 1024UL) |
7bfe0b71 | 31 | |
9114fe8c PH |
32 | /* |
33 | * We default to dicing tty buffer allocations to this many characters | |
34 | * in order to avoid multiple page allocations. We know the size of | |
35 | * tty_buffer itself but it must also be taken into account that the | |
dadc1049 XT |
36 | * buffer is 256 byte aligned. See tty_buffer_find for the allocation |
37 | * logic this must match. | |
9114fe8c PH |
38 | */ |
39 | ||
15730dc4 | 40 | #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~TTYB_ALIGN_MASK) |
9114fe8c | 41 | |
a7c8d58c | 42 | /** |
bc17b723 JS |
43 | * tty_buffer_lock_exclusive - gain exclusive access to buffer |
44 | * @port: tty port owning the flip buffer | |
a7c8d58c | 45 | * |
bc17b723 JS |
46 | * Guarantees safe use of the &tty_ldisc_ops.receive_buf() method by excluding |
47 | * the buffer work and any pending flush from using the flip buffer. Data can | |
48 | * continue to be added concurrently to the flip buffer from the driver side. | |
a7c8d58c | 49 | * |
bc17b723 | 50 | * See also tty_buffer_unlock_exclusive(). |
a7c8d58c | 51 | */ |
a7c8d58c PH |
52 | void tty_buffer_lock_exclusive(struct tty_port *port) |
53 | { | |
54 | struct tty_bufhead *buf = &port->buf; | |
55 | ||
56 | atomic_inc(&buf->priority); | |
57 | mutex_lock(&buf->lock); | |
58 | } | |
28a821c3 | 59 | EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive); |
a7c8d58c | 60 | |
bc17b723 JS |
61 | /** |
62 | * tty_buffer_unlock_exclusive - release exclusive access | |
63 | * @port: tty port owning the flip buffer | |
64 | * | |
65 | * The buffer work is restarted if there is data in the flip buffer. | |
66 | * | |
67 | * See also tty_buffer_lock_exclusive(). | |
68 | */ | |
a7c8d58c PH |
69 | void tty_buffer_unlock_exclusive(struct tty_port *port) |
70 | { | |
71 | struct tty_bufhead *buf = &port->buf; | |
083cfcf3 | 72 | bool restart = buf->head->commit != buf->head->read; |
a7c8d58c PH |
73 | |
74 | atomic_dec(&buf->priority); | |
75 | mutex_unlock(&buf->lock); | |
083cfcf3 | 76 | |
a7c8d58c PH |
77 | if (restart) |
78 | queue_work(system_unbound_wq, &buf->work); | |
79 | } | |
28a821c3 | 80 | EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive); |
a7c8d58c | 81 | |
7bfe0b71 | 82 | /** |
bc17b723 JS |
83 | * tty_buffer_space_avail - return unused buffer space |
84 | * @port: tty port owning the flip buffer | |
7bfe0b71 | 85 | * |
bc17b723 JS |
86 | * Returns: the # of bytes which can be written by the driver without reaching |
87 | * the buffer limit. | |
7bfe0b71 | 88 | * |
bc17b723 JS |
89 | * Note: this does not guarantee that memory is available to write the returned |
90 | * # of bytes (use tty_prepare_flip_string() to pre-allocate if memory | |
91 | * guarantee is required). | |
7bfe0b71 | 92 | */ |
9a33fbf9 | 93 | unsigned int tty_buffer_space_avail(struct tty_port *port) |
7bfe0b71 | 94 | { |
5dda4ca5 | 95 | int space = port->buf.mem_limit - atomic_read(&port->buf.mem_used); |
993c67b1 | 96 | |
7bfe0b71 PH |
97 | return max(space, 0); |
98 | } | |
c4a8dab5 | 99 | EXPORT_SYMBOL_GPL(tty_buffer_space_avail); |
7bfe0b71 | 100 | |
9dd5139f PH |
101 | static void tty_buffer_reset(struct tty_buffer *p, size_t size) |
102 | { | |
103 | p->used = 0; | |
104 | p->size = size; | |
105 | p->next = NULL; | |
106 | p->commit = 0; | |
6bb6fa69 | 107 | p->lookahead = 0; |
9dd5139f | 108 | p->read = 0; |
2e2b4b89 | 109 | p->flags = true; |
9dd5139f PH |
110 | } |
111 | ||
e0495736 | 112 | /** |
bc17b723 JS |
113 | * tty_buffer_free_all - free buffers used by a tty |
114 | * @port: tty port to free from | |
e0495736 | 115 | * |
bc17b723 JS |
116 | * Remove all the buffers pending on a tty whether queued with data or in the |
117 | * free ring. Must be called when the tty is no longer in use. | |
e0495736 | 118 | */ |
ecbbfd44 | 119 | void tty_buffer_free_all(struct tty_port *port) |
e0495736 | 120 | { |
ecbbfd44 | 121 | struct tty_bufhead *buf = &port->buf; |
809850b7 PH |
122 | struct tty_buffer *p, *next; |
123 | struct llist_node *llist; | |
feacbecb JS |
124 | unsigned int freed = 0; |
125 | int still_used; | |
5cff39c6 | 126 | |
2cf7b67e PH |
127 | while ((p = buf->head) != NULL) { |
128 | buf->head = p->next; | |
feacbecb | 129 | freed += p->size; |
7391ee16 PH |
130 | if (p->size > 0) |
131 | kfree(p); | |
e0495736 | 132 | } |
809850b7 PH |
133 | llist = llist_del_all(&buf->free); |
134 | llist_for_each_entry_safe(p, next, llist, free) | |
2cf7b67e | 135 | kfree(p); |
809850b7 | 136 | |
7391ee16 PH |
137 | tty_buffer_reset(&buf->sentinel, 0); |
138 | buf->head = &buf->sentinel; | |
139 | buf->tail = &buf->sentinel; | |
7bfe0b71 | 140 | |
feacbecb JS |
141 | still_used = atomic_xchg(&buf->mem_used, 0); |
142 | WARN(still_used != freed, "we still have not freed %d bytes!", | |
143 | still_used - freed); | |
e0495736 AC |
144 | } |
145 | ||
146 | /** | |
bc17b723 JS |
147 | * tty_buffer_alloc - allocate a tty buffer |
148 | * @port: tty port | |
149 | * @size: desired size (characters) | |
150 | * | |
151 | * Allocate a new tty buffer to hold the desired number of characters. We | |
152 | * round our buffers off in 256 character chunks to get better allocation | |
153 | * behaviour. | |
154 | * | |
155 | * Returns: %NULL if out of memory or the allocation would exceed the per | |
156 | * device queue. | |
e0495736 | 157 | */ |
ecbbfd44 | 158 | static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) |
e0495736 | 159 | { |
809850b7 | 160 | struct llist_node *free; |
e0495736 AC |
161 | struct tty_buffer *p; |
162 | ||
11b9faa4 PH |
163 | /* Round the buffer size out */ |
164 | size = __ALIGN_MASK(size, TTYB_ALIGN_MASK); | |
165 | ||
166 | if (size <= MIN_TTYB_SIZE) { | |
809850b7 PH |
167 | free = llist_del_first(&port->buf.free); |
168 | if (free) { | |
169 | p = llist_entry(free, struct tty_buffer, free); | |
11b9faa4 PH |
170 | goto found; |
171 | } | |
172 | } | |
173 | ||
174 | /* Should possibly check if this fails for the largest buffer we | |
80e3fce1 XT |
175 | * have queued and recycle that ? |
176 | */ | |
5dda4ca5 | 177 | if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit) |
e0495736 | 178 | return NULL; |
46bc78c8 | 179 | p = kmalloc(struct_size(p, data, 2 * size), GFP_ATOMIC | __GFP_NOWARN); |
e0495736 AC |
180 | if (p == NULL) |
181 | return NULL; | |
9dd5139f | 182 | |
11b9faa4 | 183 | found: |
9dd5139f | 184 | tty_buffer_reset(p, size); |
5dda4ca5 | 185 | atomic_add(size, &port->buf.mem_used); |
e0495736 AC |
186 | return p; |
187 | } | |
188 | ||
189 | /** | |
bc17b723 JS |
190 | * tty_buffer_free - free a tty buffer |
191 | * @port: tty port owning the buffer | |
192 | * @b: the buffer to free | |
e0495736 | 193 | * |
bc17b723 JS |
194 | * Free a tty buffer, or add it to the free list according to our internal |
195 | * strategy. | |
e0495736 | 196 | */ |
ecbbfd44 | 197 | static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b) |
e0495736 | 198 | { |
ecbbfd44 | 199 | struct tty_bufhead *buf = &port->buf; |
5cff39c6 | 200 | |
e0495736 | 201 | /* Dumb strategy for now - should keep some stats */ |
5dda4ca5 | 202 | WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0); |
e0495736 | 203 | |
1cef50e3 | 204 | if (b->size > MIN_TTYB_SIZE) |
e0495736 | 205 | kfree(b); |
7391ee16 | 206 | else if (b->size > 0) |
809850b7 | 207 | llist_add(&b->free, &buf->free); |
e0495736 AC |
208 | } |
209 | ||
e0495736 | 210 | /** |
bc17b723 JS |
211 | * tty_buffer_flush - flush full tty buffers |
212 | * @tty: tty to flush | |
213 | * @ld: optional ldisc ptr (must be referenced) | |
e0495736 | 214 | * |
bc17b723 JS |
215 | * Flush all the buffers containing receive data. If @ld != %NULL, flush the |
216 | * ldisc input buffer. | |
e0495736 | 217 | * |
bc17b723 | 218 | * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. |
e0495736 | 219 | */ |
86c80a8e | 220 | void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) |
e0495736 | 221 | { |
2fc20661 | 222 | struct tty_port *port = tty->port; |
ecbbfd44 | 223 | struct tty_bufhead *buf = &port->buf; |
47aa658a | 224 | struct tty_buffer *next; |
e0495736 | 225 | |
a7c8d58c | 226 | atomic_inc(&buf->priority); |
e9975fde | 227 | |
a7c8d58c | 228 | mutex_lock(&buf->lock); |
9e6b7cd7 DV |
229 | /* paired w/ release in __tty_buffer_request_room; ensures there are |
230 | * no pending memory accesses to the freed buffer | |
231 | */ | |
232 | while ((next = smp_load_acquire(&buf->head->next)) != NULL) { | |
47aa658a PH |
233 | tty_buffer_free(port, buf->head); |
234 | buf->head = next; | |
235 | } | |
236 | buf->head->read = buf->head->commit; | |
6bb6fa69 | 237 | buf->head->lookahead = buf->head->read; |
86c80a8e PH |
238 | |
239 | if (ld && ld->ops->flush_buffer) | |
240 | ld->ops->flush_buffer(tty); | |
241 | ||
a7c8d58c PH |
242 | atomic_dec(&buf->priority); |
243 | mutex_unlock(&buf->lock); | |
e0495736 AC |
244 | } |
245 | ||
e0495736 | 246 | /** |
bc17b723 JS |
247 | * __tty_buffer_request_room - grow tty buffer if needed |
248 | * @port: tty port | |
249 | * @size: size desired | |
2e2b4b89 | 250 | * @flags: buffer has to store flags along character data |
bc17b723 JS |
251 | * |
252 | * Make at least @size bytes of linear space available for the tty buffer. | |
e0495736 | 253 | * |
bc17b723 JS |
254 | * Will change over to a new buffer if the current buffer is encoded as |
255 | * %TTY_NORMAL (so has no flags buffer) and the new buffer requires a flags | |
256 | * buffer. | |
acc0f67f | 257 | * |
bc17b723 | 258 | * Returns: the size we managed to find. |
e0495736 | 259 | */ |
acc0f67f | 260 | static int __tty_buffer_request_room(struct tty_port *port, size_t size, |
2e2b4b89 | 261 | bool flags) |
e0495736 | 262 | { |
ecbbfd44 | 263 | struct tty_bufhead *buf = &port->buf; |
035197c9 JSS |
264 | struct tty_buffer *n, *b = buf->tail; |
265 | size_t left = (b->flags ? 1 : 2) * b->size - b->used; | |
266 | bool change = !b->flags && flags; | |
e8437d7e | 267 | |
ebee41c8 JSS |
268 | if (!change && left >= size) |
269 | return size; | |
270 | ||
271 | /* This is the slow path - looking for new buffers to use */ | |
272 | n = tty_buffer_alloc(port, size); | |
273 | if (n == NULL) | |
274 | return change ? 0 : left; | |
275 | ||
276 | n->flags = flags; | |
277 | buf->tail = n; | |
278 | /* | |
279 | * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() | |
280 | * ensures they see all buffer data. | |
281 | */ | |
282 | smp_store_release(&b->commit, b->used); | |
283 | /* | |
284 | * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() | |
285 | * ensures the latest commit value can be read before the head | |
286 | * is advanced to the next buffer. | |
287 | */ | |
288 | smp_store_release(&b->next, n); | |
289 | ||
e0495736 AC |
290 | return size; |
291 | } | |
acc0f67f PH |
292 | |
293 | int tty_buffer_request_room(struct tty_port *port, size_t size) | |
294 | { | |
2e2b4b89 | 295 | return __tty_buffer_request_room(port, size, true); |
acc0f67f | 296 | } |
e0495736 AC |
297 | EXPORT_SYMBOL_GPL(tty_buffer_request_room); |
298 | ||
6144922e JSS |
299 | size_t __tty_insert_flip_string_flags(struct tty_port *port, const u8 *chars, |
300 | const u8 *flags, bool mutable_flags, | |
301 | size_t size) | |
e0495736 | 302 | { |
c26405fd | 303 | bool need_flags = mutable_flags || flags[0] != TTY_NORMAL; |
6144922e | 304 | size_t copied = 0; |
993c67b1 | 305 | |
e0495736 | 306 | do { |
6144922e JSS |
307 | size_t goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); |
308 | size_t space = __tty_buffer_request_room(port, goal, need_flags); | |
64325a3b | 309 | struct tty_buffer *tb = port->buf.tail; |
993c67b1 | 310 | |
7391ee16 | 311 | if (unlikely(space == 0)) |
e0495736 | 312 | break; |
e0495736 | 313 | |
c26405fd | 314 | memcpy(char_buf_ptr(tb, tb->used), chars, space); |
993c67b1 | 315 | |
c26405fd JSS |
316 | if (mutable_flags) { |
317 | memcpy(flag_buf_ptr(tb, tb->used), flags, space); | |
318 | flags += space; | |
319 | } else if (tb->flags) { | |
320 | memset(flag_buf_ptr(tb, tb->used), flags[0], space); | |
4a8d99a4 JSS |
321 | } else { |
322 | /* tb->flags should be available once requested */ | |
323 | WARN_ON_ONCE(need_flags); | |
c26405fd | 324 | } |
993c67b1 | 325 | |
e0495736 AC |
326 | tb->used += space; |
327 | copied += space; | |
328 | chars += space; | |
c26405fd | 329 | |
e0495736 | 330 | /* There is a small chance that we need to split the data over |
80e3fce1 XT |
331 | * several buffers. If this is the case we must loop. |
332 | */ | |
e0495736 | 333 | } while (unlikely(size > copied)); |
c26405fd | 334 | |
e0495736 AC |
335 | return copied; |
336 | } | |
c26405fd | 337 | EXPORT_SYMBOL(__tty_insert_flip_string_flags); |
e0495736 | 338 | |
e0495736 | 339 | /** |
bc17b723 JS |
340 | * tty_prepare_flip_string - make room for characters |
341 | * @port: tty port | |
342 | * @chars: return pointer for character write area | |
343 | * @size: desired size | |
344 | * | |
345 | * Prepare a block of space in the buffer for data. | |
346 | * | |
347 | * This is used for drivers that need their own block copy routines into the | |
348 | * buffer. There is no guarantee the buffer is a DMA target! | |
349 | * | |
350 | * Returns: the length available and buffer pointer (@chars) to the space which | |
351 | * is now allocated and accounted for as ready for normal characters. | |
e0495736 | 352 | */ |
2ce2983c | 353 | size_t tty_prepare_flip_string(struct tty_port *port, u8 **chars, size_t size) |
e0495736 | 354 | { |
2ce2983c | 355 | size_t space = __tty_buffer_request_room(port, size, false); |
993c67b1 | 356 | |
e0495736 | 357 | if (likely(space)) { |
64325a3b | 358 | struct tty_buffer *tb = port->buf.tail; |
993c67b1 | 359 | |
1fc359fc | 360 | *chars = char_buf_ptr(tb, tb->used); |
2e2b4b89 | 361 | if (tb->flags) |
acc0f67f | 362 | memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space); |
e0495736 AC |
363 | tb->used += space; |
364 | } | |
2ce2983c | 365 | |
e0495736 AC |
366 | return space; |
367 | } | |
368 | EXPORT_SYMBOL_GPL(tty_prepare_flip_string); | |
369 | ||
8d082cd3 | 370 | /** |
bc17b723 JS |
371 | * tty_ldisc_receive_buf - forward data to line discipline |
372 | * @ld: line discipline to process input | |
373 | * @p: char buffer | |
374 | * @f: %TTY_NORMAL, %TTY_BREAK, etc. flags buffer | |
375 | * @count: number of bytes to process | |
8d082cd3 | 376 | * |
bc17b723 JS |
377 | * Callers other than flush_to_ldisc() need to exclude the kworker from |
378 | * concurrent use of the line discipline, see paste_selection(). | |
8d082cd3 | 379 | * |
bc17b723 | 380 | * Returns: the number of bytes processed. |
8d082cd3 | 381 | */ |
892bc209 | 382 | size_t tty_ldisc_receive_buf(struct tty_ldisc *ld, const u8 *p, const u8 *f, |
a8d9cd23 | 383 | size_t count) |
8d082cd3 PH |
384 | { |
385 | if (ld->ops->receive_buf2) | |
386 | count = ld->ops->receive_buf2(ld->tty, p, f, count); | |
387 | else { | |
8d9526f9 | 388 | count = min_t(size_t, count, ld->tty->receive_room); |
8d082cd3 PH |
389 | if (count && ld->ops->receive_buf) |
390 | ld->ops->receive_buf(ld->tty, p, f, count); | |
391 | } | |
392 | return count; | |
393 | } | |
394 | EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf); | |
e0495736 | 395 | |
6bb6fa69 IJ |
396 | static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) |
397 | { | |
398 | head->lookahead = max(head->lookahead, head->read); | |
399 | ||
400 | while (head) { | |
401 | struct tty_buffer *next; | |
6bb6fa69 IJ |
402 | unsigned int count; |
403 | ||
404 | /* | |
405 | * Paired w/ release in __tty_buffer_request_room(); | |
406 | * ensures commit value read is not stale if the head | |
407 | * is advancing to the next buffer. | |
408 | */ | |
409 | next = smp_load_acquire(&head->next); | |
410 | /* | |
411 | * Paired w/ release in __tty_buffer_request_room() or in | |
412 | * tty_buffer_flush(); ensures we see the committed buffer data. | |
413 | */ | |
414 | count = smp_load_acquire(&head->commit) - head->lookahead; | |
415 | if (!count) { | |
416 | head = next; | |
417 | continue; | |
418 | } | |
419 | ||
56c14fb4 | 420 | if (port->client_ops->lookahead_buf) { |
a8d9cd23 | 421 | u8 *p, *f = NULL; |
56c14fb4 VW |
422 | |
423 | p = char_buf_ptr(head, head->lookahead); | |
2e2b4b89 | 424 | if (head->flags) |
56c14fb4 VW |
425 | f = flag_buf_ptr(head, head->lookahead); |
426 | ||
427 | port->client_ops->lookahead_buf(port, p, f, count); | |
428 | } | |
6bb6fa69 | 429 | |
6bb6fa69 IJ |
430 | head->lookahead += count; |
431 | } | |
432 | } | |
433 | ||
201560af JSS |
434 | static size_t |
435 | receive_buf(struct tty_port *port, struct tty_buffer *head, size_t count) | |
da261e7f | 436 | { |
0b7a2b28 JSS |
437 | u8 *p = char_buf_ptr(head, head->read); |
438 | const u8 *f = NULL; | |
201560af | 439 | size_t n; |
acc0f67f | 440 | |
2e2b4b89 | 441 | if (head->flags) |
acc0f67f | 442 | f = flag_buf_ptr(head, head->read); |
da261e7f | 443 | |
c9a8e5fc LT |
444 | n = port->client_ops->receive_buf(port, p, f, count); |
445 | if (n > 0) | |
446 | memset(p, 0, n); | |
447 | return n; | |
da261e7f | 448 | } |
e0495736 AC |
449 | |
450 | /** | |
bc17b723 JS |
451 | * flush_to_ldisc - flush data from buffer to ldisc |
452 | * @work: tty structure passed from work queue. | |
e0495736 | 453 | * |
bc17b723 JS |
454 | * This routine is called out of the software interrupt to flush data from the |
455 | * buffer chain to the line discipline. | |
e0495736 | 456 | * |
bc17b723 | 457 | * The receive_buf() method is single threaded for each tty instance. |
e9975fde | 458 | * |
bc17b723 | 459 | * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. |
e0495736 | 460 | */ |
e0495736 AC |
461 | static void flush_to_ldisc(struct work_struct *work) |
462 | { | |
ecbbfd44 JS |
463 | struct tty_port *port = container_of(work, struct tty_port, buf.work); |
464 | struct tty_bufhead *buf = &port->buf; | |
e0495736 | 465 | |
a7c8d58c | 466 | mutex_lock(&buf->lock); |
45242006 | 467 | |
d7a68be4 PH |
468 | while (1) { |
469 | struct tty_buffer *head = buf->head; | |
62a0d8d7 | 470 | struct tty_buffer *next; |
201560af | 471 | size_t count, rcvd; |
d7a68be4 | 472 | |
a7c8d58c PH |
473 | /* Ldisc or user is trying to gain exclusive access */ |
474 | if (atomic_read(&buf->priority)) | |
d7a68be4 PH |
475 | break; |
476 | ||
069f38b4 | 477 | /* paired w/ release in __tty_buffer_request_room(); |
62a0d8d7 PH |
478 | * ensures commit value read is not stale if the head |
479 | * is advancing to the next buffer | |
480 | */ | |
069f38b4 | 481 | next = smp_load_acquire(&head->next); |
facd885c DV |
482 | /* paired w/ release in __tty_buffer_request_room() or in |
483 | * tty_buffer_flush(); ensures we see the committed buffer data | |
484 | */ | |
485 | count = smp_load_acquire(&head->commit) - head->read; | |
d7a68be4 | 486 | if (!count) { |
0f40fbbc | 487 | if (next == NULL) |
da261e7f | 488 | break; |
62a0d8d7 | 489 | buf->head = next; |
d7a68be4 PH |
490 | tty_buffer_free(port, head); |
491 | continue; | |
e0495736 | 492 | } |
d7a68be4 | 493 | |
6bb6fa69 IJ |
494 | rcvd = receive_buf(port, head, count); |
495 | head->read += rcvd; | |
496 | if (rcvd < count) | |
497 | lookahead_bufs(port, head); | |
498 | if (!rcvd) | |
d7a68be4 | 499 | break; |
3968ddcf GF |
500 | |
501 | if (need_resched()) | |
502 | cond_resched(); | |
e0495736 | 503 | } |
45242006 | 504 | |
a7c8d58c | 505 | mutex_unlock(&buf->lock); |
e0495736 | 506 | |
e0495736 AC |
507 | } |
508 | ||
716b1058 JS |
509 | static inline void tty_flip_buffer_commit(struct tty_buffer *tail) |
510 | { | |
511 | /* | |
512 | * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees | |
513 | * buffer data. | |
514 | */ | |
515 | smp_store_release(&tail->commit, tail->used); | |
516 | } | |
517 | ||
e0495736 | 518 | /** |
bc17b723 JS |
519 | * tty_flip_buffer_push - push terminal buffers |
520 | * @port: tty port to push | |
e0495736 | 521 | * |
bc17b723 JS |
522 | * Queue a push of the terminal flip buffers to the line discipline. Can be |
523 | * called from IRQ/atomic context. | |
e0495736 | 524 | * |
bc17b723 JS |
525 | * In the event of the queue being busy for flipping the work will be held off |
526 | * and retried later. | |
e0495736 | 527 | */ |
2e124b4a | 528 | void tty_flip_buffer_push(struct tty_port *port) |
e0495736 | 529 | { |
5db96ef2 JS |
530 | struct tty_bufhead *buf = &port->buf; |
531 | ||
716b1058 | 532 | tty_flip_buffer_commit(buf->tail); |
5db96ef2 | 533 | queue_work(system_unbound_wq, &buf->work); |
e0495736 AC |
534 | } |
535 | EXPORT_SYMBOL(tty_flip_buffer_push); | |
536 | ||
a501ab75 JS |
537 | /** |
538 | * tty_insert_flip_string_and_push_buffer - add characters to the tty buffer and | |
539 | * push | |
540 | * @port: tty port | |
541 | * @chars: characters | |
542 | * @size: size | |
543 | * | |
544 | * The function combines tty_insert_flip_string() and tty_flip_buffer_push() | |
545 | * with the exception of properly holding the @port->lock. | |
546 | * | |
547 | * To be used only internally (by pty currently). | |
548 | * | |
549 | * Returns: the number added. | |
550 | */ | |
551 | int tty_insert_flip_string_and_push_buffer(struct tty_port *port, | |
a8d9cd23 | 552 | const u8 *chars, size_t size) |
a501ab75 JS |
553 | { |
554 | struct tty_bufhead *buf = &port->buf; | |
555 | unsigned long flags; | |
556 | ||
557 | spin_lock_irqsave(&port->lock, flags); | |
558 | size = tty_insert_flip_string(port, chars, size); | |
559 | if (size) | |
560 | tty_flip_buffer_commit(buf->tail); | |
561 | spin_unlock_irqrestore(&port->lock, flags); | |
562 | ||
563 | queue_work(system_unbound_wq, &buf->work); | |
564 | ||
565 | return size; | |
566 | } | |
567 | ||
e0495736 | 568 | /** |
bc17b723 JS |
569 | * tty_buffer_init - prepare a tty buffer structure |
570 | * @port: tty port to initialise | |
e0495736 | 571 | * |
bc17b723 JS |
572 | * Set up the initial state of the buffer management for a tty device. Must be |
573 | * called before the other tty buffer functions are used. | |
e0495736 | 574 | */ |
ecbbfd44 | 575 | void tty_buffer_init(struct tty_port *port) |
e0495736 | 576 | { |
ecbbfd44 | 577 | struct tty_bufhead *buf = &port->buf; |
5cff39c6 | 578 | |
a7c8d58c | 579 | mutex_init(&buf->lock); |
7391ee16 PH |
580 | tty_buffer_reset(&buf->sentinel, 0); |
581 | buf->head = &buf->sentinel; | |
582 | buf->tail = &buf->sentinel; | |
809850b7 | 583 | init_llist_head(&buf->free); |
5dda4ca5 | 584 | atomic_set(&buf->mem_used, 0); |
a7c8d58c | 585 | atomic_set(&buf->priority, 0); |
5cff39c6 | 586 | INIT_WORK(&buf->work, flush_to_ldisc); |
4d18e6ef | 587 | buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT; |
e0495736 | 588 | } |
4d18e6ef PH |
589 | |
590 | /** | |
bc17b723 JS |
591 | * tty_buffer_set_limit - change the tty buffer memory limit |
592 | * @port: tty port to change | |
593 | * @limit: memory limit to set | |
594 | * | |
595 | * Change the tty buffer memory limit. | |
4d18e6ef | 596 | * |
bc17b723 | 597 | * Must be called before the other tty buffer functions are used. |
4d18e6ef | 598 | */ |
4d18e6ef PH |
599 | int tty_buffer_set_limit(struct tty_port *port, int limit) |
600 | { | |
601 | if (limit < MIN_TTYB_SIZE) | |
602 | return -EINVAL; | |
603 | port->buf.mem_limit = limit; | |
604 | return 0; | |
605 | } | |
606 | EXPORT_SYMBOL_GPL(tty_buffer_set_limit); | |
1d1d14da PH |
607 | |
608 | /* slave ptys can claim nested buffer lock when handling BRK and INTR */ | |
609 | void tty_buffer_set_lock_subclass(struct tty_port *port) | |
610 | { | |
611 | lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE); | |
612 | } | |
e176058f PH |
613 | |
614 | bool tty_buffer_restart_work(struct tty_port *port) | |
615 | { | |
616 | return queue_work(system_unbound_wq, &port->buf.work); | |
617 | } | |
618 | ||
619 | bool tty_buffer_cancel_work(struct tty_port *port) | |
620 | { | |
621 | return cancel_work_sync(&port->buf.work); | |
622 | } | |
0f40fbbc BB |
623 | |
624 | void tty_buffer_flush_work(struct tty_port *port) | |
625 | { | |
626 | flush_work(&port->buf.work); | |
627 | } |