Commit | Line | Data |
---|---|---|
c3c4e307 | 1 | // SPDX-License-Identifier: GPL-2.0 |
c89f2750 | 2 | /* |
c89f2750 DDT |
3 | * Copyright (C) 2012 Intel, Inc. |
4 | * Copyright (C) 2013 Intel, Inc. | |
2f3be882 | 5 | * Copyright (C) 2014 Linaro Limited |
726ea1a8 | 6 | * Copyright (C) 2011-2016 Google, Inc. |
c89f2750 DDT |
7 | * |
8 | * This software is licensed under the terms of the GNU General Public | |
9 | * License version 2, as published by the Free Software Foundation, and | |
10 | * may be copied, distributed, and modified under those terms. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | */ | |
18 | ||
19 | /* This source file contains the implementation of a special device driver | |
20 | * that intends to provide a *very* fast communication channel between the | |
21 | * guest system and the QEMU emulator. | |
22 | * | |
23 | * Usage from the guest is simply the following (error handling simplified): | |
24 | * | |
25 | * int fd = open("/dev/qemu_pipe",O_RDWR); | |
26 | * .... write() or read() through the pipe. | |
27 | * | |
28 | * This driver doesn't deal with the exact protocol used during the session. | |
29 | * It is intended to be as simple as something like: | |
30 | * | |
31 | * // do this _just_ after opening the fd to connect to a specific | |
32 | * // emulator service. | |
33 | * const char* msg = "<pipename>"; | |
34 | * if (write(fd, msg, strlen(msg)+1) < 0) { | |
35 | * ... could not connect to <pipename> service | |
36 | * close(fd); | |
37 | * } | |
38 | * | |
39 | * // after this, simply read() and write() to communicate with the | |
40 | * // service. Exact protocol details left as an exercise to the reader. | |
41 | * | |
42 | * This driver is very fast because it doesn't copy any data through | |
43 | * intermediate buffers, since the emulator is capable of translating | |
44 | * guest user addresses into host ones. | |
45 | * | |
46 | * Note that we must however ensure that each user page involved in the | |
47 | * exchange is properly mapped during a transfer. | |
48 | */ | |
49 | ||
50 | #include <linux/module.h> | |
ac316725 | 51 | #include <linux/mod_devicetable.h> |
c89f2750 DDT |
52 | #include <linux/interrupt.h> |
53 | #include <linux/kernel.h> | |
54 | #include <linux/spinlock.h> | |
55 | #include <linux/miscdevice.h> | |
56 | #include <linux/platform_device.h> | |
57 | #include <linux/poll.h> | |
58 | #include <linux/sched.h> | |
59 | #include <linux/bitops.h> | |
60 | #include <linux/slab.h> | |
61 | #include <linux/io.h> | |
1d427da1 | 62 | #include <linux/dma-mapping.h> |
2f3be882 | 63 | #include <linux/mm.h> |
d62f324b | 64 | #include <linux/acpi.h> |
d23069a5 | 65 | #include <linux/bug.h> |
95577010 | 66 | #include "goldfish_pipe_qemu.h" |
c89f2750 | 67 | |
726ea1a8 JQ |
68 | /* |
69 | * Update this when something changes in the driver's behavior so the host | |
70 | * can benefit from knowing it | |
71 | */ | |
72 | enum { | |
73 | PIPE_DRIVER_VERSION = 2, | |
74 | PIPE_CURRENT_DEVICE_VERSION = 2 | |
75 | }; | |
76 | ||
726ea1a8 JQ |
77 | enum { |
78 | MAX_BUFFERS_PER_COMMAND = 336, | |
79 | MAX_SIGNALLED_PIPES = 64, | |
80 | INITIAL_PIPES_CAPACITY = 64 | |
81 | }; | |
82 | ||
83 | struct goldfish_pipe_dev; | |
726ea1a8 JQ |
84 | |
85 | /* A per-pipe command structure, shared with the host */ | |
86 | struct goldfish_pipe_command { | |
ed824215 RK |
87 | s32 cmd; /* PipeCmdCode, guest -> host */ |
88 | s32 id; /* pipe id, guest -> host */ | |
89 | s32 status; /* command execution status, host -> guest */ | |
726ea1a8 JQ |
90 | s32 reserved; /* to pad to 64-bit boundary */ |
91 | union { | |
92 | /* Parameters for PIPE_CMD_{READ,WRITE} */ | |
93 | struct { | |
94 | /* number of buffers, guest -> host */ | |
95 | u32 buffers_count; | |
96 | /* number of consumed bytes, host -> guest */ | |
97 | s32 consumed_size; | |
98 | /* buffer pointers, guest -> host */ | |
99 | u64 ptrs[MAX_BUFFERS_PER_COMMAND]; | |
100 | /* buffer sizes, guest -> host */ | |
101 | u32 sizes[MAX_BUFFERS_PER_COMMAND]; | |
102 | } rw_params; | |
103 | }; | |
104 | }; | |
105 | ||
106 | /* A single signalled pipe information */ | |
107 | struct signalled_pipe_buffer { | |
108 | u32 id; | |
c89f2750 DDT |
109 | u32 flags; |
110 | }; | |
111 | ||
726ea1a8 JQ |
112 | /* Parameters for the PIPE_CMD_OPEN command */ |
113 | struct open_command_param { | |
114 | u64 command_buffer_ptr; | |
115 | u32 rw_params_max_count; | |
c89f2750 DDT |
116 | }; |
117 | ||
726ea1a8 JQ |
118 | /* Device-level set of buffers shared with the host */ |
119 | struct goldfish_pipe_dev_buffers { | |
120 | struct open_command_param open_command_params; | |
562a74de RK |
121 | struct signalled_pipe_buffer |
122 | signalled_pipe_buffers[MAX_SIGNALLED_PIPES]; | |
726ea1a8 | 123 | }; |
c89f2750 DDT |
124 | |
125 | /* This data type models a given pipe instance */ | |
126 | struct goldfish_pipe { | |
726ea1a8 JQ |
127 | /* pipe ID - index into goldfish_pipe_dev::pipes array */ |
128 | u32 id; | |
46928cc6 | 129 | |
726ea1a8 JQ |
130 | /* The wake flags pipe is waiting for |
131 | * Note: not protected with any lock, uses atomic operations | |
132 | * and barriers to make it thread-safe. | |
133 | */ | |
c89f2750 | 134 | unsigned long flags; |
46928cc6 | 135 | |
726ea1a8 JQ |
136 | /* wake flags host have signalled, |
137 | * - protected by goldfish_pipe_dev::lock | |
138 | */ | |
139 | unsigned long signalled_flags; | |
140 | ||
141 | /* A pointer to command buffer */ | |
142 | struct goldfish_pipe_command *command_buffer; | |
143 | ||
144 | /* doubly linked list of signalled pipes, protected by | |
145 | * goldfish_pipe_dev::lock | |
146 | */ | |
147 | struct goldfish_pipe *prev_signalled; | |
148 | struct goldfish_pipe *next_signalled; | |
149 | ||
150 | /* | |
151 | * A pipe's own lock. Protects the following: | |
152 | * - *command_buffer - makes sure a command can safely write its | |
153 | * parameters to the host and read the results back. | |
154 | */ | |
155 | struct mutex lock; | |
156 | ||
157 | /* A wake queue for sleeping until host signals an event */ | |
c89f2750 | 158 | wait_queue_head_t wake_queue; |
46928cc6 | 159 | |
726ea1a8 JQ |
160 | /* Pointer to the parent goldfish_pipe_dev instance */ |
161 | struct goldfish_pipe_dev *dev; | |
48a2d422 RK |
162 | |
163 | /* A buffer of pages, too large to fit into a stack frame */ | |
164 | struct page *pages[MAX_BUFFERS_PER_COMMAND]; | |
c89f2750 DDT |
165 | }; |
166 | ||
726ea1a8 JQ |
167 | /* The global driver data. Holds a reference to the i/o page used to |
168 | * communicate with the emulator, and a wake queue for blocked tasks | |
169 | * waiting to be awoken. | |
170 | */ | |
171 | struct goldfish_pipe_dev { | |
08360e26 RK |
172 | /* A magic number to check if this is an instance of this struct */ |
173 | void *magic; | |
174 | ||
726ea1a8 JQ |
175 | /* |
176 | * Global device spinlock. Protects the following members: | |
177 | * - pipes, pipes_capacity | |
178 | * - [*pipes, *pipes + pipes_capacity) - array data | |
179 | * - first_signalled_pipe, | |
180 | * goldfish_pipe::prev_signalled, | |
181 | * goldfish_pipe::next_signalled, | |
182 | * goldfish_pipe::signalled_flags - all singnalled-related fields, | |
183 | * in all allocated pipes | |
184 | * - open_command_params - PIPE_CMD_OPEN-related buffers | |
185 | * | |
186 | * It looks like a lot of different fields, but the trick is that | |
187 | * the only operation that happens often is the signalled pipes array | |
188 | * manipulation. That's why it's OK for now to keep the rest of the | |
189 | * fields under the same lock. If we notice too much contention because | |
190 | * of PIPE_CMD_OPEN, then we should add a separate lock there. | |
191 | */ | |
192 | spinlock_t lock; | |
c89f2750 | 193 | |
726ea1a8 JQ |
194 | /* |
195 | * Array of the pipes of |pipes_capacity| elements, | |
196 | * indexed by goldfish_pipe::id | |
197 | */ | |
198 | struct goldfish_pipe **pipes; | |
199 | u32 pipes_capacity; | |
200 | ||
201 | /* Pointers to the buffers host uses for interaction with this driver */ | |
202 | struct goldfish_pipe_dev_buffers *buffers; | |
203 | ||
204 | /* Head of a doubly linked list of signalled pipes */ | |
205 | struct goldfish_pipe *first_signalled_pipe; | |
206 | ||
25b97d57 RK |
207 | /* ptr to platform device's device struct */ |
208 | struct device *pdev_dev; | |
209 | ||
726ea1a8 JQ |
210 | /* Some device-specific data */ |
211 | int irq; | |
212 | int version; | |
213 | unsigned char __iomem *base; | |
c394cc3b RK |
214 | |
215 | /* an irq tasklet to run goldfish_interrupt_task */ | |
216 | struct tasklet_struct irq_tasklet; | |
43c2cc28 RK |
217 | |
218 | struct miscdevice miscdev; | |
c89f2750 DDT |
219 | }; |
220 | ||
92c320b9 RK |
221 | static int goldfish_pipe_cmd_locked(struct goldfish_pipe *pipe, |
222 | enum PipeCmdCode cmd) | |
a99698fa | 223 | { |
726ea1a8 JQ |
224 | pipe->command_buffer->cmd = cmd; |
225 | /* failure by default */ | |
226 | pipe->command_buffer->status = PIPE_ERROR_INVAL; | |
227 | writel(pipe->id, pipe->dev->base + PIPE_REG_CMD); | |
228 | return pipe->command_buffer->status; | |
c89f2750 DDT |
229 | } |
230 | ||
92c320b9 | 231 | static int goldfish_pipe_cmd(struct goldfish_pipe *pipe, enum PipeCmdCode cmd) |
a99698fa | 232 | { |
726ea1a8 | 233 | int status; |
c89f2750 | 234 | |
726ea1a8 JQ |
235 | if (mutex_lock_interruptible(&pipe->lock)) |
236 | return PIPE_ERROR_IO; | |
92c320b9 | 237 | status = goldfish_pipe_cmd_locked(pipe, cmd); |
726ea1a8 JQ |
238 | mutex_unlock(&pipe->lock); |
239 | return status; | |
c89f2750 DDT |
240 | } |
241 | ||
726ea1a8 JQ |
242 | /* |
243 | * This function converts an error code returned by the emulator through | |
c89f2750 DDT |
244 | * the PIPE_REG_STATUS i/o register into a valid negative errno value. |
245 | */ | |
246 | static int goldfish_pipe_error_convert(int status) | |
247 | { | |
248 | switch (status) { | |
249 | case PIPE_ERROR_AGAIN: | |
250 | return -EAGAIN; | |
251 | case PIPE_ERROR_NOMEM: | |
252 | return -ENOMEM; | |
253 | case PIPE_ERROR_IO: | |
254 | return -EIO; | |
255 | default: | |
256 | return -EINVAL; | |
257 | } | |
258 | } | |
259 | ||
52bcc7d9 RK |
260 | static int pin_user_pages(unsigned long first_page, |
261 | unsigned long last_page, | |
262 | unsigned int last_page_size, | |
263 | int is_write, | |
264 | struct page *pages[MAX_BUFFERS_PER_COMMAND], | |
265 | unsigned int *iter_last_page_size) | |
c89f2750 | 266 | { |
726ea1a8 JQ |
267 | int ret; |
268 | int requested_pages = ((last_page - first_page) >> PAGE_SHIFT) + 1; | |
269 | ||
270 | if (requested_pages > MAX_BUFFERS_PER_COMMAND) { | |
271 | requested_pages = MAX_BUFFERS_PER_COMMAND; | |
272 | *iter_last_page_size = PAGE_SIZE; | |
273 | } else { | |
274 | *iter_last_page_size = last_page_size; | |
275 | } | |
276 | ||
52bcc7d9 RK |
277 | ret = get_user_pages_fast(first_page, requested_pages, !is_write, |
278 | pages); | |
726ea1a8 JQ |
279 | if (ret <= 0) |
280 | return -EFAULT; | |
281 | if (ret < requested_pages) | |
282 | *iter_last_page_size = PAGE_SIZE; | |
c89f2750 | 283 | |
1d1021a0 | 284 | return ret; |
c89f2750 DDT |
285 | } |
286 | ||
726ea1a8 | 287 | static void release_user_pages(struct page **pages, int pages_count, |
52bcc7d9 | 288 | int is_write, s32 consumed_size) |
c89f2750 | 289 | { |
726ea1a8 | 290 | int i; |
c89f2750 | 291 | |
726ea1a8 JQ |
292 | for (i = 0; i < pages_count; i++) { |
293 | if (!is_write && consumed_size > 0) | |
294 | set_page_dirty(pages[i]); | |
295 | put_page(pages[i]); | |
296 | } | |
297 | } | |
298 | ||
299 | /* Populate the call parameters, merging adjacent pages together */ | |
52bcc7d9 RK |
300 | static void populate_rw_params(struct page **pages, |
301 | int pages_count, | |
302 | unsigned long address, | |
303 | unsigned long address_end, | |
304 | unsigned long first_page, | |
305 | unsigned long last_page, | |
306 | unsigned int iter_last_page_size, | |
307 | int is_write, | |
308 | struct goldfish_pipe_command *command) | |
726ea1a8 JQ |
309 | { |
310 | /* | |
311 | * Process the first page separately - it's the only page that | |
312 | * needs special handling for its start address. | |
313 | */ | |
314 | unsigned long xaddr = page_to_phys(pages[0]); | |
315 | unsigned long xaddr_prev = xaddr; | |
316 | int buffer_idx = 0; | |
317 | int i = 1; | |
318 | int size_on_page = first_page == last_page | |
319 | ? (int)(address_end - address) | |
320 | : (PAGE_SIZE - (address & ~PAGE_MASK)); | |
321 | command->rw_params.ptrs[0] = (u64)(xaddr | (address & ~PAGE_MASK)); | |
322 | command->rw_params.sizes[0] = size_on_page; | |
323 | for (; i < pages_count; ++i) { | |
324 | xaddr = page_to_phys(pages[i]); | |
325 | size_on_page = (i == pages_count - 1) ? | |
326 | iter_last_page_size : PAGE_SIZE; | |
327 | if (xaddr == xaddr_prev + PAGE_SIZE) { | |
328 | command->rw_params.sizes[buffer_idx] += size_on_page; | |
329 | } else { | |
330 | ++buffer_idx; | |
331 | command->rw_params.ptrs[buffer_idx] = (u64)xaddr; | |
332 | command->rw_params.sizes[buffer_idx] = size_on_page; | |
333 | } | |
334 | xaddr_prev = xaddr; | |
335 | } | |
336 | command->rw_params.buffers_count = buffer_idx + 1; | |
337 | } | |
c89f2750 | 338 | |
726ea1a8 | 339 | static int transfer_max_buffers(struct goldfish_pipe *pipe, |
52bcc7d9 RK |
340 | unsigned long address, |
341 | unsigned long address_end, | |
342 | int is_write, | |
343 | unsigned long last_page, | |
344 | unsigned int last_page_size, | |
345 | s32 *consumed_size, | |
346 | int *status) | |
726ea1a8 | 347 | { |
726ea1a8 JQ |
348 | unsigned long first_page = address & PAGE_MASK; |
349 | unsigned int iter_last_page_size; | |
48a2d422 | 350 | int pages_count; |
726ea1a8 JQ |
351 | |
352 | /* Serialize access to the pipe command buffers */ | |
353 | if (mutex_lock_interruptible(&pipe->lock)) | |
354 | return -ERESTARTSYS; | |
355 | ||
48a2d422 RK |
356 | pages_count = pin_user_pages(first_page, last_page, |
357 | last_page_size, is_write, | |
358 | pipe->pages, &iter_last_page_size); | |
359 | if (pages_count < 0) { | |
360 | mutex_unlock(&pipe->lock); | |
361 | return pages_count; | |
362 | } | |
363 | ||
364 | populate_rw_params(pipe->pages, pages_count, address, address_end, | |
52bcc7d9 RK |
365 | first_page, last_page, iter_last_page_size, is_write, |
366 | pipe->command_buffer); | |
726ea1a8 JQ |
367 | |
368 | /* Transfer the data */ | |
92c320b9 | 369 | *status = goldfish_pipe_cmd_locked(pipe, |
726ea1a8 JQ |
370 | is_write ? PIPE_CMD_WRITE : PIPE_CMD_READ); |
371 | ||
372 | *consumed_size = pipe->command_buffer->rw_params.consumed_size; | |
373 | ||
48a2d422 | 374 | release_user_pages(pipe->pages, pages_count, is_write, *consumed_size); |
726ea1a8 | 375 | |
f563dab4 | 376 | mutex_unlock(&pipe->lock); |
726ea1a8 | 377 | return 0; |
c89f2750 DDT |
378 | } |
379 | ||
726ea1a8 | 380 | static int wait_for_host_signal(struct goldfish_pipe *pipe, int is_write) |
c89f2750 | 381 | { |
61b38f02 | 382 | u32 wake_bit = is_write ? BIT_WAKE_ON_WRITE : BIT_WAKE_ON_READ; |
c89f2750 | 383 | |
61b38f02 | 384 | set_bit(wake_bit, &pipe->flags); |
726ea1a8 JQ |
385 | |
386 | /* Tell the emulator we're going to wait for a wake event */ | |
92c320b9 | 387 | goldfish_pipe_cmd(pipe, |
726ea1a8 JQ |
388 | is_write ? PIPE_CMD_WAKE_ON_WRITE : PIPE_CMD_WAKE_ON_READ); |
389 | ||
61b38f02 | 390 | while (test_bit(wake_bit, &pipe->flags)) { |
562a74de | 391 | if (wait_event_interruptible(pipe->wake_queue, |
52bcc7d9 | 392 | !test_bit(wake_bit, &pipe->flags))) |
726ea1a8 JQ |
393 | return -ERESTARTSYS; |
394 | ||
395 | if (test_bit(BIT_CLOSED_ON_HOST, &pipe->flags)) | |
396 | return -EIO; | |
397 | } | |
c89f2750 | 398 | |
c89f2750 DDT |
399 | return 0; |
400 | } | |
401 | ||
726ea1a8 | 402 | static ssize_t goldfish_pipe_read_write(struct file *filp, |
52bcc7d9 RK |
403 | char __user *buffer, |
404 | size_t bufflen, | |
405 | int is_write) | |
c89f2750 | 406 | { |
c89f2750 | 407 | struct goldfish_pipe *pipe = filp->private_data; |
2f3be882 | 408 | int count = 0, ret = -EINVAL; |
726ea1a8 JQ |
409 | unsigned long address, address_end, last_page; |
410 | unsigned int last_page_size; | |
c89f2750 DDT |
411 | |
412 | /* If the emulator already closed the pipe, no need to go further */ | |
726ea1a8 | 413 | if (unlikely(test_bit(BIT_CLOSED_ON_HOST, &pipe->flags))) |
c89f2750 | 414 | return -EIO; |
c89f2750 | 415 | /* Null reads or writes succeeds */ |
3411d035 | 416 | if (unlikely(bufflen == 0)) |
c89f2750 | 417 | return 0; |
c89f2750 | 418 | /* Check the buffer range for access */ |
96d4f267 | 419 | if (unlikely(!access_ok(buffer, bufflen))) |
c89f2750 DDT |
420 | return -EFAULT; |
421 | ||
726ea1a8 | 422 | address = (unsigned long)buffer; |
c89f2750 | 423 | address_end = address + bufflen; |
726ea1a8 JQ |
424 | last_page = (address_end - 1) & PAGE_MASK; |
425 | last_page_size = ((address_end - 1) & ~PAGE_MASK) + 1; | |
c89f2750 DDT |
426 | |
427 | while (address < address_end) { | |
726ea1a8 JQ |
428 | s32 consumed_size; |
429 | int status; | |
4f42071c | 430 | |
726ea1a8 | 431 | ret = transfer_max_buffers(pipe, address, address_end, is_write, |
52bcc7d9 RK |
432 | last_page, last_page_size, |
433 | &consumed_size, &status); | |
2f3be882 | 434 | if (ret < 0) |
5fae054c | 435 | break; |
c89f2750 | 436 | |
726ea1a8 JQ |
437 | if (consumed_size > 0) { |
438 | /* No matter what's the status, we've transferred | |
439 | * something. | |
4f42071c | 440 | */ |
726ea1a8 JQ |
441 | count += consumed_size; |
442 | address += consumed_size; | |
c89f2750 | 443 | } |
726ea1a8 | 444 | if (status > 0) |
c89f2750 | 445 | continue; |
726ea1a8 JQ |
446 | if (status == 0) { |
447 | /* EOF */ | |
2f3be882 | 448 | ret = 0; |
c89f2750 | 449 | break; |
726ea1a8 JQ |
450 | } |
451 | if (count > 0) { | |
2f3be882 | 452 | /* |
726ea1a8 JQ |
453 | * An error occurred, but we already transferred |
454 | * something on one of the previous iterations. | |
2f3be882 CD |
455 | * Just return what we already copied and log this |
456 | * err. | |
2f3be882 | 457 | */ |
25dd0f40 | 458 | if (status != PIPE_ERROR_AGAIN) |
25b97d57 RK |
459 | dev_err_ratelimited(pipe->dev->pdev_dev, |
460 | "backend error %d on %s\n", | |
2f3be882 | 461 | status, is_write ? "write" : "read"); |
c89f2750 | 462 | break; |
2f3be882 | 463 | } |
c89f2750 | 464 | |
2f3be882 | 465 | /* |
726ea1a8 | 466 | * If the error is not PIPE_ERROR_AGAIN, or if we are in |
2f3be882 CD |
467 | * non-blocking mode, just return the error code. |
468 | */ | |
c89f2750 DDT |
469 | if (status != PIPE_ERROR_AGAIN || |
470 | (filp->f_flags & O_NONBLOCK) != 0) { | |
471 | ret = goldfish_pipe_error_convert(status); | |
472 | break; | |
473 | } | |
474 | ||
726ea1a8 JQ |
475 | status = wait_for_host_signal(pipe, is_write); |
476 | if (status < 0) | |
477 | return status; | |
c89f2750 | 478 | } |
2f3be882 | 479 | |
726ea1a8 | 480 | if (count > 0) |
2f3be882 | 481 | return count; |
726ea1a8 | 482 | return ret; |
c89f2750 DDT |
483 | } |
484 | ||
485 | static ssize_t goldfish_pipe_read(struct file *filp, char __user *buffer, | |
52bcc7d9 | 486 | size_t bufflen, loff_t *ppos) |
c89f2750 | 487 | { |
726ea1a8 | 488 | return goldfish_pipe_read_write(filp, buffer, bufflen, |
52bcc7d9 | 489 | /* is_write */ 0); |
c89f2750 DDT |
490 | } |
491 | ||
492 | static ssize_t goldfish_pipe_write(struct file *filp, | |
52bcc7d9 RK |
493 | const char __user *buffer, size_t bufflen, |
494 | loff_t *ppos) | |
c89f2750 | 495 | { |
52bcc7d9 RK |
496 | /* cast away the const */ |
497 | char __user *no_const_buffer = (char __user *)buffer; | |
498 | ||
499 | return goldfish_pipe_read_write(filp, no_const_buffer, bufflen, | |
500 | /* is_write */ 1); | |
c89f2750 DDT |
501 | } |
502 | ||
afc9a42b | 503 | static __poll_t goldfish_pipe_poll(struct file *filp, poll_table *wait) |
c89f2750 DDT |
504 | { |
505 | struct goldfish_pipe *pipe = filp->private_data; | |
afc9a42b | 506 | __poll_t mask = 0; |
c89f2750 DDT |
507 | int status; |
508 | ||
c89f2750 DDT |
509 | poll_wait(filp, &pipe->wake_queue, wait); |
510 | ||
92c320b9 | 511 | status = goldfish_pipe_cmd(pipe, PIPE_CMD_POLL); |
726ea1a8 JQ |
512 | if (status < 0) |
513 | return -ERESTARTSYS; | |
c89f2750 DDT |
514 | |
515 | if (status & PIPE_POLL_IN) | |
a9a08845 | 516 | mask |= EPOLLIN | EPOLLRDNORM; |
c89f2750 | 517 | if (status & PIPE_POLL_OUT) |
a9a08845 | 518 | mask |= EPOLLOUT | EPOLLWRNORM; |
c89f2750 | 519 | if (status & PIPE_POLL_HUP) |
a9a08845 | 520 | mask |= EPOLLHUP; |
c89f2750 | 521 | if (test_bit(BIT_CLOSED_ON_HOST, &pipe->flags)) |
a9a08845 | 522 | mask |= EPOLLERR; |
c89f2750 DDT |
523 | |
524 | return mask; | |
525 | } | |
526 | ||
726ea1a8 | 527 | static void signalled_pipes_add_locked(struct goldfish_pipe_dev *dev, |
52bcc7d9 | 528 | u32 id, u32 flags) |
c89f2750 | 529 | { |
726ea1a8 | 530 | struct goldfish_pipe *pipe; |
c89f2750 | 531 | |
726ea1a8 JQ |
532 | if (WARN_ON(id >= dev->pipes_capacity)) |
533 | return; | |
534 | ||
535 | pipe = dev->pipes[id]; | |
536 | if (!pipe) | |
537 | return; | |
538 | pipe->signalled_flags |= flags; | |
539 | ||
cc14057f RK |
540 | if (pipe->prev_signalled || pipe->next_signalled || |
541 | dev->first_signalled_pipe == pipe) | |
726ea1a8 JQ |
542 | return; /* already in the list */ |
543 | pipe->next_signalled = dev->first_signalled_pipe; | |
544 | if (dev->first_signalled_pipe) | |
545 | dev->first_signalled_pipe->prev_signalled = pipe; | |
546 | dev->first_signalled_pipe = pipe; | |
547 | } | |
49a75c44 | 548 | |
726ea1a8 | 549 | static void signalled_pipes_remove_locked(struct goldfish_pipe_dev *dev, |
52bcc7d9 | 550 | struct goldfish_pipe *pipe) |
53bdf668 | 551 | { |
726ea1a8 JQ |
552 | if (pipe->prev_signalled) |
553 | pipe->prev_signalled->next_signalled = pipe->next_signalled; | |
554 | if (pipe->next_signalled) | |
555 | pipe->next_signalled->prev_signalled = pipe->prev_signalled; | |
556 | if (pipe == dev->first_signalled_pipe) | |
557 | dev->first_signalled_pipe = pipe->next_signalled; | |
558 | pipe->prev_signalled = NULL; | |
559 | pipe->next_signalled = NULL; | |
560 | } | |
25c72c78 | 561 | |
726ea1a8 JQ |
562 | static struct goldfish_pipe *signalled_pipes_pop_front( |
563 | struct goldfish_pipe_dev *dev, int *wakes) | |
564 | { | |
565 | struct goldfish_pipe *pipe; | |
566 | unsigned long flags; | |
c89f2750 | 567 | |
726ea1a8 | 568 | spin_lock_irqsave(&dev->lock, flags); |
c89f2750 | 569 | |
726ea1a8 JQ |
570 | pipe = dev->first_signalled_pipe; |
571 | if (pipe) { | |
572 | *wakes = pipe->signalled_flags; | |
573 | pipe->signalled_flags = 0; | |
574 | /* | |
575 | * This is an optimized version of | |
576 | * signalled_pipes_remove_locked() | |
577 | * - We want to make it as fast as possible to | |
578 | * wake the sleeping pipe operations faster. | |
579 | */ | |
580 | dev->first_signalled_pipe = pipe->next_signalled; | |
581 | if (dev->first_signalled_pipe) | |
582 | dev->first_signalled_pipe->prev_signalled = NULL; | |
583 | pipe->next_signalled = NULL; | |
584 | } | |
c89f2750 | 585 | |
726ea1a8 JQ |
586 | spin_unlock_irqrestore(&dev->lock, flags); |
587 | return pipe; | |
588 | } | |
589 | ||
c394cc3b | 590 | static void goldfish_interrupt_task(unsigned long dev_addr) |
726ea1a8 | 591 | { |
726ea1a8 | 592 | /* Iterate over the signalled pipes and wake them one by one */ |
c394cc3b | 593 | struct goldfish_pipe_dev *dev = (struct goldfish_pipe_dev *)dev_addr; |
726ea1a8 JQ |
594 | struct goldfish_pipe *pipe; |
595 | int wakes; | |
596 | ||
c394cc3b | 597 | while ((pipe = signalled_pipes_pop_front(dev, &wakes)) != NULL) { |
c89f2750 | 598 | if (wakes & PIPE_WAKE_CLOSED) { |
726ea1a8 JQ |
599 | pipe->flags = 1 << BIT_CLOSED_ON_HOST; |
600 | } else { | |
601 | if (wakes & PIPE_WAKE_READ) | |
602 | clear_bit(BIT_WAKE_ON_READ, &pipe->flags); | |
603 | if (wakes & PIPE_WAKE_WRITE) | |
604 | clear_bit(BIT_WAKE_ON_WRITE, &pipe->flags); | |
c89f2750 | 605 | } |
726ea1a8 JQ |
606 | /* |
607 | * wake_up_interruptible() implies a write barrier, so don't | |
608 | * explicitly add another one here. | |
609 | */ | |
c89f2750 | 610 | wake_up_interruptible(&pipe->wake_queue); |
c89f2750 | 611 | } |
726ea1a8 | 612 | } |
c89f2750 | 613 | |
08360e26 RK |
614 | static void goldfish_pipe_device_deinit(struct platform_device *pdev, |
615 | struct goldfish_pipe_dev *dev); | |
616 | ||
726ea1a8 JQ |
617 | /* |
618 | * The general idea of the interrupt handling: | |
619 | * | |
620 | * 1. device raises an interrupt if there's at least one signalled pipe | |
621 | * 2. IRQ handler reads the signalled pipes and their count from the device | |
622 | * 3. device writes them into a shared buffer and returns the count | |
623 | * it only resets the IRQ if it has returned all signalled pipes, | |
624 | * otherwise it leaves it raised, so IRQ handler will be called | |
625 | * again for the next chunk | |
626 | * 4. IRQ handler adds all returned pipes to the device's signalled pipes list | |
627 | * 5. IRQ handler launches a tasklet to process the signalled pipes from the | |
628 | * list in a separate context | |
629 | */ | |
630 | static irqreturn_t goldfish_pipe_interrupt(int irq, void *dev_id) | |
631 | { | |
632 | u32 count; | |
633 | u32 i; | |
634 | unsigned long flags; | |
635 | struct goldfish_pipe_dev *dev = dev_id; | |
636 | ||
08360e26 | 637 | if (dev->magic != &goldfish_pipe_device_deinit) |
726ea1a8 JQ |
638 | return IRQ_NONE; |
639 | ||
640 | /* Request the signalled pipes from the device */ | |
641 | spin_lock_irqsave(&dev->lock, flags); | |
642 | ||
643 | count = readl(dev->base + PIPE_REG_GET_SIGNALLED); | |
644 | if (count == 0) { | |
645 | spin_unlock_irqrestore(&dev->lock, flags); | |
646 | return IRQ_NONE; | |
647 | } | |
648 | if (count > MAX_SIGNALLED_PIPES) | |
649 | count = MAX_SIGNALLED_PIPES; | |
650 | ||
651 | for (i = 0; i < count; ++i) | |
652 | signalled_pipes_add_locked(dev, | |
653 | dev->buffers->signalled_pipe_buffers[i].id, | |
654 | dev->buffers->signalled_pipe_buffers[i].flags); | |
655 | ||
656 | spin_unlock_irqrestore(&dev->lock, flags); | |
657 | ||
c394cc3b | 658 | tasklet_schedule(&dev->irq_tasklet); |
726ea1a8 JQ |
659 | return IRQ_HANDLED; |
660 | } | |
661 | ||
662 | static int get_free_pipe_id_locked(struct goldfish_pipe_dev *dev) | |
663 | { | |
664 | int id; | |
665 | ||
666 | for (id = 0; id < dev->pipes_capacity; ++id) | |
667 | if (!dev->pipes[id]) | |
668 | return id; | |
669 | ||
670 | { | |
84ae527a RK |
671 | /* Reallocate the array. |
672 | * Since get_free_pipe_id_locked runs with interrupts disabled, | |
673 | * we don't want to make calls that could lead to sleep. | |
674 | */ | |
726ea1a8 JQ |
675 | u32 new_capacity = 2 * dev->pipes_capacity; |
676 | struct goldfish_pipe **pipes = | |
3eff8ecd | 677 | kcalloc(new_capacity, sizeof(*pipes), GFP_ATOMIC); |
726ea1a8 JQ |
678 | if (!pipes) |
679 | return -ENOMEM; | |
680 | memcpy(pipes, dev->pipes, sizeof(*pipes) * dev->pipes_capacity); | |
681 | kfree(dev->pipes); | |
682 | dev->pipes = pipes; | |
683 | id = dev->pipes_capacity; | |
684 | dev->pipes_capacity = new_capacity; | |
685 | } | |
686 | return id; | |
c89f2750 DDT |
687 | } |
688 | ||
08360e26 RK |
689 | /* A helper function to get the instance of goldfish_pipe_dev from file */ |
690 | static struct goldfish_pipe_dev *to_goldfish_pipe_dev(struct file *file) | |
691 | { | |
692 | struct miscdevice *miscdev = file->private_data; | |
693 | ||
694 | return container_of(miscdev, struct goldfish_pipe_dev, miscdev); | |
695 | } | |
696 | ||
c89f2750 | 697 | /** |
726ea1a8 | 698 | * goldfish_pipe_open - open a channel to the AVD |
c89f2750 DDT |
699 | * @inode: inode of device |
700 | * @file: file struct of opener | |
701 | * | |
702 | * Create a new pipe link between the emulator and the use application. | |
703 | * Each new request produces a new pipe. | |
704 | * | |
705 | * Note: we use the pipe ID as a mux. All goldfish emulations are 32bit | |
706 | * right now so this is fine. A move to 64bit will need this addressing | |
707 | */ | |
708 | static int goldfish_pipe_open(struct inode *inode, struct file *file) | |
709 | { | |
08360e26 | 710 | struct goldfish_pipe_dev *dev = to_goldfish_pipe_dev(file); |
726ea1a8 JQ |
711 | unsigned long flags; |
712 | int id; | |
713 | int status; | |
c89f2750 DDT |
714 | |
715 | /* Allocate new pipe kernel object */ | |
726ea1a8 | 716 | struct goldfish_pipe *pipe = kzalloc(sizeof(*pipe), GFP_KERNEL); |
6d7d725b | 717 | |
bfb8e838 | 718 | if (!pipe) |
c89f2750 DDT |
719 | return -ENOMEM; |
720 | ||
721 | pipe->dev = dev; | |
722 | mutex_init(&pipe->lock); | |
723 | init_waitqueue_head(&pipe->wake_queue); | |
724 | ||
725 | /* | |
726ea1a8 JQ |
726 | * Command buffer needs to be allocated on its own page to make sure |
727 | * it is physically contiguous in host's address space. | |
c89f2750 | 728 | */ |
d23069a5 | 729 | BUILD_BUG_ON(sizeof(struct goldfish_pipe_command) > PAGE_SIZE); |
726ea1a8 JQ |
730 | pipe->command_buffer = |
731 | (struct goldfish_pipe_command *)__get_free_page(GFP_KERNEL); | |
732 | if (!pipe->command_buffer) { | |
733 | status = -ENOMEM; | |
734 | goto err_pipe; | |
735 | } | |
c89f2750 | 736 | |
726ea1a8 JQ |
737 | spin_lock_irqsave(&dev->lock, flags); |
738 | ||
739 | id = get_free_pipe_id_locked(dev); | |
740 | if (id < 0) { | |
741 | status = id; | |
742 | goto err_id_locked; | |
c89f2750 DDT |
743 | } |
744 | ||
726ea1a8 JQ |
745 | dev->pipes[id] = pipe; |
746 | pipe->id = id; | |
747 | pipe->command_buffer->id = id; | |
748 | ||
749 | /* Now tell the emulator we're opening a new pipe. */ | |
750 | dev->buffers->open_command_params.rw_params_max_count = | |
751 | MAX_BUFFERS_PER_COMMAND; | |
752 | dev->buffers->open_command_params.command_buffer_ptr = | |
753 | (u64)(unsigned long)__pa(pipe->command_buffer); | |
92c320b9 | 754 | status = goldfish_pipe_cmd_locked(pipe, PIPE_CMD_OPEN); |
726ea1a8 JQ |
755 | spin_unlock_irqrestore(&dev->lock, flags); |
756 | if (status < 0) | |
757 | goto err_cmd; | |
c89f2750 DDT |
758 | /* All is done, save the pipe into the file's private data field */ |
759 | file->private_data = pipe; | |
760 | return 0; | |
726ea1a8 JQ |
761 | |
762 | err_cmd: | |
763 | spin_lock_irqsave(&dev->lock, flags); | |
764 | dev->pipes[id] = NULL; | |
765 | err_id_locked: | |
766 | spin_unlock_irqrestore(&dev->lock, flags); | |
767 | free_page((unsigned long)pipe->command_buffer); | |
768 | err_pipe: | |
769 | kfree(pipe); | |
770 | return status; | |
c89f2750 DDT |
771 | } |
772 | ||
773 | static int goldfish_pipe_release(struct inode *inode, struct file *filp) | |
774 | { | |
726ea1a8 | 775 | unsigned long flags; |
c89f2750 | 776 | struct goldfish_pipe *pipe = filp->private_data; |
726ea1a8 | 777 | struct goldfish_pipe_dev *dev = pipe->dev; |
c89f2750 DDT |
778 | |
779 | /* The guest is closing the channel, so tell the emulator right now */ | |
92c320b9 | 780 | goldfish_pipe_cmd(pipe, PIPE_CMD_CLOSE); |
726ea1a8 JQ |
781 | |
782 | spin_lock_irqsave(&dev->lock, flags); | |
783 | dev->pipes[pipe->id] = NULL; | |
784 | signalled_pipes_remove_locked(dev, pipe); | |
785 | spin_unlock_irqrestore(&dev->lock, flags); | |
786 | ||
c89f2750 | 787 | filp->private_data = NULL; |
726ea1a8 JQ |
788 | free_page((unsigned long)pipe->command_buffer); |
789 | kfree(pipe); | |
c89f2750 DDT |
790 | return 0; |
791 | } | |
792 | ||
793 | static const struct file_operations goldfish_pipe_fops = { | |
794 | .owner = THIS_MODULE, | |
795 | .read = goldfish_pipe_read, | |
796 | .write = goldfish_pipe_write, | |
797 | .poll = goldfish_pipe_poll, | |
798 | .open = goldfish_pipe_open, | |
799 | .release = goldfish_pipe_release, | |
800 | }; | |
801 | ||
43c2cc28 RK |
802 | static void init_miscdevice(struct miscdevice *miscdev) |
803 | { | |
804 | memset(miscdev, 0, sizeof(*miscdev)); | |
805 | ||
806 | miscdev->minor = MISC_DYNAMIC_MINOR; | |
807 | miscdev->name = "goldfish_pipe"; | |
808 | miscdev->fops = &goldfish_pipe_fops; | |
809 | } | |
c89f2750 | 810 | |
610a72b7 RK |
811 | static void write_pa_addr(void *addr, void __iomem *portl, void __iomem *porth) |
812 | { | |
813 | const unsigned long paddr = __pa(addr); | |
814 | ||
815 | writel(upper_32_bits(paddr), porth); | |
816 | writel(lower_32_bits(paddr), portl); | |
817 | } | |
818 | ||
08360e26 RK |
819 | static int goldfish_pipe_device_init(struct platform_device *pdev, |
820 | struct goldfish_pipe_dev *dev) | |
726ea1a8 | 821 | { |
c394cc3b RK |
822 | int err; |
823 | ||
824 | tasklet_init(&dev->irq_tasklet, &goldfish_interrupt_task, | |
825 | (unsigned long)dev); | |
826 | ||
827 | err = devm_request_irq(&pdev->dev, dev->irq, | |
828 | goldfish_pipe_interrupt, | |
829 | IRQF_SHARED, "goldfish_pipe", dev); | |
726ea1a8 JQ |
830 | if (err) { |
831 | dev_err(&pdev->dev, "unable to allocate IRQ for v2\n"); | |
832 | return err; | |
833 | } | |
834 | ||
43c2cc28 RK |
835 | init_miscdevice(&dev->miscdev); |
836 | err = misc_register(&dev->miscdev); | |
726ea1a8 JQ |
837 | if (err) { |
838 | dev_err(&pdev->dev, "unable to register v2 device\n"); | |
839 | return err; | |
840 | } | |
841 | ||
25b97d57 | 842 | dev->pdev_dev = &pdev->dev; |
726ea1a8 JQ |
843 | dev->first_signalled_pipe = NULL; |
844 | dev->pipes_capacity = INITIAL_PIPES_CAPACITY; | |
845 | dev->pipes = kcalloc(dev->pipes_capacity, sizeof(*dev->pipes), | |
52bcc7d9 | 846 | GFP_KERNEL); |
60a6e523 RK |
847 | if (!dev->pipes) { |
848 | misc_deregister(&dev->miscdev); | |
726ea1a8 | 849 | return -ENOMEM; |
60a6e523 | 850 | } |
726ea1a8 JQ |
851 | |
852 | /* | |
853 | * We're going to pass two buffers, open_command_params and | |
854 | * signalled_pipe_buffers, to the host. This means each of those buffers | |
855 | * needs to be contained in a single physical page. The easiest choice | |
856 | * is to just allocate a page and place the buffers in it. | |
857 | */ | |
d23069a5 | 858 | BUILD_BUG_ON(sizeof(struct goldfish_pipe_dev_buffers) > PAGE_SIZE); |
2ed43e53 RK |
859 | dev->buffers = (struct goldfish_pipe_dev_buffers *) |
860 | __get_free_page(GFP_KERNEL); | |
861 | if (!dev->buffers) { | |
726ea1a8 | 862 | kfree(dev->pipes); |
60a6e523 | 863 | misc_deregister(&dev->miscdev); |
726ea1a8 JQ |
864 | return -ENOMEM; |
865 | } | |
726ea1a8 JQ |
866 | |
867 | /* Send the buffer addresses to the host */ | |
610a72b7 RK |
868 | write_pa_addr(&dev->buffers->signalled_pipe_buffers, |
869 | dev->base + PIPE_REG_SIGNAL_BUFFER, | |
870 | dev->base + PIPE_REG_SIGNAL_BUFFER_HIGH); | |
871 | ||
468e62f9 | 872 | writel(MAX_SIGNALLED_PIPES, |
610a72b7 RK |
873 | dev->base + PIPE_REG_SIGNAL_BUFFER_COUNT); |
874 | ||
875 | write_pa_addr(&dev->buffers->open_command_params, | |
876 | dev->base + PIPE_REG_OPEN_BUFFER, | |
877 | dev->base + PIPE_REG_OPEN_BUFFER_HIGH); | |
878 | ||
08360e26 | 879 | platform_set_drvdata(pdev, dev); |
726ea1a8 JQ |
880 | return 0; |
881 | } | |
882 | ||
08360e26 RK |
883 | static void goldfish_pipe_device_deinit(struct platform_device *pdev, |
884 | struct goldfish_pipe_dev *dev) | |
726ea1a8 | 885 | { |
08360e26 RK |
886 | misc_deregister(&dev->miscdev); |
887 | tasklet_kill(&dev->irq_tasklet); | |
888 | kfree(dev->pipes); | |
889 | free_page((unsigned long)dev->buffers); | |
726ea1a8 JQ |
890 | } |
891 | ||
c89f2750 DDT |
892 | static int goldfish_pipe_probe(struct platform_device *pdev) |
893 | { | |
c89f2750 | 894 | struct resource *r; |
08360e26 | 895 | struct goldfish_pipe_dev *dev; |
c89f2750 | 896 | |
08360e26 RK |
897 | dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); |
898 | if (!dev) | |
899 | return -ENOMEM; | |
c89f2750 | 900 | |
08360e26 | 901 | dev->magic = &goldfish_pipe_device_deinit; |
c89f2750 DDT |
902 | spin_lock_init(&dev->lock); |
903 | ||
904 | r = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
bfb8e838 | 905 | if (!r || resource_size(r) < PAGE_SIZE) { |
c89f2750 DDT |
906 | dev_err(&pdev->dev, "can't allocate i/o page\n"); |
907 | return -EINVAL; | |
908 | } | |
909 | dev->base = devm_ioremap(&pdev->dev, r->start, PAGE_SIZE); | |
bfb8e838 | 910 | if (!dev->base) { |
c89f2750 DDT |
911 | dev_err(&pdev->dev, "ioremap failed\n"); |
912 | return -EINVAL; | |
913 | } | |
914 | ||
915 | r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); | |
08360e26 RK |
916 | if (!r) |
917 | return -EINVAL; | |
918 | ||
c89f2750 DDT |
919 | dev->irq = r->start; |
920 | ||
726ea1a8 JQ |
921 | /* |
922 | * Exchange the versions with the host device | |
923 | * | |
924 | * Note: v1 driver used to not report its version, so we write it before | |
925 | * reading device version back: this allows the host implementation to | |
926 | * detect the old driver (if there was no version write before read). | |
4f42071c | 927 | */ |
e6fb3193 | 928 | writel(PIPE_DRIVER_VERSION, dev->base + PIPE_REG_VERSION); |
4f42071c | 929 | dev->version = readl(dev->base + PIPE_REG_VERSION); |
726ea1a8 JQ |
930 | if (WARN_ON(dev->version < PIPE_CURRENT_DEVICE_VERSION)) |
931 | return -EINVAL; | |
932 | ||
08360e26 | 933 | return goldfish_pipe_device_init(pdev, dev); |
c89f2750 DDT |
934 | } |
935 | ||
936 | static int goldfish_pipe_remove(struct platform_device *pdev) | |
937 | { | |
08360e26 RK |
938 | struct goldfish_pipe_dev *dev = platform_get_drvdata(pdev); |
939 | ||
940 | goldfish_pipe_device_deinit(pdev, dev); | |
c89f2750 DDT |
941 | return 0; |
942 | } | |
943 | ||
d62f324b JH |
944 | static const struct acpi_device_id goldfish_pipe_acpi_match[] = { |
945 | { "GFSH0003", 0 }, | |
946 | { }, | |
947 | }; | |
948 | MODULE_DEVICE_TABLE(acpi, goldfish_pipe_acpi_match); | |
949 | ||
91a18a41 GH |
950 | static const struct of_device_id goldfish_pipe_of_match[] = { |
951 | { .compatible = "google,android-pipe", }, | |
952 | {}, | |
953 | }; | |
954 | MODULE_DEVICE_TABLE(of, goldfish_pipe_of_match); | |
955 | ||
726ea1a8 | 956 | static struct platform_driver goldfish_pipe_driver = { |
c89f2750 DDT |
957 | .probe = goldfish_pipe_probe, |
958 | .remove = goldfish_pipe_remove, | |
959 | .driver = { | |
91a18a41 | 960 | .name = "goldfish_pipe", |
91a18a41 | 961 | .of_match_table = goldfish_pipe_of_match, |
d62f324b | 962 | .acpi_match_table = ACPI_PTR(goldfish_pipe_acpi_match), |
c89f2750 DDT |
963 | } |
964 | }; | |
965 | ||
726ea1a8 | 966 | module_platform_driver(goldfish_pipe_driver); |
c89f2750 | 967 | MODULE_AUTHOR("David Turner <digit@google.com>"); |
c3c4e307 | 968 | MODULE_LICENSE("GPL v2"); |