Commit | Line | Data |
---|---|---|
1f327613 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
b530cc79 | 2 | /* |
fea511a6 | 3 | * The Virtio 9p transport driver |
b530cc79 | 4 | * |
e2735b77 EVH |
5 | * This is a block based transport driver based on the lguest block driver |
6 | * code. | |
b530cc79 | 7 | * |
fea511a6 | 8 | * Copyright (C) 2007, 2008 Eric Van Hensbergen, IBM Corporation |
b530cc79 EVH |
9 | * |
10 | * Based on virtio console driver | |
11 | * Copyright (C) 2006, 2007 Rusty Russell, IBM Corporation | |
b530cc79 EVH |
12 | */ |
13 | ||
5d385153 JP |
14 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
15 | ||
b530cc79 EVH |
16 | #include <linux/in.h> |
17 | #include <linux/module.h> | |
18 | #include <linux/net.h> | |
19 | #include <linux/ipv6.h> | |
20 | #include <linux/errno.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/un.h> | |
23 | #include <linux/uaccess.h> | |
24 | #include <linux/inet.h> | |
25 | #include <linux/idr.h> | |
26 | #include <linux/file.h> | |
b9cdc88d | 27 | #include <linux/highmem.h> |
5a0e3ad6 | 28 | #include <linux/slab.h> |
b530cc79 EVH |
29 | #include <net/9p/9p.h> |
30 | #include <linux/parser.h> | |
8b81ef58 | 31 | #include <net/9p/client.h> |
b530cc79 EVH |
32 | #include <net/9p/transport.h> |
33 | #include <linux/scatterlist.h> | |
68da9ba4 | 34 | #include <linux/swap.h> |
b530cc79 EVH |
35 | #include <linux/virtio.h> |
36 | #include <linux/virtio_9p.h> | |
4038866d | 37 | #include "trans_common.h" |
b530cc79 | 38 | |
e2735b77 EVH |
39 | #define VIRTQUEUE_NUM 128 |
40 | ||
b530cc79 | 41 | /* a single mutex to manage channel initialization and attachment */ |
c1549497 | 42 | static DEFINE_MUTEX(virtio_9p_lock); |
68da9ba4 VJJ |
43 | static DECLARE_WAIT_QUEUE_HEAD(vp_wq); |
44 | static atomic_t vp_pinned = ATOMIC_INIT(0); | |
b530cc79 | 45 | |
ee443996 EVH |
46 | /** |
47 | * struct virtio_chan - per-instance transport information | |
ee443996 EVH |
48 | * @inuse: whether the channel is in use |
49 | * @lock: protects multiple elements within this structure | |
0e15597e | 50 | * @client: client instance |
ee443996 EVH |
51 | * @vdev: virtio dev associated with this channel |
52 | * @vq: virtio queue associated with this channel | |
760b3d61 AL |
53 | * @ring_bufs_avail: flag to indicate there is some available in the ring buf |
54 | * @vc_wq: wait queue for waiting for thing to be added to ring buf | |
55 | * @p9_max_pages: maximum number of pinned pages | |
ee443996 | 56 | * @sg: scatter gather list which is used to pack a request (protected?) |
760b3d61 | 57 | * @chan_list: linked list of channels |
ee443996 EVH |
58 | * |
59 | * We keep all per-channel information in a structure. | |
b530cc79 EVH |
60 | * This structure is allocated within the devices dev->mem space. |
61 | * A pointer to the structure will get put in the transport private. | |
ee443996 | 62 | * |
b530cc79 | 63 | */ |
ee443996 | 64 | |
37c1209d | 65 | struct virtio_chan { |
ee443996 | 66 | bool inuse; |
b530cc79 | 67 | |
e2735b77 EVH |
68 | spinlock_t lock; |
69 | ||
fea511a6 | 70 | struct p9_client *client; |
b530cc79 | 71 | struct virtio_device *vdev; |
e2735b77 | 72 | struct virtqueue *vq; |
52f44e0d VJJ |
73 | int ring_bufs_avail; |
74 | wait_queue_head_t *vc_wq; | |
68da9ba4 VJJ |
75 | /* This is global limit. Since we don't have a global structure, |
76 | * will be placing it in each channel. | |
77 | */ | |
7293bfba | 78 | unsigned long p9_max_pages; |
e2735b77 EVH |
79 | /* Scatterlist: can be too big for stack. */ |
80 | struct scatterlist sg[VIRTQUEUE_NUM]; | |
760b3d61 AL |
81 | /** |
82 | * @tag: name to identify a mount null terminated | |
97ee9b02 AK |
83 | */ |
84 | char *tag; | |
85 | ||
37c1209d AK |
86 | struct list_head chan_list; |
87 | }; | |
88 | ||
89 | static struct list_head virtio_chan_list; | |
b530cc79 EVH |
90 | |
91 | /* How many bytes left in this page. */ | |
92 | static unsigned int rest_of_page(void *data) | |
93 | { | |
222e4ade | 94 | return PAGE_SIZE - offset_in_page(data); |
b530cc79 EVH |
95 | } |
96 | ||
ee443996 EVH |
97 | /** |
98 | * p9_virtio_close - reclaim resources of a channel | |
0e15597e | 99 | * @client: client instance |
ee443996 EVH |
100 | * |
101 | * This reclaims a channel by freeing its resources and | |
8ab1784d | 102 | * resetting its inuse flag. |
ee443996 EVH |
103 | * |
104 | */ | |
105 | ||
8b81ef58 | 106 | static void p9_virtio_close(struct p9_client *client) |
e2735b77 | 107 | { |
8b81ef58 | 108 | struct virtio_chan *chan = client->trans; |
b530cc79 | 109 | |
c1549497 | 110 | mutex_lock(&virtio_9p_lock); |
fb786100 AK |
111 | if (chan) |
112 | chan->inuse = false; | |
c1549497 | 113 | mutex_unlock(&virtio_9p_lock); |
b530cc79 EVH |
114 | } |
115 | ||
ee443996 EVH |
116 | /** |
117 | * req_done - callback which signals activity from the server | |
118 | * @vq: virtio queue activity was received on | |
119 | * | |
120 | * This notifies us that the server has triggered some activity | |
121 | * on the virtio channel - most likely a response to request we | |
122 | * sent. Figure out which requests now have responses and wake up | |
123 | * those threads. | |
124 | * | |
125 | * Bugs: could do with some additional sanity checking, but appears to work. | |
126 | * | |
127 | */ | |
128 | ||
e2735b77 | 129 | static void req_done(struct virtqueue *vq) |
b530cc79 | 130 | { |
e2735b77 | 131 | struct virtio_chan *chan = vq->vdev->priv; |
e2735b77 | 132 | unsigned int len; |
e2735b77 | 133 | struct p9_req_t *req; |
31934da8 | 134 | bool need_wakeup = false; |
419b3956 | 135 | unsigned long flags; |
e2735b77 | 136 | |
5d385153 | 137 | p9_debug(P9_DEBUG_TRANS, ": request done\n"); |
91b8534f | 138 | |
31934da8 | 139 | spin_lock_irqsave(&chan->lock, flags); |
140 | while ((req = virtqueue_get_buf(chan->vq, &len)) != NULL) { | |
141 | if (!chan->ring_bufs_avail) { | |
142 | chan->ring_bufs_avail = 1; | |
143 | need_wakeup = true; | |
a01a9840 | 144 | } |
31934da8 | 145 | |
f984579a | 146 | if (len) { |
523adb6c | 147 | req->rc.size = len; |
26d99834 | 148 | p9_client_cb(chan->client, req, REQ_STATUS_RCVD); |
f984579a | 149 | } |
a01a9840 | 150 | } |
31934da8 | 151 | spin_unlock_irqrestore(&chan->lock, flags); |
152 | /* Wakeup if anyone waiting for VirtIO ring space. */ | |
153 | if (need_wakeup) | |
154 | wake_up(chan->vc_wq); | |
e2735b77 | 155 | } |
b530cc79 | 156 | |
ee443996 EVH |
157 | /** |
158 | * pack_sg_list - pack a scatter gather list from a linear buffer | |
159 | * @sg: scatter/gather list to pack into | |
160 | * @start: which segment of the sg_list to start at | |
161 | * @limit: maximum segment to pack data to | |
162 | * @data: data to pack into scatter/gather list | |
163 | * @count: amount of data to pack into the scatter/gather list | |
164 | * | |
165 | * sg_lists have multiple segments of various sizes. This will pack | |
166 | * arbitrary data into an existing scatter gather list, segmenting the | |
167 | * data as necessary within constraints. | |
168 | * | |
169 | */ | |
170 | ||
abfa034e AK |
171 | static int pack_sg_list(struct scatterlist *sg, int start, |
172 | int limit, char *data, int count) | |
e2735b77 EVH |
173 | { |
174 | int s; | |
175 | int index = start; | |
176 | ||
177 | while (count) { | |
178 | s = rest_of_page(data); | |
179 | if (s > count) | |
180 | s = count; | |
23cba9cb | 181 | BUG_ON(index >= limit); |
0b36f1ad RR |
182 | /* Make sure we don't terminate early. */ |
183 | sg_unmark_end(&sg[index]); | |
e2735b77 EVH |
184 | sg_set_buf(&sg[index++], data, s); |
185 | count -= s; | |
186 | data += s; | |
e2735b77 | 187 | } |
0b36f1ad RR |
188 | if (index-start) |
189 | sg_mark_end(&sg[index - 1]); | |
e2735b77 | 190 | return index-start; |
b530cc79 EVH |
191 | } |
192 | ||
91b8534f EVH |
193 | /* We don't currently allow canceling of virtio requests */ |
194 | static int p9_virtio_cancel(struct p9_client *client, struct p9_req_t *req) | |
195 | { | |
196 | return 1; | |
197 | } | |
198 | ||
728356de TB |
199 | /* Reply won't come, so drop req ref */ |
200 | static int p9_virtio_cancelled(struct p9_client *client, struct p9_req_t *req) | |
201 | { | |
8b11ff09 | 202 | p9_req_put(client, req); |
728356de TB |
203 | return 0; |
204 | } | |
205 | ||
4038866d VJJ |
206 | /** |
207 | * pack_sg_list_p - Just like pack_sg_list. Instead of taking a buffer, | |
208 | * this takes a list of pages. | |
209 | * @sg: scatter/gather list to pack into | |
210 | * @start: which segment of the sg_list to start at | |
760b3d61 | 211 | * @limit: maximum number of pages in sg list. |
2c53040f | 212 | * @pdata: a list of pages to add into sg. |
abfa034e | 213 | * @nr_pages: number of pages to pack into the scatter/gather list |
4f3b35c1 | 214 | * @offs: amount of data in the beginning of first page _not_ to pack |
4038866d VJJ |
215 | * @count: amount of data to pack into the scatter/gather list |
216 | */ | |
217 | static int | |
abfa034e | 218 | pack_sg_list_p(struct scatterlist *sg, int start, int limit, |
4f3b35c1 | 219 | struct page **pdata, int nr_pages, size_t offs, int count) |
4038866d | 220 | { |
abfa034e | 221 | int i = 0, s; |
4f3b35c1 | 222 | int data_off = offs; |
4038866d VJJ |
223 | int index = start; |
224 | ||
abfa034e AK |
225 | BUG_ON(nr_pages > (limit - start)); |
226 | /* | |
227 | * if the first page doesn't start at | |
228 | * page boundary find the offset | |
229 | */ | |
abfa034e | 230 | while (nr_pages) { |
4f3b35c1 | 231 | s = PAGE_SIZE - data_off; |
abfa034e AK |
232 | if (s > count) |
233 | s = count; | |
23cba9cb | 234 | BUG_ON(index >= limit); |
0b36f1ad RR |
235 | /* Make sure we don't terminate early. */ |
236 | sg_unmark_end(&sg[index]); | |
abfa034e AK |
237 | sg_set_page(&sg[index++], pdata[i++], s, data_off); |
238 | data_off = 0; | |
4038866d | 239 | count -= s; |
abfa034e | 240 | nr_pages--; |
4038866d | 241 | } |
0b36f1ad RR |
242 | |
243 | if (index-start) | |
244 | sg_mark_end(&sg[index - 1]); | |
abfa034e | 245 | return index - start; |
4038866d VJJ |
246 | } |
247 | ||
ee443996 | 248 | /** |
91b8534f | 249 | * p9_virtio_request - issue a request |
0e15597e AK |
250 | * @client: client instance issuing the request |
251 | * @req: request to be issued | |
ee443996 EVH |
252 | * |
253 | */ | |
254 | ||
e2735b77 | 255 | static int |
91b8534f | 256 | p9_virtio_request(struct p9_client *client, struct p9_req_t *req) |
b530cc79 | 257 | { |
abfa034e | 258 | int err; |
0b36f1ad | 259 | int in, out, out_sgs, in_sgs; |
419b3956 | 260 | unsigned long flags; |
abfa034e | 261 | struct virtio_chan *chan = client->trans; |
0b36f1ad | 262 | struct scatterlist *sgs[2]; |
b530cc79 | 263 | |
5d385153 | 264 | p9_debug(P9_DEBUG_TRANS, "9p debug: virtio request\n"); |
b530cc79 | 265 | |
419b3956 | 266 | req->status = REQ_STATUS_SENT; |
abfa034e AK |
267 | req_retry: |
268 | spin_lock_irqsave(&chan->lock, flags); | |
269 | ||
0b36f1ad | 270 | out_sgs = in_sgs = 0; |
abfa034e AK |
271 | /* Handle out VirtIO ring buffers */ |
272 | out = pack_sg_list(chan->sg, 0, | |
523adb6c | 273 | VIRTQUEUE_NUM, req->tc.sdata, req->tc.size); |
0b36f1ad RR |
274 | if (out) |
275 | sgs[out_sgs++] = chan->sg; | |
419b3956 | 276 | |
abfa034e | 277 | in = pack_sg_list(chan->sg, out, |
523adb6c | 278 | VIRTQUEUE_NUM, req->rc.sdata, req->rc.capacity); |
0b36f1ad RR |
279 | if (in) |
280 | sgs[out_sgs + in_sgs++] = chan->sg + out; | |
4038866d | 281 | |
474fe9f7 | 282 | err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req, |
f96fde41 | 283 | GFP_ATOMIC); |
abfa034e AK |
284 | if (err < 0) { |
285 | if (err == -ENOSPC) { | |
286 | chan->ring_bufs_avail = 0; | |
287 | spin_unlock_irqrestore(&chan->lock, flags); | |
9523feac TT |
288 | err = wait_event_killable(*chan->vc_wq, |
289 | chan->ring_bufs_avail); | |
68da9ba4 VJJ |
290 | if (err == -ERESTARTSYS) |
291 | return err; | |
68da9ba4 | 292 | |
5d385153 | 293 | p9_debug(P9_DEBUG_TRANS, "Retry virtio request\n"); |
abfa034e | 294 | goto req_retry; |
4038866d | 295 | } else { |
abfa034e | 296 | spin_unlock_irqrestore(&chan->lock, flags); |
5d385153 | 297 | p9_debug(P9_DEBUG_TRANS, |
0b36f1ad | 298 | "virtio rpc add_sgs returned failure\n"); |
abfa034e | 299 | return -EIO; |
4038866d | 300 | } |
abfa034e AK |
301 | } |
302 | virtqueue_kick(chan->vq); | |
303 | spin_unlock_irqrestore(&chan->lock, flags); | |
4038866d | 304 | |
5d385153 | 305 | p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n"); |
abfa034e AK |
306 | return 0; |
307 | } | |
308 | ||
309 | static int p9_get_mapped_pages(struct virtio_chan *chan, | |
4f3b35c1 AV |
310 | struct page ***pages, |
311 | struct iov_iter *data, | |
312 | int count, | |
313 | size_t *offs, | |
314 | int *need_drop) | |
abfa034e | 315 | { |
4f3b35c1 | 316 | int nr_pages; |
abfa034e | 317 | int err; |
4f3b35c1 AV |
318 | |
319 | if (!iov_iter_count(data)) | |
320 | return 0; | |
321 | ||
2cbfdf4d | 322 | if (!iov_iter_is_kvec(data)) { |
4f3b35c1 | 323 | int n; |
abfa034e AK |
324 | /* |
325 | * We allow only p9_max_pages pinned. We wait for the | |
326 | * Other zc request to finish here | |
327 | */ | |
328 | if (atomic_read(&vp_pinned) >= chan->p9_max_pages) { | |
9523feac | 329 | err = wait_event_killable(vp_wq, |
abfa034e AK |
330 | (atomic_read(&vp_pinned) < chan->p9_max_pages)); |
331 | if (err == -ERESTARTSYS) | |
332 | return err; | |
333 | } | |
7f024647 | 334 | n = iov_iter_get_pages_alloc2(data, pages, count, offs); |
4f3b35c1 AV |
335 | if (n < 0) |
336 | return n; | |
337 | *need_drop = 1; | |
338 | nr_pages = DIV_ROUND_UP(n + *offs, PAGE_SIZE); | |
abfa034e | 339 | atomic_add(nr_pages, &vp_pinned); |
4f3b35c1 | 340 | return n; |
abfa034e AK |
341 | } else { |
342 | /* kernel buffer, no need to pin pages */ | |
4f3b35c1 AV |
343 | int index; |
344 | size_t len; | |
345 | void *p; | |
346 | ||
347 | /* we'd already checked that it's non-empty */ | |
348 | while (1) { | |
349 | len = iov_iter_single_seg_count(data); | |
350 | if (likely(len)) { | |
351 | p = data->kvec->iov_base + data->iov_offset; | |
352 | break; | |
353 | } | |
354 | iov_iter_advance(data, 0); | |
355 | } | |
356 | if (len > count) | |
357 | len = count; | |
358 | ||
359 | nr_pages = DIV_ROUND_UP((unsigned long)p + len, PAGE_SIZE) - | |
360 | (unsigned long)p / PAGE_SIZE; | |
361 | ||
6da2ec56 KC |
362 | *pages = kmalloc_array(nr_pages, sizeof(struct page *), |
363 | GFP_NOFS); | |
4f3b35c1 AV |
364 | if (!*pages) |
365 | return -ENOMEM; | |
366 | ||
367 | *need_drop = 0; | |
222e4ade | 368 | p -= (*offs = offset_in_page(p)); |
4f3b35c1 AV |
369 | for (index = 0; index < nr_pages; index++) { |
370 | if (is_vmalloc_addr(p)) | |
371 | (*pages)[index] = vmalloc_to_page(p); | |
b6f52ae2 | 372 | else |
4f3b35c1 AV |
373 | (*pages)[index] = kmap_to_page(p); |
374 | p += PAGE_SIZE; | |
4038866d | 375 | } |
7f024647 | 376 | iov_iter_advance(data, len); |
4f3b35c1 | 377 | return len; |
4038866d | 378 | } |
abfa034e | 379 | } |
4038866d | 380 | |
f615625a AV |
381 | static void handle_rerror(struct p9_req_t *req, int in_hdr_len, |
382 | size_t offs, struct page **pages) | |
383 | { | |
384 | unsigned size, n; | |
385 | void *to = req->rc.sdata + in_hdr_len; | |
386 | ||
387 | // Fits entirely into the static data? Nothing to do. | |
388 | if (req->rc.size < in_hdr_len) | |
389 | return; | |
390 | ||
391 | // Really long error message? Tough, truncate the reply. Might get | |
392 | // rejected (we can't be arsed to adjust the size encoded in header, | |
393 | // or string size for that matter), but it wouldn't be anything valid | |
394 | // anyway. | |
395 | if (unlikely(req->rc.size > P9_ZC_HDR_SZ)) | |
396 | req->rc.size = P9_ZC_HDR_SZ; | |
397 | ||
398 | // data won't span more than two pages | |
399 | size = req->rc.size - in_hdr_len; | |
400 | n = PAGE_SIZE - offs; | |
401 | if (size > n) { | |
402 | memcpy_from_page(to, *pages++, offs, n); | |
403 | offs = 0; | |
404 | to += n; | |
405 | size -= n; | |
406 | } | |
407 | memcpy_from_page(to, *pages, offs, size); | |
408 | } | |
409 | ||
abfa034e AK |
410 | /** |
411 | * p9_virtio_zc_request - issue a zero copy request | |
412 | * @client: client instance issuing the request | |
413 | * @req: request to be issued | |
c7ebbae7 | 414 | * @uidata: user buffer that should be used for zero copy read |
415 | * @uodata: user buffer that should be used for zero copy write | |
abfa034e | 416 | * @inlen: read buffer size |
4a026da9 SL |
417 | * @outlen: write buffer size |
418 | * @in_hdr_len: reader header size, This is the size of response protocol data | |
abfa034e AK |
419 | * |
420 | */ | |
421 | static int | |
422 | p9_virtio_zc_request(struct p9_client *client, struct p9_req_t *req, | |
4f3b35c1 AV |
423 | struct iov_iter *uidata, struct iov_iter *uodata, |
424 | int inlen, int outlen, int in_hdr_len) | |
abfa034e | 425 | { |
0b36f1ad | 426 | int in, out, err, out_sgs, in_sgs; |
abfa034e AK |
427 | unsigned long flags; |
428 | int in_nr_pages = 0, out_nr_pages = 0; | |
429 | struct page **in_pages = NULL, **out_pages = NULL; | |
430 | struct virtio_chan *chan = client->trans; | |
0b36f1ad | 431 | struct scatterlist *sgs[4]; |
4f3b35c1 AV |
432 | size_t offs; |
433 | int need_drop = 0; | |
728356de | 434 | int kicked = 0; |
4038866d | 435 | |
5d385153 | 436 | p9_debug(P9_DEBUG_TRANS, "virtio request\n"); |
abfa034e AK |
437 | |
438 | if (uodata) { | |
d28c756c | 439 | __le32 sz; |
4f3b35c1 AV |
440 | int n = p9_get_mapped_pages(chan, &out_pages, uodata, |
441 | outlen, &offs, &need_drop); | |
728356de TB |
442 | if (n < 0) { |
443 | err = n; | |
444 | goto err_out; | |
445 | } | |
4f3b35c1 AV |
446 | out_nr_pages = DIV_ROUND_UP(n + offs, PAGE_SIZE); |
447 | if (n != outlen) { | |
448 | __le32 v = cpu_to_le32(n); | |
523adb6c | 449 | memcpy(&req->tc.sdata[req->tc.size - 4], &v, 4); |
4f3b35c1 | 450 | outlen = n; |
4038866d | 451 | } |
d28c756c CE |
452 | /* The size field of the message must include the length of the |
453 | * header and the length of the data. We didn't actually know | |
454 | * the length of the data until this point so add it in now. | |
455 | */ | |
523adb6c DM |
456 | sz = cpu_to_le32(req->tc.size + outlen); |
457 | memcpy(&req->tc.sdata[0], &sz, sizeof(sz)); | |
4f3b35c1 AV |
458 | } else if (uidata) { |
459 | int n = p9_get_mapped_pages(chan, &in_pages, uidata, | |
460 | inlen, &offs, &need_drop); | |
728356de TB |
461 | if (n < 0) { |
462 | err = n; | |
463 | goto err_out; | |
464 | } | |
4f3b35c1 AV |
465 | in_nr_pages = DIV_ROUND_UP(n + offs, PAGE_SIZE); |
466 | if (n != inlen) { | |
467 | __le32 v = cpu_to_le32(n); | |
523adb6c | 468 | memcpy(&req->tc.sdata[req->tc.size - 4], &v, 4); |
4f3b35c1 | 469 | inlen = n; |
4038866d | 470 | } |
4038866d | 471 | } |
abfa034e AK |
472 | req->status = REQ_STATUS_SENT; |
473 | req_retry_pinned: | |
474 | spin_lock_irqsave(&chan->lock, flags); | |
0b36f1ad RR |
475 | |
476 | out_sgs = in_sgs = 0; | |
477 | ||
abfa034e AK |
478 | /* out data */ |
479 | out = pack_sg_list(chan->sg, 0, | |
523adb6c | 480 | VIRTQUEUE_NUM, req->tc.sdata, req->tc.size); |
abfa034e | 481 | |
0b36f1ad RR |
482 | if (out) |
483 | sgs[out_sgs++] = chan->sg; | |
484 | ||
485 | if (out_pages) { | |
486 | sgs[out_sgs++] = chan->sg + out; | |
abfa034e | 487 | out += pack_sg_list_p(chan->sg, out, VIRTQUEUE_NUM, |
4f3b35c1 | 488 | out_pages, out_nr_pages, offs, outlen); |
0b36f1ad | 489 | } |
c69f297d | 490 | |
abfa034e AK |
491 | /* |
492 | * Take care of in data | |
493 | * For example TREAD have 11. | |
494 | * 11 is the read/write header = PDU Header(7) + IO Size (4). | |
495 | * Arrange in such a way that server places header in the | |
8ab1784d | 496 | * allocated memory and payload onto the user buffer. |
abfa034e AK |
497 | */ |
498 | in = pack_sg_list(chan->sg, out, | |
523adb6c | 499 | VIRTQUEUE_NUM, req->rc.sdata, in_hdr_len); |
0b36f1ad RR |
500 | if (in) |
501 | sgs[out_sgs + in_sgs++] = chan->sg + out; | |
502 | ||
503 | if (in_pages) { | |
504 | sgs[out_sgs + in_sgs++] = chan->sg + out + in; | |
abfa034e | 505 | in += pack_sg_list_p(chan->sg, out + in, VIRTQUEUE_NUM, |
4f3b35c1 | 506 | in_pages, in_nr_pages, offs, inlen); |
0b36f1ad | 507 | } |
b530cc79 | 508 | |
0b36f1ad | 509 | BUG_ON(out_sgs + in_sgs > ARRAY_SIZE(sgs)); |
474fe9f7 | 510 | err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req, |
f96fde41 | 511 | GFP_ATOMIC); |
419b3956 | 512 | if (err < 0) { |
52f44e0d VJJ |
513 | if (err == -ENOSPC) { |
514 | chan->ring_bufs_avail = 0; | |
515 | spin_unlock_irqrestore(&chan->lock, flags); | |
9523feac TT |
516 | err = wait_event_killable(*chan->vc_wq, |
517 | chan->ring_bufs_avail); | |
52f44e0d | 518 | if (err == -ERESTARTSYS) |
abfa034e | 519 | goto err_out; |
52f44e0d | 520 | |
5d385153 | 521 | p9_debug(P9_DEBUG_TRANS, "Retry virtio request\n"); |
316ad550 | 522 | goto req_retry_pinned; |
52f44e0d VJJ |
523 | } else { |
524 | spin_unlock_irqrestore(&chan->lock, flags); | |
5d385153 | 525 | p9_debug(P9_DEBUG_TRANS, |
0b36f1ad | 526 | "virtio rpc add_sgs returned failure\n"); |
abfa034e AK |
527 | err = -EIO; |
528 | goto err_out; | |
52f44e0d | 529 | } |
e2735b77 | 530 | } |
dc3f5e68 | 531 | virtqueue_kick(chan->vq); |
419b3956 | 532 | spin_unlock_irqrestore(&chan->lock, flags); |
728356de | 533 | kicked = 1; |
5d385153 | 534 | p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n"); |
2557d0c5 | 535 | err = wait_event_killable(req->wq, req->status >= REQ_STATUS_RCVD); |
f615625a AV |
536 | // RERROR needs reply (== error string) in static data |
537 | if (req->status == REQ_STATUS_RCVD && | |
538 | unlikely(req->rc.sdata[4] == P9_RERROR)) | |
539 | handle_rerror(req, in_hdr_len, offs, in_pages); | |
540 | ||
abfa034e AK |
541 | /* |
542 | * Non kernel buffers are pinned, unpin them | |
543 | */ | |
544 | err_out: | |
4f3b35c1 | 545 | if (need_drop) { |
abfa034e AK |
546 | if (in_pages) { |
547 | p9_release_pages(in_pages, in_nr_pages); | |
548 | atomic_sub(in_nr_pages, &vp_pinned); | |
549 | } | |
550 | if (out_pages) { | |
551 | p9_release_pages(out_pages, out_nr_pages); | |
552 | atomic_sub(out_nr_pages, &vp_pinned); | |
553 | } | |
554 | /* wakeup anybody waiting for slots to pin pages */ | |
555 | wake_up(&vp_wq); | |
556 | } | |
1b8553c0 VN |
557 | kvfree(in_pages); |
558 | kvfree(out_pages); | |
728356de TB |
559 | if (!kicked) { |
560 | /* reply won't come */ | |
8b11ff09 | 561 | p9_req_put(client, req); |
728356de | 562 | } |
abfa034e | 563 | return err; |
b530cc79 EVH |
564 | } |
565 | ||
86c84373 AK |
566 | static ssize_t p9_mount_tag_show(struct device *dev, |
567 | struct device_attribute *attr, char *buf) | |
568 | { | |
569 | struct virtio_chan *chan; | |
570 | struct virtio_device *vdev; | |
edcd9d97 | 571 | int tag_len; |
86c84373 AK |
572 | |
573 | vdev = dev_to_virtio(dev); | |
574 | chan = vdev->priv; | |
edcd9d97 | 575 | tag_len = strlen(chan->tag); |
86c84373 | 576 | |
edcd9d97 | 577 | memcpy(buf, chan->tag, tag_len + 1); |
179a5bc4 | 578 | |
edcd9d97 | 579 | return tag_len + 1; |
86c84373 AK |
580 | } |
581 | ||
582 | static DEVICE_ATTR(mount_tag, 0444, p9_mount_tag_show, NULL); | |
583 | ||
ee443996 EVH |
584 | /** |
585 | * p9_virtio_probe - probe for existence of 9P virtio channels | |
586 | * @vdev: virtio device to probe | |
587 | * | |
37c1209d | 588 | * This probes for existing virtio channels. |
ee443996 EVH |
589 | * |
590 | */ | |
591 | ||
e2735b77 | 592 | static int p9_virtio_probe(struct virtio_device *vdev) |
b530cc79 | 593 | { |
97ee9b02 AK |
594 | __u16 tag_len; |
595 | char *tag; | |
b530cc79 EVH |
596 | int err; |
597 | struct virtio_chan *chan; | |
b530cc79 | 598 | |
7754f53e MT |
599 | if (!vdev->config->get) { |
600 | dev_err(&vdev->dev, "%s failure: config access disabled\n", | |
601 | __func__); | |
602 | return -EINVAL; | |
603 | } | |
604 | ||
37c1209d AK |
605 | chan = kmalloc(sizeof(struct virtio_chan), GFP_KERNEL); |
606 | if (!chan) { | |
5d385153 | 607 | pr_err("Failed to allocate virtio 9P channel\n"); |
b530cc79 EVH |
608 | err = -ENOMEM; |
609 | goto fail; | |
610 | } | |
611 | ||
e2735b77 | 612 | chan->vdev = vdev; |
b530cc79 | 613 | |
e2735b77 | 614 | /* We expect one virtqueue, for requests. */ |
d2a7ddda | 615 | chan->vq = virtio_find_single_vq(vdev, req_done, "requests"); |
e2735b77 EVH |
616 | if (IS_ERR(chan->vq)) { |
617 | err = PTR_ERR(chan->vq); | |
92aef467 | 618 | goto out_free_chan; |
b530cc79 | 619 | } |
e2735b77 EVH |
620 | chan->vq->vdev->priv = chan; |
621 | spin_lock_init(&chan->lock); | |
b530cc79 | 622 | |
e2735b77 | 623 | sg_init_table(chan->sg, VIRTQUEUE_NUM); |
b530cc79 | 624 | |
b530cc79 | 625 | chan->inuse = false; |
97ee9b02 | 626 | if (virtio_has_feature(vdev, VIRTIO_9P_MOUNT_TAG)) { |
855e0c52 | 627 | virtio_cread(vdev, struct virtio_9p_config, tag_len, &tag_len); |
97ee9b02 AK |
628 | } else { |
629 | err = -EINVAL; | |
630 | goto out_free_vq; | |
631 | } | |
edcd9d97 | 632 | tag = kzalloc(tag_len + 1, GFP_KERNEL); |
97ee9b02 AK |
633 | if (!tag) { |
634 | err = -ENOMEM; | |
635 | goto out_free_vq; | |
636 | } | |
855e0c52 RR |
637 | |
638 | virtio_cread_bytes(vdev, offsetof(struct virtio_9p_config, tag), | |
639 | tag, tag_len); | |
97ee9b02 | 640 | chan->tag = tag; |
86c84373 AK |
641 | err = sysfs_create_file(&(vdev->dev.kobj), &dev_attr_mount_tag.attr); |
642 | if (err) { | |
52f44e0d | 643 | goto out_free_tag; |
86c84373 | 644 | } |
52f44e0d VJJ |
645 | chan->vc_wq = kmalloc(sizeof(wait_queue_head_t), GFP_KERNEL); |
646 | if (!chan->vc_wq) { | |
647 | err = -ENOMEM; | |
f997ea3b | 648 | goto out_remove_file; |
52f44e0d VJJ |
649 | } |
650 | init_waitqueue_head(chan->vc_wq); | |
651 | chan->ring_bufs_avail = 1; | |
68da9ba4 VJJ |
652 | /* Ceiling limit to avoid denial of service attacks */ |
653 | chan->p9_max_pages = nr_free_buffer_pages()/4; | |
52f44e0d | 654 | |
64b4cc39 MT |
655 | virtio_device_ready(vdev); |
656 | ||
37c1209d AK |
657 | mutex_lock(&virtio_9p_lock); |
658 | list_add_tail(&chan->chan_list, &virtio_chan_list); | |
659 | mutex_unlock(&virtio_9p_lock); | |
e0d6cb9c MM |
660 | |
661 | /* Let udev rules use the new mount_tag attribute. */ | |
662 | kobject_uevent(&(vdev->dev.kobj), KOBJ_CHANGE); | |
663 | ||
b530cc79 EVH |
664 | return 0; |
665 | ||
f997ea3b XY |
666 | out_remove_file: |
667 | sysfs_remove_file(&vdev->dev.kobj, &dev_attr_mount_tag.attr); | |
52f44e0d VJJ |
668 | out_free_tag: |
669 | kfree(tag); | |
e2735b77 | 670 | out_free_vq: |
d2a7ddda | 671 | vdev->config->del_vqs(vdev); |
92aef467 | 672 | out_free_chan: |
37c1209d | 673 | kfree(chan); |
b530cc79 | 674 | fail: |
b530cc79 EVH |
675 | return err; |
676 | } | |
677 | ||
ee443996 EVH |
678 | |
679 | /** | |
680 | * p9_virtio_create - allocate a new virtio channel | |
8b81ef58 | 681 | * @client: client instance invoking this transport |
ee443996 EVH |
682 | * @devname: string identifying the channel to connect to (unused) |
683 | * @args: args passed from sys_mount() for per-transport options (unused) | |
ee443996 EVH |
684 | * |
685 | * This sets up a transport channel for 9p communication. Right now | |
f2745669 | 686 | * we only match the first available channel, but eventually we could look up |
b530cc79 EVH |
687 | * alternate channels by matching devname versus a virtio_config entry. |
688 | * We use a simple reference count mechanism to ensure that only a single | |
ee443996 EVH |
689 | * mount has a channel open at a time. |
690 | * | |
ee443996 EVH |
691 | */ |
692 | ||
8b81ef58 EVH |
693 | static int |
694 | p9_virtio_create(struct p9_client *client, const char *devname, char *args) | |
b530cc79 | 695 | { |
37c1209d | 696 | struct virtio_chan *chan; |
c1a7c226 | 697 | int ret = -ENOENT; |
37c1209d | 698 | int found = 0; |
b530cc79 | 699 | |
10aa1452 TB |
700 | if (devname == NULL) |
701 | return -EINVAL; | |
702 | ||
c1549497 | 703 | mutex_lock(&virtio_9p_lock); |
37c1209d | 704 | list_for_each_entry(chan, &virtio_chan_list, chan_list) { |
edcd9d97 | 705 | if (!strcmp(devname, chan->tag)) { |
f75580c4 AK |
706 | if (!chan->inuse) { |
707 | chan->inuse = true; | |
37c1209d | 708 | found = 1; |
f75580c4 AK |
709 | break; |
710 | } | |
c1a7c226 | 711 | ret = -EBUSY; |
b530cc79 EVH |
712 | } |
713 | } | |
c1549497 | 714 | mutex_unlock(&virtio_9p_lock); |
b530cc79 | 715 | |
37c1209d | 716 | if (!found) { |
c7c72c5a | 717 | pr_err("no channels available for device %s\n", devname); |
c1a7c226 | 718 | return ret; |
e2735b77 EVH |
719 | } |
720 | ||
8b81ef58 | 721 | client->trans = (void *)chan; |
562ada61 | 722 | client->status = Connected; |
fea511a6 | 723 | chan->client = client; |
b530cc79 | 724 | |
8b81ef58 | 725 | return 0; |
b530cc79 EVH |
726 | } |
727 | ||
ee443996 EVH |
728 | /** |
729 | * p9_virtio_remove - clean up resources associated with a virtio device | |
730 | * @vdev: virtio device to remove | |
731 | * | |
732 | */ | |
733 | ||
f3933545 EVH |
734 | static void p9_virtio_remove(struct virtio_device *vdev) |
735 | { | |
736 | struct virtio_chan *chan = vdev->priv; | |
8051a2a5 | 737 | unsigned long warning_time; |
37c1209d AK |
738 | |
739 | mutex_lock(&virtio_9p_lock); | |
8051a2a5 MT |
740 | |
741 | /* Remove self from list so we don't get new users. */ | |
37c1209d | 742 | list_del(&chan->chan_list); |
8051a2a5 MT |
743 | warning_time = jiffies; |
744 | ||
745 | /* Wait for existing users to close. */ | |
746 | while (chan->inuse) { | |
747 | mutex_unlock(&virtio_9p_lock); | |
748 | msleep(250); | |
749 | if (time_after(jiffies, warning_time + 10 * HZ)) { | |
750 | dev_emerg(&vdev->dev, | |
751 | "p9_virtio_remove: waiting for device in use.\n"); | |
752 | warning_time = jiffies; | |
753 | } | |
754 | mutex_lock(&virtio_9p_lock); | |
755 | } | |
756 | ||
37c1209d | 757 | mutex_unlock(&virtio_9p_lock); |
8051a2a5 | 758 | |
d9679d00 | 759 | virtio_reset_device(vdev); |
8051a2a5 MT |
760 | vdev->config->del_vqs(vdev); |
761 | ||
86c84373 | 762 | sysfs_remove_file(&(vdev->dev.kobj), &dev_attr_mount_tag.attr); |
e0d6cb9c | 763 | kobject_uevent(&(vdev->dev.kobj), KOBJ_CHANGE); |
97ee9b02 | 764 | kfree(chan->tag); |
52f44e0d | 765 | kfree(chan->vc_wq); |
37c1209d | 766 | kfree(chan); |
f3933545 | 767 | |
f3933545 EVH |
768 | } |
769 | ||
b530cc79 EVH |
770 | static struct virtio_device_id id_table[] = { |
771 | { VIRTIO_ID_9P, VIRTIO_DEV_ANY_ID }, | |
772 | { 0 }, | |
773 | }; | |
774 | ||
97ee9b02 AK |
775 | static unsigned int features[] = { |
776 | VIRTIO_9P_MOUNT_TAG, | |
777 | }; | |
778 | ||
b530cc79 EVH |
779 | /* The standard "struct lguest_driver": */ |
780 | static struct virtio_driver p9_virtio_drv = { | |
97ee9b02 AK |
781 | .feature_table = features, |
782 | .feature_table_size = ARRAY_SIZE(features), | |
783 | .driver.name = KBUILD_MODNAME, | |
784 | .driver.owner = THIS_MODULE, | |
785 | .id_table = id_table, | |
786 | .probe = p9_virtio_probe, | |
787 | .remove = p9_virtio_remove, | |
b530cc79 EVH |
788 | }; |
789 | ||
790 | static struct p9_trans_module p9_virtio_trans = { | |
791 | .name = "virtio", | |
792 | .create = p9_virtio_create, | |
8b81ef58 | 793 | .close = p9_virtio_close, |
91b8534f | 794 | .request = p9_virtio_request, |
dc893e19 | 795 | .zc_request = p9_virtio_zc_request, |
91b8534f | 796 | .cancel = p9_virtio_cancel, |
728356de | 797 | .cancelled = p9_virtio_cancelled, |
b49d8b5d AK |
798 | /* |
799 | * We leave one entry for input and one entry for response | |
8ab1784d | 800 | * headers. We also skip one more entry to accommodate, address |
b49d8b5d AK |
801 | * that are not at page boundary, that can result in an extra |
802 | * page in zero copy. | |
803 | */ | |
804 | .maxsize = PAGE_SIZE * (VIRTQUEUE_NUM - 3), | |
f94741fd | 805 | .def = 1, |
72029fe8 | 806 | .owner = THIS_MODULE, |
b530cc79 EVH |
807 | }; |
808 | ||
809 | /* The standard init function */ | |
810 | static int __init p9_virtio_init(void) | |
811 | { | |
d4548543 Y |
812 | int rc; |
813 | ||
37c1209d | 814 | INIT_LIST_HEAD(&virtio_chan_list); |
b530cc79 EVH |
815 | |
816 | v9fs_register_trans(&p9_virtio_trans); | |
d4548543 Y |
817 | rc = register_virtio_driver(&p9_virtio_drv); |
818 | if (rc) | |
819 | v9fs_unregister_trans(&p9_virtio_trans); | |
820 | ||
821 | return rc; | |
b530cc79 EVH |
822 | } |
823 | ||
f3933545 EVH |
824 | static void __exit p9_virtio_cleanup(void) |
825 | { | |
826 | unregister_virtio_driver(&p9_virtio_drv); | |
72029fe8 | 827 | v9fs_unregister_trans(&p9_virtio_trans); |
f3933545 EVH |
828 | } |
829 | ||
b530cc79 | 830 | module_init(p9_virtio_init); |
f3933545 | 831 | module_exit(p9_virtio_cleanup); |
4cd82a5b | 832 | MODULE_ALIAS_9P("virtio"); |
b530cc79 EVH |
833 | |
834 | MODULE_DEVICE_TABLE(virtio, id_table); | |
835 | MODULE_AUTHOR("Eric Van Hensbergen <ericvh@gmail.com>"); | |
836 | MODULE_DESCRIPTION("Virtio 9p Transport"); | |
837 | MODULE_LICENSE("GPL"); |