Commit | Line | Data |
---|---|---|
61e9c905 SD |
1 | /* |
2 | * Intel MIC Platform Software Stack (MPSS) | |
3 | * | |
4 | * Copyright(c) 2016 Intel Corporation. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License, version 2, as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License for more details. | |
14 | * | |
15 | * The full GNU General Public License is included in this distribution in | |
16 | * the file called "COPYING". | |
17 | * | |
18 | * Intel Virtio Over PCIe (VOP) driver. | |
19 | * | |
20 | */ | |
21 | #include <linux/sched.h> | |
22 | #include <linux/poll.h> | |
23 | #include <linux/dma-mapping.h> | |
24 | ||
25 | #include <linux/mic_common.h> | |
26 | #include "../common/mic_dev.h" | |
27 | ||
28 | #include <linux/mic_ioctl.h> | |
29 | #include "vop_main.h" | |
30 | ||
31 | /* Helper API to obtain the VOP PCIe device */ | |
32 | static inline struct device *vop_dev(struct vop_vdev *vdev) | |
33 | { | |
34 | return vdev->vpdev->dev.parent; | |
35 | } | |
36 | ||
37 | /* Helper API to check if a virtio device is initialized */ | |
38 | static inline int vop_vdev_inited(struct vop_vdev *vdev) | |
39 | { | |
40 | if (!vdev) | |
41 | return -EINVAL; | |
42 | /* Device has not been created yet */ | |
43 | if (!vdev->dd || !vdev->dd->type) { | |
44 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
45 | __func__, __LINE__, -EINVAL); | |
46 | return -EINVAL; | |
47 | } | |
48 | /* Device has been removed/deleted */ | |
49 | if (vdev->dd->type == -1) { | |
50 | dev_dbg(vop_dev(vdev), "%s %d err %d\n", | |
51 | __func__, __LINE__, -ENODEV); | |
52 | return -ENODEV; | |
53 | } | |
54 | return 0; | |
55 | } | |
56 | ||
57 | static void _vop_notify(struct vringh *vrh) | |
58 | { | |
59 | struct vop_vringh *vvrh = container_of(vrh, struct vop_vringh, vrh); | |
60 | struct vop_vdev *vdev = vvrh->vdev; | |
61 | struct vop_device *vpdev = vdev->vpdev; | |
62 | s8 db = vdev->dc->h2c_vdev_db; | |
63 | ||
64 | if (db != -1) | |
65 | vpdev->hw_ops->send_intr(vpdev, db); | |
66 | } | |
67 | ||
68 | static void vop_virtio_init_post(struct vop_vdev *vdev) | |
69 | { | |
70 | struct mic_vqconfig *vqconfig = mic_vq_config(vdev->dd); | |
71 | struct vop_device *vpdev = vdev->vpdev; | |
72 | int i, used_size; | |
73 | ||
74 | for (i = 0; i < vdev->dd->num_vq; i++) { | |
75 | used_size = PAGE_ALIGN(sizeof(u16) * 3 + | |
76 | sizeof(struct vring_used_elem) * | |
77 | le16_to_cpu(vqconfig->num)); | |
78 | if (!le64_to_cpu(vqconfig[i].used_address)) { | |
79 | dev_warn(vop_dev(vdev), "used_address zero??\n"); | |
80 | continue; | |
81 | } | |
82 | vdev->vvr[i].vrh.vring.used = | |
83 | (void __force *)vpdev->hw_ops->ioremap( | |
84 | vpdev, | |
85 | le64_to_cpu(vqconfig[i].used_address), | |
86 | used_size); | |
87 | } | |
88 | ||
89 | vdev->dc->used_address_updated = 0; | |
90 | ||
91 | dev_info(vop_dev(vdev), "%s: device type %d LINKUP\n", | |
92 | __func__, vdev->virtio_id); | |
93 | } | |
94 | ||
95 | static inline void vop_virtio_device_reset(struct vop_vdev *vdev) | |
96 | { | |
97 | int i; | |
98 | ||
99 | dev_dbg(vop_dev(vdev), "%s: status %d device type %d RESET\n", | |
100 | __func__, vdev->dd->status, vdev->virtio_id); | |
101 | ||
102 | for (i = 0; i < vdev->dd->num_vq; i++) | |
103 | /* | |
104 | * Avoid lockdep false positive. The + 1 is for the vop | |
105 | * mutex which is held in the reset devices code path. | |
106 | */ | |
107 | mutex_lock_nested(&vdev->vvr[i].vr_mutex, i + 1); | |
108 | ||
109 | /* 0 status means "reset" */ | |
110 | vdev->dd->status = 0; | |
111 | vdev->dc->vdev_reset = 0; | |
112 | vdev->dc->host_ack = 1; | |
113 | ||
114 | for (i = 0; i < vdev->dd->num_vq; i++) { | |
115 | struct vringh *vrh = &vdev->vvr[i].vrh; | |
116 | ||
117 | vdev->vvr[i].vring.info->avail_idx = 0; | |
118 | vrh->completed = 0; | |
119 | vrh->last_avail_idx = 0; | |
120 | vrh->last_used_idx = 0; | |
121 | } | |
122 | ||
123 | for (i = 0; i < vdev->dd->num_vq; i++) | |
124 | mutex_unlock(&vdev->vvr[i].vr_mutex); | |
125 | } | |
126 | ||
127 | static void vop_virtio_reset_devices(struct vop_info *vi) | |
128 | { | |
129 | struct list_head *pos, *tmp; | |
130 | struct vop_vdev *vdev; | |
131 | ||
132 | list_for_each_safe(pos, tmp, &vi->vdev_list) { | |
133 | vdev = list_entry(pos, struct vop_vdev, list); | |
134 | vop_virtio_device_reset(vdev); | |
135 | vdev->poll_wake = 1; | |
136 | wake_up(&vdev->waitq); | |
137 | } | |
138 | } | |
139 | ||
140 | static void vop_bh_handler(struct work_struct *work) | |
141 | { | |
142 | struct vop_vdev *vdev = container_of(work, struct vop_vdev, | |
143 | virtio_bh_work); | |
144 | ||
145 | if (vdev->dc->used_address_updated) | |
146 | vop_virtio_init_post(vdev); | |
147 | ||
148 | if (vdev->dc->vdev_reset) | |
149 | vop_virtio_device_reset(vdev); | |
150 | ||
151 | vdev->poll_wake = 1; | |
152 | wake_up(&vdev->waitq); | |
153 | } | |
154 | ||
155 | static irqreturn_t _vop_virtio_intr_handler(int irq, void *data) | |
156 | { | |
157 | struct vop_vdev *vdev = data; | |
158 | struct vop_device *vpdev = vdev->vpdev; | |
159 | ||
160 | vpdev->hw_ops->ack_interrupt(vpdev, vdev->virtio_db); | |
161 | schedule_work(&vdev->virtio_bh_work); | |
162 | return IRQ_HANDLED; | |
163 | } | |
164 | ||
165 | static int vop_virtio_config_change(struct vop_vdev *vdev, void *argp) | |
166 | { | |
167 | DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake); | |
168 | int ret = 0, retry, i; | |
169 | struct vop_device *vpdev = vdev->vpdev; | |
170 | struct vop_info *vi = dev_get_drvdata(&vpdev->dev); | |
171 | struct mic_bootparam *bootparam = vpdev->hw_ops->get_dp(vpdev); | |
172 | s8 db = bootparam->h2c_config_db; | |
173 | ||
174 | mutex_lock(&vi->vop_mutex); | |
175 | for (i = 0; i < vdev->dd->num_vq; i++) | |
176 | mutex_lock_nested(&vdev->vvr[i].vr_mutex, i + 1); | |
177 | ||
178 | if (db == -1 || vdev->dd->type == -1) { | |
179 | ret = -EIO; | |
180 | goto exit; | |
181 | } | |
182 | ||
183 | memcpy(mic_vq_configspace(vdev->dd), argp, vdev->dd->config_len); | |
184 | vdev->dc->config_change = MIC_VIRTIO_PARAM_CONFIG_CHANGED; | |
185 | vpdev->hw_ops->send_intr(vpdev, db); | |
186 | ||
187 | for (retry = 100; retry--;) { | |
188 | ret = wait_event_timeout(wake, vdev->dc->guest_ack, | |
189 | msecs_to_jiffies(100)); | |
190 | if (ret) | |
191 | break; | |
192 | } | |
193 | ||
194 | dev_dbg(vop_dev(vdev), | |
195 | "%s %d retry: %d\n", __func__, __LINE__, retry); | |
196 | vdev->dc->config_change = 0; | |
197 | vdev->dc->guest_ack = 0; | |
198 | exit: | |
199 | for (i = 0; i < vdev->dd->num_vq; i++) | |
200 | mutex_unlock(&vdev->vvr[i].vr_mutex); | |
201 | mutex_unlock(&vi->vop_mutex); | |
202 | return ret; | |
203 | } | |
204 | ||
205 | static int vop_copy_dp_entry(struct vop_vdev *vdev, | |
206 | struct mic_device_desc *argp, __u8 *type, | |
207 | struct mic_device_desc **devpage) | |
208 | { | |
209 | struct vop_device *vpdev = vdev->vpdev; | |
210 | struct mic_device_desc *devp; | |
211 | struct mic_vqconfig *vqconfig; | |
212 | int ret = 0, i; | |
213 | bool slot_found = false; | |
214 | ||
215 | vqconfig = mic_vq_config(argp); | |
216 | for (i = 0; i < argp->num_vq; i++) { | |
217 | if (le16_to_cpu(vqconfig[i].num) > MIC_MAX_VRING_ENTRIES) { | |
218 | ret = -EINVAL; | |
219 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
220 | __func__, __LINE__, ret); | |
221 | goto exit; | |
222 | } | |
223 | } | |
224 | ||
225 | /* Find the first free device page entry */ | |
226 | for (i = sizeof(struct mic_bootparam); | |
227 | i < MIC_DP_SIZE - mic_total_desc_size(argp); | |
228 | i += mic_total_desc_size(devp)) { | |
229 | devp = vpdev->hw_ops->get_dp(vpdev) + i; | |
230 | if (devp->type == 0 || devp->type == -1) { | |
231 | slot_found = true; | |
232 | break; | |
233 | } | |
234 | } | |
235 | if (!slot_found) { | |
236 | ret = -EINVAL; | |
237 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
238 | __func__, __LINE__, ret); | |
239 | goto exit; | |
240 | } | |
241 | /* | |
242 | * Save off the type before doing the memcpy. Type will be set in the | |
243 | * end after completing all initialization for the new device. | |
244 | */ | |
245 | *type = argp->type; | |
246 | argp->type = 0; | |
247 | memcpy(devp, argp, mic_desc_size(argp)); | |
248 | ||
249 | *devpage = devp; | |
250 | exit: | |
251 | return ret; | |
252 | } | |
253 | ||
254 | static void vop_init_device_ctrl(struct vop_vdev *vdev, | |
255 | struct mic_device_desc *devpage) | |
256 | { | |
257 | struct mic_device_ctrl *dc; | |
258 | ||
259 | dc = (void *)devpage + mic_aligned_desc_size(devpage); | |
260 | ||
261 | dc->config_change = 0; | |
262 | dc->guest_ack = 0; | |
263 | dc->vdev_reset = 0; | |
264 | dc->host_ack = 0; | |
265 | dc->used_address_updated = 0; | |
266 | dc->c2h_vdev_db = -1; | |
267 | dc->h2c_vdev_db = -1; | |
268 | vdev->dc = dc; | |
269 | } | |
270 | ||
271 | static int vop_virtio_add_device(struct vop_vdev *vdev, | |
272 | struct mic_device_desc *argp) | |
273 | { | |
274 | struct vop_info *vi = vdev->vi; | |
275 | struct vop_device *vpdev = vi->vpdev; | |
276 | struct mic_device_desc *dd = NULL; | |
277 | struct mic_vqconfig *vqconfig; | |
278 | int vr_size, i, j, ret; | |
279 | u8 type = 0; | |
280 | s8 db = -1; | |
281 | char irqname[16]; | |
282 | struct mic_bootparam *bootparam; | |
283 | u16 num; | |
284 | dma_addr_t vr_addr; | |
285 | ||
286 | bootparam = vpdev->hw_ops->get_dp(vpdev); | |
287 | init_waitqueue_head(&vdev->waitq); | |
288 | INIT_LIST_HEAD(&vdev->list); | |
289 | vdev->vpdev = vpdev; | |
290 | ||
291 | ret = vop_copy_dp_entry(vdev, argp, &type, &dd); | |
292 | if (ret) { | |
61e9c905 SD |
293 | dev_err(vop_dev(vdev), "%s %d err %d\n", |
294 | __func__, __LINE__, ret); | |
6ddf2f0c | 295 | kfree(vdev); |
61e9c905 SD |
296 | return ret; |
297 | } | |
298 | ||
299 | vop_init_device_ctrl(vdev, dd); | |
300 | ||
301 | vdev->dd = dd; | |
302 | vdev->virtio_id = type; | |
303 | vqconfig = mic_vq_config(dd); | |
304 | INIT_WORK(&vdev->virtio_bh_work, vop_bh_handler); | |
305 | ||
306 | for (i = 0; i < dd->num_vq; i++) { | |
307 | struct vop_vringh *vvr = &vdev->vvr[i]; | |
308 | struct mic_vring *vr = &vdev->vvr[i].vring; | |
309 | ||
310 | num = le16_to_cpu(vqconfig[i].num); | |
311 | mutex_init(&vvr->vr_mutex); | |
312 | vr_size = PAGE_ALIGN(vring_size(num, MIC_VIRTIO_RING_ALIGN) + | |
313 | sizeof(struct _mic_vring_info)); | |
314 | vr->va = (void *) | |
315 | __get_free_pages(GFP_KERNEL | __GFP_ZERO, | |
316 | get_order(vr_size)); | |
317 | if (!vr->va) { | |
318 | ret = -ENOMEM; | |
319 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
320 | __func__, __LINE__, ret); | |
321 | goto err; | |
322 | } | |
323 | vr->len = vr_size; | |
324 | vr->info = vr->va + vring_size(num, MIC_VIRTIO_RING_ALIGN); | |
325 | vr->info->magic = cpu_to_le32(MIC_MAGIC + vdev->virtio_id + i); | |
326 | vr_addr = dma_map_single(&vpdev->dev, vr->va, vr_size, | |
327 | DMA_BIDIRECTIONAL); | |
328 | if (dma_mapping_error(&vpdev->dev, vr_addr)) { | |
329 | free_pages((unsigned long)vr->va, get_order(vr_size)); | |
330 | ret = -ENOMEM; | |
331 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
332 | __func__, __LINE__, ret); | |
333 | goto err; | |
334 | } | |
335 | vqconfig[i].address = cpu_to_le64(vr_addr); | |
336 | ||
337 | vring_init(&vr->vr, num, vr->va, MIC_VIRTIO_RING_ALIGN); | |
338 | ret = vringh_init_kern(&vvr->vrh, | |
339 | *(u32 *)mic_vq_features(vdev->dd), | |
340 | num, false, vr->vr.desc, vr->vr.avail, | |
341 | vr->vr.used); | |
342 | if (ret) { | |
343 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
344 | __func__, __LINE__, ret); | |
345 | goto err; | |
346 | } | |
347 | vringh_kiov_init(&vvr->riov, NULL, 0); | |
348 | vringh_kiov_init(&vvr->wiov, NULL, 0); | |
349 | vvr->head = USHRT_MAX; | |
350 | vvr->vdev = vdev; | |
351 | vvr->vrh.notify = _vop_notify; | |
352 | dev_dbg(&vpdev->dev, | |
353 | "%s %d index %d va %p info %p vr_size 0x%x\n", | |
354 | __func__, __LINE__, i, vr->va, vr->info, vr_size); | |
355 | vvr->buf = (void *)__get_free_pages(GFP_KERNEL, | |
356 | get_order(VOP_INT_DMA_BUF_SIZE)); | |
357 | vvr->buf_da = dma_map_single(&vpdev->dev, | |
358 | vvr->buf, VOP_INT_DMA_BUF_SIZE, | |
359 | DMA_BIDIRECTIONAL); | |
360 | } | |
361 | ||
362 | snprintf(irqname, sizeof(irqname), "vop%dvirtio%d", vpdev->index, | |
363 | vdev->virtio_id); | |
364 | vdev->virtio_db = vpdev->hw_ops->next_db(vpdev); | |
365 | vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev, | |
366 | _vop_virtio_intr_handler, irqname, vdev, | |
367 | vdev->virtio_db); | |
368 | if (IS_ERR(vdev->virtio_cookie)) { | |
369 | ret = PTR_ERR(vdev->virtio_cookie); | |
370 | dev_dbg(&vpdev->dev, "request irq failed\n"); | |
371 | goto err; | |
372 | } | |
373 | ||
374 | vdev->dc->c2h_vdev_db = vdev->virtio_db; | |
375 | ||
376 | /* | |
377 | * Order the type update with previous stores. This write barrier | |
378 | * is paired with the corresponding read barrier before the uncached | |
379 | * system memory read of the type, on the card while scanning the | |
380 | * device page. | |
381 | */ | |
382 | smp_wmb(); | |
383 | dd->type = type; | |
384 | argp->type = type; | |
385 | ||
386 | if (bootparam) { | |
387 | db = bootparam->h2c_config_db; | |
388 | if (db != -1) | |
389 | vpdev->hw_ops->send_intr(vpdev, db); | |
390 | } | |
391 | dev_dbg(&vpdev->dev, "Added virtio id %d db %d\n", dd->type, db); | |
392 | return 0; | |
393 | err: | |
394 | vqconfig = mic_vq_config(dd); | |
395 | for (j = 0; j < i; j++) { | |
396 | struct vop_vringh *vvr = &vdev->vvr[j]; | |
397 | ||
398 | dma_unmap_single(&vpdev->dev, le64_to_cpu(vqconfig[j].address), | |
399 | vvr->vring.len, DMA_BIDIRECTIONAL); | |
400 | free_pages((unsigned long)vvr->vring.va, | |
401 | get_order(vvr->vring.len)); | |
402 | } | |
403 | return ret; | |
404 | } | |
405 | ||
406 | static void vop_dev_remove(struct vop_info *pvi, struct mic_device_ctrl *devp, | |
407 | struct vop_device *vpdev) | |
408 | { | |
409 | struct mic_bootparam *bootparam = vpdev->hw_ops->get_dp(vpdev); | |
410 | s8 db; | |
411 | int ret, retry; | |
412 | DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake); | |
413 | ||
414 | devp->config_change = MIC_VIRTIO_PARAM_DEV_REMOVE; | |
415 | db = bootparam->h2c_config_db; | |
416 | if (db != -1) | |
417 | vpdev->hw_ops->send_intr(vpdev, db); | |
418 | else | |
419 | goto done; | |
420 | for (retry = 15; retry--;) { | |
421 | ret = wait_event_timeout(wake, devp->guest_ack, | |
422 | msecs_to_jiffies(1000)); | |
423 | if (ret) | |
424 | break; | |
425 | } | |
426 | done: | |
427 | devp->config_change = 0; | |
428 | devp->guest_ack = 0; | |
429 | } | |
430 | ||
431 | static void vop_virtio_del_device(struct vop_vdev *vdev) | |
432 | { | |
433 | struct vop_info *vi = vdev->vi; | |
434 | struct vop_device *vpdev = vdev->vpdev; | |
435 | int i; | |
436 | struct mic_vqconfig *vqconfig; | |
437 | struct mic_bootparam *bootparam = vpdev->hw_ops->get_dp(vpdev); | |
438 | ||
439 | if (!bootparam) | |
440 | goto skip_hot_remove; | |
441 | vop_dev_remove(vi, vdev->dc, vpdev); | |
442 | skip_hot_remove: | |
443 | vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev); | |
444 | flush_work(&vdev->virtio_bh_work); | |
445 | vqconfig = mic_vq_config(vdev->dd); | |
446 | for (i = 0; i < vdev->dd->num_vq; i++) { | |
447 | struct vop_vringh *vvr = &vdev->vvr[i]; | |
448 | ||
449 | dma_unmap_single(&vpdev->dev, | |
450 | vvr->buf_da, VOP_INT_DMA_BUF_SIZE, | |
451 | DMA_BIDIRECTIONAL); | |
452 | free_pages((unsigned long)vvr->buf, | |
453 | get_order(VOP_INT_DMA_BUF_SIZE)); | |
454 | vringh_kiov_cleanup(&vvr->riov); | |
455 | vringh_kiov_cleanup(&vvr->wiov); | |
456 | dma_unmap_single(&vpdev->dev, le64_to_cpu(vqconfig[i].address), | |
457 | vvr->vring.len, DMA_BIDIRECTIONAL); | |
458 | free_pages((unsigned long)vvr->vring.va, | |
459 | get_order(vvr->vring.len)); | |
460 | } | |
461 | /* | |
462 | * Order the type update with previous stores. This write barrier | |
463 | * is paired with the corresponding read barrier before the uncached | |
464 | * system memory read of the type, on the card while scanning the | |
465 | * device page. | |
466 | */ | |
467 | smp_wmb(); | |
468 | vdev->dd->type = -1; | |
469 | } | |
470 | ||
471 | /* | |
472 | * vop_sync_dma - Wrapper for synchronous DMAs. | |
473 | * | |
474 | * @dev - The address of the pointer to the device instance used | |
475 | * for DMA registration. | |
476 | * @dst - destination DMA address. | |
477 | * @src - source DMA address. | |
478 | * @len - size of the transfer. | |
479 | * | |
480 | * Return DMA_SUCCESS on success | |
481 | */ | |
482 | static int vop_sync_dma(struct vop_vdev *vdev, dma_addr_t dst, dma_addr_t src, | |
483 | size_t len) | |
484 | { | |
485 | int err = 0; | |
486 | struct dma_device *ddev; | |
487 | struct dma_async_tx_descriptor *tx; | |
488 | struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev); | |
489 | struct dma_chan *vop_ch = vi->dma_ch; | |
490 | ||
491 | if (!vop_ch) { | |
492 | err = -EBUSY; | |
493 | goto error; | |
494 | } | |
495 | ddev = vop_ch->device; | |
496 | tx = ddev->device_prep_dma_memcpy(vop_ch, dst, src, len, | |
497 | DMA_PREP_FENCE); | |
498 | if (!tx) { | |
499 | err = -ENOMEM; | |
500 | goto error; | |
501 | } else { | |
502 | dma_cookie_t cookie; | |
503 | ||
504 | cookie = tx->tx_submit(tx); | |
505 | if (dma_submit_error(cookie)) { | |
506 | err = -ENOMEM; | |
507 | goto error; | |
508 | } | |
509 | dma_async_issue_pending(vop_ch); | |
510 | err = dma_sync_wait(vop_ch, cookie); | |
511 | } | |
512 | error: | |
513 | if (err) | |
514 | dev_err(&vi->vpdev->dev, "%s %d err %d\n", | |
515 | __func__, __LINE__, err); | |
516 | return err; | |
517 | } | |
518 | ||
519 | #define VOP_USE_DMA true | |
520 | ||
521 | /* | |
522 | * Initiates the copies across the PCIe bus from card memory to a user | |
523 | * space buffer. When transfers are done using DMA, source/destination | |
524 | * addresses and transfer length must follow the alignment requirements of | |
525 | * the MIC DMA engine. | |
526 | */ | |
527 | static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf, | |
528 | size_t len, u64 daddr, size_t dlen, | |
529 | int vr_idx) | |
530 | { | |
531 | struct vop_device *vpdev = vdev->vpdev; | |
532 | void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len); | |
533 | struct vop_vringh *vvr = &vdev->vvr[vr_idx]; | |
534 | struct vop_info *vi = dev_get_drvdata(&vpdev->dev); | |
535 | size_t dma_alignment = 1 << vi->dma_ch->device->copy_align; | |
536 | bool x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1); | |
537 | size_t dma_offset, partlen; | |
538 | int err; | |
539 | ||
540 | if (!VOP_USE_DMA) { | |
541 | if (copy_to_user(ubuf, (void __force *)dbuf, len)) { | |
542 | err = -EFAULT; | |
543 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
544 | __func__, __LINE__, err); | |
545 | goto err; | |
546 | } | |
547 | vdev->in_bytes += len; | |
548 | err = 0; | |
549 | goto err; | |
550 | } | |
551 | ||
552 | dma_offset = daddr - round_down(daddr, dma_alignment); | |
553 | daddr -= dma_offset; | |
554 | len += dma_offset; | |
555 | /* | |
556 | * X100 uses DMA addresses as seen by the card so adding | |
557 | * the aperture base is not required for DMA. However x200 | |
558 | * requires DMA addresses to be an offset into the bar so | |
559 | * add the aperture base for x200. | |
560 | */ | |
561 | if (x200) | |
562 | daddr += vpdev->aper->pa; | |
563 | while (len) { | |
564 | partlen = min_t(size_t, len, VOP_INT_DMA_BUF_SIZE); | |
565 | err = vop_sync_dma(vdev, vvr->buf_da, daddr, | |
566 | ALIGN(partlen, dma_alignment)); | |
567 | if (err) { | |
568 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
569 | __func__, __LINE__, err); | |
570 | goto err; | |
571 | } | |
572 | if (copy_to_user(ubuf, vvr->buf + dma_offset, | |
573 | partlen - dma_offset)) { | |
574 | err = -EFAULT; | |
575 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
576 | __func__, __LINE__, err); | |
577 | goto err; | |
578 | } | |
579 | daddr += partlen; | |
580 | ubuf += partlen; | |
581 | dbuf += partlen; | |
582 | vdev->in_bytes_dma += partlen; | |
583 | vdev->in_bytes += partlen; | |
584 | len -= partlen; | |
585 | dma_offset = 0; | |
586 | } | |
587 | err = 0; | |
588 | err: | |
589 | vpdev->hw_ops->iounmap(vpdev, dbuf); | |
590 | dev_dbg(vop_dev(vdev), | |
591 | "%s: ubuf %p dbuf %p len 0x%lx vr_idx 0x%x\n", | |
592 | __func__, ubuf, dbuf, len, vr_idx); | |
593 | return err; | |
594 | } | |
595 | ||
596 | /* | |
597 | * Initiates copies across the PCIe bus from a user space buffer to card | |
598 | * memory. When transfers are done using DMA, source/destination addresses | |
599 | * and transfer length must follow the alignment requirements of the MIC | |
600 | * DMA engine. | |
601 | */ | |
602 | static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf, | |
603 | size_t len, u64 daddr, size_t dlen, | |
604 | int vr_idx) | |
605 | { | |
606 | struct vop_device *vpdev = vdev->vpdev; | |
607 | void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len); | |
608 | struct vop_vringh *vvr = &vdev->vvr[vr_idx]; | |
609 | struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev); | |
610 | size_t dma_alignment = 1 << vi->dma_ch->device->copy_align; | |
611 | bool x200 = is_dma_copy_aligned(vi->dma_ch->device, 1, 1, 1); | |
612 | size_t partlen; | |
613 | bool dma = VOP_USE_DMA; | |
614 | int err = 0; | |
615 | ||
616 | if (daddr & (dma_alignment - 1)) { | |
617 | vdev->tx_dst_unaligned += len; | |
618 | dma = false; | |
619 | } else if (ALIGN(len, dma_alignment) > dlen) { | |
620 | vdev->tx_len_unaligned += len; | |
621 | dma = false; | |
622 | } | |
623 | ||
624 | if (!dma) | |
625 | goto memcpy; | |
626 | ||
627 | /* | |
628 | * X100 uses DMA addresses as seen by the card so adding | |
629 | * the aperture base is not required for DMA. However x200 | |
630 | * requires DMA addresses to be an offset into the bar so | |
631 | * add the aperture base for x200. | |
632 | */ | |
633 | if (x200) | |
634 | daddr += vpdev->aper->pa; | |
635 | while (len) { | |
636 | partlen = min_t(size_t, len, VOP_INT_DMA_BUF_SIZE); | |
637 | ||
638 | if (copy_from_user(vvr->buf, ubuf, partlen)) { | |
639 | err = -EFAULT; | |
640 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
641 | __func__, __LINE__, err); | |
642 | goto err; | |
643 | } | |
644 | err = vop_sync_dma(vdev, daddr, vvr->buf_da, | |
645 | ALIGN(partlen, dma_alignment)); | |
646 | if (err) { | |
647 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
648 | __func__, __LINE__, err); | |
649 | goto err; | |
650 | } | |
651 | daddr += partlen; | |
652 | ubuf += partlen; | |
653 | dbuf += partlen; | |
654 | vdev->out_bytes_dma += partlen; | |
655 | vdev->out_bytes += partlen; | |
656 | len -= partlen; | |
657 | } | |
658 | memcpy: | |
659 | /* | |
660 | * We are copying to IO below and should ideally use something | |
661 | * like copy_from_user_toio(..) if it existed. | |
662 | */ | |
663 | if (copy_from_user((void __force *)dbuf, ubuf, len)) { | |
664 | err = -EFAULT; | |
665 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
666 | __func__, __LINE__, err); | |
667 | goto err; | |
668 | } | |
669 | vdev->out_bytes += len; | |
670 | err = 0; | |
671 | err: | |
672 | vpdev->hw_ops->iounmap(vpdev, dbuf); | |
673 | dev_dbg(vop_dev(vdev), | |
674 | "%s: ubuf %p dbuf %p len 0x%lx vr_idx 0x%x\n", | |
675 | __func__, ubuf, dbuf, len, vr_idx); | |
676 | return err; | |
677 | } | |
678 | ||
679 | #define MIC_VRINGH_READ true | |
680 | ||
681 | /* Determine the total number of bytes consumed in a VRINGH KIOV */ | |
682 | static inline u32 vop_vringh_iov_consumed(struct vringh_kiov *iov) | |
683 | { | |
684 | int i; | |
685 | u32 total = iov->consumed; | |
686 | ||
687 | for (i = 0; i < iov->i; i++) | |
688 | total += iov->iov[i].iov_len; | |
689 | return total; | |
690 | } | |
691 | ||
692 | /* | |
693 | * Traverse the VRINGH KIOV and issue the APIs to trigger the copies. | |
694 | * This API is heavily based on the vringh_iov_xfer(..) implementation | |
695 | * in vringh.c. The reason we cannot reuse vringh_iov_pull_kern(..) | |
696 | * and vringh_iov_push_kern(..) directly is because there is no | |
697 | * way to override the VRINGH xfer(..) routines as of v3.10. | |
698 | */ | |
699 | static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov, | |
700 | void __user *ubuf, size_t len, bool read, int vr_idx, | |
701 | size_t *out_len) | |
702 | { | |
703 | int ret = 0; | |
704 | size_t partlen, tot_len = 0; | |
705 | ||
706 | while (len && iov->i < iov->used) { | |
707 | struct kvec *kiov = &iov->iov[iov->i]; | |
708 | ||
709 | partlen = min(kiov->iov_len, len); | |
710 | if (read) | |
711 | ret = vop_virtio_copy_to_user(vdev, ubuf, partlen, | |
712 | (u64)kiov->iov_base, | |
713 | kiov->iov_len, | |
714 | vr_idx); | |
715 | else | |
716 | ret = vop_virtio_copy_from_user(vdev, ubuf, partlen, | |
717 | (u64)kiov->iov_base, | |
718 | kiov->iov_len, | |
719 | vr_idx); | |
720 | if (ret) { | |
721 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
722 | __func__, __LINE__, ret); | |
723 | break; | |
724 | } | |
725 | len -= partlen; | |
726 | ubuf += partlen; | |
727 | tot_len += partlen; | |
728 | iov->consumed += partlen; | |
729 | kiov->iov_len -= partlen; | |
730 | kiov->iov_base += partlen; | |
731 | if (!kiov->iov_len) { | |
732 | /* Fix up old iov element then increment. */ | |
733 | kiov->iov_len = iov->consumed; | |
734 | kiov->iov_base -= iov->consumed; | |
735 | ||
736 | iov->consumed = 0; | |
737 | iov->i++; | |
738 | } | |
739 | } | |
740 | *out_len = tot_len; | |
741 | return ret; | |
742 | } | |
743 | ||
744 | /* | |
745 | * Use the standard VRINGH infrastructure in the kernel to fetch new | |
746 | * descriptors, initiate the copies and update the used ring. | |
747 | */ | |
748 | static int _vop_virtio_copy(struct vop_vdev *vdev, struct mic_copy_desc *copy) | |
749 | { | |
750 | int ret = 0; | |
751 | u32 iovcnt = copy->iovcnt; | |
752 | struct iovec iov; | |
753 | struct iovec __user *u_iov = copy->iov; | |
754 | void __user *ubuf = NULL; | |
755 | struct vop_vringh *vvr = &vdev->vvr[copy->vr_idx]; | |
756 | struct vringh_kiov *riov = &vvr->riov; | |
757 | struct vringh_kiov *wiov = &vvr->wiov; | |
758 | struct vringh *vrh = &vvr->vrh; | |
759 | u16 *head = &vvr->head; | |
760 | struct mic_vring *vr = &vvr->vring; | |
761 | size_t len = 0, out_len; | |
762 | ||
763 | copy->out_len = 0; | |
764 | /* Fetch a new IOVEC if all previous elements have been processed */ | |
765 | if (riov->i == riov->used && wiov->i == wiov->used) { | |
766 | ret = vringh_getdesc_kern(vrh, riov, wiov, | |
767 | head, GFP_KERNEL); | |
768 | /* Check if there are available descriptors */ | |
769 | if (ret <= 0) | |
770 | return ret; | |
771 | } | |
772 | while (iovcnt) { | |
773 | if (!len) { | |
774 | /* Copy over a new iovec from user space. */ | |
775 | ret = copy_from_user(&iov, u_iov, sizeof(*u_iov)); | |
776 | if (ret) { | |
777 | ret = -EINVAL; | |
778 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
779 | __func__, __LINE__, ret); | |
780 | break; | |
781 | } | |
782 | len = iov.iov_len; | |
783 | ubuf = iov.iov_base; | |
784 | } | |
785 | /* Issue all the read descriptors first */ | |
786 | ret = vop_vringh_copy(vdev, riov, ubuf, len, | |
787 | MIC_VRINGH_READ, copy->vr_idx, &out_len); | |
788 | if (ret) { | |
789 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
790 | __func__, __LINE__, ret); | |
791 | break; | |
792 | } | |
793 | len -= out_len; | |
794 | ubuf += out_len; | |
795 | copy->out_len += out_len; | |
796 | /* Issue the write descriptors next */ | |
797 | ret = vop_vringh_copy(vdev, wiov, ubuf, len, | |
798 | !MIC_VRINGH_READ, copy->vr_idx, &out_len); | |
799 | if (ret) { | |
800 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
801 | __func__, __LINE__, ret); | |
802 | break; | |
803 | } | |
804 | len -= out_len; | |
805 | ubuf += out_len; | |
806 | copy->out_len += out_len; | |
807 | if (!len) { | |
808 | /* One user space iovec is now completed */ | |
809 | iovcnt--; | |
810 | u_iov++; | |
811 | } | |
812 | /* Exit loop if all elements in KIOVs have been processed. */ | |
813 | if (riov->i == riov->used && wiov->i == wiov->used) | |
814 | break; | |
815 | } | |
816 | /* | |
817 | * Update the used ring if a descriptor was available and some data was | |
818 | * copied in/out and the user asked for a used ring update. | |
819 | */ | |
820 | if (*head != USHRT_MAX && copy->out_len && copy->update_used) { | |
821 | u32 total = 0; | |
822 | ||
823 | /* Determine the total data consumed */ | |
824 | total += vop_vringh_iov_consumed(riov); | |
825 | total += vop_vringh_iov_consumed(wiov); | |
826 | vringh_complete_kern(vrh, *head, total); | |
827 | *head = USHRT_MAX; | |
828 | if (vringh_need_notify_kern(vrh) > 0) | |
829 | vringh_notify(vrh); | |
830 | vringh_kiov_cleanup(riov); | |
831 | vringh_kiov_cleanup(wiov); | |
832 | /* Update avail idx for user space */ | |
833 | vr->info->avail_idx = vrh->last_avail_idx; | |
834 | } | |
835 | return ret; | |
836 | } | |
837 | ||
838 | static inline int vop_verify_copy_args(struct vop_vdev *vdev, | |
839 | struct mic_copy_desc *copy) | |
840 | { | |
841 | if (!vdev || copy->vr_idx >= vdev->dd->num_vq) | |
842 | return -EINVAL; | |
843 | return 0; | |
844 | } | |
845 | ||
846 | /* Copy a specified number of virtio descriptors in a chain */ | |
847 | static int vop_virtio_copy_desc(struct vop_vdev *vdev, | |
848 | struct mic_copy_desc *copy) | |
849 | { | |
850 | int err; | |
59ea2590 | 851 | struct vop_vringh *vvr; |
61e9c905 SD |
852 | |
853 | err = vop_verify_copy_args(vdev, copy); | |
854 | if (err) | |
855 | return err; | |
856 | ||
59ea2590 | 857 | vvr = &vdev->vvr[copy->vr_idx]; |
61e9c905 SD |
858 | mutex_lock(&vvr->vr_mutex); |
859 | if (!vop_vdevup(vdev)) { | |
860 | err = -ENODEV; | |
861 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
862 | __func__, __LINE__, err); | |
863 | goto err; | |
864 | } | |
865 | err = _vop_virtio_copy(vdev, copy); | |
866 | if (err) { | |
867 | dev_err(vop_dev(vdev), "%s %d err %d\n", | |
868 | __func__, __LINE__, err); | |
869 | } | |
870 | err: | |
871 | mutex_unlock(&vvr->vr_mutex); | |
872 | return err; | |
873 | } | |
874 | ||
875 | static int vop_open(struct inode *inode, struct file *f) | |
876 | { | |
877 | struct vop_vdev *vdev; | |
878 | struct vop_info *vi = container_of(f->private_data, | |
879 | struct vop_info, miscdev); | |
880 | ||
881 | vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); | |
882 | if (!vdev) | |
883 | return -ENOMEM; | |
884 | vdev->vi = vi; | |
885 | mutex_init(&vdev->vdev_mutex); | |
886 | f->private_data = vdev; | |
887 | init_completion(&vdev->destroy); | |
888 | complete(&vdev->destroy); | |
889 | return 0; | |
890 | } | |
891 | ||
892 | static int vop_release(struct inode *inode, struct file *f) | |
893 | { | |
894 | struct vop_vdev *vdev = f->private_data, *vdev_tmp; | |
895 | struct vop_info *vi = vdev->vi; | |
896 | struct list_head *pos, *tmp; | |
897 | bool found = false; | |
898 | ||
899 | mutex_lock(&vdev->vdev_mutex); | |
900 | if (vdev->deleted) | |
901 | goto unlock; | |
902 | mutex_lock(&vi->vop_mutex); | |
903 | list_for_each_safe(pos, tmp, &vi->vdev_list) { | |
904 | vdev_tmp = list_entry(pos, struct vop_vdev, list); | |
905 | if (vdev == vdev_tmp) { | |
906 | vop_virtio_del_device(vdev); | |
907 | list_del(pos); | |
908 | found = true; | |
909 | break; | |
910 | } | |
911 | } | |
912 | mutex_unlock(&vi->vop_mutex); | |
913 | unlock: | |
914 | mutex_unlock(&vdev->vdev_mutex); | |
915 | if (!found) | |
916 | wait_for_completion(&vdev->destroy); | |
917 | f->private_data = NULL; | |
918 | kfree(vdev); | |
919 | return 0; | |
920 | } | |
921 | ||
922 | static long vop_ioctl(struct file *f, unsigned int cmd, unsigned long arg) | |
923 | { | |
924 | struct vop_vdev *vdev = f->private_data; | |
925 | struct vop_info *vi = vdev->vi; | |
926 | void __user *argp = (void __user *)arg; | |
927 | int ret; | |
928 | ||
929 | switch (cmd) { | |
930 | case MIC_VIRTIO_ADD_DEVICE: | |
931 | { | |
932 | struct mic_device_desc dd, *dd_config; | |
933 | ||
934 | if (copy_from_user(&dd, argp, sizeof(dd))) | |
935 | return -EFAULT; | |
936 | ||
937 | if (mic_aligned_desc_size(&dd) > MIC_MAX_DESC_BLK_SIZE || | |
938 | dd.num_vq > MIC_MAX_VRINGS) | |
939 | return -EINVAL; | |
940 | ||
941 | dd_config = kzalloc(mic_desc_size(&dd), GFP_KERNEL); | |
942 | if (!dd_config) | |
943 | return -ENOMEM; | |
944 | if (copy_from_user(dd_config, argp, mic_desc_size(&dd))) { | |
945 | ret = -EFAULT; | |
946 | goto free_ret; | |
947 | } | |
82dc4afd AD |
948 | /* Ensure desc has not changed between the two reads */ |
949 | if (memcmp(&dd, dd_config, sizeof(dd))) { | |
950 | ret = -EINVAL; | |
951 | goto free_ret; | |
952 | } | |
61e9c905 SD |
953 | mutex_lock(&vdev->vdev_mutex); |
954 | mutex_lock(&vi->vop_mutex); | |
955 | ret = vop_virtio_add_device(vdev, dd_config); | |
956 | if (ret) | |
957 | goto unlock_ret; | |
958 | list_add_tail(&vdev->list, &vi->vdev_list); | |
959 | unlock_ret: | |
960 | mutex_unlock(&vi->vop_mutex); | |
961 | mutex_unlock(&vdev->vdev_mutex); | |
962 | free_ret: | |
963 | kfree(dd_config); | |
964 | return ret; | |
965 | } | |
966 | case MIC_VIRTIO_COPY_DESC: | |
967 | { | |
968 | struct mic_copy_desc copy; | |
969 | ||
970 | mutex_lock(&vdev->vdev_mutex); | |
971 | ret = vop_vdev_inited(vdev); | |
972 | if (ret) | |
973 | goto _unlock_ret; | |
974 | ||
975 | if (copy_from_user(©, argp, sizeof(copy))) { | |
976 | ret = -EFAULT; | |
977 | goto _unlock_ret; | |
978 | } | |
979 | ||
980 | ret = vop_virtio_copy_desc(vdev, ©); | |
981 | if (ret < 0) | |
982 | goto _unlock_ret; | |
983 | if (copy_to_user( | |
984 | &((struct mic_copy_desc __user *)argp)->out_len, | |
985 | ©.out_len, sizeof(copy.out_len))) | |
986 | ret = -EFAULT; | |
987 | _unlock_ret: | |
988 | mutex_unlock(&vdev->vdev_mutex); | |
989 | return ret; | |
990 | } | |
991 | case MIC_VIRTIO_CONFIG_CHANGE: | |
992 | { | |
993 | void *buf; | |
994 | ||
995 | mutex_lock(&vdev->vdev_mutex); | |
996 | ret = vop_vdev_inited(vdev); | |
997 | if (ret) | |
998 | goto __unlock_ret; | |
999 | buf = kzalloc(vdev->dd->config_len, GFP_KERNEL); | |
1000 | if (!buf) { | |
1001 | ret = -ENOMEM; | |
1002 | goto __unlock_ret; | |
1003 | } | |
1004 | if (copy_from_user(buf, argp, vdev->dd->config_len)) { | |
1005 | ret = -EFAULT; | |
1006 | goto done; | |
1007 | } | |
1008 | ret = vop_virtio_config_change(vdev, buf); | |
1009 | done: | |
1010 | kfree(buf); | |
1011 | __unlock_ret: | |
1012 | mutex_unlock(&vdev->vdev_mutex); | |
1013 | return ret; | |
1014 | } | |
1015 | default: | |
1016 | return -ENOIOCTLCMD; | |
1017 | }; | |
1018 | return 0; | |
1019 | } | |
1020 | ||
1021 | /* | |
1022 | * We return POLLIN | POLLOUT from poll when new buffers are enqueued, and | |
1023 | * not when previously enqueued buffers may be available. This means that | |
1024 | * in the card->host (TX) path, when userspace is unblocked by poll it | |
1025 | * must drain all available descriptors or it can stall. | |
1026 | */ | |
1027 | static unsigned int vop_poll(struct file *f, poll_table *wait) | |
1028 | { | |
1029 | struct vop_vdev *vdev = f->private_data; | |
1030 | int mask = 0; | |
1031 | ||
1032 | mutex_lock(&vdev->vdev_mutex); | |
1033 | if (vop_vdev_inited(vdev)) { | |
1034 | mask = POLLERR; | |
1035 | goto done; | |
1036 | } | |
1037 | poll_wait(f, &vdev->waitq, wait); | |
1038 | if (vop_vdev_inited(vdev)) { | |
1039 | mask = POLLERR; | |
1040 | } else if (vdev->poll_wake) { | |
1041 | vdev->poll_wake = 0; | |
1042 | mask = POLLIN | POLLOUT; | |
1043 | } | |
1044 | done: | |
1045 | mutex_unlock(&vdev->vdev_mutex); | |
1046 | return mask; | |
1047 | } | |
1048 | ||
1049 | static inline int | |
1050 | vop_query_offset(struct vop_vdev *vdev, unsigned long offset, | |
1051 | unsigned long *size, unsigned long *pa) | |
1052 | { | |
1053 | struct vop_device *vpdev = vdev->vpdev; | |
1054 | unsigned long start = MIC_DP_SIZE; | |
1055 | int i; | |
1056 | ||
1057 | /* | |
1058 | * MMAP interface is as follows: | |
1059 | * offset region | |
1060 | * 0x0 virtio device_page | |
1061 | * 0x1000 first vring | |
1062 | * 0x1000 + size of 1st vring second vring | |
1063 | * .... | |
1064 | */ | |
1065 | if (!offset) { | |
1066 | *pa = virt_to_phys(vpdev->hw_ops->get_dp(vpdev)); | |
1067 | *size = MIC_DP_SIZE; | |
1068 | return 0; | |
1069 | } | |
1070 | ||
1071 | for (i = 0; i < vdev->dd->num_vq; i++) { | |
1072 | struct vop_vringh *vvr = &vdev->vvr[i]; | |
1073 | ||
1074 | if (offset == start) { | |
1075 | *pa = virt_to_phys(vvr->vring.va); | |
1076 | *size = vvr->vring.len; | |
1077 | return 0; | |
1078 | } | |
1079 | start += vvr->vring.len; | |
1080 | } | |
1081 | return -1; | |
1082 | } | |
1083 | ||
1084 | /* | |
1085 | * Maps the device page and virtio rings to user space for readonly access. | |
1086 | */ | |
1087 | static int vop_mmap(struct file *f, struct vm_area_struct *vma) | |
1088 | { | |
1089 | struct vop_vdev *vdev = f->private_data; | |
1090 | unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; | |
1091 | unsigned long pa, size = vma->vm_end - vma->vm_start, size_rem = size; | |
1092 | int i, err; | |
1093 | ||
1094 | err = vop_vdev_inited(vdev); | |
1095 | if (err) | |
1096 | goto ret; | |
1097 | if (vma->vm_flags & VM_WRITE) { | |
1098 | err = -EACCES; | |
1099 | goto ret; | |
1100 | } | |
1101 | while (size_rem) { | |
1102 | i = vop_query_offset(vdev, offset, &size, &pa); | |
1103 | if (i < 0) { | |
1104 | err = -EINVAL; | |
1105 | goto ret; | |
1106 | } | |
1107 | err = remap_pfn_range(vma, vma->vm_start + offset, | |
1108 | pa >> PAGE_SHIFT, size, | |
1109 | vma->vm_page_prot); | |
1110 | if (err) | |
1111 | goto ret; | |
1112 | size_rem -= size; | |
1113 | offset += size; | |
1114 | } | |
1115 | ret: | |
1116 | return err; | |
1117 | } | |
1118 | ||
1119 | static const struct file_operations vop_fops = { | |
1120 | .open = vop_open, | |
1121 | .release = vop_release, | |
1122 | .unlocked_ioctl = vop_ioctl, | |
1123 | .poll = vop_poll, | |
1124 | .mmap = vop_mmap, | |
1125 | .owner = THIS_MODULE, | |
1126 | }; | |
1127 | ||
1128 | int vop_host_init(struct vop_info *vi) | |
1129 | { | |
1130 | int rc; | |
1131 | struct miscdevice *mdev; | |
1132 | struct vop_device *vpdev = vi->vpdev; | |
1133 | ||
1134 | INIT_LIST_HEAD(&vi->vdev_list); | |
1135 | vi->dma_ch = vpdev->dma_ch; | |
1136 | mdev = &vi->miscdev; | |
1137 | mdev->minor = MISC_DYNAMIC_MINOR; | |
1138 | snprintf(vi->name, sizeof(vi->name), "vop_virtio%d", vpdev->index); | |
1139 | mdev->name = vi->name; | |
1140 | mdev->fops = &vop_fops; | |
1141 | mdev->parent = &vpdev->dev; | |
1142 | ||
1143 | rc = misc_register(mdev); | |
1144 | if (rc) | |
1145 | dev_err(&vpdev->dev, "%s failed rc %d\n", __func__, rc); | |
1146 | return rc; | |
1147 | } | |
1148 | ||
1149 | void vop_host_uninit(struct vop_info *vi) | |
1150 | { | |
1151 | struct list_head *pos, *tmp; | |
1152 | struct vop_vdev *vdev; | |
1153 | ||
1154 | mutex_lock(&vi->vop_mutex); | |
1155 | vop_virtio_reset_devices(vi); | |
1156 | list_for_each_safe(pos, tmp, &vi->vdev_list) { | |
1157 | vdev = list_entry(pos, struct vop_vdev, list); | |
1158 | list_del(pos); | |
1159 | reinit_completion(&vdev->destroy); | |
1160 | mutex_unlock(&vi->vop_mutex); | |
1161 | mutex_lock(&vdev->vdev_mutex); | |
1162 | vop_virtio_del_device(vdev); | |
1163 | vdev->deleted = true; | |
1164 | mutex_unlock(&vdev->vdev_mutex); | |
1165 | complete(&vdev->destroy); | |
1166 | mutex_lock(&vi->vop_mutex); | |
1167 | } | |
1168 | mutex_unlock(&vi->vop_mutex); | |
1169 | misc_deregister(&vi->miscdev); | |
1170 | } |