Commit | Line | Data |
---|---|---|
1fcf0512 MT |
1 | /* |
2 | * Virtio PCI driver - modern (virtio 1.0) device support | |
3 | * | |
4 | * This module allows virtio devices to be used over a virtual PCI device. | |
5 | * This can be used with QEMU based VMMs like KVM or Xen. | |
6 | * | |
7 | * Copyright IBM Corp. 2007 | |
8 | * Copyright Red Hat, Inc. 2014 | |
9 | * | |
10 | * Authors: | |
11 | * Anthony Liguori <aliguori@us.ibm.com> | |
12 | * Rusty Russell <rusty@rustcorp.com.au> | |
13 | * Michael S. Tsirkin <mst@redhat.com> | |
14 | * | |
15 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
16 | * See the COPYING file in the top-level directory. | |
17 | * | |
18 | */ | |
19 | ||
05dbcb43 | 20 | #include <linux/delay.h> |
1fcf0512 MT |
21 | #define VIRTIO_PCI_NO_LEGACY |
22 | #include "virtio_pci_common.h" | |
23 | ||
c5d4c2c9 MT |
24 | /* |
25 | * Type-safe wrappers for io accesses. | |
26 | * Use these to enforce at compile time the following spec requirement: | |
27 | * | |
28 | * The driver MUST access each field using the “natural” access | |
29 | * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses | |
30 | * for 16-bit fields and 8-bit accesses for 8-bit fields. | |
31 | */ | |
32 | static inline u8 vp_ioread8(u8 __iomem *addr) | |
33 | { | |
34 | return ioread8(addr); | |
35 | } | |
61bd405f | 36 | static inline u16 vp_ioread16 (__le16 __iomem *addr) |
c5d4c2c9 MT |
37 | { |
38 | return ioread16(addr); | |
39 | } | |
40 | ||
61bd405f | 41 | static inline u32 vp_ioread32(__le32 __iomem *addr) |
c5d4c2c9 MT |
42 | { |
43 | return ioread32(addr); | |
44 | } | |
45 | ||
46 | static inline void vp_iowrite8(u8 value, u8 __iomem *addr) | |
47 | { | |
48 | iowrite8(value, addr); | |
49 | } | |
50 | ||
61bd405f | 51 | static inline void vp_iowrite16(u16 value, __le16 __iomem *addr) |
c5d4c2c9 MT |
52 | { |
53 | iowrite16(value, addr); | |
54 | } | |
55 | ||
61bd405f | 56 | static inline void vp_iowrite32(u32 value, __le32 __iomem *addr) |
c5d4c2c9 MT |
57 | { |
58 | iowrite32(value, addr); | |
59 | } | |
60 | ||
a8557d32 MT |
61 | static void vp_iowrite64_twopart(u64 val, |
62 | __le32 __iomem *lo, __le32 __iomem *hi) | |
63 | { | |
64 | vp_iowrite32((u32)val, lo); | |
65 | vp_iowrite32(val >> 32, hi); | |
66 | } | |
67 | ||
1fcf0512 MT |
68 | static void __iomem *map_capability(struct pci_dev *dev, int off, |
69 | size_t minlen, | |
70 | u32 align, | |
71 | u32 start, u32 size, | |
72 | size_t *len) | |
73 | { | |
74 | u8 bar; | |
75 | u32 offset, length; | |
76 | void __iomem *p; | |
77 | ||
78 | pci_read_config_byte(dev, off + offsetof(struct virtio_pci_cap, | |
79 | bar), | |
80 | &bar); | |
81 | pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, offset), | |
82 | &offset); | |
83 | pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length), | |
84 | &length); | |
85 | ||
86 | if (length <= start) { | |
87 | dev_err(&dev->dev, | |
88 | "virtio_pci: bad capability len %u (>%u expected)\n", | |
89 | length, start); | |
90 | return NULL; | |
91 | } | |
92 | ||
93 | if (length - start < minlen) { | |
94 | dev_err(&dev->dev, | |
95 | "virtio_pci: bad capability len %u (>=%zu expected)\n", | |
96 | length, minlen); | |
97 | return NULL; | |
98 | } | |
99 | ||
100 | length -= start; | |
101 | ||
102 | if (start + offset < offset) { | |
103 | dev_err(&dev->dev, | |
104 | "virtio_pci: map wrap-around %u+%u\n", | |
105 | start, offset); | |
106 | return NULL; | |
107 | } | |
108 | ||
109 | offset += start; | |
110 | ||
111 | if (offset & (align - 1)) { | |
112 | dev_err(&dev->dev, | |
113 | "virtio_pci: offset %u not aligned to %u\n", | |
114 | offset, align); | |
115 | return NULL; | |
116 | } | |
117 | ||
118 | if (length > size) | |
119 | length = size; | |
120 | ||
121 | if (len) | |
122 | *len = length; | |
123 | ||
124 | if (minlen + offset < minlen || | |
125 | minlen + offset > pci_resource_len(dev, bar)) { | |
126 | dev_err(&dev->dev, | |
127 | "virtio_pci: map virtio %zu@%u " | |
128 | "out of range on bar %i length %lu\n", | |
129 | minlen, offset, | |
130 | bar, (unsigned long)pci_resource_len(dev, bar)); | |
131 | return NULL; | |
132 | } | |
133 | ||
134 | p = pci_iomap_range(dev, bar, offset, length); | |
135 | if (!p) | |
136 | dev_err(&dev->dev, | |
137 | "virtio_pci: unable to map virtio %u@%u on bar %i\n", | |
138 | length, offset, bar); | |
139 | return p; | |
140 | } | |
141 | ||
1fcf0512 MT |
142 | /* virtio config->get_features() implementation */ |
143 | static u64 vp_get_features(struct virtio_device *vdev) | |
144 | { | |
145 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
146 | u64 features; | |
147 | ||
a8557d32 MT |
148 | vp_iowrite32(0, &vp_dev->common->device_feature_select); |
149 | features = vp_ioread32(&vp_dev->common->device_feature); | |
150 | vp_iowrite32(1, &vp_dev->common->device_feature_select); | |
151 | features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32); | |
1fcf0512 MT |
152 | |
153 | return features; | |
154 | } | |
155 | ||
156 | /* virtio config->finalize_features() implementation */ | |
157 | static int vp_finalize_features(struct virtio_device *vdev) | |
158 | { | |
159 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
160 | ||
161 | /* Give virtio_ring a chance to accept features. */ | |
162 | vring_transport_features(vdev); | |
163 | ||
164 | if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { | |
165 | dev_err(&vdev->dev, "virtio: device uses modern interface " | |
166 | "but does not have VIRTIO_F_VERSION_1\n"); | |
167 | return -EINVAL; | |
168 | } | |
169 | ||
a8557d32 MT |
170 | vp_iowrite32(0, &vp_dev->common->guest_feature_select); |
171 | vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature); | |
172 | vp_iowrite32(1, &vp_dev->common->guest_feature_select); | |
173 | vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature); | |
1fcf0512 MT |
174 | |
175 | return 0; | |
176 | } | |
177 | ||
178 | /* virtio config->get() implementation */ | |
179 | static void vp_get(struct virtio_device *vdev, unsigned offset, | |
180 | void *buf, unsigned len) | |
181 | { | |
182 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
183 | u8 b; | |
184 | __le16 w; | |
185 | __le32 l; | |
186 | ||
187 | BUG_ON(offset + len > vp_dev->device_len); | |
188 | ||
189 | switch (len) { | |
190 | case 1: | |
191 | b = ioread8(vp_dev->device + offset); | |
192 | memcpy(buf, &b, sizeof b); | |
193 | break; | |
194 | case 2: | |
195 | w = cpu_to_le16(ioread16(vp_dev->device + offset)); | |
196 | memcpy(buf, &w, sizeof w); | |
197 | break; | |
198 | case 4: | |
199 | l = cpu_to_le32(ioread32(vp_dev->device + offset)); | |
200 | memcpy(buf, &l, sizeof l); | |
201 | break; | |
202 | case 8: | |
203 | l = cpu_to_le32(ioread32(vp_dev->device + offset)); | |
204 | memcpy(buf, &l, sizeof l); | |
205 | l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l)); | |
206 | memcpy(buf + sizeof l, &l, sizeof l); | |
207 | break; | |
208 | default: | |
209 | BUG(); | |
210 | } | |
211 | } | |
212 | ||
213 | /* the config->set() implementation. it's symmetric to the config->get() | |
214 | * implementation */ | |
215 | static void vp_set(struct virtio_device *vdev, unsigned offset, | |
216 | const void *buf, unsigned len) | |
217 | { | |
218 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
219 | u8 b; | |
220 | __le16 w; | |
221 | __le32 l; | |
222 | ||
223 | BUG_ON(offset + len > vp_dev->device_len); | |
224 | ||
225 | switch (len) { | |
226 | case 1: | |
227 | memcpy(&b, buf, sizeof b); | |
228 | iowrite8(b, vp_dev->device + offset); | |
229 | break; | |
230 | case 2: | |
231 | memcpy(&w, buf, sizeof w); | |
232 | iowrite16(le16_to_cpu(w), vp_dev->device + offset); | |
233 | break; | |
234 | case 4: | |
235 | memcpy(&l, buf, sizeof l); | |
236 | iowrite32(le32_to_cpu(l), vp_dev->device + offset); | |
237 | break; | |
238 | case 8: | |
239 | memcpy(&l, buf, sizeof l); | |
240 | iowrite32(le32_to_cpu(l), vp_dev->device + offset); | |
241 | memcpy(&l, buf + sizeof l, sizeof l); | |
242 | iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l); | |
243 | break; | |
244 | default: | |
245 | BUG(); | |
246 | } | |
247 | } | |
248 | ||
249 | static u32 vp_generation(struct virtio_device *vdev) | |
250 | { | |
251 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
a8557d32 | 252 | return vp_ioread8(&vp_dev->common->config_generation); |
1fcf0512 MT |
253 | } |
254 | ||
255 | /* config->{get,set}_status() implementations */ | |
256 | static u8 vp_get_status(struct virtio_device *vdev) | |
257 | { | |
258 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
a8557d32 | 259 | return vp_ioread8(&vp_dev->common->device_status); |
1fcf0512 MT |
260 | } |
261 | ||
262 | static void vp_set_status(struct virtio_device *vdev, u8 status) | |
263 | { | |
264 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
265 | /* We should never be setting status to 0. */ | |
266 | BUG_ON(status == 0); | |
a8557d32 | 267 | vp_iowrite8(status, &vp_dev->common->device_status); |
1fcf0512 MT |
268 | } |
269 | ||
270 | static void vp_reset(struct virtio_device *vdev) | |
271 | { | |
272 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
273 | /* 0 status means a reset. */ | |
a8557d32 | 274 | vp_iowrite8(0, &vp_dev->common->device_status); |
05dbcb43 MT |
275 | /* After writing 0 to device_status, the driver MUST wait for a read of |
276 | * device_status to return 0 before reinitializing the device. | |
277 | * This will flush out the status write, and flush in device writes, | |
278 | * including MSI-X interrupts, if any. | |
279 | */ | |
280 | while (vp_ioread8(&vp_dev->common->device_status)) | |
281 | msleep(1); | |
1fcf0512 MT |
282 | /* Flush pending VQ/configuration callbacks. */ |
283 | vp_synchronize_vectors(vdev); | |
284 | } | |
285 | ||
286 | static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) | |
287 | { | |
288 | /* Setup the vector used for configuration events */ | |
a8557d32 | 289 | vp_iowrite16(vector, &vp_dev->common->msix_config); |
1fcf0512 MT |
290 | /* Verify we had enough resources to assign the vector */ |
291 | /* Will also flush the write out to device */ | |
a8557d32 | 292 | return vp_ioread16(&vp_dev->common->msix_config); |
1fcf0512 MT |
293 | } |
294 | ||
1fcf0512 | 295 | static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, |
1fcf0512 MT |
296 | unsigned index, |
297 | void (*callback)(struct virtqueue *vq), | |
298 | const char *name, | |
299 | u16 msix_vec) | |
300 | { | |
301 | struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common; | |
302 | struct virtqueue *vq; | |
303 | u16 num, off; | |
304 | int err; | |
305 | ||
a8557d32 | 306 | if (index >= vp_ioread16(&cfg->num_queues)) |
1fcf0512 MT |
307 | return ERR_PTR(-ENOENT); |
308 | ||
309 | /* Select the queue we're interested in */ | |
a8557d32 | 310 | vp_iowrite16(index, &cfg->queue_select); |
1fcf0512 MT |
311 | |
312 | /* Check if queue is either not available or already active. */ | |
a8557d32 MT |
313 | num = vp_ioread16(&cfg->queue_size); |
314 | if (!num || vp_ioread16(&cfg->queue_enable)) | |
1fcf0512 MT |
315 | return ERR_PTR(-ENOENT); |
316 | ||
317 | if (num & (num - 1)) { | |
318 | dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num); | |
319 | return ERR_PTR(-EINVAL); | |
320 | } | |
321 | ||
322 | /* get offset of notification word for this vq */ | |
a8557d32 | 323 | off = vp_ioread16(&cfg->queue_notify_off); |
1fcf0512 | 324 | |
1fcf0512 | 325 | /* create the vring */ |
7a5589b2 AL |
326 | vq = vring_create_virtqueue(index, num, |
327 | SMP_CACHE_BYTES, &vp_dev->vdev, | |
328 | true, true, vp_notify, callback, name); | |
329 | if (!vq) | |
330 | return ERR_PTR(-ENOMEM); | |
1fcf0512 MT |
331 | |
332 | /* activate the queue */ | |
7a5589b2 AL |
333 | vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size); |
334 | vp_iowrite64_twopart(virtqueue_get_desc_addr(vq), | |
a8557d32 | 335 | &cfg->queue_desc_lo, &cfg->queue_desc_hi); |
7a5589b2 | 336 | vp_iowrite64_twopart(virtqueue_get_avail_addr(vq), |
a8557d32 | 337 | &cfg->queue_avail_lo, &cfg->queue_avail_hi); |
7a5589b2 | 338 | vp_iowrite64_twopart(virtqueue_get_used_addr(vq), |
a8557d32 | 339 | &cfg->queue_used_lo, &cfg->queue_used_hi); |
1fcf0512 | 340 | |
3909213c MT |
341 | if (vp_dev->notify_base) { |
342 | /* offset should not wrap */ | |
343 | if ((u64)off * vp_dev->notify_offset_multiplier + 2 | |
344 | > vp_dev->notify_len) { | |
345 | dev_warn(&vp_dev->pci_dev->dev, | |
346 | "bad notification offset %u (x %u) " | |
347 | "for queue %u > %zd", | |
348 | off, vp_dev->notify_offset_multiplier, | |
349 | index, vp_dev->notify_len); | |
350 | err = -EINVAL; | |
351 | goto err_map_notify; | |
352 | } | |
353 | vq->priv = (void __force *)vp_dev->notify_base + | |
354 | off * vp_dev->notify_offset_multiplier; | |
355 | } else { | |
356 | vq->priv = (void __force *)map_capability(vp_dev->pci_dev, | |
357 | vp_dev->notify_map_cap, 2, 2, | |
358 | off * vp_dev->notify_offset_multiplier, 2, | |
359 | NULL); | |
360 | } | |
1fcf0512 MT |
361 | |
362 | if (!vq->priv) { | |
363 | err = -ENOMEM; | |
364 | goto err_map_notify; | |
365 | } | |
366 | ||
367 | if (msix_vec != VIRTIO_MSI_NO_VECTOR) { | |
a8557d32 MT |
368 | vp_iowrite16(msix_vec, &cfg->queue_msix_vector); |
369 | msix_vec = vp_ioread16(&cfg->queue_msix_vector); | |
1fcf0512 MT |
370 | if (msix_vec == VIRTIO_MSI_NO_VECTOR) { |
371 | err = -EBUSY; | |
372 | goto err_assign_vector; | |
373 | } | |
374 | } | |
375 | ||
376 | return vq; | |
377 | ||
378 | err_assign_vector: | |
3909213c MT |
379 | if (!vp_dev->notify_base) |
380 | pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv); | |
1fcf0512 MT |
381 | err_map_notify: |
382 | vring_del_virtqueue(vq); | |
1fcf0512 MT |
383 | return ERR_PTR(err); |
384 | } | |
385 | ||
386 | static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, | |
fb5e31d9 CH |
387 | struct virtqueue *vqs[], vq_callback_t *callbacks[], |
388 | const char * const names[], struct irq_affinity *desc) | |
1fcf0512 MT |
389 | { |
390 | struct virtio_pci_device *vp_dev = to_vp_device(vdev); | |
391 | struct virtqueue *vq; | |
fb5e31d9 | 392 | int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, desc); |
1fcf0512 MT |
393 | |
394 | if (rc) | |
395 | return rc; | |
396 | ||
397 | /* Select and activate all queues. Has to be done last: once we do | |
398 | * this, there's no way to go back except reset. | |
399 | */ | |
400 | list_for_each_entry(vq, &vdev->vqs, list) { | |
a8557d32 MT |
401 | vp_iowrite16(vq->index, &vp_dev->common->queue_select); |
402 | vp_iowrite16(1, &vp_dev->common->queue_enable); | |
1fcf0512 MT |
403 | } |
404 | ||
405 | return 0; | |
406 | } | |
407 | ||
5c34d002 | 408 | static void del_vq(struct virtqueue *vq) |
1fcf0512 | 409 | { |
1fcf0512 MT |
410 | struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); |
411 | ||
a8557d32 | 412 | vp_iowrite16(vq->index, &vp_dev->common->queue_select); |
1fcf0512 | 413 | |
2008c154 | 414 | if (vp_dev->msix_enabled) { |
a8557d32 MT |
415 | vp_iowrite16(VIRTIO_MSI_NO_VECTOR, |
416 | &vp_dev->common->queue_msix_vector); | |
1fcf0512 | 417 | /* Flush the write out to device */ |
a8557d32 | 418 | vp_ioread16(&vp_dev->common->queue_msix_vector); |
1fcf0512 MT |
419 | } |
420 | ||
3909213c MT |
421 | if (!vp_dev->notify_base) |
422 | pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv); | |
1fcf0512 MT |
423 | |
424 | vring_del_virtqueue(vq); | |
1fcf0512 MT |
425 | } |
426 | ||
d3f5f065 MT |
427 | static const struct virtio_config_ops virtio_pci_config_nodev_ops = { |
428 | .get = NULL, | |
429 | .set = NULL, | |
430 | .generation = vp_generation, | |
431 | .get_status = vp_get_status, | |
432 | .set_status = vp_set_status, | |
433 | .reset = vp_reset, | |
434 | .find_vqs = vp_modern_find_vqs, | |
435 | .del_vqs = vp_del_vqs, | |
436 | .get_features = vp_get_features, | |
437 | .finalize_features = vp_finalize_features, | |
438 | .bus_name = vp_bus_name, | |
439 | .set_vq_affinity = vp_set_vq_affinity, | |
bbaba479 | 440 | .get_vq_affinity = vp_get_vq_affinity, |
d3f5f065 MT |
441 | }; |
442 | ||
1fcf0512 MT |
443 | static const struct virtio_config_ops virtio_pci_config_ops = { |
444 | .get = vp_get, | |
445 | .set = vp_set, | |
446 | .generation = vp_generation, | |
447 | .get_status = vp_get_status, | |
448 | .set_status = vp_set_status, | |
449 | .reset = vp_reset, | |
450 | .find_vqs = vp_modern_find_vqs, | |
451 | .del_vqs = vp_del_vqs, | |
452 | .get_features = vp_get_features, | |
453 | .finalize_features = vp_finalize_features, | |
454 | .bus_name = vp_bus_name, | |
455 | .set_vq_affinity = vp_set_vq_affinity, | |
bbaba479 | 456 | .get_vq_affinity = vp_get_vq_affinity, |
1fcf0512 MT |
457 | }; |
458 | ||
459 | /** | |
460 | * virtio_pci_find_capability - walk capabilities to find device info. | |
461 | * @dev: the pci device | |
462 | * @cfg_type: the VIRTIO_PCI_CAP_* value we seek | |
463 | * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. | |
464 | * | |
465 | * Returns offset of the capability, or 0. | |
466 | */ | |
467 | static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, | |
59a5b0f7 | 468 | u32 ioresource_types, int *bars) |
1fcf0512 MT |
469 | { |
470 | int pos; | |
471 | ||
472 | for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); | |
473 | pos > 0; | |
474 | pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { | |
475 | u8 type, bar; | |
476 | pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, | |
477 | cfg_type), | |
478 | &type); | |
479 | pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, | |
480 | bar), | |
481 | &bar); | |
482 | ||
483 | /* Ignore structures with reserved BAR values */ | |
484 | if (bar > 0x5) | |
485 | continue; | |
486 | ||
487 | if (type == cfg_type) { | |
488 | if (pci_resource_len(dev, bar) && | |
59a5b0f7 GH |
489 | pci_resource_flags(dev, bar) & ioresource_types) { |
490 | *bars |= (1 << bar); | |
1fcf0512 | 491 | return pos; |
59a5b0f7 | 492 | } |
1fcf0512 MT |
493 | } |
494 | } | |
495 | return 0; | |
496 | } | |
497 | ||
89461c4a | 498 | /* This is part of the ABI. Don't screw with it. */ |
1fcf0512 MT |
499 | static inline void check_offsets(void) |
500 | { | |
89461c4a RR |
501 | /* Note: disk space was harmed in compilation of this function. */ |
502 | BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != | |
503 | offsetof(struct virtio_pci_cap, cap_vndr)); | |
504 | BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != | |
505 | offsetof(struct virtio_pci_cap, cap_next)); | |
506 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != | |
507 | offsetof(struct virtio_pci_cap, cap_len)); | |
508 | BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != | |
509 | offsetof(struct virtio_pci_cap, cfg_type)); | |
510 | BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != | |
511 | offsetof(struct virtio_pci_cap, bar)); | |
512 | BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != | |
513 | offsetof(struct virtio_pci_cap, offset)); | |
514 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != | |
515 | offsetof(struct virtio_pci_cap, length)); | |
516 | BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != | |
517 | offsetof(struct virtio_pci_notify_cap, | |
518 | notify_off_multiplier)); | |
519 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != | |
520 | offsetof(struct virtio_pci_common_cfg, | |
521 | device_feature_select)); | |
522 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != | |
523 | offsetof(struct virtio_pci_common_cfg, device_feature)); | |
524 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != | |
525 | offsetof(struct virtio_pci_common_cfg, | |
526 | guest_feature_select)); | |
527 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != | |
528 | offsetof(struct virtio_pci_common_cfg, guest_feature)); | |
529 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != | |
530 | offsetof(struct virtio_pci_common_cfg, msix_config)); | |
531 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != | |
532 | offsetof(struct virtio_pci_common_cfg, num_queues)); | |
533 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != | |
534 | offsetof(struct virtio_pci_common_cfg, device_status)); | |
535 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != | |
536 | offsetof(struct virtio_pci_common_cfg, config_generation)); | |
537 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != | |
538 | offsetof(struct virtio_pci_common_cfg, queue_select)); | |
539 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != | |
540 | offsetof(struct virtio_pci_common_cfg, queue_size)); | |
541 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != | |
542 | offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); | |
543 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != | |
544 | offsetof(struct virtio_pci_common_cfg, queue_enable)); | |
545 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != | |
546 | offsetof(struct virtio_pci_common_cfg, queue_notify_off)); | |
547 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != | |
548 | offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); | |
549 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != | |
550 | offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); | |
551 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != | |
552 | offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); | |
553 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != | |
554 | offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); | |
555 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != | |
556 | offsetof(struct virtio_pci_common_cfg, queue_used_lo)); | |
557 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != | |
558 | offsetof(struct virtio_pci_common_cfg, queue_used_hi)); | |
1fcf0512 MT |
559 | } |
560 | ||
561 | /* the PCI probing function */ | |
562 | int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) | |
563 | { | |
564 | struct pci_dev *pci_dev = vp_dev->pci_dev; | |
565 | int err, common, isr, notify, device; | |
566 | u32 notify_length; | |
3909213c | 567 | u32 notify_offset; |
1fcf0512 MT |
568 | |
569 | check_offsets(); | |
570 | ||
571 | /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ | |
572 | if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) | |
573 | return -ENODEV; | |
574 | ||
575 | if (pci_dev->device < 0x1040) { | |
576 | /* Transitional devices: use the PCI subsystem device id as | |
577 | * virtio device id, same as legacy driver always did. | |
578 | */ | |
579 | vp_dev->vdev.id.device = pci_dev->subsystem_device; | |
580 | } else { | |
581 | /* Modern devices: simply use PCI device id, but start from 0x1040. */ | |
582 | vp_dev->vdev.id.device = pci_dev->device - 0x1040; | |
583 | } | |
584 | vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; | |
585 | ||
1fcf0512 MT |
586 | /* check for a common config: if not, use legacy mode (bar 0). */ |
587 | common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, | |
59a5b0f7 GH |
588 | IORESOURCE_IO | IORESOURCE_MEM, |
589 | &vp_dev->modern_bars); | |
1fcf0512 MT |
590 | if (!common) { |
591 | dev_info(&pci_dev->dev, | |
592 | "virtio_pci: leaving for legacy driver\n"); | |
593 | return -ENODEV; | |
594 | } | |
595 | ||
596 | /* If common is there, these should be too... */ | |
597 | isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG, | |
59a5b0f7 GH |
598 | IORESOURCE_IO | IORESOURCE_MEM, |
599 | &vp_dev->modern_bars); | |
1fcf0512 | 600 | notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, |
59a5b0f7 GH |
601 | IORESOURCE_IO | IORESOURCE_MEM, |
602 | &vp_dev->modern_bars); | |
1fcf0512 MT |
603 | if (!isr || !notify) { |
604 | dev_err(&pci_dev->dev, | |
605 | "virtio_pci: missing capabilities %i/%i/%i\n", | |
606 | common, isr, notify); | |
607 | return -EINVAL; | |
608 | } | |
609 | ||
7a5589b2 AL |
610 | err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); |
611 | if (err) | |
612 | err = dma_set_mask_and_coherent(&pci_dev->dev, | |
613 | DMA_BIT_MASK(32)); | |
614 | if (err) | |
615 | dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); | |
616 | ||
1fcf0512 MT |
617 | /* Device capability is only mandatory for devices that have |
618 | * device-specific configuration. | |
619 | */ | |
620 | device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, | |
59a5b0f7 GH |
621 | IORESOURCE_IO | IORESOURCE_MEM, |
622 | &vp_dev->modern_bars); | |
623 | ||
624 | err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars, | |
625 | "virtio-pci-modern"); | |
626 | if (err) | |
627 | return err; | |
1fcf0512 MT |
628 | |
629 | err = -EINVAL; | |
630 | vp_dev->common = map_capability(pci_dev, common, | |
631 | sizeof(struct virtio_pci_common_cfg), 4, | |
632 | 0, sizeof(struct virtio_pci_common_cfg), | |
633 | NULL); | |
634 | if (!vp_dev->common) | |
635 | goto err_map_common; | |
636 | vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1, | |
637 | 0, 1, | |
638 | NULL); | |
639 | if (!vp_dev->isr) | |
640 | goto err_map_isr; | |
641 | ||
642 | /* Read notify_off_multiplier from config space. */ | |
643 | pci_read_config_dword(pci_dev, | |
644 | notify + offsetof(struct virtio_pci_notify_cap, | |
645 | notify_off_multiplier), | |
646 | &vp_dev->notify_offset_multiplier); | |
3909213c | 647 | /* Read notify length and offset from config space. */ |
1fcf0512 MT |
648 | pci_read_config_dword(pci_dev, |
649 | notify + offsetof(struct virtio_pci_notify_cap, | |
650 | cap.length), | |
651 | ¬ify_length); | |
652 | ||
3909213c MT |
653 | pci_read_config_dword(pci_dev, |
654 | notify + offsetof(struct virtio_pci_notify_cap, | |
4e94ebdd | 655 | cap.offset), |
3909213c MT |
656 | ¬ify_offset); |
657 | ||
658 | /* We don't know how many VQs we'll map, ahead of the time. | |
659 | * If notify length is small, map it all now. | |
660 | * Otherwise, map each VQ individually later. | |
661 | */ | |
662 | if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { | |
663 | vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2, | |
664 | 0, notify_length, | |
665 | &vp_dev->notify_len); | |
666 | if (!vp_dev->notify_base) | |
667 | goto err_map_notify; | |
668 | } else { | |
669 | vp_dev->notify_map_cap = notify; | |
670 | } | |
1fcf0512 MT |
671 | |
672 | /* Again, we don't know how much we should map, but PAGE_SIZE | |
673 | * is more than enough for all existing devices. | |
674 | */ | |
675 | if (device) { | |
676 | vp_dev->device = map_capability(pci_dev, device, 0, 4, | |
677 | 0, PAGE_SIZE, | |
678 | &vp_dev->device_len); | |
679 | if (!vp_dev->device) | |
680 | goto err_map_device; | |
1fcf0512 | 681 | |
d3f5f065 MT |
682 | vp_dev->vdev.config = &virtio_pci_config_ops; |
683 | } else { | |
684 | vp_dev->vdev.config = &virtio_pci_config_nodev_ops; | |
685 | } | |
1fcf0512 MT |
686 | |
687 | vp_dev->config_vector = vp_config_vector; | |
688 | vp_dev->setup_vq = setup_vq; | |
689 | vp_dev->del_vq = del_vq; | |
690 | ||
691 | return 0; | |
692 | ||
693 | err_map_device: | |
3909213c MT |
694 | if (vp_dev->notify_base) |
695 | pci_iounmap(pci_dev, vp_dev->notify_base); | |
696 | err_map_notify: | |
1fcf0512 MT |
697 | pci_iounmap(pci_dev, vp_dev->isr); |
698 | err_map_isr: | |
699 | pci_iounmap(pci_dev, vp_dev->common); | |
700 | err_map_common: | |
701 | return err; | |
702 | } | |
703 | ||
704 | void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) | |
705 | { | |
706 | struct pci_dev *pci_dev = vp_dev->pci_dev; | |
707 | ||
708 | if (vp_dev->device) | |
709 | pci_iounmap(pci_dev, vp_dev->device); | |
3909213c MT |
710 | if (vp_dev->notify_base) |
711 | pci_iounmap(pci_dev, vp_dev->notify_base); | |
1fcf0512 MT |
712 | pci_iounmap(pci_dev, vp_dev->isr); |
713 | pci_iounmap(pci_dev, vp_dev->common); | |
59a5b0f7 | 714 | pci_release_selected_regions(pci_dev, vp_dev->modern_bars); |
1fcf0512 | 715 | } |