Commit | Line | Data |
---|---|---|
1357dfd7 LS |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* | |
3 | * Mellanox BlueField SoC TmFifo driver | |
4 | * | |
5 | * Copyright (C) 2019 Mellanox Technologies | |
6 | */ | |
7 | ||
8 | #include <linux/acpi.h> | |
9 | #include <linux/bitfield.h> | |
10 | #include <linux/circ_buf.h> | |
11 | #include <linux/efi.h> | |
12 | #include <linux/irq.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/mutex.h> | |
15 | #include <linux/platform_device.h> | |
16 | #include <linux/types.h> | |
17 | ||
18 | #include <linux/virtio_config.h> | |
19 | #include <linux/virtio_console.h> | |
20 | #include <linux/virtio_ids.h> | |
21 | #include <linux/virtio_net.h> | |
22 | #include <linux/virtio_ring.h> | |
23 | ||
24 | #include "mlxbf-tmfifo-regs.h" | |
25 | ||
26 | /* Vring size. */ | |
27 | #define MLXBF_TMFIFO_VRING_SIZE SZ_1K | |
28 | ||
29 | /* Console Tx buffer size. */ | |
30 | #define MLXBF_TMFIFO_CON_TX_BUF_SIZE SZ_32K | |
31 | ||
32 | /* Console Tx buffer reserved space. */ | |
33 | #define MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE 8 | |
34 | ||
35 | /* House-keeping timer interval. */ | |
36 | #define MLXBF_TMFIFO_TIMER_INTERVAL (HZ / 10) | |
37 | ||
38 | /* Virtual devices sharing the TM FIFO. */ | |
39 | #define MLXBF_TMFIFO_VDEV_MAX (VIRTIO_ID_CONSOLE + 1) | |
40 | ||
41 | /* | |
42 | * Reserve 1/16 of TmFifo space, so console messages are not starved by | |
43 | * the networking traffic. | |
44 | */ | |
45 | #define MLXBF_TMFIFO_RESERVE_RATIO 16 | |
46 | ||
47 | /* Message with data needs at least two words (for header & data). */ | |
48 | #define MLXBF_TMFIFO_DATA_MIN_WORDS 2 | |
49 | ||
50 | struct mlxbf_tmfifo; | |
51 | ||
52 | /** | |
53 | * mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring | |
54 | * @va: virtual address of the ring | |
55 | * @dma: dma address of the ring | |
56 | * @vq: pointer to the virtio virtqueue | |
57 | * @desc: current descriptor of the pending packet | |
58 | * @desc_head: head descriptor of the pending packet | |
59 | * @cur_len: processed length of the current descriptor | |
60 | * @rem_len: remaining length of the pending packet | |
61 | * @pkt_len: total length of the pending packet | |
62 | * @next_avail: next avail descriptor id | |
63 | * @num: vring size (number of descriptors) | |
64 | * @align: vring alignment size | |
65 | * @index: vring index | |
66 | * @vdev_id: vring virtio id (VIRTIO_ID_xxx) | |
67 | * @fifo: pointer to the tmfifo structure | |
68 | */ | |
69 | struct mlxbf_tmfifo_vring { | |
70 | void *va; | |
71 | dma_addr_t dma; | |
72 | struct virtqueue *vq; | |
73 | struct vring_desc *desc; | |
74 | struct vring_desc *desc_head; | |
75 | int cur_len; | |
76 | int rem_len; | |
77 | u32 pkt_len; | |
78 | u16 next_avail; | |
79 | int num; | |
80 | int align; | |
81 | int index; | |
82 | int vdev_id; | |
83 | struct mlxbf_tmfifo *fifo; | |
84 | }; | |
85 | ||
86 | /* Interrupt types. */ | |
87 | enum { | |
88 | MLXBF_TM_RX_LWM_IRQ, | |
89 | MLXBF_TM_RX_HWM_IRQ, | |
90 | MLXBF_TM_TX_LWM_IRQ, | |
91 | MLXBF_TM_TX_HWM_IRQ, | |
92 | MLXBF_TM_MAX_IRQ | |
93 | }; | |
94 | ||
95 | /* Ring types (Rx & Tx). */ | |
96 | enum { | |
97 | MLXBF_TMFIFO_VRING_RX, | |
98 | MLXBF_TMFIFO_VRING_TX, | |
99 | MLXBF_TMFIFO_VRING_MAX | |
100 | }; | |
101 | ||
102 | /** | |
103 | * mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device | |
104 | * @vdev: virtio device, in which the vdev.id.device field has the | |
105 | * VIRTIO_ID_xxx id to distinguish the virtual device. | |
106 | * @status: status of the device | |
107 | * @features: supported features of the device | |
108 | * @vrings: array of tmfifo vrings of this device | |
109 | * @config.cons: virtual console config - | |
110 | * select if vdev.id.device is VIRTIO_ID_CONSOLE | |
111 | * @config.net: virtual network config - | |
112 | * select if vdev.id.device is VIRTIO_ID_NET | |
113 | * @tx_buf: tx buffer used to buffer data before writing into the FIFO | |
114 | */ | |
115 | struct mlxbf_tmfifo_vdev { | |
116 | struct virtio_device vdev; | |
117 | u8 status; | |
118 | u64 features; | |
119 | struct mlxbf_tmfifo_vring vrings[MLXBF_TMFIFO_VRING_MAX]; | |
120 | union { | |
121 | struct virtio_console_config cons; | |
122 | struct virtio_net_config net; | |
123 | } config; | |
124 | struct circ_buf tx_buf; | |
125 | }; | |
126 | ||
127 | /** | |
128 | * mlxbf_tmfifo_irq_info - Structure of the interrupt information | |
129 | * @fifo: pointer to the tmfifo structure | |
130 | * @irq: interrupt number | |
131 | * @index: index into the interrupt array | |
132 | */ | |
133 | struct mlxbf_tmfifo_irq_info { | |
134 | struct mlxbf_tmfifo *fifo; | |
135 | int irq; | |
136 | int index; | |
137 | }; | |
138 | ||
139 | /** | |
140 | * mlxbf_tmfifo - Structure of the TmFifo | |
141 | * @vdev: array of the virtual devices running over the TmFifo | |
142 | * @lock: lock to protect the TmFifo access | |
143 | * @rx_base: mapped register base address for the Rx FIFO | |
144 | * @tx_base: mapped register base address for the Tx FIFO | |
145 | * @rx_fifo_size: number of entries of the Rx FIFO | |
146 | * @tx_fifo_size: number of entries of the Tx FIFO | |
147 | * @pend_events: pending bits for deferred events | |
148 | * @irq_info: interrupt information | |
149 | * @work: work struct for deferred process | |
150 | * @timer: background timer | |
151 | * @vring: Tx/Rx ring | |
152 | * @spin_lock: spin lock | |
153 | * @is_ready: ready flag | |
154 | */ | |
155 | struct mlxbf_tmfifo { | |
156 | struct mlxbf_tmfifo_vdev *vdev[MLXBF_TMFIFO_VDEV_MAX]; | |
157 | struct mutex lock; /* TmFifo lock */ | |
158 | void __iomem *rx_base; | |
159 | void __iomem *tx_base; | |
160 | int rx_fifo_size; | |
161 | int tx_fifo_size; | |
162 | unsigned long pend_events; | |
163 | struct mlxbf_tmfifo_irq_info irq_info[MLXBF_TM_MAX_IRQ]; | |
164 | struct work_struct work; | |
165 | struct timer_list timer; | |
166 | struct mlxbf_tmfifo_vring *vring[2]; | |
167 | spinlock_t spin_lock; /* spin lock */ | |
168 | bool is_ready; | |
169 | }; | |
170 | ||
171 | /** | |
172 | * mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header | |
173 | * @type: message type | |
174 | * @len: payload length in network byte order. Messages sent into the FIFO | |
175 | * will be read by the other side as data stream in the same byte order. | |
176 | * The length needs to be encoded into network order so both sides | |
177 | * could understand it. | |
178 | */ | |
179 | struct mlxbf_tmfifo_msg_hdr { | |
180 | u8 type; | |
181 | __be16 len; | |
182 | u8 unused[5]; | |
183 | } __packed __aligned(sizeof(u64)); | |
184 | ||
185 | /* | |
186 | * Default MAC. | |
187 | * This MAC address will be read from EFI persistent variable if configured. | |
188 | * It can also be reconfigured with standard Linux tools. | |
189 | */ | |
190 | static u8 mlxbf_tmfifo_net_default_mac[ETH_ALEN] = { | |
191 | 0x00, 0x1A, 0xCA, 0xFF, 0xFF, 0x01 | |
192 | }; | |
193 | ||
194 | /* EFI variable name of the MAC address. */ | |
195 | static efi_char16_t mlxbf_tmfifo_efi_name[] = L"RshimMacAddr"; | |
196 | ||
197 | /* Maximum L2 header length. */ | |
198 | #define MLXBF_TMFIFO_NET_L2_OVERHEAD 36 | |
199 | ||
200 | /* Supported virtio-net features. */ | |
201 | #define MLXBF_TMFIFO_NET_FEATURES \ | |
202 | (BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_STATUS) | \ | |
203 | BIT_ULL(VIRTIO_NET_F_MAC)) | |
204 | ||
205 | #define mlxbf_vdev_to_tmfifo(d) container_of(d, struct mlxbf_tmfifo_vdev, vdev) | |
206 | ||
207 | /* Free vrings of the FIFO device. */ | |
208 | static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo, | |
209 | struct mlxbf_tmfifo_vdev *tm_vdev) | |
210 | { | |
211 | struct mlxbf_tmfifo_vring *vring; | |
212 | int i, size; | |
213 | ||
214 | for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { | |
215 | vring = &tm_vdev->vrings[i]; | |
216 | if (vring->va) { | |
217 | size = vring_size(vring->num, vring->align); | |
218 | dma_free_coherent(tm_vdev->vdev.dev.parent, size, | |
219 | vring->va, vring->dma); | |
220 | vring->va = NULL; | |
221 | if (vring->vq) { | |
222 | vring_del_virtqueue(vring->vq); | |
223 | vring->vq = NULL; | |
224 | } | |
225 | } | |
226 | } | |
227 | } | |
228 | ||
229 | /* Allocate vrings for the FIFO. */ | |
230 | static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo, | |
231 | struct mlxbf_tmfifo_vdev *tm_vdev) | |
232 | { | |
233 | struct mlxbf_tmfifo_vring *vring; | |
234 | struct device *dev; | |
235 | dma_addr_t dma; | |
236 | int i, size; | |
237 | void *va; | |
238 | ||
239 | for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { | |
240 | vring = &tm_vdev->vrings[i]; | |
241 | vring->fifo = fifo; | |
242 | vring->num = MLXBF_TMFIFO_VRING_SIZE; | |
243 | vring->align = SMP_CACHE_BYTES; | |
244 | vring->index = i; | |
245 | vring->vdev_id = tm_vdev->vdev.id.device; | |
246 | dev = &tm_vdev->vdev.dev; | |
247 | ||
248 | size = vring_size(vring->num, vring->align); | |
249 | va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL); | |
250 | if (!va) { | |
251 | mlxbf_tmfifo_free_vrings(fifo, tm_vdev); | |
252 | dev_err(dev->parent, "dma_alloc_coherent failed\n"); | |
253 | return -ENOMEM; | |
254 | } | |
255 | ||
256 | vring->va = va; | |
257 | vring->dma = dma; | |
258 | } | |
259 | ||
260 | return 0; | |
261 | } | |
262 | ||
263 | /* Disable interrupts of the FIFO device. */ | |
264 | static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo) | |
265 | { | |
266 | int i, irq; | |
267 | ||
268 | for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) { | |
269 | irq = fifo->irq_info[i].irq; | |
270 | fifo->irq_info[i].irq = 0; | |
271 | disable_irq(irq); | |
272 | } | |
273 | } | |
274 | ||
275 | /* Interrupt handler. */ | |
276 | static irqreturn_t mlxbf_tmfifo_irq_handler(int irq, void *arg) | |
277 | { | |
278 | struct mlxbf_tmfifo_irq_info *irq_info = arg; | |
279 | ||
280 | if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events)) | |
281 | schedule_work(&irq_info->fifo->work); | |
282 | ||
283 | return IRQ_HANDLED; | |
284 | } | |
285 | ||
286 | /* Get the next packet descriptor from the vring. */ | |
287 | static struct vring_desc * | |
288 | mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring) | |
289 | { | |
290 | const struct vring *vr = virtqueue_get_vring(vring->vq); | |
291 | struct virtio_device *vdev = vring->vq->vdev; | |
292 | unsigned int idx, head; | |
293 | ||
294 | if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx)) | |
295 | return NULL; | |
296 | ||
297 | idx = vring->next_avail % vr->num; | |
298 | head = virtio16_to_cpu(vdev, vr->avail->ring[idx]); | |
299 | if (WARN_ON(head >= vr->num)) | |
300 | return NULL; | |
301 | ||
302 | vring->next_avail++; | |
303 | ||
304 | return &vr->desc[head]; | |
305 | } | |
306 | ||
307 | /* Release virtio descriptor. */ | |
308 | static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring, | |
309 | struct vring_desc *desc, u32 len) | |
310 | { | |
311 | const struct vring *vr = virtqueue_get_vring(vring->vq); | |
312 | struct virtio_device *vdev = vring->vq->vdev; | |
313 | u16 idx, vr_idx; | |
314 | ||
315 | vr_idx = virtio16_to_cpu(vdev, vr->used->idx); | |
316 | idx = vr_idx % vr->num; | |
317 | vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc); | |
318 | vr->used->ring[idx].len = cpu_to_virtio32(vdev, len); | |
319 | ||
320 | /* | |
321 | * Virtio could poll and check the 'idx' to decide whether the desc is | |
322 | * done or not. Add a memory barrier here to make sure the update above | |
323 | * completes before updating the idx. | |
324 | */ | |
325 | mb(); | |
326 | vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1); | |
327 | } | |
328 | ||
329 | /* Get the total length of the descriptor chain. */ | |
330 | static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring, | |
331 | struct vring_desc *desc) | |
332 | { | |
333 | const struct vring *vr = virtqueue_get_vring(vring->vq); | |
334 | struct virtio_device *vdev = vring->vq->vdev; | |
335 | u32 len = 0, idx; | |
336 | ||
337 | while (desc) { | |
338 | len += virtio32_to_cpu(vdev, desc->len); | |
339 | if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) | |
340 | break; | |
341 | idx = virtio16_to_cpu(vdev, desc->next); | |
342 | desc = &vr->desc[idx]; | |
343 | } | |
344 | ||
345 | return len; | |
346 | } | |
347 | ||
348 | static void mlxbf_tmfifo_release_pending_pkt(struct mlxbf_tmfifo_vring *vring) | |
349 | { | |
350 | struct vring_desc *desc_head; | |
351 | u32 len = 0; | |
352 | ||
353 | if (vring->desc_head) { | |
354 | desc_head = vring->desc_head; | |
355 | len = vring->pkt_len; | |
356 | } else { | |
357 | desc_head = mlxbf_tmfifo_get_next_desc(vring); | |
358 | len = mlxbf_tmfifo_get_pkt_len(vring, desc_head); | |
359 | } | |
360 | ||
361 | if (desc_head) | |
362 | mlxbf_tmfifo_release_desc(vring, desc_head, len); | |
363 | ||
364 | vring->pkt_len = 0; | |
365 | vring->desc = NULL; | |
366 | vring->desc_head = NULL; | |
367 | } | |
368 | ||
369 | static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring, | |
370 | struct vring_desc *desc, bool is_rx) | |
371 | { | |
372 | struct virtio_device *vdev = vring->vq->vdev; | |
373 | struct virtio_net_hdr *net_hdr; | |
374 | ||
375 | net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); | |
376 | memset(net_hdr, 0, sizeof(*net_hdr)); | |
377 | } | |
378 | ||
379 | /* Get and initialize the next packet. */ | |
380 | static struct vring_desc * | |
381 | mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx) | |
382 | { | |
383 | struct vring_desc *desc; | |
384 | ||
385 | desc = mlxbf_tmfifo_get_next_desc(vring); | |
386 | if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET) | |
387 | mlxbf_tmfifo_init_net_desc(vring, desc, is_rx); | |
388 | ||
389 | vring->desc_head = desc; | |
390 | vring->desc = desc; | |
391 | ||
392 | return desc; | |
393 | } | |
394 | ||
395 | /* House-keeping timer. */ | |
396 | static void mlxbf_tmfifo_timer(struct timer_list *t) | |
397 | { | |
398 | struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer); | |
399 | int rx, tx; | |
400 | ||
401 | rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events); | |
402 | tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events); | |
403 | ||
404 | if (rx || tx) | |
405 | schedule_work(&fifo->work); | |
406 | ||
407 | mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); | |
408 | } | |
409 | ||
410 | /* Copy one console packet into the output buffer. */ | |
411 | static void mlxbf_tmfifo_console_output_one(struct mlxbf_tmfifo_vdev *cons, | |
412 | struct mlxbf_tmfifo_vring *vring, | |
413 | struct vring_desc *desc) | |
414 | { | |
415 | const struct vring *vr = virtqueue_get_vring(vring->vq); | |
416 | struct virtio_device *vdev = &cons->vdev; | |
417 | u32 len, idx, seg; | |
418 | void *addr; | |
419 | ||
420 | while (desc) { | |
421 | addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); | |
422 | len = virtio32_to_cpu(vdev, desc->len); | |
423 | ||
424 | seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail, | |
425 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); | |
426 | if (len <= seg) { | |
427 | memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len); | |
428 | } else { | |
429 | memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg); | |
430 | addr += seg; | |
431 | memcpy(cons->tx_buf.buf, addr, len - seg); | |
432 | } | |
433 | cons->tx_buf.head = (cons->tx_buf.head + len) % | |
434 | MLXBF_TMFIFO_CON_TX_BUF_SIZE; | |
435 | ||
436 | if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) | |
437 | break; | |
438 | idx = virtio16_to_cpu(vdev, desc->next); | |
439 | desc = &vr->desc[idx]; | |
440 | } | |
441 | } | |
442 | ||
443 | /* Copy console data into the output buffer. */ | |
444 | static void mlxbf_tmfifo_console_output(struct mlxbf_tmfifo_vdev *cons, | |
445 | struct mlxbf_tmfifo_vring *vring) | |
446 | { | |
447 | struct vring_desc *desc; | |
448 | u32 len, avail; | |
449 | ||
450 | desc = mlxbf_tmfifo_get_next_desc(vring); | |
451 | while (desc) { | |
452 | /* Release the packet if not enough space. */ | |
453 | len = mlxbf_tmfifo_get_pkt_len(vring, desc); | |
454 | avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail, | |
455 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); | |
456 | if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) { | |
457 | mlxbf_tmfifo_release_desc(vring, desc, len); | |
458 | break; | |
459 | } | |
460 | ||
461 | mlxbf_tmfifo_console_output_one(cons, vring, desc); | |
462 | mlxbf_tmfifo_release_desc(vring, desc, len); | |
463 | desc = mlxbf_tmfifo_get_next_desc(vring); | |
464 | } | |
465 | } | |
466 | ||
467 | /* Get the number of available words in Rx FIFO for receiving. */ | |
468 | static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo) | |
469 | { | |
470 | u64 sts; | |
471 | ||
472 | sts = readq(fifo->rx_base + MLXBF_TMFIFO_RX_STS); | |
473 | return FIELD_GET(MLXBF_TMFIFO_RX_STS__COUNT_MASK, sts); | |
474 | } | |
475 | ||
476 | /* Get the number of available words in the TmFifo for sending. */ | |
477 | static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id) | |
478 | { | |
479 | int tx_reserve; | |
480 | u32 count; | |
481 | u64 sts; | |
482 | ||
483 | /* Reserve some room in FIFO for console messages. */ | |
484 | if (vdev_id == VIRTIO_ID_NET) | |
485 | tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO; | |
486 | else | |
487 | tx_reserve = 1; | |
488 | ||
489 | sts = readq(fifo->tx_base + MLXBF_TMFIFO_TX_STS); | |
490 | count = FIELD_GET(MLXBF_TMFIFO_TX_STS__COUNT_MASK, sts); | |
491 | return fifo->tx_fifo_size - tx_reserve - count; | |
492 | } | |
493 | ||
494 | /* Console Tx (move data from the output buffer into the TmFifo). */ | |
495 | static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail) | |
496 | { | |
497 | struct mlxbf_tmfifo_msg_hdr hdr; | |
498 | struct mlxbf_tmfifo_vdev *cons; | |
499 | unsigned long flags; | |
500 | int size, seg; | |
501 | void *addr; | |
502 | u64 data; | |
503 | ||
504 | /* Return if not enough space available. */ | |
505 | if (avail < MLXBF_TMFIFO_DATA_MIN_WORDS) | |
506 | return; | |
507 | ||
508 | cons = fifo->vdev[VIRTIO_ID_CONSOLE]; | |
509 | if (!cons || !cons->tx_buf.buf) | |
510 | return; | |
511 | ||
512 | /* Return if no data to send. */ | |
513 | size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail, | |
514 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); | |
515 | if (size == 0) | |
516 | return; | |
517 | ||
518 | /* Adjust the size to available space. */ | |
519 | if (size + sizeof(hdr) > avail * sizeof(u64)) | |
520 | size = avail * sizeof(u64) - sizeof(hdr); | |
521 | ||
522 | /* Write header. */ | |
523 | hdr.type = VIRTIO_ID_CONSOLE; | |
524 | hdr.len = htons(size); | |
525 | writeq(*(u64 *)&hdr, fifo->tx_base + MLXBF_TMFIFO_TX_DATA); | |
526 | ||
527 | /* Use spin-lock to protect the 'cons->tx_buf'. */ | |
528 | spin_lock_irqsave(&fifo->spin_lock, flags); | |
529 | ||
530 | while (size > 0) { | |
531 | addr = cons->tx_buf.buf + cons->tx_buf.tail; | |
532 | ||
533 | seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail, | |
534 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); | |
535 | if (seg >= sizeof(u64)) { | |
536 | memcpy(&data, addr, sizeof(u64)); | |
537 | } else { | |
538 | memcpy(&data, addr, seg); | |
539 | memcpy((u8 *)&data + seg, cons->tx_buf.buf, | |
540 | sizeof(u64) - seg); | |
541 | } | |
542 | writeq(data, fifo->tx_base + MLXBF_TMFIFO_TX_DATA); | |
543 | ||
544 | if (size >= sizeof(u64)) { | |
545 | cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) % | |
546 | MLXBF_TMFIFO_CON_TX_BUF_SIZE; | |
547 | size -= sizeof(u64); | |
548 | } else { | |
549 | cons->tx_buf.tail = (cons->tx_buf.tail + size) % | |
550 | MLXBF_TMFIFO_CON_TX_BUF_SIZE; | |
551 | size = 0; | |
552 | } | |
553 | } | |
554 | ||
555 | spin_unlock_irqrestore(&fifo->spin_lock, flags); | |
556 | } | |
557 | ||
558 | /* Rx/Tx one word in the descriptor buffer. */ | |
559 | static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring, | |
560 | struct vring_desc *desc, | |
561 | bool is_rx, int len) | |
562 | { | |
563 | struct virtio_device *vdev = vring->vq->vdev; | |
564 | struct mlxbf_tmfifo *fifo = vring->fifo; | |
565 | void *addr; | |
566 | u64 data; | |
567 | ||
568 | /* Get the buffer address of this desc. */ | |
569 | addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); | |
570 | ||
571 | /* Read a word from FIFO for Rx. */ | |
572 | if (is_rx) | |
573 | data = readq(fifo->rx_base + MLXBF_TMFIFO_RX_DATA); | |
574 | ||
575 | if (vring->cur_len + sizeof(u64) <= len) { | |
576 | /* The whole word. */ | |
577 | if (is_rx) | |
578 | memcpy(addr + vring->cur_len, &data, sizeof(u64)); | |
579 | else | |
580 | memcpy(&data, addr + vring->cur_len, sizeof(u64)); | |
581 | vring->cur_len += sizeof(u64); | |
582 | } else { | |
583 | /* Leftover bytes. */ | |
584 | if (is_rx) | |
585 | memcpy(addr + vring->cur_len, &data, | |
586 | len - vring->cur_len); | |
587 | else | |
588 | memcpy(&data, addr + vring->cur_len, | |
589 | len - vring->cur_len); | |
590 | vring->cur_len = len; | |
591 | } | |
592 | ||
593 | /* Write the word into FIFO for Tx. */ | |
594 | if (!is_rx) | |
595 | writeq(data, fifo->tx_base + MLXBF_TMFIFO_TX_DATA); | |
596 | } | |
597 | ||
598 | /* | |
599 | * Rx/Tx packet header. | |
600 | * | |
601 | * In Rx case, the packet might be found to belong to a different vring since | |
602 | * the TmFifo is shared by different services. In such case, the 'vring_change' | |
603 | * flag is set. | |
604 | */ | |
605 | static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring, | |
606 | struct vring_desc *desc, | |
607 | bool is_rx, bool *vring_change) | |
608 | { | |
609 | struct mlxbf_tmfifo *fifo = vring->fifo; | |
610 | struct virtio_net_config *config; | |
611 | struct mlxbf_tmfifo_msg_hdr hdr; | |
612 | int vdev_id, hdr_len; | |
613 | ||
614 | /* Read/Write packet header. */ | |
615 | if (is_rx) { | |
616 | /* Drain one word from the FIFO. */ | |
617 | *(u64 *)&hdr = readq(fifo->rx_base + MLXBF_TMFIFO_RX_DATA); | |
618 | ||
619 | /* Skip the length 0 packets (keepalive). */ | |
620 | if (hdr.len == 0) | |
621 | return; | |
622 | ||
623 | /* Check packet type. */ | |
624 | if (hdr.type == VIRTIO_ID_NET) { | |
625 | vdev_id = VIRTIO_ID_NET; | |
626 | hdr_len = sizeof(struct virtio_net_hdr); | |
627 | config = &fifo->vdev[vdev_id]->config.net; | |
628 | if (ntohs(hdr.len) > config->mtu + | |
629 | MLXBF_TMFIFO_NET_L2_OVERHEAD) | |
630 | return; | |
631 | } else { | |
632 | vdev_id = VIRTIO_ID_CONSOLE; | |
633 | hdr_len = 0; | |
634 | } | |
635 | ||
636 | /* | |
637 | * Check whether the new packet still belongs to this vring. | |
638 | * If not, update the pkt_len of the new vring. | |
639 | */ | |
640 | if (vdev_id != vring->vdev_id) { | |
641 | struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id]; | |
642 | ||
643 | if (!tm_dev2) | |
644 | return; | |
645 | vring->desc = desc; | |
646 | vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX]; | |
647 | *vring_change = true; | |
648 | } | |
649 | vring->pkt_len = ntohs(hdr.len) + hdr_len; | |
650 | } else { | |
651 | /* Network virtio has an extra header. */ | |
652 | hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ? | |
653 | sizeof(struct virtio_net_hdr) : 0; | |
654 | vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc); | |
655 | hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ? | |
656 | VIRTIO_ID_NET : VIRTIO_ID_CONSOLE; | |
657 | hdr.len = htons(vring->pkt_len - hdr_len); | |
658 | writeq(*(u64 *)&hdr, fifo->tx_base + MLXBF_TMFIFO_TX_DATA); | |
659 | } | |
660 | ||
661 | vring->cur_len = hdr_len; | |
662 | vring->rem_len = vring->pkt_len; | |
663 | fifo->vring[is_rx] = vring; | |
664 | } | |
665 | ||
666 | /* | |
667 | * Rx/Tx one descriptor. | |
668 | * | |
669 | * Return true to indicate more data available. | |
670 | */ | |
671 | static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring, | |
672 | bool is_rx, int *avail) | |
673 | { | |
674 | const struct vring *vr = virtqueue_get_vring(vring->vq); | |
675 | struct mlxbf_tmfifo *fifo = vring->fifo; | |
676 | struct virtio_device *vdev; | |
677 | bool vring_change = false; | |
678 | struct vring_desc *desc; | |
679 | unsigned long flags; | |
680 | u32 len, idx; | |
681 | ||
682 | vdev = &fifo->vdev[vring->vdev_id]->vdev; | |
683 | ||
684 | /* Get the descriptor of the next packet. */ | |
685 | if (!vring->desc) { | |
686 | desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx); | |
687 | if (!desc) | |
688 | return false; | |
689 | } else { | |
690 | desc = vring->desc; | |
691 | } | |
692 | ||
693 | /* Beginning of a packet. Start to Rx/Tx packet header. */ | |
694 | if (vring->pkt_len == 0) { | |
695 | mlxbf_tmfifo_rxtx_header(vring, desc, is_rx, &vring_change); | |
696 | (*avail)--; | |
697 | ||
698 | /* Return if new packet is for another ring. */ | |
699 | if (vring_change) | |
700 | return false; | |
701 | goto mlxbf_tmfifo_desc_done; | |
702 | } | |
703 | ||
704 | /* Get the length of this desc. */ | |
705 | len = virtio32_to_cpu(vdev, desc->len); | |
706 | if (len > vring->rem_len) | |
707 | len = vring->rem_len; | |
708 | ||
709 | /* Rx/Tx one word (8 bytes) if not done. */ | |
710 | if (vring->cur_len < len) { | |
711 | mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len); | |
712 | (*avail)--; | |
713 | } | |
714 | ||
715 | /* Check again whether it's done. */ | |
716 | if (vring->cur_len == len) { | |
717 | vring->cur_len = 0; | |
718 | vring->rem_len -= len; | |
719 | ||
720 | /* Get the next desc on the chain. */ | |
721 | if (vring->rem_len > 0 && | |
722 | (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) { | |
723 | idx = virtio16_to_cpu(vdev, desc->next); | |
724 | desc = &vr->desc[idx]; | |
725 | goto mlxbf_tmfifo_desc_done; | |
726 | } | |
727 | ||
728 | /* Done and release the pending packet. */ | |
729 | mlxbf_tmfifo_release_pending_pkt(vring); | |
730 | desc = NULL; | |
731 | fifo->vring[is_rx] = NULL; | |
732 | ||
733 | /* Notify upper layer that packet is done. */ | |
734 | spin_lock_irqsave(&fifo->spin_lock, flags); | |
735 | vring_interrupt(0, vring->vq); | |
736 | spin_unlock_irqrestore(&fifo->spin_lock, flags); | |
737 | } | |
738 | ||
739 | mlxbf_tmfifo_desc_done: | |
740 | /* Save the current desc. */ | |
741 | vring->desc = desc; | |
742 | ||
743 | return true; | |
744 | } | |
745 | ||
746 | /* Rx & Tx processing of a queue. */ | |
747 | static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx) | |
748 | { | |
749 | int avail = 0, devid = vring->vdev_id; | |
750 | struct mlxbf_tmfifo *fifo; | |
751 | bool more; | |
752 | ||
753 | fifo = vring->fifo; | |
754 | ||
755 | /* Return if vdev is not ready. */ | |
756 | if (!fifo->vdev[devid]) | |
757 | return; | |
758 | ||
759 | /* Return if another vring is running. */ | |
760 | if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring) | |
761 | return; | |
762 | ||
763 | /* Only handle console and network for now. */ | |
764 | if (WARN_ON(devid != VIRTIO_ID_NET && devid != VIRTIO_ID_CONSOLE)) | |
765 | return; | |
766 | ||
767 | do { | |
768 | /* Get available FIFO space. */ | |
769 | if (avail == 0) { | |
770 | if (is_rx) | |
771 | avail = mlxbf_tmfifo_get_rx_avail(fifo); | |
772 | else | |
773 | avail = mlxbf_tmfifo_get_tx_avail(fifo, devid); | |
774 | if (avail <= 0) | |
775 | break; | |
776 | } | |
777 | ||
778 | /* Console output always comes from the Tx buffer. */ | |
779 | if (!is_rx && devid == VIRTIO_ID_CONSOLE) { | |
780 | mlxbf_tmfifo_console_tx(fifo, avail); | |
781 | break; | |
782 | } | |
783 | ||
784 | /* Handle one descriptor. */ | |
785 | more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail); | |
786 | } while (more); | |
787 | } | |
788 | ||
789 | /* Handle Rx or Tx queues. */ | |
790 | static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id, | |
791 | int irq_id, bool is_rx) | |
792 | { | |
793 | struct mlxbf_tmfifo_vdev *tm_vdev; | |
794 | struct mlxbf_tmfifo_vring *vring; | |
795 | int i; | |
796 | ||
797 | if (!test_and_clear_bit(irq_id, &fifo->pend_events) || | |
798 | !fifo->irq_info[irq_id].irq) | |
799 | return; | |
800 | ||
801 | for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) { | |
802 | tm_vdev = fifo->vdev[i]; | |
803 | if (tm_vdev) { | |
804 | vring = &tm_vdev->vrings[queue_id]; | |
805 | if (vring->vq) | |
806 | mlxbf_tmfifo_rxtx(vring, is_rx); | |
807 | } | |
808 | } | |
809 | } | |
810 | ||
811 | /* Work handler for Rx and Tx case. */ | |
812 | static void mlxbf_tmfifo_work_handler(struct work_struct *work) | |
813 | { | |
814 | struct mlxbf_tmfifo *fifo; | |
815 | ||
816 | fifo = container_of(work, struct mlxbf_tmfifo, work); | |
817 | if (!fifo->is_ready) | |
818 | return; | |
819 | ||
820 | mutex_lock(&fifo->lock); | |
821 | ||
822 | /* Tx (Send data to the TmFifo). */ | |
823 | mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_TX, | |
824 | MLXBF_TM_TX_LWM_IRQ, false); | |
825 | ||
826 | /* Rx (Receive data from the TmFifo). */ | |
827 | mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_RX, | |
828 | MLXBF_TM_RX_HWM_IRQ, true); | |
829 | ||
830 | mutex_unlock(&fifo->lock); | |
831 | } | |
832 | ||
833 | /* The notify function is called when new buffers are posted. */ | |
834 | static bool mlxbf_tmfifo_virtio_notify(struct virtqueue *vq) | |
835 | { | |
836 | struct mlxbf_tmfifo_vring *vring = vq->priv; | |
837 | struct mlxbf_tmfifo_vdev *tm_vdev; | |
838 | struct mlxbf_tmfifo *fifo; | |
839 | unsigned long flags; | |
840 | ||
841 | fifo = vring->fifo; | |
842 | ||
843 | /* | |
844 | * Virtio maintains vrings in pairs, even number ring for Rx | |
845 | * and odd number ring for Tx. | |
846 | */ | |
847 | if (vring->index & BIT(0)) { | |
848 | /* | |
849 | * Console could make blocking call with interrupts disabled. | |
850 | * In such case, the vring needs to be served right away. For | |
851 | * other cases, just set the TX LWM bit to start Tx in the | |
852 | * worker handler. | |
853 | */ | |
854 | if (vring->vdev_id == VIRTIO_ID_CONSOLE) { | |
855 | spin_lock_irqsave(&fifo->spin_lock, flags); | |
856 | tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE]; | |
857 | mlxbf_tmfifo_console_output(tm_vdev, vring); | |
858 | spin_unlock_irqrestore(&fifo->spin_lock, flags); | |
859 | } else if (test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, | |
860 | &fifo->pend_events)) { | |
861 | return true; | |
862 | } | |
863 | } else { | |
864 | if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events)) | |
865 | return true; | |
866 | } | |
867 | ||
868 | schedule_work(&fifo->work); | |
869 | ||
870 | return true; | |
871 | } | |
872 | ||
873 | /* Get the array of feature bits for this device. */ | |
874 | static u64 mlxbf_tmfifo_virtio_get_features(struct virtio_device *vdev) | |
875 | { | |
876 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
877 | ||
878 | return tm_vdev->features; | |
879 | } | |
880 | ||
881 | /* Confirm device features to use. */ | |
882 | static int mlxbf_tmfifo_virtio_finalize_features(struct virtio_device *vdev) | |
883 | { | |
884 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
885 | ||
886 | tm_vdev->features = vdev->features; | |
887 | ||
888 | return 0; | |
889 | } | |
890 | ||
891 | /* Free virtqueues found by find_vqs(). */ | |
892 | static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev) | |
893 | { | |
894 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
895 | struct mlxbf_tmfifo_vring *vring; | |
896 | struct virtqueue *vq; | |
897 | int i; | |
898 | ||
899 | for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { | |
900 | vring = &tm_vdev->vrings[i]; | |
901 | ||
902 | /* Release the pending packet. */ | |
903 | if (vring->desc) | |
904 | mlxbf_tmfifo_release_pending_pkt(vring); | |
905 | vq = vring->vq; | |
906 | if (vq) { | |
907 | vring->vq = NULL; | |
908 | vring_del_virtqueue(vq); | |
909 | } | |
910 | } | |
911 | } | |
912 | ||
913 | /* Create and initialize the virtual queues. */ | |
914 | static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, | |
915 | unsigned int nvqs, | |
916 | struct virtqueue *vqs[], | |
917 | vq_callback_t *callbacks[], | |
918 | const char * const names[], | |
919 | const bool *ctx, | |
920 | struct irq_affinity *desc) | |
921 | { | |
922 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
923 | struct mlxbf_tmfifo_vring *vring; | |
924 | struct virtqueue *vq; | |
925 | int i, ret, size; | |
926 | ||
927 | if (nvqs > ARRAY_SIZE(tm_vdev->vrings)) | |
928 | return -EINVAL; | |
929 | ||
930 | for (i = 0; i < nvqs; ++i) { | |
931 | if (!names[i]) { | |
932 | ret = -EINVAL; | |
933 | goto error; | |
934 | } | |
935 | vring = &tm_vdev->vrings[i]; | |
936 | ||
937 | /* zero vring */ | |
938 | size = vring_size(vring->num, vring->align); | |
939 | memset(vring->va, 0, size); | |
940 | vq = vring_new_virtqueue(i, vring->num, vring->align, vdev, | |
941 | false, false, vring->va, | |
942 | mlxbf_tmfifo_virtio_notify, | |
943 | callbacks[i], names[i]); | |
944 | if (!vq) { | |
945 | dev_err(&vdev->dev, "vring_new_virtqueue failed\n"); | |
946 | ret = -ENOMEM; | |
947 | goto error; | |
948 | } | |
949 | ||
950 | vqs[i] = vq; | |
951 | vring->vq = vq; | |
952 | vq->priv = vring; | |
953 | } | |
954 | ||
955 | return 0; | |
956 | ||
957 | error: | |
958 | mlxbf_tmfifo_virtio_del_vqs(vdev); | |
959 | return ret; | |
960 | } | |
961 | ||
962 | /* Read the status byte. */ | |
963 | static u8 mlxbf_tmfifo_virtio_get_status(struct virtio_device *vdev) | |
964 | { | |
965 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
966 | ||
967 | return tm_vdev->status; | |
968 | } | |
969 | ||
970 | /* Write the status byte. */ | |
971 | static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev, | |
972 | u8 status) | |
973 | { | |
974 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
975 | ||
976 | tm_vdev->status = status; | |
977 | } | |
978 | ||
979 | /* Reset the device. Not much here for now. */ | |
980 | static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev) | |
981 | { | |
982 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
983 | ||
984 | tm_vdev->status = 0; | |
985 | } | |
986 | ||
987 | /* Read the value of a configuration field. */ | |
988 | static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev, | |
989 | unsigned int offset, | |
990 | void *buf, | |
991 | unsigned int len) | |
992 | { | |
993 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
994 | ||
995 | if ((u64)offset + len > sizeof(tm_vdev->config)) | |
996 | return; | |
997 | ||
998 | memcpy(buf, (u8 *)&tm_vdev->config + offset, len); | |
999 | } | |
1000 | ||
1001 | /* Write the value of a configuration field. */ | |
1002 | static void mlxbf_tmfifo_virtio_set(struct virtio_device *vdev, | |
1003 | unsigned int offset, | |
1004 | const void *buf, | |
1005 | unsigned int len) | |
1006 | { | |
1007 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
1008 | ||
1009 | if ((u64)offset + len > sizeof(tm_vdev->config)) | |
1010 | return; | |
1011 | ||
1012 | memcpy((u8 *)&tm_vdev->config + offset, buf, len); | |
1013 | } | |
1014 | ||
1015 | static void tmfifo_virtio_dev_release(struct device *device) | |
1016 | { | |
1017 | struct virtio_device *vdev = | |
1018 | container_of(device, struct virtio_device, dev); | |
1019 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); | |
1020 | ||
1021 | kfree(tm_vdev); | |
1022 | } | |
1023 | ||
1024 | /* Virtio config operations. */ | |
1025 | static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = { | |
1026 | .get_features = mlxbf_tmfifo_virtio_get_features, | |
1027 | .finalize_features = mlxbf_tmfifo_virtio_finalize_features, | |
1028 | .find_vqs = mlxbf_tmfifo_virtio_find_vqs, | |
1029 | .del_vqs = mlxbf_tmfifo_virtio_del_vqs, | |
1030 | .reset = mlxbf_tmfifo_virtio_reset, | |
1031 | .set_status = mlxbf_tmfifo_virtio_set_status, | |
1032 | .get_status = mlxbf_tmfifo_virtio_get_status, | |
1033 | .get = mlxbf_tmfifo_virtio_get, | |
1034 | .set = mlxbf_tmfifo_virtio_set, | |
1035 | }; | |
1036 | ||
1037 | /* Create vdev for the FIFO. */ | |
1038 | static int mlxbf_tmfifo_create_vdev(struct device *dev, | |
1039 | struct mlxbf_tmfifo *fifo, | |
1040 | int vdev_id, u64 features, | |
1041 | void *config, u32 size) | |
1042 | { | |
1043 | struct mlxbf_tmfifo_vdev *tm_vdev, *reg_dev = NULL; | |
1044 | int ret; | |
1045 | ||
1046 | mutex_lock(&fifo->lock); | |
1047 | ||
1048 | tm_vdev = fifo->vdev[vdev_id]; | |
1049 | if (tm_vdev) { | |
1050 | dev_err(dev, "vdev %d already exists\n", vdev_id); | |
1051 | ret = -EEXIST; | |
1052 | goto fail; | |
1053 | } | |
1054 | ||
1055 | tm_vdev = kzalloc(sizeof(*tm_vdev), GFP_KERNEL); | |
1056 | if (!tm_vdev) { | |
1057 | ret = -ENOMEM; | |
1058 | goto fail; | |
1059 | } | |
1060 | ||
1061 | tm_vdev->vdev.id.device = vdev_id; | |
1062 | tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops; | |
1063 | tm_vdev->vdev.dev.parent = dev; | |
1064 | tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release; | |
1065 | tm_vdev->features = features; | |
1066 | if (config) | |
1067 | memcpy(&tm_vdev->config, config, size); | |
1068 | ||
1069 | if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) { | |
1070 | dev_err(dev, "unable to allocate vring\n"); | |
1071 | ret = -ENOMEM; | |
1072 | goto vdev_fail; | |
1073 | } | |
1074 | ||
1075 | /* Allocate an output buffer for the console device. */ | |
1076 | if (vdev_id == VIRTIO_ID_CONSOLE) | |
1077 | tm_vdev->tx_buf.buf = devm_kmalloc(dev, | |
1078 | MLXBF_TMFIFO_CON_TX_BUF_SIZE, | |
1079 | GFP_KERNEL); | |
1080 | fifo->vdev[vdev_id] = tm_vdev; | |
1081 | ||
1082 | /* Register the virtio device. */ | |
1083 | ret = register_virtio_device(&tm_vdev->vdev); | |
1084 | reg_dev = tm_vdev; | |
1085 | if (ret) { | |
1086 | dev_err(dev, "register_virtio_device failed\n"); | |
1087 | goto vdev_fail; | |
1088 | } | |
1089 | ||
1090 | mutex_unlock(&fifo->lock); | |
1091 | return 0; | |
1092 | ||
1093 | vdev_fail: | |
1094 | mlxbf_tmfifo_free_vrings(fifo, tm_vdev); | |
1095 | fifo->vdev[vdev_id] = NULL; | |
1096 | if (reg_dev) | |
1097 | put_device(&tm_vdev->vdev.dev); | |
1098 | else | |
1099 | kfree(tm_vdev); | |
1100 | fail: | |
1101 | mutex_unlock(&fifo->lock); | |
1102 | return ret; | |
1103 | } | |
1104 | ||
1105 | /* Delete vdev for the FIFO. */ | |
1106 | static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id) | |
1107 | { | |
1108 | struct mlxbf_tmfifo_vdev *tm_vdev; | |
1109 | ||
1110 | mutex_lock(&fifo->lock); | |
1111 | ||
1112 | /* Unregister vdev. */ | |
1113 | tm_vdev = fifo->vdev[vdev_id]; | |
1114 | if (tm_vdev) { | |
1115 | unregister_virtio_device(&tm_vdev->vdev); | |
1116 | mlxbf_tmfifo_free_vrings(fifo, tm_vdev); | |
1117 | fifo->vdev[vdev_id] = NULL; | |
1118 | } | |
1119 | ||
1120 | mutex_unlock(&fifo->lock); | |
1121 | ||
1122 | return 0; | |
1123 | } | |
1124 | ||
1125 | /* Read the configured network MAC address from efi variable. */ | |
1126 | static void mlxbf_tmfifo_get_cfg_mac(u8 *mac) | |
1127 | { | |
1128 | efi_guid_t guid = EFI_GLOBAL_VARIABLE_GUID; | |
1129 | unsigned long size = ETH_ALEN; | |
1130 | u8 buf[ETH_ALEN]; | |
1131 | efi_status_t rc; | |
1132 | ||
1133 | rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf); | |
1134 | if (rc == EFI_SUCCESS && size == ETH_ALEN) | |
1135 | ether_addr_copy(mac, buf); | |
1136 | else | |
1137 | ether_addr_copy(mac, mlxbf_tmfifo_net_default_mac); | |
1138 | } | |
1139 | ||
1140 | /* Set TmFifo thresolds which is used to trigger interrupts. */ | |
1141 | static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo) | |
1142 | { | |
1143 | u64 ctl; | |
1144 | ||
1145 | /* Get Tx FIFO size and set the low/high watermark. */ | |
1146 | ctl = readq(fifo->tx_base + MLXBF_TMFIFO_TX_CTL); | |
1147 | fifo->tx_fifo_size = | |
1148 | FIELD_GET(MLXBF_TMFIFO_TX_CTL__MAX_ENTRIES_MASK, ctl); | |
1149 | ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__LWM_MASK) | | |
1150 | FIELD_PREP(MLXBF_TMFIFO_TX_CTL__LWM_MASK, | |
1151 | fifo->tx_fifo_size / 2); | |
1152 | ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__HWM_MASK) | | |
1153 | FIELD_PREP(MLXBF_TMFIFO_TX_CTL__HWM_MASK, | |
1154 | fifo->tx_fifo_size - 1); | |
1155 | writeq(ctl, fifo->tx_base + MLXBF_TMFIFO_TX_CTL); | |
1156 | ||
1157 | /* Get Rx FIFO size and set the low/high watermark. */ | |
1158 | ctl = readq(fifo->rx_base + MLXBF_TMFIFO_RX_CTL); | |
1159 | fifo->rx_fifo_size = | |
1160 | FIELD_GET(MLXBF_TMFIFO_RX_CTL__MAX_ENTRIES_MASK, ctl); | |
1161 | ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__LWM_MASK) | | |
1162 | FIELD_PREP(MLXBF_TMFIFO_RX_CTL__LWM_MASK, 0); | |
1163 | ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__HWM_MASK) | | |
1164 | FIELD_PREP(MLXBF_TMFIFO_RX_CTL__HWM_MASK, 1); | |
1165 | writeq(ctl, fifo->rx_base + MLXBF_TMFIFO_RX_CTL); | |
1166 | } | |
1167 | ||
1168 | static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo) | |
1169 | { | |
1170 | int i; | |
1171 | ||
1172 | fifo->is_ready = false; | |
1173 | del_timer_sync(&fifo->timer); | |
1174 | mlxbf_tmfifo_disable_irqs(fifo); | |
1175 | cancel_work_sync(&fifo->work); | |
1176 | for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) | |
1177 | mlxbf_tmfifo_delete_vdev(fifo, i); | |
1178 | } | |
1179 | ||
1180 | /* Probe the TMFIFO. */ | |
1181 | static int mlxbf_tmfifo_probe(struct platform_device *pdev) | |
1182 | { | |
1183 | struct virtio_net_config net_config; | |
1184 | struct device *dev = &pdev->dev; | |
1185 | struct mlxbf_tmfifo *fifo; | |
1186 | int i, rc; | |
1187 | ||
1188 | fifo = devm_kzalloc(dev, sizeof(*fifo), GFP_KERNEL); | |
1189 | if (!fifo) | |
1190 | return -ENOMEM; | |
1191 | ||
1192 | spin_lock_init(&fifo->spin_lock); | |
1193 | INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler); | |
1194 | mutex_init(&fifo->lock); | |
1195 | ||
1196 | /* Get the resource of the Rx FIFO. */ | |
1197 | fifo->rx_base = devm_platform_ioremap_resource(pdev, 0); | |
1198 | if (IS_ERR(fifo->rx_base)) | |
1199 | return PTR_ERR(fifo->rx_base); | |
1200 | ||
1201 | /* Get the resource of the Tx FIFO. */ | |
1202 | fifo->tx_base = devm_platform_ioremap_resource(pdev, 1); | |
1203 | if (IS_ERR(fifo->tx_base)) | |
1204 | return PTR_ERR(fifo->tx_base); | |
1205 | ||
1206 | platform_set_drvdata(pdev, fifo); | |
1207 | ||
1208 | timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0); | |
1209 | ||
1210 | for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) { | |
1211 | fifo->irq_info[i].index = i; | |
1212 | fifo->irq_info[i].fifo = fifo; | |
1213 | fifo->irq_info[i].irq = platform_get_irq(pdev, i); | |
1214 | rc = devm_request_irq(dev, fifo->irq_info[i].irq, | |
1215 | mlxbf_tmfifo_irq_handler, 0, | |
1216 | "tmfifo", &fifo->irq_info[i]); | |
1217 | if (rc) { | |
1218 | dev_err(dev, "devm_request_irq failed\n"); | |
1219 | fifo->irq_info[i].irq = 0; | |
1220 | return rc; | |
1221 | } | |
1222 | } | |
1223 | ||
1224 | mlxbf_tmfifo_set_threshold(fifo); | |
1225 | ||
1226 | /* Create the console vdev. */ | |
1227 | rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, 0, NULL, 0); | |
1228 | if (rc) | |
1229 | goto fail; | |
1230 | ||
1231 | /* Create the network vdev. */ | |
1232 | memset(&net_config, 0, sizeof(net_config)); | |
1233 | net_config.mtu = ETH_DATA_LEN; | |
1234 | net_config.status = VIRTIO_NET_S_LINK_UP; | |
1235 | mlxbf_tmfifo_get_cfg_mac(net_config.mac); | |
1236 | rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET, | |
1237 | MLXBF_TMFIFO_NET_FEATURES, &net_config, | |
1238 | sizeof(net_config)); | |
1239 | if (rc) | |
1240 | goto fail; | |
1241 | ||
1242 | mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); | |
1243 | ||
1244 | fifo->is_ready = true; | |
1245 | return 0; | |
1246 | ||
1247 | fail: | |
1248 | mlxbf_tmfifo_cleanup(fifo); | |
1249 | return rc; | |
1250 | } | |
1251 | ||
1252 | /* Device remove function. */ | |
1253 | static int mlxbf_tmfifo_remove(struct platform_device *pdev) | |
1254 | { | |
1255 | struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev); | |
1256 | ||
1257 | mlxbf_tmfifo_cleanup(fifo); | |
1258 | ||
1259 | return 0; | |
1260 | } | |
1261 | ||
1262 | static const struct acpi_device_id mlxbf_tmfifo_acpi_match[] = { | |
1263 | { "MLNXBF01", 0 }, | |
1264 | {} | |
1265 | }; | |
1266 | MODULE_DEVICE_TABLE(acpi, mlxbf_tmfifo_acpi_match); | |
1267 | ||
1268 | static struct platform_driver mlxbf_tmfifo_driver = { | |
1269 | .probe = mlxbf_tmfifo_probe, | |
1270 | .remove = mlxbf_tmfifo_remove, | |
1271 | .driver = { | |
1272 | .name = "bf-tmfifo", | |
1273 | .acpi_match_table = mlxbf_tmfifo_acpi_match, | |
1274 | }, | |
1275 | }; | |
1276 | ||
1277 | module_platform_driver(mlxbf_tmfifo_driver); | |
1278 | ||
1279 | MODULE_DESCRIPTION("Mellanox BlueField SoC TmFifo Driver"); | |
1280 | MODULE_LICENSE("GPL v2"); | |
1281 | MODULE_AUTHOR("Mellanox Technologies"); |