Commit | Line | Data |
---|---|---|
ca9c54d2 DC |
1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | /* Copyright (c) 2021, Microsoft Corporation. */ | |
3 | ||
4 | #include <linux/module.h> | |
5 | #include <linux/pci.h> | |
3c37f357 DC |
6 | #include <linux/utsname.h> |
7 | #include <linux/version.h> | |
ca9c54d2 | 8 | |
fd325cd6 | 9 | #include <net/mana/mana.h> |
ca9c54d2 DC |
10 | |
11 | static u32 mana_gd_r32(struct gdma_context *g, u64 offset) | |
12 | { | |
13 | return readl(g->bar0_va + offset); | |
14 | } | |
15 | ||
16 | static u64 mana_gd_r64(struct gdma_context *g, u64 offset) | |
17 | { | |
18 | return readq(g->bar0_va + offset); | |
19 | } | |
20 | ||
1566e7d6 DC |
21 | static void mana_gd_init_pf_regs(struct pci_dev *pdev) |
22 | { | |
23 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
24 | void __iomem *sriov_base_va; | |
25 | u64 sriov_base_off; | |
26 | ||
27 | gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF; | |
28 | gc->db_page_base = gc->bar0_va + | |
29 | mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); | |
30 | ||
31 | sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF); | |
32 | ||
33 | sriov_base_va = gc->bar0_va + sriov_base_off; | |
34 | gc->shm_base = sriov_base_va + | |
35 | mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF); | |
36 | } | |
37 | ||
38 | static void mana_gd_init_vf_regs(struct pci_dev *pdev) | |
ca9c54d2 DC |
39 | { |
40 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
41 | ||
42 | gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; | |
43 | ||
44 | gc->db_page_base = gc->bar0_va + | |
45 | mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); | |
46 | ||
f3dc0962 LL |
47 | gc->phys_db_page_base = gc->bar0_pa + |
48 | mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); | |
49 | ||
ca9c54d2 DC |
50 | gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET); |
51 | } | |
52 | ||
1566e7d6 DC |
53 | static void mana_gd_init_registers(struct pci_dev *pdev) |
54 | { | |
55 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
56 | ||
57 | if (gc->is_pf) | |
58 | mana_gd_init_pf_regs(pdev); | |
59 | else | |
60 | mana_gd_init_vf_regs(pdev); | |
61 | } | |
62 | ||
ca9c54d2 DC |
63 | static int mana_gd_query_max_resources(struct pci_dev *pdev) |
64 | { | |
65 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
66 | struct gdma_query_max_resources_resp resp = {}; | |
67 | struct gdma_general_req req = {}; | |
68 | int err; | |
69 | ||
70 | mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, | |
71 | sizeof(req), sizeof(resp)); | |
72 | ||
73 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
74 | if (err || resp.hdr.status) { | |
75 | dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n", | |
76 | err, resp.hdr.status); | |
77 | return err ? err : -EPROTO; | |
78 | } | |
79 | ||
80 | if (gc->num_msix_usable > resp.max_msix) | |
81 | gc->num_msix_usable = resp.max_msix; | |
82 | ||
83 | if (gc->num_msix_usable <= 1) | |
84 | return -ENOSPC; | |
85 | ||
86 | gc->max_num_queues = num_online_cpus(); | |
87 | if (gc->max_num_queues > MANA_MAX_NUM_QUEUES) | |
88 | gc->max_num_queues = MANA_MAX_NUM_QUEUES; | |
89 | ||
90 | if (gc->max_num_queues > resp.max_eq) | |
91 | gc->max_num_queues = resp.max_eq; | |
92 | ||
93 | if (gc->max_num_queues > resp.max_cq) | |
94 | gc->max_num_queues = resp.max_cq; | |
95 | ||
96 | if (gc->max_num_queues > resp.max_sq) | |
97 | gc->max_num_queues = resp.max_sq; | |
98 | ||
99 | if (gc->max_num_queues > resp.max_rq) | |
100 | gc->max_num_queues = resp.max_rq; | |
101 | ||
1e2d0824 HZ |
102 | /* The Hardware Channel (HWC) used 1 MSI-X */ |
103 | if (gc->max_num_queues > gc->num_msix_usable - 1) | |
104 | gc->max_num_queues = gc->num_msix_usable - 1; | |
105 | ||
ca9c54d2 DC |
106 | return 0; |
107 | } | |
108 | ||
109 | static int mana_gd_detect_devices(struct pci_dev *pdev) | |
110 | { | |
111 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
112 | struct gdma_list_devices_resp resp = {}; | |
113 | struct gdma_general_req req = {}; | |
114 | struct gdma_dev_id dev; | |
115 | u32 i, max_num_devs; | |
116 | u16 dev_type; | |
117 | int err; | |
118 | ||
119 | mana_gd_init_req_hdr(&req.hdr, GDMA_LIST_DEVICES, sizeof(req), | |
120 | sizeof(resp)); | |
121 | ||
122 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
123 | if (err || resp.hdr.status) { | |
124 | dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err, | |
125 | resp.hdr.status); | |
126 | return err ? err : -EPROTO; | |
127 | } | |
128 | ||
129 | max_num_devs = min_t(u32, MAX_NUM_GDMA_DEVICES, resp.num_of_devs); | |
130 | ||
131 | for (i = 0; i < max_num_devs; i++) { | |
132 | dev = resp.devs[i]; | |
133 | dev_type = dev.type; | |
134 | ||
135 | /* HWC is already detected in mana_hwc_create_channel(). */ | |
136 | if (dev_type == GDMA_DEVICE_HWC) | |
137 | continue; | |
138 | ||
139 | if (dev_type == GDMA_DEVICE_MANA) { | |
140 | gc->mana.gdma_context = gc; | |
141 | gc->mana.dev_id = dev; | |
142 | } | |
143 | } | |
144 | ||
145 | return gc->mana.dev_id.type == 0 ? -ENODEV : 0; | |
146 | } | |
147 | ||
148 | int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, | |
149 | u32 resp_len, void *resp) | |
150 | { | |
151 | struct hw_channel_context *hwc = gc->hwc.driver_data; | |
152 | ||
153 | return mana_hwc_send_request(hwc, req_len, req, resp_len, resp); | |
154 | } | |
4c0ff7a1 | 155 | EXPORT_SYMBOL_NS(mana_gd_send_request, NET_MANA); |
ca9c54d2 DC |
156 | |
157 | int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, | |
158 | struct gdma_mem_info *gmi) | |
159 | { | |
160 | dma_addr_t dma_handle; | |
161 | void *buf; | |
162 | ||
163 | if (length < PAGE_SIZE || !is_power_of_2(length)) | |
164 | return -EINVAL; | |
165 | ||
166 | gmi->dev = gc->dev; | |
167 | buf = dma_alloc_coherent(gmi->dev, length, &dma_handle, GFP_KERNEL); | |
168 | if (!buf) | |
169 | return -ENOMEM; | |
170 | ||
171 | gmi->dma_handle = dma_handle; | |
172 | gmi->virt_addr = buf; | |
173 | gmi->length = length; | |
174 | ||
175 | return 0; | |
176 | } | |
177 | ||
178 | void mana_gd_free_memory(struct gdma_mem_info *gmi) | |
179 | { | |
180 | dma_free_coherent(gmi->dev, gmi->length, gmi->virt_addr, | |
181 | gmi->dma_handle); | |
182 | } | |
183 | ||
184 | static int mana_gd_create_hw_eq(struct gdma_context *gc, | |
185 | struct gdma_queue *queue) | |
186 | { | |
187 | struct gdma_create_queue_resp resp = {}; | |
188 | struct gdma_create_queue_req req = {}; | |
189 | int err; | |
190 | ||
191 | if (queue->type != GDMA_EQ) | |
192 | return -EINVAL; | |
193 | ||
194 | mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_QUEUE, | |
195 | sizeof(req), sizeof(resp)); | |
196 | ||
197 | req.hdr.dev_id = queue->gdma_dev->dev_id; | |
198 | req.type = queue->type; | |
199 | req.pdid = queue->gdma_dev->pdid; | |
200 | req.doolbell_id = queue->gdma_dev->doorbell; | |
28c66cfa | 201 | req.gdma_region = queue->mem_info.dma_region_handle; |
ca9c54d2 DC |
202 | req.queue_size = queue->queue_size; |
203 | req.log2_throttle_limit = queue->eq.log2_throttle_limit; | |
204 | req.eq_pci_msix_index = queue->eq.msix_index; | |
205 | ||
206 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
207 | if (err || resp.hdr.status) { | |
208 | dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err, | |
209 | resp.hdr.status); | |
210 | return err ? err : -EPROTO; | |
211 | } | |
212 | ||
213 | queue->id = resp.queue_index; | |
214 | queue->eq.disable_needed = true; | |
28c66cfa | 215 | queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; |
ca9c54d2 DC |
216 | return 0; |
217 | } | |
218 | ||
219 | static int mana_gd_disable_queue(struct gdma_queue *queue) | |
220 | { | |
221 | struct gdma_context *gc = queue->gdma_dev->gdma_context; | |
222 | struct gdma_disable_queue_req req = {}; | |
223 | struct gdma_general_resp resp = {}; | |
224 | int err; | |
225 | ||
226 | WARN_ON(queue->type != GDMA_EQ); | |
227 | ||
228 | mana_gd_init_req_hdr(&req.hdr, GDMA_DISABLE_QUEUE, | |
229 | sizeof(req), sizeof(resp)); | |
230 | ||
231 | req.hdr.dev_id = queue->gdma_dev->dev_id; | |
232 | req.type = queue->type; | |
233 | req.queue_index = queue->id; | |
234 | req.alloc_res_id_on_creation = 1; | |
235 | ||
236 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
237 | if (err || resp.hdr.status) { | |
238 | dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err, | |
239 | resp.hdr.status); | |
240 | return err ? err : -EPROTO; | |
241 | } | |
242 | ||
243 | return 0; | |
244 | } | |
245 | ||
246 | #define DOORBELL_OFFSET_SQ 0x0 | |
247 | #define DOORBELL_OFFSET_RQ 0x400 | |
248 | #define DOORBELL_OFFSET_CQ 0x800 | |
249 | #define DOORBELL_OFFSET_EQ 0xFF8 | |
250 | ||
251 | static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, | |
252 | enum gdma_queue_type q_type, u32 qid, | |
253 | u32 tail_ptr, u8 num_req) | |
254 | { | |
255 | void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; | |
256 | union gdma_doorbell_entry e = {}; | |
257 | ||
258 | switch (q_type) { | |
259 | case GDMA_EQ: | |
260 | e.eq.id = qid; | |
261 | e.eq.tail_ptr = tail_ptr; | |
262 | e.eq.arm = num_req; | |
263 | ||
264 | addr += DOORBELL_OFFSET_EQ; | |
265 | break; | |
266 | ||
267 | case GDMA_CQ: | |
268 | e.cq.id = qid; | |
269 | e.cq.tail_ptr = tail_ptr; | |
270 | e.cq.arm = num_req; | |
271 | ||
272 | addr += DOORBELL_OFFSET_CQ; | |
273 | break; | |
274 | ||
275 | case GDMA_RQ: | |
276 | e.rq.id = qid; | |
277 | e.rq.tail_ptr = tail_ptr; | |
278 | e.rq.wqe_cnt = num_req; | |
279 | ||
280 | addr += DOORBELL_OFFSET_RQ; | |
281 | break; | |
282 | ||
283 | case GDMA_SQ: | |
284 | e.sq.id = qid; | |
285 | e.sq.tail_ptr = tail_ptr; | |
286 | ||
287 | addr += DOORBELL_OFFSET_SQ; | |
288 | break; | |
289 | ||
290 | default: | |
291 | WARN_ON(1); | |
292 | return; | |
293 | } | |
294 | ||
295 | /* Ensure all writes are done before ring doorbell */ | |
296 | wmb(); | |
297 | ||
298 | writeq(e.as_uint64, addr); | |
299 | } | |
300 | ||
301 | void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) | |
302 | { | |
303 | mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, | |
304 | queue->id, queue->head * GDMA_WQE_BU_SIZE, 1); | |
305 | } | |
306 | ||
e1b5683f | 307 | void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) |
ca9c54d2 DC |
308 | { |
309 | struct gdma_context *gc = cq->gdma_dev->gdma_context; | |
310 | ||
311 | u32 num_cqe = cq->queue_size / GDMA_CQE_SIZE; | |
312 | ||
313 | u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); | |
314 | ||
315 | mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id, | |
e1b5683f | 316 | head, arm_bit); |
ca9c54d2 DC |
317 | } |
318 | ||
319 | static void mana_gd_process_eqe(struct gdma_queue *eq) | |
320 | { | |
321 | u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); | |
322 | struct gdma_context *gc = eq->gdma_dev->gdma_context; | |
323 | struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr; | |
324 | union gdma_eqe_info eqe_info; | |
325 | enum gdma_eqe_type type; | |
326 | struct gdma_event event; | |
327 | struct gdma_queue *cq; | |
328 | struct gdma_eqe *eqe; | |
329 | u32 cq_id; | |
330 | ||
331 | eqe = &eq_eqe_ptr[head]; | |
332 | eqe_info.as_uint32 = eqe->eqe_info; | |
333 | type = eqe_info.type; | |
334 | ||
335 | switch (type) { | |
336 | case GDMA_EQE_COMPLETION: | |
337 | cq_id = eqe->details[0] & 0xFFFFFF; | |
338 | if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs)) | |
339 | break; | |
340 | ||
341 | cq = gc->cq_table[cq_id]; | |
342 | if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id)) | |
343 | break; | |
344 | ||
345 | if (cq->cq.callback) | |
346 | cq->cq.callback(cq->cq.context, cq); | |
347 | ||
348 | break; | |
349 | ||
350 | case GDMA_EQE_TEST_EVENT: | |
351 | gc->test_event_eq_id = eq->id; | |
352 | complete(&gc->eq_test_event); | |
353 | break; | |
354 | ||
355 | case GDMA_EQE_HWC_INIT_EQ_ID_DB: | |
356 | case GDMA_EQE_HWC_INIT_DATA: | |
357 | case GDMA_EQE_HWC_INIT_DONE: | |
358 | if (!eq->eq.callback) | |
359 | break; | |
360 | ||
361 | event.type = type; | |
362 | memcpy(&event.details, &eqe->details, GDMA_EVENT_DATA_SIZE); | |
363 | eq->eq.callback(eq->eq.context, eq, &event); | |
364 | break; | |
365 | ||
366 | default: | |
367 | break; | |
368 | } | |
369 | } | |
370 | ||
371 | static void mana_gd_process_eq_events(void *arg) | |
372 | { | |
373 | u32 owner_bits, new_bits, old_bits; | |
374 | union gdma_eqe_info eqe_info; | |
375 | struct gdma_eqe *eq_eqe_ptr; | |
376 | struct gdma_queue *eq = arg; | |
377 | struct gdma_context *gc; | |
378 | struct gdma_eqe *eqe; | |
ca9c54d2 DC |
379 | u32 head, num_eqe; |
380 | int i; | |
381 | ||
382 | gc = eq->gdma_dev->gdma_context; | |
383 | ||
384 | num_eqe = eq->queue_size / GDMA_EQE_SIZE; | |
385 | eq_eqe_ptr = eq->queue_mem_ptr; | |
386 | ||
387 | /* Process up to 5 EQEs at a time, and update the HW head. */ | |
388 | for (i = 0; i < 5; i++) { | |
389 | eqe = &eq_eqe_ptr[eq->head % num_eqe]; | |
390 | eqe_info.as_uint32 = eqe->eqe_info; | |
391 | owner_bits = eqe_info.owner_bits; | |
392 | ||
393 | old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK; | |
394 | /* No more entries */ | |
395 | if (owner_bits == old_bits) | |
396 | break; | |
397 | ||
398 | new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK; | |
399 | if (owner_bits != new_bits) { | |
400 | dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id); | |
401 | break; | |
402 | } | |
403 | ||
6fd2c68d HZ |
404 | /* Per GDMA spec, rmb is necessary after checking owner_bits, before |
405 | * reading eqe. | |
406 | */ | |
407 | rmb(); | |
408 | ||
ca9c54d2 DC |
409 | mana_gd_process_eqe(eq); |
410 | ||
411 | eq->head++; | |
412 | } | |
413 | ||
ca9c54d2 DC |
414 | head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); |
415 | ||
416 | mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id, | |
e1b5683f | 417 | head, SET_ARM_BIT); |
ca9c54d2 DC |
418 | } |
419 | ||
420 | static int mana_gd_register_irq(struct gdma_queue *queue, | |
421 | const struct gdma_queue_spec *spec) | |
422 | { | |
423 | struct gdma_dev *gd = queue->gdma_dev; | |
ca9c54d2 DC |
424 | struct gdma_irq_context *gic; |
425 | struct gdma_context *gc; | |
426 | struct gdma_resource *r; | |
427 | unsigned int msi_index; | |
428 | unsigned long flags; | |
1e2d0824 HZ |
429 | struct device *dev; |
430 | int err = 0; | |
ca9c54d2 DC |
431 | |
432 | gc = gd->gdma_context; | |
433 | r = &gc->msix_resource; | |
1e2d0824 | 434 | dev = gc->dev; |
ca9c54d2 DC |
435 | |
436 | spin_lock_irqsave(&r->lock, flags); | |
437 | ||
438 | msi_index = find_first_zero_bit(r->map, r->size); | |
1e2d0824 | 439 | if (msi_index >= r->size || msi_index >= gc->num_msix_usable) { |
ca9c54d2 DC |
440 | err = -ENOSPC; |
441 | } else { | |
442 | bitmap_set(r->map, msi_index, 1); | |
443 | queue->eq.msix_index = msi_index; | |
ca9c54d2 DC |
444 | } |
445 | ||
446 | spin_unlock_irqrestore(&r->lock, flags); | |
447 | ||
1e2d0824 HZ |
448 | if (err) { |
449 | dev_err(dev, "Register IRQ err:%d, msi:%u rsize:%u, nMSI:%u", | |
450 | err, msi_index, r->size, gc->num_msix_usable); | |
ca9c54d2 | 451 | |
1e2d0824 HZ |
452 | return err; |
453 | } | |
ca9c54d2 DC |
454 | |
455 | gic = &gc->irq_contexts[msi_index]; | |
456 | ||
ca9c54d2 DC |
457 | WARN_ON(gic->handler || gic->arg); |
458 | ||
459 | gic->arg = queue; | |
460 | ||
e1b5683f | 461 | gic->handler = mana_gd_process_eq_events; |
ca9c54d2 DC |
462 | |
463 | return 0; | |
464 | } | |
465 | ||
466 | static void mana_gd_deregiser_irq(struct gdma_queue *queue) | |
467 | { | |
468 | struct gdma_dev *gd = queue->gdma_dev; | |
469 | struct gdma_irq_context *gic; | |
470 | struct gdma_context *gc; | |
471 | struct gdma_resource *r; | |
472 | unsigned int msix_index; | |
473 | unsigned long flags; | |
474 | ||
475 | gc = gd->gdma_context; | |
476 | r = &gc->msix_resource; | |
477 | ||
478 | /* At most num_online_cpus() + 1 interrupts are used. */ | |
479 | msix_index = queue->eq.msix_index; | |
480 | if (WARN_ON(msix_index >= gc->num_msix_usable)) | |
481 | return; | |
482 | ||
483 | gic = &gc->irq_contexts[msix_index]; | |
484 | gic->handler = NULL; | |
485 | gic->arg = NULL; | |
486 | ||
487 | spin_lock_irqsave(&r->lock, flags); | |
488 | bitmap_clear(r->map, msix_index, 1); | |
489 | spin_unlock_irqrestore(&r->lock, flags); | |
490 | ||
491 | queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; | |
492 | } | |
493 | ||
494 | int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq) | |
495 | { | |
496 | struct gdma_generate_test_event_req req = {}; | |
497 | struct gdma_general_resp resp = {}; | |
498 | struct device *dev = gc->dev; | |
499 | int err; | |
500 | ||
501 | mutex_lock(&gc->eq_test_event_mutex); | |
502 | ||
503 | init_completion(&gc->eq_test_event); | |
504 | gc->test_event_eq_id = INVALID_QUEUE_ID; | |
505 | ||
506 | mana_gd_init_req_hdr(&req.hdr, GDMA_GENERATE_TEST_EQE, | |
507 | sizeof(req), sizeof(resp)); | |
508 | ||
509 | req.hdr.dev_id = eq->gdma_dev->dev_id; | |
510 | req.queue_index = eq->id; | |
511 | ||
512 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
513 | if (err) { | |
514 | dev_err(dev, "test_eq failed: %d\n", err); | |
515 | goto out; | |
516 | } | |
517 | ||
518 | err = -EPROTO; | |
519 | ||
520 | if (resp.hdr.status) { | |
521 | dev_err(dev, "test_eq failed: 0x%x\n", resp.hdr.status); | |
522 | goto out; | |
523 | } | |
524 | ||
525 | if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) { | |
526 | dev_err(dev, "test_eq timed out on queue %d\n", eq->id); | |
527 | goto out; | |
528 | } | |
529 | ||
530 | if (eq->id != gc->test_event_eq_id) { | |
531 | dev_err(dev, "test_eq got an event on wrong queue %d (%d)\n", | |
532 | gc->test_event_eq_id, eq->id); | |
533 | goto out; | |
534 | } | |
535 | ||
536 | err = 0; | |
537 | out: | |
538 | mutex_unlock(&gc->eq_test_event_mutex); | |
539 | return err; | |
540 | } | |
541 | ||
542 | static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets, | |
543 | struct gdma_queue *queue) | |
544 | { | |
545 | int err; | |
546 | ||
547 | if (flush_evenets) { | |
548 | err = mana_gd_test_eq(gc, queue); | |
549 | if (err) | |
550 | dev_warn(gc->dev, "Failed to flush EQ: %d\n", err); | |
551 | } | |
552 | ||
553 | mana_gd_deregiser_irq(queue); | |
554 | ||
ca9c54d2 DC |
555 | if (queue->eq.disable_needed) |
556 | mana_gd_disable_queue(queue); | |
557 | } | |
558 | ||
559 | static int mana_gd_create_eq(struct gdma_dev *gd, | |
560 | const struct gdma_queue_spec *spec, | |
561 | bool create_hwq, struct gdma_queue *queue) | |
562 | { | |
563 | struct gdma_context *gc = gd->gdma_context; | |
564 | struct device *dev = gc->dev; | |
565 | u32 log2_num_entries; | |
566 | int err; | |
567 | ||
568 | queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; | |
569 | ||
570 | log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE); | |
571 | ||
572 | if (spec->eq.log2_throttle_limit > log2_num_entries) { | |
573 | dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n", | |
574 | spec->eq.log2_throttle_limit, log2_num_entries); | |
575 | return -EINVAL; | |
576 | } | |
577 | ||
578 | err = mana_gd_register_irq(queue, spec); | |
579 | if (err) { | |
580 | dev_err(dev, "Failed to register irq: %d\n", err); | |
581 | return err; | |
582 | } | |
583 | ||
584 | queue->eq.callback = spec->eq.callback; | |
585 | queue->eq.context = spec->eq.context; | |
586 | queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); | |
587 | queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1; | |
588 | ||
589 | if (create_hwq) { | |
590 | err = mana_gd_create_hw_eq(gc, queue); | |
591 | if (err) | |
592 | goto out; | |
593 | ||
594 | err = mana_gd_test_eq(gc, queue); | |
595 | if (err) | |
596 | goto out; | |
597 | } | |
598 | ||
599 | return 0; | |
600 | out: | |
601 | dev_err(dev, "Failed to create EQ: %d\n", err); | |
602 | mana_gd_destroy_eq(gc, false, queue); | |
603 | return err; | |
604 | } | |
605 | ||
606 | static void mana_gd_create_cq(const struct gdma_queue_spec *spec, | |
607 | struct gdma_queue *queue) | |
608 | { | |
609 | u32 log2_num_entries = ilog2(spec->queue_size / GDMA_CQE_SIZE); | |
610 | ||
611 | queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); | |
612 | queue->cq.parent = spec->cq.parent_eq; | |
613 | queue->cq.context = spec->cq.context; | |
614 | queue->cq.callback = spec->cq.callback; | |
615 | } | |
616 | ||
617 | static void mana_gd_destroy_cq(struct gdma_context *gc, | |
618 | struct gdma_queue *queue) | |
619 | { | |
620 | u32 id = queue->id; | |
621 | ||
622 | if (id >= gc->max_num_cqs) | |
623 | return; | |
624 | ||
625 | if (!gc->cq_table[id]) | |
626 | return; | |
627 | ||
628 | gc->cq_table[id] = NULL; | |
629 | } | |
630 | ||
631 | int mana_gd_create_hwc_queue(struct gdma_dev *gd, | |
632 | const struct gdma_queue_spec *spec, | |
633 | struct gdma_queue **queue_ptr) | |
634 | { | |
635 | struct gdma_context *gc = gd->gdma_context; | |
636 | struct gdma_mem_info *gmi; | |
637 | struct gdma_queue *queue; | |
638 | int err; | |
639 | ||
640 | queue = kzalloc(sizeof(*queue), GFP_KERNEL); | |
641 | if (!queue) | |
642 | return -ENOMEM; | |
643 | ||
644 | gmi = &queue->mem_info; | |
645 | err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); | |
646 | if (err) | |
647 | goto free_q; | |
648 | ||
649 | queue->head = 0; | |
650 | queue->tail = 0; | |
651 | queue->queue_mem_ptr = gmi->virt_addr; | |
652 | queue->queue_size = spec->queue_size; | |
653 | queue->monitor_avl_buf = spec->monitor_avl_buf; | |
654 | queue->type = spec->type; | |
655 | queue->gdma_dev = gd; | |
656 | ||
657 | if (spec->type == GDMA_EQ) | |
658 | err = mana_gd_create_eq(gd, spec, false, queue); | |
659 | else if (spec->type == GDMA_CQ) | |
660 | mana_gd_create_cq(spec, queue); | |
661 | ||
662 | if (err) | |
663 | goto out; | |
664 | ||
665 | *queue_ptr = queue; | |
666 | return 0; | |
667 | out: | |
668 | mana_gd_free_memory(gmi); | |
669 | free_q: | |
670 | kfree(queue); | |
671 | return err; | |
672 | } | |
673 | ||
3574cfdc | 674 | int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle) |
ca9c54d2 DC |
675 | { |
676 | struct gdma_destroy_dma_region_req req = {}; | |
677 | struct gdma_general_resp resp = {}; | |
678 | int err; | |
679 | ||
28c66cfa AS |
680 | if (dma_region_handle == GDMA_INVALID_DMA_REGION) |
681 | return 0; | |
ca9c54d2 DC |
682 | |
683 | mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_DMA_REGION, sizeof(req), | |
684 | sizeof(resp)); | |
28c66cfa | 685 | req.dma_region_handle = dma_region_handle; |
ca9c54d2 DC |
686 | |
687 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
28c66cfa | 688 | if (err || resp.hdr.status) { |
ca9c54d2 DC |
689 | dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n", |
690 | err, resp.hdr.status); | |
28c66cfa AS |
691 | return -EPROTO; |
692 | } | |
693 | ||
694 | return 0; | |
ca9c54d2 | 695 | } |
28c66cfa | 696 | EXPORT_SYMBOL_NS(mana_gd_destroy_dma_region, NET_MANA); |
ca9c54d2 DC |
697 | |
698 | static int mana_gd_create_dma_region(struct gdma_dev *gd, | |
699 | struct gdma_mem_info *gmi) | |
700 | { | |
701 | unsigned int num_page = gmi->length / PAGE_SIZE; | |
702 | struct gdma_create_dma_region_req *req = NULL; | |
703 | struct gdma_create_dma_region_resp resp = {}; | |
704 | struct gdma_context *gc = gd->gdma_context; | |
705 | struct hw_channel_context *hwc; | |
706 | u32 length = gmi->length; | |
10cdc794 | 707 | size_t req_msg_size; |
ca9c54d2 DC |
708 | int err; |
709 | int i; | |
710 | ||
711 | if (length < PAGE_SIZE || !is_power_of_2(length)) | |
712 | return -EINVAL; | |
713 | ||
714 | if (offset_in_page(gmi->virt_addr) != 0) | |
715 | return -EINVAL; | |
716 | ||
717 | hwc = gc->hwc.driver_data; | |
10cdc794 | 718 | req_msg_size = struct_size(req, page_addr_list, num_page); |
ca9c54d2 DC |
719 | if (req_msg_size > hwc->max_req_msg_size) |
720 | return -EINVAL; | |
721 | ||
722 | req = kzalloc(req_msg_size, GFP_KERNEL); | |
723 | if (!req) | |
724 | return -ENOMEM; | |
725 | ||
726 | mana_gd_init_req_hdr(&req->hdr, GDMA_CREATE_DMA_REGION, | |
727 | req_msg_size, sizeof(resp)); | |
728 | req->length = length; | |
729 | req->offset_in_page = 0; | |
730 | req->gdma_page_type = GDMA_PAGE_TYPE_4K; | |
731 | req->page_count = num_page; | |
732 | req->page_addr_list_len = num_page; | |
733 | ||
734 | for (i = 0; i < num_page; i++) | |
735 | req->page_addr_list[i] = gmi->dma_handle + i * PAGE_SIZE; | |
736 | ||
737 | err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp); | |
738 | if (err) | |
739 | goto out; | |
740 | ||
28c66cfa AS |
741 | if (resp.hdr.status || |
742 | resp.dma_region_handle == GDMA_INVALID_DMA_REGION) { | |
ca9c54d2 DC |
743 | dev_err(gc->dev, "Failed to create DMA region: 0x%x\n", |
744 | resp.hdr.status); | |
745 | err = -EPROTO; | |
746 | goto out; | |
747 | } | |
748 | ||
28c66cfa | 749 | gmi->dma_region_handle = resp.dma_region_handle; |
ca9c54d2 DC |
750 | out: |
751 | kfree(req); | |
752 | return err; | |
753 | } | |
754 | ||
755 | int mana_gd_create_mana_eq(struct gdma_dev *gd, | |
756 | const struct gdma_queue_spec *spec, | |
757 | struct gdma_queue **queue_ptr) | |
758 | { | |
759 | struct gdma_context *gc = gd->gdma_context; | |
760 | struct gdma_mem_info *gmi; | |
761 | struct gdma_queue *queue; | |
762 | int err; | |
763 | ||
764 | if (spec->type != GDMA_EQ) | |
765 | return -EINVAL; | |
766 | ||
767 | queue = kzalloc(sizeof(*queue), GFP_KERNEL); | |
768 | if (!queue) | |
769 | return -ENOMEM; | |
770 | ||
771 | gmi = &queue->mem_info; | |
772 | err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); | |
773 | if (err) | |
774 | goto free_q; | |
775 | ||
776 | err = mana_gd_create_dma_region(gd, gmi); | |
777 | if (err) | |
778 | goto out; | |
779 | ||
780 | queue->head = 0; | |
781 | queue->tail = 0; | |
782 | queue->queue_mem_ptr = gmi->virt_addr; | |
783 | queue->queue_size = spec->queue_size; | |
784 | queue->monitor_avl_buf = spec->monitor_avl_buf; | |
785 | queue->type = spec->type; | |
786 | queue->gdma_dev = gd; | |
787 | ||
788 | err = mana_gd_create_eq(gd, spec, true, queue); | |
789 | if (err) | |
790 | goto out; | |
791 | ||
792 | *queue_ptr = queue; | |
793 | return 0; | |
794 | out: | |
795 | mana_gd_free_memory(gmi); | |
796 | free_q: | |
797 | kfree(queue); | |
798 | return err; | |
799 | } | |
800 | ||
801 | int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, | |
802 | const struct gdma_queue_spec *spec, | |
803 | struct gdma_queue **queue_ptr) | |
804 | { | |
805 | struct gdma_context *gc = gd->gdma_context; | |
806 | struct gdma_mem_info *gmi; | |
807 | struct gdma_queue *queue; | |
808 | int err; | |
809 | ||
810 | if (spec->type != GDMA_CQ && spec->type != GDMA_SQ && | |
811 | spec->type != GDMA_RQ) | |
812 | return -EINVAL; | |
813 | ||
814 | queue = kzalloc(sizeof(*queue), GFP_KERNEL); | |
815 | if (!queue) | |
816 | return -ENOMEM; | |
817 | ||
818 | gmi = &queue->mem_info; | |
819 | err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); | |
820 | if (err) | |
821 | goto free_q; | |
822 | ||
823 | err = mana_gd_create_dma_region(gd, gmi); | |
824 | if (err) | |
825 | goto out; | |
826 | ||
827 | queue->head = 0; | |
828 | queue->tail = 0; | |
829 | queue->queue_mem_ptr = gmi->virt_addr; | |
830 | queue->queue_size = spec->queue_size; | |
831 | queue->monitor_avl_buf = spec->monitor_avl_buf; | |
832 | queue->type = spec->type; | |
833 | queue->gdma_dev = gd; | |
834 | ||
835 | if (spec->type == GDMA_CQ) | |
836 | mana_gd_create_cq(spec, queue); | |
837 | ||
838 | *queue_ptr = queue; | |
839 | return 0; | |
840 | out: | |
841 | mana_gd_free_memory(gmi); | |
842 | free_q: | |
843 | kfree(queue); | |
844 | return err; | |
845 | } | |
846 | ||
847 | void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) | |
848 | { | |
849 | struct gdma_mem_info *gmi = &queue->mem_info; | |
850 | ||
851 | switch (queue->type) { | |
852 | case GDMA_EQ: | |
853 | mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue); | |
854 | break; | |
855 | ||
856 | case GDMA_CQ: | |
857 | mana_gd_destroy_cq(gc, queue); | |
858 | break; | |
859 | ||
860 | case GDMA_RQ: | |
861 | break; | |
862 | ||
863 | case GDMA_SQ: | |
864 | break; | |
865 | ||
866 | default: | |
867 | dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n", | |
868 | queue->type); | |
869 | return; | |
870 | } | |
871 | ||
28c66cfa | 872 | mana_gd_destroy_dma_region(gc, gmi->dma_region_handle); |
ca9c54d2 DC |
873 | mana_gd_free_memory(gmi); |
874 | kfree(queue); | |
875 | } | |
876 | ||
877 | int mana_gd_verify_vf_version(struct pci_dev *pdev) | |
878 | { | |
879 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
880 | struct gdma_verify_ver_resp resp = {}; | |
881 | struct gdma_verify_ver_req req = {}; | |
882 | int err; | |
883 | ||
884 | mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION, | |
885 | sizeof(req), sizeof(resp)); | |
886 | ||
887 | req.protocol_ver_min = GDMA_PROTOCOL_FIRST; | |
888 | req.protocol_ver_max = GDMA_PROTOCOL_LAST; | |
889 | ||
1e2d0824 HZ |
890 | req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1; |
891 | req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2; | |
892 | req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3; | |
893 | req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4; | |
894 | ||
3c37f357 DC |
895 | req.drv_ver = 0; /* Unused*/ |
896 | req.os_type = 0x10; /* Linux */ | |
897 | req.os_ver_major = LINUX_VERSION_MAJOR; | |
898 | req.os_ver_minor = LINUX_VERSION_PATCHLEVEL; | |
899 | req.os_ver_build = LINUX_VERSION_SUBLEVEL; | |
900 | strscpy(req.os_ver_str1, utsname()->sysname, sizeof(req.os_ver_str1)); | |
901 | strscpy(req.os_ver_str2, utsname()->release, sizeof(req.os_ver_str2)); | |
902 | strscpy(req.os_ver_str3, utsname()->version, sizeof(req.os_ver_str3)); | |
903 | ||
ca9c54d2 DC |
904 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
905 | if (err || resp.hdr.status) { | |
906 | dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n", | |
907 | err, resp.hdr.status); | |
908 | return err ? err : -EPROTO; | |
909 | } | |
910 | ||
911 | return 0; | |
912 | } | |
913 | ||
914 | int mana_gd_register_device(struct gdma_dev *gd) | |
915 | { | |
916 | struct gdma_context *gc = gd->gdma_context; | |
917 | struct gdma_register_device_resp resp = {}; | |
918 | struct gdma_general_req req = {}; | |
919 | int err; | |
920 | ||
921 | gd->pdid = INVALID_PDID; | |
922 | gd->doorbell = INVALID_DOORBELL; | |
923 | gd->gpa_mkey = INVALID_MEM_KEY; | |
924 | ||
925 | mana_gd_init_req_hdr(&req.hdr, GDMA_REGISTER_DEVICE, sizeof(req), | |
926 | sizeof(resp)); | |
927 | ||
928 | req.hdr.dev_id = gd->dev_id; | |
929 | ||
930 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
931 | if (err || resp.hdr.status) { | |
932 | dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n", | |
933 | err, resp.hdr.status); | |
934 | return err ? err : -EPROTO; | |
935 | } | |
936 | ||
937 | gd->pdid = resp.pdid; | |
938 | gd->gpa_mkey = resp.gpa_mkey; | |
939 | gd->doorbell = resp.db_id; | |
940 | ||
941 | return 0; | |
942 | } | |
943 | ||
944 | int mana_gd_deregister_device(struct gdma_dev *gd) | |
945 | { | |
946 | struct gdma_context *gc = gd->gdma_context; | |
947 | struct gdma_general_resp resp = {}; | |
948 | struct gdma_general_req req = {}; | |
949 | int err; | |
950 | ||
951 | if (gd->pdid == INVALID_PDID) | |
952 | return -EINVAL; | |
953 | ||
954 | mana_gd_init_req_hdr(&req.hdr, GDMA_DEREGISTER_DEVICE, sizeof(req), | |
955 | sizeof(resp)); | |
956 | ||
957 | req.hdr.dev_id = gd->dev_id; | |
958 | ||
959 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); | |
960 | if (err || resp.hdr.status) { | |
961 | dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n", | |
962 | err, resp.hdr.status); | |
963 | if (!err) | |
964 | err = -EPROTO; | |
965 | } | |
966 | ||
967 | gd->pdid = INVALID_PDID; | |
968 | gd->doorbell = INVALID_DOORBELL; | |
969 | gd->gpa_mkey = INVALID_MEM_KEY; | |
970 | ||
971 | return err; | |
972 | } | |
973 | ||
974 | u32 mana_gd_wq_avail_space(struct gdma_queue *wq) | |
975 | { | |
976 | u32 used_space = (wq->head - wq->tail) * GDMA_WQE_BU_SIZE; | |
977 | u32 wq_size = wq->queue_size; | |
978 | ||
979 | WARN_ON_ONCE(used_space > wq_size); | |
980 | ||
981 | return wq_size - used_space; | |
982 | } | |
983 | ||
984 | u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset) | |
985 | { | |
986 | u32 offset = (wqe_offset * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); | |
987 | ||
988 | WARN_ON_ONCE((offset + GDMA_WQE_BU_SIZE) > wq->queue_size); | |
989 | ||
990 | return wq->queue_mem_ptr + offset; | |
991 | } | |
992 | ||
993 | static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req, | |
994 | enum gdma_queue_type q_type, | |
995 | u32 client_oob_size, u32 sgl_data_size, | |
996 | u8 *wqe_ptr) | |
997 | { | |
998 | bool oob_in_sgl = !!(wqe_req->flags & GDMA_WR_OOB_IN_SGL); | |
999 | bool pad_data = !!(wqe_req->flags & GDMA_WR_PAD_BY_SGE0); | |
1000 | struct gdma_wqe *header = (struct gdma_wqe *)wqe_ptr; | |
1001 | u8 *ptr; | |
1002 | ||
1003 | memset(header, 0, sizeof(struct gdma_wqe)); | |
1004 | header->num_sge = wqe_req->num_sge; | |
1005 | header->inline_oob_size_div4 = client_oob_size / sizeof(u32); | |
1006 | ||
1007 | if (oob_in_sgl) { | |
1008 | WARN_ON_ONCE(!pad_data || wqe_req->num_sge < 2); | |
1009 | ||
1010 | header->client_oob_in_sgl = 1; | |
1011 | ||
1012 | if (pad_data) | |
1013 | header->last_vbytes = wqe_req->sgl[0].size; | |
1014 | } | |
1015 | ||
1016 | if (q_type == GDMA_SQ) | |
1017 | header->client_data_unit = wqe_req->client_data_unit; | |
1018 | ||
1019 | /* The size of gdma_wqe + client_oob_size must be less than or equal | |
1020 | * to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond | |
1021 | * the queue memory buffer boundary. | |
1022 | */ | |
1023 | ptr = wqe_ptr + sizeof(header); | |
1024 | ||
1025 | if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) { | |
1026 | memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size); | |
1027 | ||
1028 | if (client_oob_size > wqe_req->inline_oob_size) | |
1029 | memset(ptr + wqe_req->inline_oob_size, 0, | |
1030 | client_oob_size - wqe_req->inline_oob_size); | |
1031 | } | |
1032 | ||
1033 | return sizeof(header) + client_oob_size; | |
1034 | } | |
1035 | ||
1036 | static void mana_gd_write_sgl(struct gdma_queue *wq, u8 *wqe_ptr, | |
1037 | const struct gdma_wqe_request *wqe_req) | |
1038 | { | |
1039 | u32 sgl_size = sizeof(struct gdma_sge) * wqe_req->num_sge; | |
1040 | const u8 *address = (u8 *)wqe_req->sgl; | |
1041 | u8 *base_ptr, *end_ptr; | |
1042 | u32 size_to_end; | |
1043 | ||
1044 | base_ptr = wq->queue_mem_ptr; | |
1045 | end_ptr = base_ptr + wq->queue_size; | |
1046 | size_to_end = (u32)(end_ptr - wqe_ptr); | |
1047 | ||
1048 | if (size_to_end < sgl_size) { | |
1049 | memcpy(wqe_ptr, address, size_to_end); | |
1050 | ||
1051 | wqe_ptr = base_ptr; | |
1052 | address += size_to_end; | |
1053 | sgl_size -= size_to_end; | |
1054 | } | |
1055 | ||
1056 | memcpy(wqe_ptr, address, sgl_size); | |
1057 | } | |
1058 | ||
1059 | int mana_gd_post_work_request(struct gdma_queue *wq, | |
1060 | const struct gdma_wqe_request *wqe_req, | |
1061 | struct gdma_posted_wqe_info *wqe_info) | |
1062 | { | |
1063 | u32 client_oob_size = wqe_req->inline_oob_size; | |
1064 | struct gdma_context *gc; | |
1065 | u32 sgl_data_size; | |
1066 | u32 max_wqe_size; | |
1067 | u32 wqe_size; | |
1068 | u8 *wqe_ptr; | |
1069 | ||
1070 | if (wqe_req->num_sge == 0) | |
1071 | return -EINVAL; | |
1072 | ||
1073 | if (wq->type == GDMA_RQ) { | |
1074 | if (client_oob_size != 0) | |
1075 | return -EINVAL; | |
1076 | ||
1077 | client_oob_size = INLINE_OOB_SMALL_SIZE; | |
1078 | ||
1079 | max_wqe_size = GDMA_MAX_RQE_SIZE; | |
1080 | } else { | |
1081 | if (client_oob_size != INLINE_OOB_SMALL_SIZE && | |
1082 | client_oob_size != INLINE_OOB_LARGE_SIZE) | |
1083 | return -EINVAL; | |
1084 | ||
1085 | max_wqe_size = GDMA_MAX_SQE_SIZE; | |
1086 | } | |
1087 | ||
1088 | sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge; | |
1089 | wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size + | |
1090 | sgl_data_size, GDMA_WQE_BU_SIZE); | |
1091 | if (wqe_size > max_wqe_size) | |
1092 | return -EINVAL; | |
1093 | ||
1094 | if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) { | |
1095 | gc = wq->gdma_dev->gdma_context; | |
1096 | dev_err(gc->dev, "unsuccessful flow control!\n"); | |
1097 | return -ENOSPC; | |
1098 | } | |
1099 | ||
1100 | if (wqe_info) | |
1101 | wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; | |
1102 | ||
1103 | wqe_ptr = mana_gd_get_wqe_ptr(wq, wq->head); | |
1104 | wqe_ptr += mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size, | |
1105 | sgl_data_size, wqe_ptr); | |
1106 | if (wqe_ptr >= (u8 *)wq->queue_mem_ptr + wq->queue_size) | |
1107 | wqe_ptr -= wq->queue_size; | |
1108 | ||
1109 | mana_gd_write_sgl(wq, wqe_ptr, wqe_req); | |
1110 | ||
1111 | wq->head += wqe_size / GDMA_WQE_BU_SIZE; | |
1112 | ||
1113 | return 0; | |
1114 | } | |
1115 | ||
1116 | int mana_gd_post_and_ring(struct gdma_queue *queue, | |
1117 | const struct gdma_wqe_request *wqe_req, | |
1118 | struct gdma_posted_wqe_info *wqe_info) | |
1119 | { | |
1120 | struct gdma_context *gc = queue->gdma_dev->gdma_context; | |
1121 | int err; | |
1122 | ||
1123 | err = mana_gd_post_work_request(queue, wqe_req, wqe_info); | |
1124 | if (err) | |
1125 | return err; | |
1126 | ||
1127 | mana_gd_wq_ring_doorbell(gc, queue); | |
1128 | ||
1129 | return 0; | |
1130 | } | |
1131 | ||
1132 | static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp) | |
1133 | { | |
1134 | unsigned int num_cqe = cq->queue_size / sizeof(struct gdma_cqe); | |
1135 | struct gdma_cqe *cq_cqe = cq->queue_mem_ptr; | |
1136 | u32 owner_bits, new_bits, old_bits; | |
1137 | struct gdma_cqe *cqe; | |
1138 | ||
1139 | cqe = &cq_cqe[cq->head % num_cqe]; | |
1140 | owner_bits = cqe->cqe_info.owner_bits; | |
1141 | ||
1142 | old_bits = (cq->head / num_cqe - 1) & GDMA_CQE_OWNER_MASK; | |
1143 | /* Return 0 if no more entries. */ | |
1144 | if (owner_bits == old_bits) | |
1145 | return 0; | |
1146 | ||
1147 | new_bits = (cq->head / num_cqe) & GDMA_CQE_OWNER_MASK; | |
1148 | /* Return -1 if overflow detected. */ | |
c1a3e9f9 | 1149 | if (WARN_ON_ONCE(owner_bits != new_bits)) |
ca9c54d2 DC |
1150 | return -1; |
1151 | ||
6fd2c68d HZ |
1152 | /* Per GDMA spec, rmb is necessary after checking owner_bits, before |
1153 | * reading completion info | |
1154 | */ | |
1155 | rmb(); | |
1156 | ||
ca9c54d2 DC |
1157 | comp->wq_num = cqe->cqe_info.wq_num; |
1158 | comp->is_sq = cqe->cqe_info.is_sq; | |
1159 | memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE); | |
1160 | ||
1161 | return 1; | |
1162 | } | |
1163 | ||
1164 | int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) | |
1165 | { | |
1166 | int cqe_idx; | |
1167 | int ret; | |
1168 | ||
1169 | for (cqe_idx = 0; cqe_idx < num_cqe; cqe_idx++) { | |
1170 | ret = mana_gd_read_cqe(cq, &comp[cqe_idx]); | |
1171 | ||
1172 | if (ret < 0) { | |
1173 | cq->head -= cqe_idx; | |
1174 | return ret; | |
1175 | } | |
1176 | ||
1177 | if (ret == 0) | |
1178 | break; | |
1179 | ||
1180 | cq->head++; | |
1181 | } | |
1182 | ||
1183 | return cqe_idx; | |
1184 | } | |
1185 | ||
1186 | static irqreturn_t mana_gd_intr(int irq, void *arg) | |
1187 | { | |
1188 | struct gdma_irq_context *gic = arg; | |
1189 | ||
1190 | if (gic->handler) | |
1191 | gic->handler(gic->arg); | |
1192 | ||
1193 | return IRQ_HANDLED; | |
1194 | } | |
1195 | ||
1196 | int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r) | |
1197 | { | |
1198 | r->map = bitmap_zalloc(res_avail, GFP_KERNEL); | |
1199 | if (!r->map) | |
1200 | return -ENOMEM; | |
1201 | ||
1202 | r->size = res_avail; | |
1203 | spin_lock_init(&r->lock); | |
1204 | ||
1205 | return 0; | |
1206 | } | |
1207 | ||
1208 | void mana_gd_free_res_map(struct gdma_resource *r) | |
1209 | { | |
1210 | bitmap_free(r->map); | |
1211 | r->map = NULL; | |
1212 | r->size = 0; | |
1213 | } | |
1214 | ||
1215 | static int mana_gd_setup_irqs(struct pci_dev *pdev) | |
1216 | { | |
1217 | unsigned int max_queues_per_port = num_online_cpus(); | |
1218 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1219 | struct gdma_irq_context *gic; | |
18a04837 | 1220 | unsigned int max_irqs, cpu; |
ca9c54d2 | 1221 | int nvec, irq; |
71fa6887 | 1222 | int err, i = 0, j; |
ca9c54d2 DC |
1223 | |
1224 | if (max_queues_per_port > MANA_MAX_NUM_QUEUES) | |
1225 | max_queues_per_port = MANA_MAX_NUM_QUEUES; | |
1226 | ||
ca9c54d2 | 1227 | /* Need 1 interrupt for the Hardware communication Channel (HWC) */ |
1e2d0824 | 1228 | max_irqs = max_queues_per_port + 1; |
ca9c54d2 DC |
1229 | |
1230 | nvec = pci_alloc_irq_vectors(pdev, 2, max_irqs, PCI_IRQ_MSIX); | |
1231 | if (nvec < 0) | |
1232 | return nvec; | |
1233 | ||
1234 | gc->irq_contexts = kcalloc(nvec, sizeof(struct gdma_irq_context), | |
1235 | GFP_KERNEL); | |
1236 | if (!gc->irq_contexts) { | |
1237 | err = -ENOMEM; | |
1238 | goto free_irq_vector; | |
1239 | } | |
1240 | ||
1241 | for (i = 0; i < nvec; i++) { | |
1242 | gic = &gc->irq_contexts[i]; | |
1243 | gic->handler = NULL; | |
1244 | gic->arg = NULL; | |
1245 | ||
20e3028c HZ |
1246 | if (!i) |
1247 | snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s", | |
1248 | pci_name(pdev)); | |
1249 | else | |
1250 | snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_q%d@pci:%s", | |
1251 | i - 1, pci_name(pdev)); | |
1252 | ||
ca9c54d2 DC |
1253 | irq = pci_irq_vector(pdev, i); |
1254 | if (irq < 0) { | |
1255 | err = irq; | |
18a04837 | 1256 | goto free_irq; |
ca9c54d2 DC |
1257 | } |
1258 | ||
20e3028c | 1259 | err = request_irq(irq, mana_gd_intr, 0, gic->name, gic); |
ca9c54d2 | 1260 | if (err) |
18a04837 HZ |
1261 | goto free_irq; |
1262 | ||
1263 | cpu = cpumask_local_spread(i, gc->numa_node); | |
1264 | irq_set_affinity_and_hint(irq, cpumask_of(cpu)); | |
ca9c54d2 DC |
1265 | } |
1266 | ||
1267 | err = mana_gd_alloc_res_map(nvec, &gc->msix_resource); | |
1268 | if (err) | |
1269 | goto free_irq; | |
1270 | ||
1271 | gc->max_num_msix = nvec; | |
1272 | gc->num_msix_usable = nvec; | |
1273 | ||
1274 | return 0; | |
1275 | ||
1276 | free_irq: | |
1277 | for (j = i - 1; j >= 0; j--) { | |
1278 | irq = pci_irq_vector(pdev, j); | |
1279 | gic = &gc->irq_contexts[j]; | |
18a04837 HZ |
1280 | |
1281 | irq_update_affinity_hint(irq, NULL); | |
ca9c54d2 DC |
1282 | free_irq(irq, gic); |
1283 | } | |
1284 | ||
1285 | kfree(gc->irq_contexts); | |
1286 | gc->irq_contexts = NULL; | |
1287 | free_irq_vector: | |
1288 | pci_free_irq_vectors(pdev); | |
1289 | return err; | |
1290 | } | |
1291 | ||
1292 | static void mana_gd_remove_irqs(struct pci_dev *pdev) | |
1293 | { | |
1294 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1295 | struct gdma_irq_context *gic; | |
1296 | int irq, i; | |
1297 | ||
1298 | if (gc->max_num_msix < 1) | |
1299 | return; | |
1300 | ||
1301 | mana_gd_free_res_map(&gc->msix_resource); | |
1302 | ||
1303 | for (i = 0; i < gc->max_num_msix; i++) { | |
1304 | irq = pci_irq_vector(pdev, i); | |
1305 | if (irq < 0) | |
1306 | continue; | |
1307 | ||
1308 | gic = &gc->irq_contexts[i]; | |
18a04837 HZ |
1309 | |
1310 | /* Need to clear the hint before free_irq */ | |
1311 | irq_update_affinity_hint(irq, NULL); | |
ca9c54d2 DC |
1312 | free_irq(irq, gic); |
1313 | } | |
1314 | ||
1315 | pci_free_irq_vectors(pdev); | |
1316 | ||
1317 | gc->max_num_msix = 0; | |
1318 | gc->num_msix_usable = 0; | |
1319 | kfree(gc->irq_contexts); | |
1320 | gc->irq_contexts = NULL; | |
1321 | } | |
1322 | ||
635096a8 DC |
1323 | static int mana_gd_setup(struct pci_dev *pdev) |
1324 | { | |
1325 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1326 | int err; | |
1327 | ||
1328 | mana_gd_init_registers(pdev); | |
1329 | mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); | |
1330 | ||
1331 | err = mana_gd_setup_irqs(pdev); | |
1332 | if (err) | |
1333 | return err; | |
1334 | ||
1335 | err = mana_hwc_create_channel(gc); | |
1336 | if (err) | |
1337 | goto remove_irq; | |
1338 | ||
1339 | err = mana_gd_verify_vf_version(pdev); | |
1340 | if (err) | |
1341 | goto destroy_hwc; | |
1342 | ||
1343 | err = mana_gd_query_max_resources(pdev); | |
1344 | if (err) | |
1345 | goto destroy_hwc; | |
1346 | ||
1347 | err = mana_gd_detect_devices(pdev); | |
1348 | if (err) | |
1349 | goto destroy_hwc; | |
1350 | ||
1351 | return 0; | |
1352 | ||
1353 | destroy_hwc: | |
1354 | mana_hwc_destroy_channel(gc); | |
1355 | remove_irq: | |
1356 | mana_gd_remove_irqs(pdev); | |
1357 | return err; | |
1358 | } | |
1359 | ||
1360 | static void mana_gd_cleanup(struct pci_dev *pdev) | |
1361 | { | |
1362 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1363 | ||
1364 | mana_hwc_destroy_channel(gc); | |
1365 | ||
1366 | mana_gd_remove_irqs(pdev); | |
1367 | } | |
1368 | ||
1566e7d6 DC |
1369 | static bool mana_is_pf(unsigned short dev_id) |
1370 | { | |
1371 | return dev_id == MANA_PF_DEVICE_ID; | |
1372 | } | |
1373 | ||
ca9c54d2 DC |
1374 | static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) |
1375 | { | |
1376 | struct gdma_context *gc; | |
1377 | void __iomem *bar0_va; | |
1378 | int bar = 0; | |
1379 | int err; | |
1380 | ||
1e2d0824 HZ |
1381 | /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ |
1382 | BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); | |
1383 | ||
ca9c54d2 DC |
1384 | err = pci_enable_device(pdev); |
1385 | if (err) | |
1386 | return -ENXIO; | |
1387 | ||
1388 | pci_set_master(pdev); | |
1389 | ||
1390 | err = pci_request_regions(pdev, "mana"); | |
1391 | if (err) | |
1392 | goto disable_dev; | |
1393 | ||
1394 | err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); | |
1395 | if (err) | |
1396 | goto release_region; | |
1397 | ||
6fe25416 AS |
1398 | err = dma_set_max_seg_size(&pdev->dev, UINT_MAX); |
1399 | if (err) { | |
1400 | dev_err(&pdev->dev, "Failed to set dma device segment size\n"); | |
1401 | goto release_region; | |
1402 | } | |
1403 | ||
ca9c54d2 DC |
1404 | err = -ENOMEM; |
1405 | gc = vzalloc(sizeof(*gc)); | |
1406 | if (!gc) | |
1407 | goto release_region; | |
1408 | ||
635096a8 DC |
1409 | mutex_init(&gc->eq_test_event_mutex); |
1410 | pci_set_drvdata(pdev, gc); | |
f3dc0962 | 1411 | gc->bar0_pa = pci_resource_start(pdev, 0); |
635096a8 | 1412 | |
ca9c54d2 DC |
1413 | bar0_va = pci_iomap(pdev, bar, 0); |
1414 | if (!bar0_va) | |
1415 | goto free_gc; | |
1416 | ||
71fa6887 | 1417 | gc->numa_node = dev_to_node(&pdev->dev); |
1566e7d6 | 1418 | gc->is_pf = mana_is_pf(pdev->device); |
ca9c54d2 DC |
1419 | gc->bar0_va = bar0_va; |
1420 | gc->dev = &pdev->dev; | |
1421 | ||
635096a8 | 1422 | err = mana_gd_setup(pdev); |
ca9c54d2 DC |
1423 | if (err) |
1424 | goto unmap_bar; | |
1425 | ||
635096a8 | 1426 | err = mana_probe(&gc->mana, false); |
ca9c54d2 | 1427 | if (err) |
635096a8 | 1428 | goto cleanup_gd; |
ca9c54d2 DC |
1429 | |
1430 | return 0; | |
1431 | ||
635096a8 DC |
1432 | cleanup_gd: |
1433 | mana_gd_cleanup(pdev); | |
ca9c54d2 DC |
1434 | unmap_bar: |
1435 | pci_iounmap(pdev, bar0_va); | |
1436 | free_gc: | |
635096a8 | 1437 | pci_set_drvdata(pdev, NULL); |
ca9c54d2 DC |
1438 | vfree(gc); |
1439 | release_region: | |
1440 | pci_release_regions(pdev); | |
1441 | disable_dev: | |
1442 | pci_clear_master(pdev); | |
1443 | pci_disable_device(pdev); | |
1444 | dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err); | |
1445 | return err; | |
1446 | } | |
1447 | ||
1448 | static void mana_gd_remove(struct pci_dev *pdev) | |
1449 | { | |
1450 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1451 | ||
635096a8 | 1452 | mana_remove(&gc->mana, false); |
ca9c54d2 | 1453 | |
635096a8 | 1454 | mana_gd_cleanup(pdev); |
ca9c54d2 DC |
1455 | |
1456 | pci_iounmap(pdev, gc->bar0_va); | |
1457 | ||
1458 | vfree(gc); | |
1459 | ||
1460 | pci_release_regions(pdev); | |
1461 | pci_clear_master(pdev); | |
1462 | pci_disable_device(pdev); | |
1463 | } | |
1464 | ||
635096a8 DC |
1465 | /* The 'state' parameter is not used. */ |
1466 | static int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) | |
1467 | { | |
1468 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1469 | ||
1470 | mana_remove(&gc->mana, true); | |
1471 | ||
1472 | mana_gd_cleanup(pdev); | |
1473 | ||
1474 | return 0; | |
1475 | } | |
1476 | ||
1477 | /* In case the NIC hardware stops working, the suspend and resume callbacks will | |
1478 | * fail -- if this happens, it's safer to just report an error than try to undo | |
1479 | * what has been done. | |
1480 | */ | |
1481 | static int mana_gd_resume(struct pci_dev *pdev) | |
1482 | { | |
1483 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1484 | int err; | |
1485 | ||
1486 | err = mana_gd_setup(pdev); | |
1487 | if (err) | |
1488 | return err; | |
1489 | ||
1490 | err = mana_probe(&gc->mana, true); | |
1491 | if (err) | |
1492 | return err; | |
1493 | ||
1494 | return 0; | |
1495 | } | |
1496 | ||
1497 | /* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ | |
1498 | static void mana_gd_shutdown(struct pci_dev *pdev) | |
1499 | { | |
1500 | struct gdma_context *gc = pci_get_drvdata(pdev); | |
1501 | ||
8f1bc38b | 1502 | dev_info(&pdev->dev, "Shutdown was called\n"); |
635096a8 DC |
1503 | |
1504 | mana_remove(&gc->mana, true); | |
1505 | ||
1506 | mana_gd_cleanup(pdev); | |
1507 | ||
1508 | pci_disable_device(pdev); | |
1509 | } | |
1510 | ||
ca9c54d2 | 1511 | static const struct pci_device_id mana_id_table[] = { |
1566e7d6 DC |
1512 | { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, |
1513 | { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, | |
ca9c54d2 DC |
1514 | { } |
1515 | }; | |
1516 | ||
1517 | static struct pci_driver mana_driver = { | |
1518 | .name = "mana", | |
1519 | .id_table = mana_id_table, | |
1520 | .probe = mana_gd_probe, | |
1521 | .remove = mana_gd_remove, | |
635096a8 DC |
1522 | .suspend = mana_gd_suspend, |
1523 | .resume = mana_gd_resume, | |
1524 | .shutdown = mana_gd_shutdown, | |
ca9c54d2 DC |
1525 | }; |
1526 | ||
1527 | module_pci_driver(mana_driver); | |
1528 | ||
1529 | MODULE_DEVICE_TABLE(pci, mana_id_table); | |
1530 | ||
1531 | MODULE_LICENSE("Dual BSD/GPL"); | |
1532 | MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver"); |