projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/mlx5: Use dev_addr_mod()
[linux-block.git] / drivers / infiniband / hw / irdma / utils.c
CommitLineData
915cc7ac
MI		 1// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2/* Copyright (c) 2015 - 2021 Intel Corporation */
3#include "main.h"
4
5/**
6 * irdma_arp_table -manage arp table
7 * @rf: RDMA PCI function
8 * @ip_addr: ip address for device
9 * @ipv4: IPv4 flag
10 * @mac_addr: mac address ptr
11 * @action: modify, delete or add
12 */
13int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
14 u8 *mac_addr, u32 action)
15{
16 unsigned long flags;
17 int arp_index;
18 u32 ip[4] = {};
19
20 if (ipv4)
21 ip[0] = *ip_addr;
22 else
23 memcpy(ip, ip_addr, sizeof(ip));
24
25 spin_lock_irqsave(&rf->arp_lock, flags);
26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
28 break;
29 }
30
31 switch (action) {
32 case IRDMA_ARP_ADD:
33 if (arp_index != rf->arp_table_size) {
34 arp_index = -1;
35 break;
36 }
37
38 arp_index = 0;
39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
40 (u32 *)&arp_index, &rf->next_arp_index)) {
41 arp_index = -1;
42 break;
43 }
44
45 memcpy(rf->arp_table[arp_index].ip_addr, ip,
46 sizeof(rf->arp_table[arp_index].ip_addr));
47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
48 break;
49 case IRDMA_ARP_RESOLVE:
50 if (arp_index == rf->arp_table_size)
51 arp_index = -1;
52 break;
53 case IRDMA_ARP_DELETE:
54 if (arp_index == rf->arp_table_size) {
55 arp_index = -1;
56 break;
57 }
58
59 memset(rf->arp_table[arp_index].ip_addr, 0,
60 sizeof(rf->arp_table[arp_index].ip_addr));
61 eth_zero_addr(rf->arp_table[arp_index].mac_addr);
62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
63 break;
64 default:
65 arp_index = -1;
66 break;
67 }
68
69 spin_unlock_irqrestore(&rf->arp_lock, flags);
70 return arp_index;
71}
72
73/**
74 * irdma_add_arp - add a new arp entry if needed
75 * @rf: RDMA function
76 * @ip: IP address
77 * @ipv4: IPv4 flag
78 * @mac: MAC address
79 */
80int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, u8 *mac)
81{
82 int arpidx;
83
84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
85 if (arpidx >= 0) {
86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
87 return arpidx;
88
89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
90 ipv4, IRDMA_ARP_DELETE);
91 }
92
93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
94
95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
96}
97
98/**
99 * wr32 - write 32 bits to hw register
100 * @hw: hardware information including registers
101 * @reg: register offset
102 * @val: value to write to register
103 */
104inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
105{
106 writel(val, hw->hw_addr + reg);
107}
108
109/**
110 * rd32 - read a 32 bit hw register
111 * @hw: hardware information including registers
112 * @reg: register offset
113 *
114 * Return value of register content
115 */
116inline u32 rd32(struct irdma_hw *hw, u32 reg)
117{
118 return readl(hw->hw_addr + reg);
119}
120
121/**
122 * rd64 - read a 64 bit hw register
123 * @hw: hardware information including registers
124 * @reg: register offset
125 *
126 * Return value of register content
127 */
128inline u64 rd64(struct irdma_hw *hw, u32 reg)
129{
130 return readq(hw->hw_addr + reg);
131}
132
133static void irdma_gid_change_event(struct ib_device *ibdev)
134{
135 struct ib_event ib_event;
136
137 ib_event.event = IB_EVENT_GID_CHANGE;
138 ib_event.device = ibdev;
139 ib_event.element.port_num = 1;
140 ib_dispatch_event(&ib_event);
141}
142
143/**
144 * irdma_inetaddr_event - system notifier for ipv4 addr events
145 * @notifier: not used
146 * @event: event for notifier
147 * @ptr: if address
148 */
149int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
150 void *ptr)
151{
152 struct in_ifaddr *ifa = ptr;
153 struct net_device *netdev = ifa->ifa_dev->dev;
154 struct irdma_device *iwdev;
155 struct ib_device *ibdev;
156 u32 local_ipaddr;
157
158 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
159 if (!ibdev)
160 return NOTIFY_DONE;
161
162 iwdev = to_iwdev(ibdev);
163 local_ipaddr = ntohl(ifa->ifa_address);
164 ibdev_dbg(&iwdev->ibdev,
165 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", netdev,
166 event, &local_ipaddr, netdev->dev_addr);
167 switch (event) {
168 case NETDEV_DOWN:
169 irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
170 &local_ipaddr, true, IRDMA_ARP_DELETE);
171 irdma_if_notify(iwdev, netdev, &local_ipaddr, true, false);
172 irdma_gid_change_event(&iwdev->ibdev);
173 break;
174 case NETDEV_UP:
175 case NETDEV_CHANGEADDR:
176 irdma_add_arp(iwdev->rf, &local_ipaddr, true, netdev->dev_addr);
177 irdma_if_notify(iwdev, netdev, &local_ipaddr, true, true);
178 irdma_gid_change_event(&iwdev->ibdev);
179 break;
180 default:
181 break;
182 }
183
184 ib_device_put(ibdev);
185
186 return NOTIFY_DONE;
187}
188
189/**
190 * irdma_inet6addr_event - system notifier for ipv6 addr events
191 * @notifier: not used
192 * @event: event for notifier
193 * @ptr: if address
194 */
195int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
196 void *ptr)
197{
198 struct inet6_ifaddr *ifa = ptr;
199 struct net_device *netdev = ifa->idev->dev;
200 struct irdma_device *iwdev;
201 struct ib_device *ibdev;
202 u32 local_ipaddr6[4];
203
204 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
205 if (!ibdev)
206 return NOTIFY_DONE;
207
208 iwdev = to_iwdev(ibdev);
209 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
210 ibdev_dbg(&iwdev->ibdev,
211 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", netdev,
212 event, local_ipaddr6, netdev->dev_addr);
213 switch (event) {
214 case NETDEV_DOWN:
215 irdma_manage_arp_cache(iwdev->rf, netdev->dev_addr,
216 local_ipaddr6, false, IRDMA_ARP_DELETE);
217 irdma_if_notify(iwdev, netdev, local_ipaddr6, false, false);
218 irdma_gid_change_event(&iwdev->ibdev);
219 break;
220 case NETDEV_UP:
221 case NETDEV_CHANGEADDR:
222 irdma_add_arp(iwdev->rf, local_ipaddr6, false,
223 netdev->dev_addr);
224 irdma_if_notify(iwdev, netdev, local_ipaddr6, false, true);
225 irdma_gid_change_event(&iwdev->ibdev);
226 break;
227 default:
228 break;
229 }
230
231 ib_device_put(ibdev);
232
233 return NOTIFY_DONE;
234}
235
236/**
237 * irdma_net_event - system notifier for net events
238 * @notifier: not used
239 * @event: event for notifier
240 * @ptr: neighbor
241 */
242int irdma_net_event(struct notifier_block *notifier, unsigned long event,
243 void *ptr)
244{
245 struct neighbour *neigh = ptr;
246 struct irdma_device *iwdev;
247 struct ib_device *ibdev;
248 __be32 *p;
249 u32 local_ipaddr[4] = {};
250 bool ipv4 = true;
251
252 ibdev = ib_device_get_by_netdev((struct net_device *)neigh->dev,
253 RDMA_DRIVER_IRDMA);
254 if (!ibdev)
255 return NOTIFY_DONE;
256
257 iwdev = to_iwdev(ibdev);
258
259 switch (event) {
260 case NETEVENT_NEIGH_UPDATE:
261 p = (__be32 *)neigh->primary_key;
262 if (neigh->tbl->family == AF_INET6) {
263 ipv4 = false;
264 irdma_copy_ip_ntohl(local_ipaddr, p);
265 } else {
266 local_ipaddr[0] = ntohl(*p);
267 }
268
269 ibdev_dbg(&iwdev->ibdev,
270 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
271 iwdev->netdev, neigh->nud_state, local_ipaddr,
272 neigh->ha);
273
274 if (neigh->nud_state & NUD_VALID)
275 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
276
277 else
278 irdma_manage_arp_cache(iwdev->rf, neigh->ha,
279 local_ipaddr, ipv4,
280 IRDMA_ARP_DELETE);
281 break;
282 default:
283 break;
284 }
285
286 ib_device_put(ibdev);
287
288 return NOTIFY_DONE;
289}
290
291/**
292 * irdma_netdevice_event - system notifier for netdev events
293 * @notifier: not used
294 * @event: event for notifier
295 * @ptr: netdev
296 */
297int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
298 void *ptr)
299{
300 struct irdma_device *iwdev;
301 struct ib_device *ibdev;
302 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
303
304 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
305 if (!ibdev)
306 return NOTIFY_DONE;
307
308 iwdev = to_iwdev(ibdev);
309 iwdev->iw_status = 1;
310 switch (event) {
311 case NETDEV_DOWN:
312 iwdev->iw_status = 0;
313 fallthrough;
314 case NETDEV_UP:
315 irdma_port_ibevent(iwdev);
316 break;
317 default:
318 break;
319 }
320 ib_device_put(ibdev);
321
322 return NOTIFY_DONE;
323}
324
325/**
326 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
327 * @iwdev: irdma device
328 */
329static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
330{
331 struct net_device *ip_dev;
332 struct inet6_dev *idev;
333 struct inet6_ifaddr *ifp, *tmp;
334 u32 local_ipaddr6[4];
335
336 rcu_read_lock();
337 for_each_netdev_rcu (&init_net, ip_dev) {
338 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
339 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
340 ip_dev == iwdev->netdev) &&
341 (READ_ONCE(ip_dev->flags) & IFF_UP)) {
342 idev = __in6_dev_get(ip_dev);
343 if (!idev) {
344 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
345 break;
346 }
347 list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
348 if_list) {
349 ibdev_dbg(&iwdev->ibdev,
350 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
351 &ifp->addr,
352 rdma_vlan_dev_vlan_id(ip_dev),
353 ip_dev->dev_addr);
354
355 irdma_copy_ip_ntohl(local_ipaddr6,
356 ifp->addr.in6_u.u6_addr32);
357 irdma_manage_arp_cache(iwdev->rf,
358 ip_dev->dev_addr,
359 local_ipaddr6, false,
360 IRDMA_ARP_ADD);
361 }
362 }
363 }
364 rcu_read_unlock();
365}
366
367/**
368 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
369 * @iwdev: irdma device
370 */
371static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
372{
373 struct net_device *dev;
374 struct in_device *idev;
375 u32 ip_addr;
376
377 rcu_read_lock();
378 for_each_netdev_rcu (&init_net, dev) {
379 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
380 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
381 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
382 const struct in_ifaddr *ifa;
383
384 idev = __in_dev_get_rcu(dev);
385 if (!idev)
386 continue;
387
388 in_dev_for_each_ifa_rcu(ifa, idev) {
389 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
390 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
391 dev->dev_addr);
392
393 ip_addr = ntohl(ifa->ifa_address);
394 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
395 &ip_addr, true,
396 IRDMA_ARP_ADD);
397 }
398 }
399 }
400 rcu_read_unlock();
401}
402
403/**
404 * irdma_add_ip - add ip addresses
405 * @iwdev: irdma device
406 *
407 * Add ipv4/ipv6 addresses to the arp cache
408 */
409void irdma_add_ip(struct irdma_device *iwdev)
410{
411 irdma_add_ipv4_addr(iwdev);
412 irdma_add_ipv6_addr(iwdev);
413}
414
415/**
416 * irdma_alloc_and_get_cqp_request - get cqp struct
417 * @cqp: device cqp ptr
418 * @wait: cqp to be used in wait mode
419 */
420struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
421 bool wait)
422{
423 struct irdma_cqp_request *cqp_request = NULL;
424 unsigned long flags;
425
426 spin_lock_irqsave(&cqp->req_lock, flags);
427 if (!list_empty(&cqp->cqp_avail_reqs)) {
6246f1cc
SS		 428 cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
429 struct irdma_cqp_request, list);
915cc7ac
MI		 430 list_del_init(&cqp_request->list);
431 }
432 spin_unlock_irqrestore(&cqp->req_lock, flags);
433 if (!cqp_request) {
434 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
435 if (cqp_request) {
436 cqp_request->dynamic = true;
437 if (wait)
438 init_waitqueue_head(&cqp_request->waitq);
439 }
440 }
441 if (!cqp_request) {
442 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
443 return NULL;
444 }
445
446 cqp_request->waiting = wait;
447 refcount_set(&cqp_request->refcnt, 1);
448 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
449
450 return cqp_request;
451}
452
453/**
454 * irdma_get_cqp_request - increase refcount for cqp_request
455 * @cqp_request: pointer to cqp_request instance
456 */
457static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
458{
459 refcount_inc(&cqp_request->refcnt);
460}
461
462/**
463 * irdma_free_cqp_request - free cqp request
464 * @cqp: cqp ptr
465 * @cqp_request: to be put back in cqp list
466 */
467void irdma_free_cqp_request(struct irdma_cqp *cqp,
468 struct irdma_cqp_request *cqp_request)
469{
470 unsigned long flags;
471
472 if (cqp_request->dynamic) {
473 kfree(cqp_request);
474 } else {
475 cqp_request->request_done = false;
476 cqp_request->callback_fcn = NULL;
477 cqp_request->waiting = false;
478
479 spin_lock_irqsave(&cqp->req_lock, flags);
480 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
481 spin_unlock_irqrestore(&cqp->req_lock, flags);
482 }
483 wake_up(&cqp->remove_wq);
484}
485
486/**
487 * irdma_put_cqp_request - dec ref count and free if 0
488 * @cqp: cqp ptr
489 * @cqp_request: to be put back in cqp list
490 */
491void irdma_put_cqp_request(struct irdma_cqp *cqp,
492 struct irdma_cqp_request *cqp_request)
493{
494 if (refcount_dec_and_test(&cqp_request->refcnt))
495 irdma_free_cqp_request(cqp, cqp_request);
496}
497
498/**
499 * irdma_free_pending_cqp_request -free pending cqp request objs
500 * @cqp: cqp ptr
501 * @cqp_request: to be put back in cqp list
502 */
503static void
504irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
505 struct irdma_cqp_request *cqp_request)
506{
507 if (cqp_request->waiting) {
508 cqp_request->compl_info.error = true;
509 cqp_request->request_done = true;
510 wake_up(&cqp_request->waitq);
511 }
512 wait_event_timeout(cqp->remove_wq,
513 refcount_read(&cqp_request->refcnt) == 1, 1000);
514 irdma_put_cqp_request(cqp, cqp_request);
515}
516
517/**
518 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
519 * completions
520 * @rf: RDMA PCI function
521 */
522void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
523{
524 struct irdma_sc_dev *dev = &rf->sc_dev;
525 struct irdma_cqp *cqp = &rf->cqp;
526 struct irdma_cqp_request *cqp_request = NULL;
527 struct cqp_cmds_info *pcmdinfo = NULL;
528 u32 i, pending_work, wqe_idx;
529
530 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
531 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
532 for (i = 0; i < pending_work; i++) {
533 cqp_request = (struct irdma_cqp_request *)(unsigned long)
534 cqp->scratch_array[wqe_idx];
535 if (cqp_request)
536 irdma_free_pending_cqp_request(cqp, cqp_request);
537 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
538 }
539
540 while (!list_empty(&dev->cqp_cmd_head)) {
541 pcmdinfo = irdma_remove_cqp_head(dev);
542 cqp_request =
543 container_of(pcmdinfo, struct irdma_cqp_request, info);
544 if (cqp_request)
545 irdma_free_pending_cqp_request(cqp, cqp_request);
546 }
547}
548
549/**
550 * irdma_wait_event - wait for completion
551 * @rf: RDMA PCI function
552 * @cqp_request: cqp request to wait
553 */
554static enum irdma_status_code irdma_wait_event(struct irdma_pci_f *rf,
555 struct irdma_cqp_request *cqp_request)
556{
557 struct irdma_cqp_timeout cqp_timeout = {};
558 bool cqp_error = false;
559 enum irdma_status_code err_code = 0;
560
561 cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
562 do {
563 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
564 if (wait_event_timeout(cqp_request->waitq,
565 cqp_request->request_done,
566 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
567 break;
568
569 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
570
571 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
572 continue;
573
574 if (!rf->reset) {
575 rf->reset = true;
576 rf->gen_ops.request_reset(rf);
577 }
578 return IRDMA_ERR_TIMEOUT;
579 } while (1);
580
581 cqp_error = cqp_request->compl_info.error;
582 if (cqp_error) {
583 err_code = IRDMA_ERR_CQP_COMPL_ERROR;
584 if (cqp_request->compl_info.maj_err_code == 0xFFFF &&
585 cqp_request->compl_info.min_err_code == 0x8029) {
586 if (!rf->reset) {
587 rf->reset = true;
588 rf->gen_ops.request_reset(rf);
589 }
590 }
591 }
592
593 return err_code;
594}
595
596static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
597 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
598 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
599 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
600 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
601 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
602 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
603 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
604 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
605 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
606 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
607 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
608 [IRDMA_OP_QP_CREATE] = "Create QP Cmd",
609 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
610 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
611 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
612 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
613 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
614 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
615 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
616 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
617 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
618 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
619 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
620 [IRDMA_OP_RESUME] = "Resume QP Cmd",
621 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
622 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
623 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
624 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
625 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
626 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
627 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
628 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
629 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
630 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
631 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
632 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
633 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
634 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
635 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
636 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
637 [IRDMA_OP_GEN_AE] = "Generate AE Cmd",
638 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
205be5dc 639 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
915cc7ac
MI		 640 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
641 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
642 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
643};
644
645static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
646 {0xffff, 0x8006, "Flush No Wqe Pending"},
647 {0xffff, 0x8007, "Modify QP Bad Close"},
648 {0xffff, 0x8009, "LLP Closed"},
649 {0xffff, 0x800a, "Reset Not Sent"}
650};
651
652/**
653 * irdma_cqp_crit_err - check if CQP error is critical
654 * @dev: pointer to dev structure
655 * @cqp_cmd: code for last CQP operation
656 * @maj_err_code: major error code
657 * @min_err_code: minot error code
658 */
659bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
660 u16 maj_err_code, u16 min_err_code)
661{
662 int i;
663
664 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
665 if (maj_err_code == irdma_noncrit_err_list[i].maj &&
666 min_err_code == irdma_noncrit_err_list[i].min) {
667 ibdev_dbg(to_ibdev(dev),
668 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
669 irdma_noncrit_err_list[i].desc,
670 irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
671 min_err_code);
672 return false;
673 }
674 }
675 return true;
676}
677
678/**
679 * irdma_handle_cqp_op - process cqp command
680 * @rf: RDMA PCI function
681 * @cqp_request: cqp request to process
682 */
683enum irdma_status_code irdma_handle_cqp_op(struct irdma_pci_f *rf,
684 struct irdma_cqp_request *cqp_request)
685{
686 struct irdma_sc_dev *dev = &rf->sc_dev;
687 struct cqp_cmds_info *info = &cqp_request->info;
688 enum irdma_status_code status;
689 bool put_cqp_request = true;
690
691 if (rf->reset)
692 return IRDMA_ERR_NOT_READY;
693
694 irdma_get_cqp_request(cqp_request);
695 status = irdma_process_cqp_cmd(dev, info);
696 if (status)
697 goto err;
698
699 if (cqp_request->waiting) {
700 put_cqp_request = false;
701 status = irdma_wait_event(rf, cqp_request);
702 if (status)
703 goto err;
704 }
705
706 return 0;
707
708err:
709 if (irdma_cqp_crit_err(dev, info->cqp_cmd,
710 cqp_request->compl_info.maj_err_code,
711 cqp_request->compl_info.min_err_code))
712 ibdev_err(&rf->iwdev->ibdev,
713 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
714 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
715 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
716 cqp_request->compl_info.min_err_code);
717
718 if (put_cqp_request)
719 irdma_put_cqp_request(&rf->cqp, cqp_request);
720
721 return status;
722}
723
724void irdma_qp_add_ref(struct ib_qp *ibqp)
725{
726 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
727
728 refcount_inc(&iwqp->refcnt);
729}
730
731void irdma_qp_rem_ref(struct ib_qp *ibqp)
732{
733 struct irdma_qp *iwqp = to_iwqp(ibqp);
734 struct irdma_device *iwdev = iwqp->iwdev;
735 u32 qp_num;
736 unsigned long flags;
737
738 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
739 if (!refcount_dec_and_test(&iwqp->refcnt)) {
740 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
741 return;
742 }
743
744 qp_num = iwqp->ibqp.qp_num;
745 iwdev->rf->qp_table[qp_num] = NULL;
746 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
747 complete(&iwqp->free_qp);
748}
749
750struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
751{
752 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
753}
754
755/**
756 * irdma_get_qp - get qp address
757 * @device: iwarp device
758 * @qpn: qp number
759 */
760struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
761{
762 struct irdma_device *iwdev = to_iwdev(device);
763
764 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
765 return NULL;
766
767 return &iwdev->rf->qp_table[qpn]->ibqp;
768}
769
915cc7ac
MI		 770/**
771 * irdma_remove_cqp_head - return head entry and remove
772 * @dev: device
773 */
774void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
775{
776 struct list_head *entry;
777 struct list_head *list = &dev->cqp_cmd_head;
778
779 if (list_empty(list))
780 return NULL;
781
782 entry = list->next;
783 list_del(entry);
784
785 return entry;
786}
787
788/**
789 * irdma_cqp_sds_cmd - create cqp command for sd
790 * @dev: hardware control device structure
791 * @sdinfo: information for sd cqp
792 *
793 */
794enum irdma_status_code irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
795 struct irdma_update_sds_info *sdinfo)
796{
797 struct irdma_cqp_request *cqp_request;
798 struct cqp_cmds_info *cqp_info;
799 struct irdma_pci_f *rf = dev_to_rf(dev);
800 enum irdma_status_code status;
801
802 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
803 if (!cqp_request)
804 return IRDMA_ERR_NO_MEMORY;
805
806 cqp_info = &cqp_request->info;
807 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
808 sizeof(cqp_info->in.u.update_pe_sds.info));
809 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
810 cqp_info->post_sq = 1;
811 cqp_info->in.u.update_pe_sds.dev = dev;
812 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
813
814 status = irdma_handle_cqp_op(rf, cqp_request);
815 irdma_put_cqp_request(&rf->cqp, cqp_request);
816
817 return status;
818}
819
820/**
821 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
822 * @qp: hardware control qp
823 * @op: suspend or resume
824 */
825enum irdma_status_code irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp,
826 u8 op)
827{
828 struct irdma_sc_dev *dev = qp->dev;
829 struct irdma_cqp_request *cqp_request;
830 struct irdma_sc_cqp *cqp = dev->cqp;
831 struct cqp_cmds_info *cqp_info;
832 struct irdma_pci_f *rf = dev_to_rf(dev);
833 enum irdma_status_code status;
834
835 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
836 if (!cqp_request)
837 return IRDMA_ERR_NO_MEMORY;
838
839 cqp_info = &cqp_request->info;
840 cqp_info->cqp_cmd = op;
841 cqp_info->in.u.suspend_resume.cqp = cqp;
842 cqp_info->in.u.suspend_resume.qp = qp;
843 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
844
845 status = irdma_handle_cqp_op(rf, cqp_request);
846 irdma_put_cqp_request(&rf->cqp, cqp_request);
847
848 return status;
849}
850
851/**
852 * irdma_term_modify_qp - modify qp for term message
853 * @qp: hardware control qp
854 * @next_state: qp's next state
855 * @term: terminate code
856 * @term_len: length
857 */
858void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
859 u8 term_len)
860{
861 struct irdma_qp *iwqp;
862
863 iwqp = qp->qp_uk.back_qp;
864 irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
865};
866
867/**
868 * irdma_terminate_done - after terminate is completed
869 * @qp: hardware control qp
870 * @timeout_occurred: indicates if terminate timer expired
871 */
872void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
873{
874 struct irdma_qp *iwqp;
875 u8 hte = 0;
876 bool first_time;
877 unsigned long flags;
878
879 iwqp = qp->qp_uk.back_qp;
880 spin_lock_irqsave(&iwqp->lock, flags);
881 if (iwqp->hte_added) {
882 iwqp->hte_added = 0;
883 hte = 1;
884 }
885 first_time = !(qp->term_flags & IRDMA_TERM_DONE);
886 qp->term_flags |= IRDMA_TERM_DONE;
887 spin_unlock_irqrestore(&iwqp->lock, flags);
888 if (first_time) {
889 if (!timeout_occurred)
890 irdma_terminate_del_timer(qp);
891
892 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
893 irdma_cm_disconn(iwqp);
894 }
895}
896
897static void irdma_terminate_timeout(struct timer_list *t)
898{
899 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
900 struct irdma_sc_qp *qp = &iwqp->sc_qp;
901
902 irdma_terminate_done(qp, 1);
903 irdma_qp_rem_ref(&iwqp->ibqp);
904}
905
906/**
907 * irdma_terminate_start_timer - start terminate timeout
908 * @qp: hardware control qp
909 */
910void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
911{
912 struct irdma_qp *iwqp;
913
914 iwqp = qp->qp_uk.back_qp;
915 irdma_qp_add_ref(&iwqp->ibqp);
916 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
917 iwqp->terminate_timer.expires = jiffies + HZ;
918
919 add_timer(&iwqp->terminate_timer);
920}
921
922/**
923 * irdma_terminate_del_timer - delete terminate timeout
924 * @qp: hardware control qp
925 */
926void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
927{
928 struct irdma_qp *iwqp;
929 int ret;
930
931 iwqp = qp->qp_uk.back_qp;
932 ret = del_timer(&iwqp->terminate_timer);
933 if (ret)
934 irdma_qp_rem_ref(&iwqp->ibqp);
935}
936
937/**
938 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
939 * @dev: function device struct
940 * @val_mem: buffer for fpm
941 * @hmc_fn_id: function id for fpm
942 */
943enum irdma_status_code
944irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
945 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
946{
947 struct irdma_cqp_request *cqp_request;
948 struct cqp_cmds_info *cqp_info;
949 struct irdma_pci_f *rf = dev_to_rf(dev);
950 enum irdma_status_code status;
951
952 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
953 if (!cqp_request)
954 return IRDMA_ERR_NO_MEMORY;
955
956 cqp_info = &cqp_request->info;
957 cqp_request->param = NULL;
958 cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
959 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
960 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
961 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
962 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
963 cqp_info->post_sq = 1;
964 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
965
966 status = irdma_handle_cqp_op(rf, cqp_request);
967 irdma_put_cqp_request(&rf->cqp, cqp_request);
968
969 return status;
970}
971
972/**
973 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
974 * @dev: hardware control device structure
975 * @val_mem: buffer with fpm values
976 * @hmc_fn_id: function id for fpm
977 */
978enum irdma_status_code
979irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
980 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
981{
982 struct irdma_cqp_request *cqp_request;
983 struct cqp_cmds_info *cqp_info;
984 struct irdma_pci_f *rf = dev_to_rf(dev);
985 enum irdma_status_code status;
986
987 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
988 if (!cqp_request)
989 return IRDMA_ERR_NO_MEMORY;
990
991 cqp_info = &cqp_request->info;
992 cqp_request->param = NULL;
993 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
994 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
995 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
996 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
997 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
998 cqp_info->post_sq = 1;
999 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
1000
1001 status = irdma_handle_cqp_op(rf, cqp_request);
1002 irdma_put_cqp_request(&rf->cqp, cqp_request);
1003
1004 return status;
1005}
1006
1007/**
1008 * irdma_cqp_cq_create_cmd - create a cq for the cqp
1009 * @dev: device pointer
1010 * @cq: pointer to created cq
1011 */
1012enum irdma_status_code irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev,
1013 struct irdma_sc_cq *cq)
1014{
1015 struct irdma_pci_f *rf = dev_to_rf(dev);
1016 struct irdma_cqp *iwcqp = &rf->cqp;
1017 struct irdma_cqp_request *cqp_request;
1018 struct cqp_cmds_info *cqp_info;
1019 enum irdma_status_code status;
1020
1021 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1022 if (!cqp_request)
1023 return IRDMA_ERR_NO_MEMORY;
1024
1025 cqp_info = &cqp_request->info;
1026 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1027 cqp_info->post_sq = 1;
1028 cqp_info->in.u.cq_create.cq = cq;
1029 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1030
1031 status = irdma_handle_cqp_op(rf, cqp_request);
1032 irdma_put_cqp_request(iwcqp, cqp_request);
1033
1034 return status;
1035}
1036
1037/**
1038 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1039 * @dev: device pointer
1040 * @qp: pointer to created qp
1041 */
1042enum irdma_status_code irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev,
1043 struct irdma_sc_qp *qp)
1044{
1045 struct irdma_pci_f *rf = dev_to_rf(dev);
1046 struct irdma_cqp *iwcqp = &rf->cqp;
1047 struct irdma_cqp_request *cqp_request;
1048 struct cqp_cmds_info *cqp_info;
1049 struct irdma_create_qp_info *qp_info;
1050 enum irdma_status_code status;
1051
1052 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1053 if (!cqp_request)
1054 return IRDMA_ERR_NO_MEMORY;
1055
1056 cqp_info = &cqp_request->info;
1057 qp_info = &cqp_request->info.in.u.qp_create.info;
1058 memset(qp_info, 0, sizeof(*qp_info));
1059 qp_info->cq_num_valid = true;
1060 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1061 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1062 cqp_info->post_sq = 1;
1063 cqp_info->in.u.qp_create.qp = qp;
1064 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1065
1066 status = irdma_handle_cqp_op(rf, cqp_request);
1067 irdma_put_cqp_request(iwcqp, cqp_request);
1068
1069 return status;
1070}
1071
1072/**
1073 * irdma_dealloc_push_page - free a push page for qp
1074 * @rf: RDMA PCI function
1075 * @qp: hardware control qp
1076 */
1077static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1078 struct irdma_sc_qp *qp)
1079{
1080 struct irdma_cqp_request *cqp_request;
1081 struct cqp_cmds_info *cqp_info;
1082 enum irdma_status_code status;
1083
1084 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1085 return;
1086
1087 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1088 if (!cqp_request)
1089 return;
1090
1091 cqp_info = &cqp_request->info;
1092 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1093 cqp_info->post_sq = 1;
1094 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1095 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1096 cqp_info->in.u.manage_push_page.info.free_page = 1;
1097 cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1098 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1099 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1100 status = irdma_handle_cqp_op(rf, cqp_request);
1101 if (!status)
1102 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1103 irdma_put_cqp_request(&rf->cqp, cqp_request);
1104}
1105
1106/**
1107 * irdma_free_qp_rsrc - free up memory resources for qp
1108 * @iwqp: qp ptr (user or kernel)
1109 */
1110void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1111{
1112 struct irdma_device *iwdev = iwqp->iwdev;
1113 struct irdma_pci_f *rf = iwdev->rf;
1114 u32 qp_num = iwqp->ibqp.qp_num;
1115
1116 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1117 irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1118 if (iwqp->sc_qp.vsi) {
1119 irdma_qp_rem_qos(&iwqp->sc_qp);
1120 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1121 iwqp->sc_qp.user_pri);
1122 }
1123
1124 if (qp_num > 2)
1125 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1126 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1127 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1128 iwqp->q2_ctx_mem.va = NULL;
1129 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1130 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1131 iwqp->kqp.dma_mem.va = NULL;
1132 kfree(iwqp->kqp.sq_wrid_mem);
915cc7ac 1133 kfree(iwqp->kqp.rq_wrid_mem);
915cc7ac
MI		 1134}
1135
1136/**
1137 * irdma_cq_wq_destroy - send cq destroy cqp
1138 * @rf: RDMA PCI function
1139 * @cq: hardware control cq
1140 */
1141void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1142{
1143 struct irdma_cqp_request *cqp_request;
1144 struct cqp_cmds_info *cqp_info;
1145
1146 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1147 if (!cqp_request)
1148 return;
1149
1150 cqp_info = &cqp_request->info;
1151 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1152 cqp_info->post_sq = 1;
1153 cqp_info->in.u.cq_destroy.cq = cq;
1154 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1155
1156 irdma_handle_cqp_op(rf, cqp_request);
1157 irdma_put_cqp_request(&rf->cqp, cqp_request);
1158}
1159
1160/**
1161 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1162 * @cqp_request: modify QP completion
1163 */
1164static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1165{
1166 struct cqp_cmds_info *cqp_info;
1167 struct irdma_qp *iwqp;
1168
1169 cqp_info = &cqp_request->info;
1170 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1171 atomic_dec(&iwqp->hw_mod_qp_pend);
1172 wake_up(&iwqp->mod_qp_waitq);
1173}
1174
1175/**
1176 * irdma_hw_modify_qp - setup cqp for modify qp
1177 * @iwdev: RDMA device
1178 * @iwqp: qp ptr (user or kernel)
1179 * @info: info for modify qp
1180 * @wait: flag to wait or not for modify qp completion
1181 */
1182enum irdma_status_code irdma_hw_modify_qp(struct irdma_device *iwdev,
1183 struct irdma_qp *iwqp,
1184 struct irdma_modify_qp_info *info,
1185 bool wait)
1186{
1187 enum irdma_status_code status;
1188 struct irdma_pci_f *rf = iwdev->rf;
1189 struct irdma_cqp_request *cqp_request;
1190 struct cqp_cmds_info *cqp_info;
1191 struct irdma_modify_qp_info *m_info;
1192
1193 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1194 if (!cqp_request)
1195 return IRDMA_ERR_NO_MEMORY;
1196
1197 if (!wait) {
1198 cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1199 atomic_inc(&iwqp->hw_mod_qp_pend);
1200 }
1201 cqp_info = &cqp_request->info;
1202 m_info = &cqp_info->in.u.qp_modify.info;
1203 memcpy(m_info, info, sizeof(*m_info));
1204 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1205 cqp_info->post_sq = 1;
1206 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1207 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1208 status = irdma_handle_cqp_op(rf, cqp_request);
1209 irdma_put_cqp_request(&rf->cqp, cqp_request);
1210 if (status) {
1211 if (rdma_protocol_roce(&iwdev->ibdev, 1))
1212 return status;
1213
1214 switch (m_info->next_iwarp_state) {
1215 struct irdma_gen_ae_info ae_info;
1216
1217 case IRDMA_QP_STATE_RTS:
1218 case IRDMA_QP_STATE_IDLE:
1219 case IRDMA_QP_STATE_TERMINATE:
1220 case IRDMA_QP_STATE_CLOSING:
1221 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1222 irdma_send_reset(iwqp->cm_node);
1223 else
1224 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1225 if (!wait) {
1226 ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1227 ae_info.ae_src = 0;
1228 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1229 } else {
1230 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1231 wait);
1232 if (!cqp_request)
1233 return IRDMA_ERR_NO_MEMORY;
1234
1235 cqp_info = &cqp_request->info;
1236 m_info = &cqp_info->in.u.qp_modify.info;
1237 memcpy(m_info, info, sizeof(*m_info));
1238 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1239 cqp_info->post_sq = 1;
1240 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1241 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1242 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1243 m_info->reset_tcp_conn = true;
1244 irdma_handle_cqp_op(rf, cqp_request);
1245 irdma_put_cqp_request(&rf->cqp, cqp_request);
1246 }
1247 break;
1248 case IRDMA_QP_STATE_ERROR:
1249 default:
1250 break;
1251 }
1252 }
1253
1254 return status;
1255}
1256
1257/**
1258 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1259 * @dev: device pointer
1260 * @cq: pointer to cq
1261 */
1262void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1263{
1264 struct irdma_pci_f *rf = dev_to_rf(dev);
1265
1266 irdma_cq_wq_destroy(rf, cq);
1267}
1268
1269/**
1270 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1271 * @dev: device pointer
1272 * @qp: pointer to qp
1273 */
1274enum irdma_status_code irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1275{
1276 struct irdma_pci_f *rf = dev_to_rf(dev);
1277 struct irdma_cqp *iwcqp = &rf->cqp;
1278 struct irdma_cqp_request *cqp_request;
1279 struct cqp_cmds_info *cqp_info;
1280 enum irdma_status_code status;
1281
1282 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1283 if (!cqp_request)
1284 return IRDMA_ERR_NO_MEMORY;
1285
1286 cqp_info = &cqp_request->info;
1287 memset(cqp_info, 0, sizeof(*cqp_info));
1288 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1289 cqp_info->post_sq = 1;
1290 cqp_info->in.u.qp_destroy.qp = qp;
1291 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1292 cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1293
1294 status = irdma_handle_cqp_op(rf, cqp_request);
1295 irdma_put_cqp_request(&rf->cqp, cqp_request);
1296
1297 return status;
1298}
1299
1300/**
1301 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1302 * @dev: hardware control device structure
1303 * @qp: hardware control qp
1304 */
1305void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1306{
1307 struct irdma_gen_ae_info info = {};
1308 struct irdma_pci_f *rf = dev_to_rf(dev);
1309
1310 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1311 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1312 info.ae_src = IRDMA_AE_SOURCE_RQ;
1313 irdma_gen_ae(rf, qp, &info, false);
1314}
1315
1316/**
1317 * irdma_init_hash_desc - initialize hash for crc calculation
1318 * @desc: cryption type
1319 */
1320enum irdma_status_code irdma_init_hash_desc(struct shash_desc **desc)
1321{
1322 struct crypto_shash *tfm;
1323 struct shash_desc *tdesc;
1324
1325 tfm = crypto_alloc_shash("crc32c", 0, 0);
1326 if (IS_ERR(tfm))
1327 return IRDMA_ERR_MPA_CRC;
1328
1329 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
1330 GFP_KERNEL);
1331 if (!tdesc) {
1332 crypto_free_shash(tfm);
1333 return IRDMA_ERR_MPA_CRC;
1334 }
1335
1336 tdesc->tfm = tfm;
1337 *desc = tdesc;
1338
1339 return 0;
1340}
1341
1342/**
1343 * irdma_free_hash_desc - free hash desc
1344 * @desc: to be freed
1345 */
1346void irdma_free_hash_desc(struct shash_desc *desc)
1347{
1348 if (desc) {
1349 crypto_free_shash(desc->tfm);
1350 kfree(desc);
1351 }
1352}
1353
1354/**
1355 * irdma_ieq_check_mpacrc - check if mpa crc is OK
1356 * @desc: desc for hash
1357 * @addr: address of buffer for crc
1358 * @len: length of buffer
1359 * @val: value to be compared
1360 */
1361enum irdma_status_code irdma_ieq_check_mpacrc(struct shash_desc *desc,
1362 void *addr, u32 len, u32 val)
1363{
1364 u32 crc = 0;
1365 int ret;
1366 enum irdma_status_code ret_code = 0;
1367
1368 crypto_shash_init(desc);
1369 ret = crypto_shash_update(desc, addr, len);
1370 if (!ret)
1371 crypto_shash_final(desc, (u8 *)&crc);
1372 if (crc != val)
1373 ret_code = IRDMA_ERR_MPA_CRC;
1374
1375 return ret_code;
1376}
1377
1378/**
1379 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1380 * @dev: hardware control device structure
1381 * @buf: receive puda buffer on exception q
1382 */
1383struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1384 struct irdma_puda_buf *buf)
1385{
1386 struct irdma_qp *iwqp;
1387 struct irdma_cm_node *cm_node;
1388 struct irdma_device *iwdev = buf->vsi->back_vsi;
1389 u32 loc_addr[4] = {};
1390 u32 rem_addr[4] = {};
1391 u16 loc_port, rem_port;
1392 struct ipv6hdr *ip6h;
1393 struct iphdr *iph = (struct iphdr *)buf->iph;
1394 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1395
1396 if (iph->version == 4) {
1397 loc_addr[0] = ntohl(iph->daddr);
1398 rem_addr[0] = ntohl(iph->saddr);
1399 } else {
1400 ip6h = (struct ipv6hdr *)buf->iph;
1401 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1402 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1403 }
1404 loc_port = ntohs(tcph->dest);
1405 rem_port = ntohs(tcph->source);
1406 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1407 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1408 if (!cm_node)
1409 return NULL;
1410
1411 iwqp = cm_node->iwqp;
1412 irdma_rem_ref_cm_node(cm_node);
1413
1414 return &iwqp->sc_qp;
1415}
1416
1417/**
1418 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1419 * @qp: qp ptr
1420 */
1421void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1422{
1423 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1424 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1425 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1426
1427 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1428 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1429
1430 irdma_send_ack(cm_node);
1431}
1432
1433/**
1434 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1435 * @qp: qp pointer
1436 * @ah_info: AH info pointer
1437 */
1438void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1439 struct irdma_ah_info *ah_info)
1440{
1441 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1442 struct iphdr *iph;
1443 struct ipv6hdr *ip6h;
1444
1445 memset(ah_info, 0, sizeof(*ah_info));
1446 ah_info->do_lpbk = true;
1447 ah_info->vlan_tag = buf->vlan_id;
1448 ah_info->insert_vlan_tag = buf->vlan_valid;
1449 ah_info->ipv4_valid = buf->ipv4;
1450 ah_info->vsi = qp->vsi;
1451
1452 if (buf->smac_valid)
1453 ether_addr_copy(ah_info->mac_addr, buf->smac);
1454
1455 if (buf->ipv4) {
1456 ah_info->ipv4_valid = true;
1457 iph = (struct iphdr *)buf->iph;
1458 ah_info->hop_ttl = iph->ttl;
1459 ah_info->tc_tos = iph->tos;
1460 ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1461 ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1462 } else {
1463 ip6h = (struct ipv6hdr *)buf->iph;
1464 ah_info->hop_ttl = ip6h->hop_limit;
1465 ah_info->tc_tos = ip6h->priority;
1466 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1467 ip6h->daddr.in6_u.u6_addr32);
1468 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1469 ip6h->saddr.in6_u.u6_addr32);
1470 }
1471
1472 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1473 ah_info->dest_ip_addr,
1474 ah_info->ipv4_valid,
1475 NULL, IRDMA_ARP_RESOLVE);
1476}
1477
1478/**
1479 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1480 * @buf: puda to update
1481 * @len: length of buffer
1482 * @seqnum: seq number for tcp
1483 */
1484static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1485 u16 len, u32 seqnum)
1486{
1487 struct tcphdr *tcph;
1488 struct iphdr *iph;
1489 u16 iphlen;
1490 u16 pktsize;
1491 u8 *addr = buf->mem.va;
1492
1493 iphlen = (buf->ipv4) ? 20 : 40;
1494 iph = (struct iphdr *)(addr + buf->maclen);
1495 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1496 pktsize = len + buf->tcphlen + iphlen;
1497 iph->tot_len = htons(pktsize);
1498 tcph->seq = htonl(seqnum);
1499}
1500
1501/**
1502 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1503 * @buf: puda to update
1504 * @len: length of buffer
1505 * @seqnum: seq number for tcp
1506 */
1507void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1508 u32 seqnum)
1509{
1510 struct tcphdr *tcph;
1511 u8 *addr;
1512
1513 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1514 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1515
1516 addr = buf->mem.va;
1517 tcph = (struct tcphdr *)addr;
1518 tcph->seq = htonl(seqnum);
1519}
1520
1521/**
1522 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1523 * buffer
1524 * @info: to get information
1525 * @buf: puda buffer
1526 */
1527static enum irdma_status_code
1528irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1529 struct irdma_puda_buf *buf)
1530{
1531 struct iphdr *iph;
1532 struct ipv6hdr *ip6h;
1533 struct tcphdr *tcph;
1534 u16 iphlen;
1535 u16 pkt_len;
1536 u8 *mem = buf->mem.va;
1537 struct ethhdr *ethh = buf->mem.va;
1538
1539 if (ethh->h_proto == htons(0x8100)) {
1540 info->vlan_valid = true;
1541 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1542 VLAN_VID_MASK;
1543 }
1544
1545 buf->maclen = (info->vlan_valid) ? 18 : 14;
1546 iphlen = (info->l3proto) ? 40 : 20;
1547 buf->ipv4 = (info->l3proto) ? false : true;
1548 buf->iph = mem + buf->maclen;
1549 iph = (struct iphdr *)buf->iph;
1550 buf->tcph = buf->iph + iphlen;
1551 tcph = (struct tcphdr *)buf->tcph;
1552
1553 if (buf->ipv4) {
1554 pkt_len = ntohs(iph->tot_len);
1555 } else {
1556 ip6h = (struct ipv6hdr *)buf->iph;
1557 pkt_len = ntohs(ip6h->payload_len) + iphlen;
1558 }
1559
1560 buf->totallen = pkt_len + buf->maclen;
1561
1562 if (info->payload_len < buf->totallen) {
1563 ibdev_dbg(to_ibdev(buf->vsi->dev),
1564 "ERR: payload_len = 0x%x totallen expected0x%x\n",
1565 info->payload_len, buf->totallen);
1566 return IRDMA_ERR_INVALID_SIZE;
1567 }
1568
1569 buf->tcphlen = tcph->doff << 2;
1570 buf->datalen = pkt_len - iphlen - buf->tcphlen;
1571 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1572 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1573 buf->seqnum = ntohl(tcph->seq);
1574
1575 return 0;
1576}
1577
1578/**
1579 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1580 * @info: to get information
1581 * @buf: puda buffer
1582 */
1583enum irdma_status_code
1584irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1585 struct irdma_puda_buf *buf)
1586{
1587 struct tcphdr *tcph;
1588 u32 pkt_len;
1589 u8 *mem;
1590
1591 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1592 return irdma_gen1_puda_get_tcpip_info(info, buf);
1593
1594 mem = buf->mem.va;
1595 buf->vlan_valid = info->vlan_valid;
1596 if (info->vlan_valid)
1597 buf->vlan_id = info->vlan;
1598
1599 buf->ipv4 = info->ipv4;
1600 if (buf->ipv4)
1601 buf->iph = mem + IRDMA_IPV4_PAD;
1602 else
1603 buf->iph = mem;
1604
1605 buf->tcph = mem + IRDMA_TCP_OFFSET;
1606 tcph = (struct tcphdr *)buf->tcph;
1607 pkt_len = info->payload_len;
1608 buf->totallen = pkt_len;
1609 buf->tcphlen = tcph->doff << 2;
1610 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1611 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1612 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1613 buf->seqnum = ntohl(tcph->seq);
1614
1615 if (info->smac_valid) {
1616 ether_addr_copy(buf->smac, info->smac);
1617 buf->smac_valid = true;
1618 }
1619
1620 return 0;
1621}
1622
1623/**
1624 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1625 * @t: timer_list pointer
1626 */
1627static void irdma_hw_stats_timeout(struct timer_list *t)
1628{
1629 struct irdma_vsi_pestat *pf_devstat =
1630 from_timer(pf_devstat, t, stats_timer);
1631 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1632
1633 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1634 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
1635 else
1636 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1637
1638 mod_timer(&pf_devstat->stats_timer,
1639 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1640}
1641
1642/**
1643 * irdma_hw_stats_start_timer - Start periodic stats timer
1644 * @vsi: vsi structure pointer
1645 */
1646void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1647{
1648 struct irdma_vsi_pestat *devstat = vsi->pestat;
1649
1650 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1651 mod_timer(&devstat->stats_timer,
1652 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1653}
1654
1655/**
1656 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1657 * @vsi: pointer to vsi structure
1658 */
1659void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1660{
1661 struct irdma_vsi_pestat *devstat = vsi->pestat;
1662
1663 del_timer_sync(&devstat->stats_timer);
1664}
1665
1666/**
1667 * irdma_process_stats - Checking for wrap and update stats
1668 * @pestat: stats structure pointer
1669 */
1670static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1671{
1672 sc_vsi_update_stats(pestat->vsi);
1673}
1674
1675/**
1676 * irdma_cqp_gather_stats_gen1 - Gather stats
1677 * @dev: pointer to device structure
1678 * @pestat: statistics structure
1679 */
1680void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1681 struct irdma_vsi_pestat *pestat)
1682{
1683 struct irdma_gather_stats *gather_stats =
1684 pestat->gather_info.gather_stats_va;
1685 u32 stats_inst_offset_32;
1686 u32 stats_inst_offset_64;
1687
1688 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1689 pestat->gather_info.stats_inst_index :
1690 pestat->hw->hmc.hmc_fn_id;
1691 stats_inst_offset_32 *= 4;
1692 stats_inst_offset_64 = stats_inst_offset_32 * 2;
1693
1694 gather_stats->rxvlanerr =
1695 rd32(dev->hw,
1696 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR]
1697 + stats_inst_offset_32);
1698 gather_stats->ip4rxdiscard =
1699 rd32(dev->hw,
1700 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD]
1701 + stats_inst_offset_32);
1702 gather_stats->ip4rxtrunc =
1703 rd32(dev->hw,
1704 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC]
1705 + stats_inst_offset_32);
1706 gather_stats->ip4txnoroute =
1707 rd32(dev->hw,
1708 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE]
1709 + stats_inst_offset_32);
1710 gather_stats->ip6rxdiscard =
1711 rd32(dev->hw,
1712 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD]
1713 + stats_inst_offset_32);
1714 gather_stats->ip6rxtrunc =
1715 rd32(dev->hw,
1716 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC]
1717 + stats_inst_offset_32);
1718 gather_stats->ip6txnoroute =
1719 rd32(dev->hw,
1720 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE]
1721 + stats_inst_offset_32);
1722 gather_stats->tcprtxseg =
1723 rd32(dev->hw,
1724 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG]
1725 + stats_inst_offset_32);
1726 gather_stats->tcprxopterr =
1727 rd32(dev->hw,
1728 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR]
1729 + stats_inst_offset_32);
1730
1731 gather_stats->ip4rxocts =
1732 rd64(dev->hw,
1733 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS]
1734 + stats_inst_offset_64);
1735 gather_stats->ip4rxpkts =
1736 rd64(dev->hw,
1737 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS]
1738 + stats_inst_offset_64);
1739 gather_stats->ip4txfrag =
1740 rd64(dev->hw,
1741 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS]
1742 + stats_inst_offset_64);
1743 gather_stats->ip4rxmcpkts =
1744 rd64(dev->hw,
1745 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS]
1746 + stats_inst_offset_64);
1747 gather_stats->ip4txocts =
1748 rd64(dev->hw,
1749 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS]
1750 + stats_inst_offset_64);
1751 gather_stats->ip4txpkts =
1752 rd64(dev->hw,
1753 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS]
1754 + stats_inst_offset_64);
1755 gather_stats->ip4txfrag =
1756 rd64(dev->hw,
1757 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS]
1758 + stats_inst_offset_64);
1759 gather_stats->ip4txmcpkts =
1760 rd64(dev->hw,
1761 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS]
1762 + stats_inst_offset_64);
1763 gather_stats->ip6rxocts =
1764 rd64(dev->hw,
1765 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS]
1766 + stats_inst_offset_64);
1767 gather_stats->ip6rxpkts =
1768 rd64(dev->hw,
1769 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS]
1770 + stats_inst_offset_64);
1771 gather_stats->ip6txfrags =
1772 rd64(dev->hw,
1773 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS]
1774 + stats_inst_offset_64);
1775 gather_stats->ip6rxmcpkts =
1776 rd64(dev->hw,
1777 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS]
1778 + stats_inst_offset_64);
1779 gather_stats->ip6txocts =
1780 rd64(dev->hw,
1781 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS]
1782 + stats_inst_offset_64);
1783 gather_stats->ip6txpkts =
1784 rd64(dev->hw,
1785 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS]
1786 + stats_inst_offset_64);
1787 gather_stats->ip6txfrags =
1788 rd64(dev->hw,
1789 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS]
1790 + stats_inst_offset_64);
1791 gather_stats->ip6txmcpkts =
1792 rd64(dev->hw,
1793 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS]
1794 + stats_inst_offset_64);
1795 gather_stats->tcprxsegs =
1796 rd64(dev->hw,
1797 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS]
1798 + stats_inst_offset_64);
1799 gather_stats->tcptxsegs =
1800 rd64(dev->hw,
1801 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG]
1802 + stats_inst_offset_64);
1803 gather_stats->rdmarxrds =
1804 rd64(dev->hw,
1805 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS]
1806 + stats_inst_offset_64);
1807 gather_stats->rdmarxsnds =
1808 rd64(dev->hw,
1809 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS]
1810 + stats_inst_offset_64);
1811 gather_stats->rdmarxwrs =
1812 rd64(dev->hw,
1813 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS]
1814 + stats_inst_offset_64);
1815 gather_stats->rdmatxrds =
1816 rd64(dev->hw,
1817 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS]
1818 + stats_inst_offset_64);
1819 gather_stats->rdmatxsnds =
1820 rd64(dev->hw,
1821 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS]
1822 + stats_inst_offset_64);
1823 gather_stats->rdmatxwrs =
1824 rd64(dev->hw,
1825 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS]
1826 + stats_inst_offset_64);
1827 gather_stats->rdmavbn =
1828 rd64(dev->hw,
1829 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND]
1830 + stats_inst_offset_64);
1831 gather_stats->rdmavinv =
1832 rd64(dev->hw,
1833 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV]
1834 + stats_inst_offset_64);
1835 gather_stats->udprxpkts =
1836 rd64(dev->hw,
1837 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS]
1838 + stats_inst_offset_64);
1839 gather_stats->udptxpkts =
1840 rd64(dev->hw,
1841 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS]
1842 + stats_inst_offset_64);
1843
1844 irdma_process_stats(pestat);
1845}
1846
1847/**
1848 * irdma_process_cqp_stats - Checking for wrap and update stats
1849 * @cqp_request: cqp_request structure pointer
1850 */
1851static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1852{
1853 struct irdma_vsi_pestat *pestat = cqp_request->param;
1854
1855 irdma_process_stats(pestat);
1856}
1857
1858/**
1859 * irdma_cqp_gather_stats_cmd - Gather stats
1860 * @dev: pointer to device structure
1861 * @pestat: pointer to stats info
1862 * @wait: flag to wait or not wait for stats
1863 */
1864enum irdma_status_code
1865irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1866 struct irdma_vsi_pestat *pestat, bool wait)
1867
1868{
1869 struct irdma_pci_f *rf = dev_to_rf(dev);
1870 struct irdma_cqp *iwcqp = &rf->cqp;
1871 struct irdma_cqp_request *cqp_request;
1872 struct cqp_cmds_info *cqp_info;
1873 enum irdma_status_code status;
1874
1875 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1876 if (!cqp_request)
1877 return IRDMA_ERR_NO_MEMORY;
1878
1879 cqp_info = &cqp_request->info;
1880 memset(cqp_info, 0, sizeof(*cqp_info));
1881 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1882 cqp_info->post_sq = 1;
1883 cqp_info->in.u.stats_gather.info = pestat->gather_info;
1884 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1885 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1886 cqp_request->param = pestat;
1887 if (!wait)
1888 cqp_request->callback_fcn = irdma_process_cqp_stats;
1889 status = irdma_handle_cqp_op(rf, cqp_request);
1890 if (wait)
1891 irdma_process_stats(pestat);
1892 irdma_put_cqp_request(&rf->cqp, cqp_request);
1893
1894 return status;
1895}
1896
1897/**
1898 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1899 * @vsi: pointer to vsi structure
1900 * @cmd: command to allocate or free
1901 * @stats_info: pointer to allocate stats info
1902 */
1903enum irdma_status_code
1904irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1905 struct irdma_stats_inst_info *stats_info)
1906{
1907 struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1908 struct irdma_cqp *iwcqp = &rf->cqp;
1909 struct irdma_cqp_request *cqp_request;
1910 struct cqp_cmds_info *cqp_info;
1911 enum irdma_status_code status;
1912 bool wait = false;
1913
1914 if (cmd == IRDMA_OP_STATS_ALLOCATE)
1915 wait = true;
1916 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1917 if (!cqp_request)
1918 return IRDMA_ERR_NO_MEMORY;
1919
1920 cqp_info = &cqp_request->info;
1921 memset(cqp_info, 0, sizeof(*cqp_info));
1922 cqp_info->cqp_cmd = cmd;
1923 cqp_info->post_sq = 1;
1924 cqp_info->in.u.stats_manage.info = *stats_info;
1925 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1926 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1927 status = irdma_handle_cqp_op(rf, cqp_request);
1928 if (wait)
1929 stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1930 irdma_put_cqp_request(iwcqp, cqp_request);
1931
1932 return status;
1933}
1934
1935/**
1936 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1937 * @dev: pointer to device info
1938 * @sc_ceq: pointer to ceq structure
1939 * @op: Create or Destroy
1940 */
1941enum irdma_status_code irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev,
1942 struct irdma_sc_ceq *sc_ceq, u8 op)
1943{
1944 struct irdma_cqp_request *cqp_request;
1945 struct cqp_cmds_info *cqp_info;
1946 struct irdma_pci_f *rf = dev_to_rf(dev);
1947 enum irdma_status_code status;
1948
1949 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1950 if (!cqp_request)
1951 return IRDMA_ERR_NO_MEMORY;
1952
1953 cqp_info = &cqp_request->info;
1954 cqp_info->post_sq = 1;
1955 cqp_info->cqp_cmd = op;
1956 cqp_info->in.u.ceq_create.ceq = sc_ceq;
1957 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1958
1959 status = irdma_handle_cqp_op(rf, cqp_request);
1960 irdma_put_cqp_request(&rf->cqp, cqp_request);
1961
1962 return status;
1963}
1964
1965/**
1966 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1967 * @dev: pointer to device info
1968 * @sc_aeq: pointer to aeq structure
1969 * @op: Create or Destroy
1970 */
1971enum irdma_status_code irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev,
1972 struct irdma_sc_aeq *sc_aeq, u8 op)
1973{
1974 struct irdma_cqp_request *cqp_request;
1975 struct cqp_cmds_info *cqp_info;
1976 struct irdma_pci_f *rf = dev_to_rf(dev);
1977 enum irdma_status_code status;
1978
1979 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1980 if (!cqp_request)
1981 return IRDMA_ERR_NO_MEMORY;
1982
1983 cqp_info = &cqp_request->info;
1984 cqp_info->post_sq = 1;
1985 cqp_info->cqp_cmd = op;
1986 cqp_info->in.u.aeq_create.aeq = sc_aeq;
1987 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1988
1989 status = irdma_handle_cqp_op(rf, cqp_request);
1990 irdma_put_cqp_request(&rf->cqp, cqp_request);
1991
1992 return status;
1993}
1994
1995/**
1996 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1997 * @dev: pointer to device structure
1998 * @cmd: Add, modify or delete
1999 * @node_info: pointer to ws node info
2000 */
2001enum irdma_status_code
2002irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
2003 struct irdma_ws_node_info *node_info)
2004{
2005 struct irdma_pci_f *rf = dev_to_rf(dev);
2006 struct irdma_cqp *iwcqp = &rf->cqp;
2007 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
2008 struct irdma_cqp_request *cqp_request;
2009 struct cqp_cmds_info *cqp_info;
2010 enum irdma_status_code status;
2011 bool poll;
2012
2013 if (!rf->sc_dev.ceq_valid)
2014 poll = true;
2015 else
2016 poll = false;
2017
2018 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
2019 if (!cqp_request)
2020 return IRDMA_ERR_NO_MEMORY;
2021
2022 cqp_info = &cqp_request->info;
2023 memset(cqp_info, 0, sizeof(*cqp_info));
2024 cqp_info->cqp_cmd = cmd;
2025 cqp_info->post_sq = 1;
2026 cqp_info->in.u.ws_node.info = *node_info;
2027 cqp_info->in.u.ws_node.cqp = cqp;
2028 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
2029 status = irdma_handle_cqp_op(rf, cqp_request);
2030 if (status)
2031 goto exit;
2032
2033 if (poll) {
2034 struct irdma_ccq_cqe_info compl_info;
2035
2036 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
2037 &compl_info);
2038 node_info->qs_handle = compl_info.op_ret_val;
2039 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
2040 compl_info.op_code, compl_info.op_ret_val);
2041 } else {
2042 node_info->qs_handle = cqp_request->compl_info.op_ret_val;
2043 }
2044
2045exit:
2046 irdma_put_cqp_request(&rf->cqp, cqp_request);
915cc7ac
MI		 2047
2048 return status;
2049}
2050
2051/**
2052 * irdma_ah_cqp_op - perform an AH cqp operation
2053 * @rf: RDMA PCI function
2054 * @sc_ah: address handle
2055 * @cmd: AH operation
2056 * @wait: wait if true
2057 * @callback_fcn: Callback function on CQP op completion
2058 * @cb_param: parameter for callback function
2059 *
2060 * returns errno
2061 */
2062int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
2063 bool wait,
2064 void (*callback_fcn)(struct irdma_cqp_request *),
2065 void *cb_param)
2066{
2067 struct irdma_cqp_request *cqp_request;
2068 struct cqp_cmds_info *cqp_info;
2069 enum irdma_status_code status;
2070
2071 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
2072 return -EINVAL;
2073
2074 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
2075 if (!cqp_request)
2076 return -ENOMEM;
2077
2078 cqp_info = &cqp_request->info;
2079 cqp_info->cqp_cmd = cmd;
2080 cqp_info->post_sq = 1;
2081 if (cmd == IRDMA_OP_AH_CREATE) {
2082 cqp_info->in.u.ah_create.info = sc_ah->ah_info;
2083 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
2084 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
2085 } else if (cmd == IRDMA_OP_AH_DESTROY) {
2086 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
2087 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
2088 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
2089 }
2090
2091 if (!wait) {
2092 cqp_request->callback_fcn = callback_fcn;
2093 cqp_request->param = cb_param;
2094 }
2095 status = irdma_handle_cqp_op(rf, cqp_request);
2096 irdma_put_cqp_request(&rf->cqp, cqp_request);
2097
2098 if (status)
2099 return -ENOMEM;
2100
2101 if (wait)
2102 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
2103
2104 return 0;
2105}
2106
2107/**
2108 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
2109 * @cqp_request: pointer to cqp_request of create AH
2110 */
2111static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
2112{
2113 struct irdma_sc_qp *qp = cqp_request->param;
2114 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
2115 unsigned long flags;
2116
2117 spin_lock_irqsave(&qp->pfpdu.lock, flags);
2118 if (!cqp_request->compl_info.op_ret_val) {
2119 sc_ah->ah_info.ah_valid = true;
2120 irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
2121 } else {
2122 sc_ah->ah_info.ah_valid = false;
2123 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
2124 }
2125 spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
2126}
2127
2128/**
2129 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
2130 * @cqp_request: pointer to cqp_request of create AH
2131 */
2132static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2133{
2134 struct irdma_cm_node *cm_node = cqp_request->param;
2135 struct irdma_sc_ah *sc_ah = cm_node->ah;
2136
2137 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2138 irdma_add_conn_est_qh(cm_node);
2139}
2140
2141/**
2142 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2143 * @dev: device pointer
2144 * @ah_info: Address handle info
2145 * @wait: When true will wait for operation to complete
2146 * @type: ILQ/IEQ
2147 * @cb_param: Callback param when not waiting
2148 * @ah_ret: Returned pointer to address handle if created
2149 *
2150 */
2151enum irdma_status_code irdma_puda_create_ah(struct irdma_sc_dev *dev,
2152 struct irdma_ah_info *ah_info,
2153 bool wait, enum puda_rsrc_type type,
2154 void *cb_param,
2155 struct irdma_sc_ah **ah_ret)
2156{
2157 struct irdma_sc_ah *ah;
2158 struct irdma_pci_f *rf = dev_to_rf(dev);
2159 int err;
2160
2161 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2162 *ah_ret = ah;
2163 if (!ah)
2164 return IRDMA_ERR_NO_MEMORY;
2165
2166 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
2167 &ah_info->ah_idx, &rf->next_ah);
2168 if (err)
2169 goto err_free;
2170
2171 ah->dev = dev;
2172 ah->ah_info = *ah_info;
2173
2174 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2175 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2176 irdma_ilq_ah_cb, cb_param);
2177 else
2178 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2179 irdma_ieq_ah_cb, cb_param);
2180
2181 if (err)
2182 goto error;
2183 return 0;
2184
2185error:
2186 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2187err_free:
2188 kfree(ah);
2189 *ah_ret = NULL;
2190 return IRDMA_ERR_NO_MEMORY;
2191}
2192
2193/**
2194 * irdma_puda_free_ah - free a puda address handle
2195 * @dev: device pointer
2196 * @ah: The address handle to free
2197 */
2198void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2199{
2200 struct irdma_pci_f *rf = dev_to_rf(dev);
2201
2202 if (!ah)
2203 return;
2204
2205 if (ah->ah_info.ah_valid) {
2206 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2207 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2208 }
2209
2210 kfree(ah);
2211}
2212
2213/**
2214 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2215 * @cqp_request: pointer to cqp_request of create AH
2216 */
2217void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2218{
2219 struct irdma_sc_ah *sc_ah = cqp_request->param;
2220
2221 if (!cqp_request->compl_info.op_ret_val)
2222 sc_ah->ah_info.ah_valid = true;
2223 else
2224 sc_ah->ah_info.ah_valid = false;
2225}
2226
2227/**
2228 * irdma_prm_add_pble_mem - add moemory to pble resources
2229 * @pprm: pble resource manager
2230 * @pchunk: chunk of memory to add
2231 */
2232enum irdma_status_code irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2233 struct irdma_chunk *pchunk)
2234{
2235 u64 sizeofbitmap;
2236
2237 if (pchunk->size & 0xfff)
2238 return IRDMA_ERR_PARAM;
2239
2240 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2241
2242 pchunk->bitmapmem.size = sizeofbitmap >> 3;
2243 pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_KERNEL);
2244
2245 if (!pchunk->bitmapmem.va)
2246 return IRDMA_ERR_NO_MEMORY;
2247
2248 pchunk->bitmapbuf = pchunk->bitmapmem.va;
2249 bitmap_zero(pchunk->bitmapbuf, sizeofbitmap);
2250
2251 pchunk->sizeofbitmap = sizeofbitmap;
2252 /* each pble is 8 bytes hence shift by 3 */
2253 pprm->total_pble_alloc += pchunk->size >> 3;
2254 pprm->free_pble_cnt += pchunk->size >> 3;
2255
2256 return 0;
2257}
2258
2259/**
2260 * irdma_prm_get_pbles - get pble's from prm
2261 * @pprm: pble resource manager
2262 * @chunkinfo: nformation about chunk where pble's were acquired
2263 * @mem_size: size of pble memory needed
2264 * @vaddr: returns virtual address of pble memory
2265 * @fpm_addr: returns fpm address of pble memory
2266 */
2267enum irdma_status_code
2268irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
1f700757 2269 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2db7b2ea 2270 u64 **vaddr, u64 *fpm_addr)
915cc7ac
MI		 2271{
2272 u64 bits_needed;
2273 u64 bit_idx = PBLE_INVALID_IDX;
2274 struct irdma_chunk *pchunk = NULL;
2275 struct list_head *chunk_entry = pprm->clist.next;
2276 u32 offset;
2277 unsigned long flags;
2db7b2ea 2278 *vaddr = NULL;
915cc7ac
MI		 2279 *fpm_addr = 0;
2280
1f700757 2281 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
915cc7ac
MI		 2282
2283 spin_lock_irqsave(&pprm->prm_lock, flags);
2284 while (chunk_entry != &pprm->clist) {
2285 pchunk = (struct irdma_chunk *)chunk_entry;
2286 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2287 pchunk->sizeofbitmap, 0,
2288 bits_needed, 0);
2289 if (bit_idx < pchunk->sizeofbitmap)
2290 break;
2291
2292 /* list.next used macro */
2293 chunk_entry = pchunk->list.next;
2294 }
2295
2296 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2297 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2298 return IRDMA_ERR_NO_MEMORY;
2299 }
2300
2301 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2302 offset = bit_idx << pprm->pble_shift;
2303 *vaddr = pchunk->vaddr + offset;
2304 *fpm_addr = pchunk->fpm_addr + offset;
2305
2306 chunkinfo->pchunk = pchunk;
2307 chunkinfo->bit_idx = bit_idx;
2308 chunkinfo->bits_used = bits_needed;
2309 /* 3 is sizeof pble divide */
2310 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2311 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2312
2313 return 0;
2314}
2315
2316/**
2317 * irdma_prm_return_pbles - return pbles back to prm
2318 * @pprm: pble resource manager
2319 * @chunkinfo: chunk where pble's were acquired and to be freed
2320 */
2321void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2322 struct irdma_pble_chunkinfo *chunkinfo)
2323{
2324 unsigned long flags;
2325
2326 spin_lock_irqsave(&pprm->prm_lock, flags);
2327 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2328 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2329 chunkinfo->bits_used);
2330 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2331}
2332
2333enum irdma_status_code irdma_map_vm_page_list(struct irdma_hw *hw, void *va,
2334 dma_addr_t *pg_dma, u32 pg_cnt)
2335{
2336 struct page *vm_page;
2337 int i;
2338 u8 *addr;
2339
2340 addr = (u8 *)(uintptr_t)va;
2341 for (i = 0; i < pg_cnt; i++) {
2342 vm_page = vmalloc_to_page(addr);
2343 if (!vm_page)
2344 goto err;
2345
2346 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2347 DMA_BIDIRECTIONAL);
2348 if (dma_mapping_error(hw->device, pg_dma[i]))
2349 goto err;
2350
2351 addr += PAGE_SIZE;
2352 }
2353
2354 return 0;
2355
2356err:
2357 irdma_unmap_vm_page_list(hw, pg_dma, i);
2358 return IRDMA_ERR_NO_MEMORY;
2359}
2360
2361void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2362{
2363 int i;
2364
2365 for (i = 0; i < pg_cnt; i++)
2366 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2367}
2368
2369/**
2370 * irdma_pble_free_paged_mem - free virtual paged memory
2371 * @chunk: chunk to free with paged memory
2372 */
2373void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2374{
2375 if (!chunk->pg_cnt)
2376 goto done;
2377
2378 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2379 chunk->pg_cnt);
2380
2381done:
2382 kfree(chunk->dmainfo.dmaaddrs);
2383 chunk->dmainfo.dmaaddrs = NULL;
2db7b2ea
SS		 2384 vfree(chunk->vaddr);
2385 chunk->vaddr = NULL;
915cc7ac
MI		 2386 chunk->type = 0;
2387}
2388
2389/**
2390 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2391 * @chunk: chunk to add for paged memory
2392 * @pg_cnt: number of pages needed
2393 */
2394enum irdma_status_code irdma_pble_get_paged_mem(struct irdma_chunk *chunk,
2395 u32 pg_cnt)
2396{
2397 u32 size;
2398 void *va;
2399
2400 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2401 if (!chunk->dmainfo.dmaaddrs)
2402 return IRDMA_ERR_NO_MEMORY;
2403
2404 size = PAGE_SIZE * pg_cnt;
2405 va = vmalloc(size);
2406 if (!va)
2407 goto err;
2408
2409 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2410 pg_cnt)) {
2411 vfree(va);
2412 goto err;
2413 }
2db7b2ea 2414 chunk->vaddr = va;
915cc7ac
MI		 2415 chunk->size = size;
2416 chunk->pg_cnt = pg_cnt;
2417 chunk->type = PBLE_SD_PAGED;
2418
2419 return 0;
2420err:
2421 kfree(chunk->dmainfo.dmaaddrs);
2422 chunk->dmainfo.dmaaddrs = NULL;
2423
2424 return IRDMA_ERR_NO_MEMORY;
2425}
2426
2427/**
2428 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2429 * @dev: device pointer
2430 */
2431u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2432{
2433 struct irdma_pci_f *rf = dev_to_rf(dev);
2434 u32 next = 1;
2435 u32 node_id;
2436
2437 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2438 &node_id, &next))
2439 return IRDMA_WS_NODE_INVALID;
2440
2441 return (u16)node_id;
2442}
2443
2444/**
2445 * irdma_free_ws_node_id - Free a tx scheduler node ID
2446 * @dev: device pointer
2447 * @node_id: Work scheduler node ID
2448 */
2449void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2450{
2451 struct irdma_pci_f *rf = dev_to_rf(dev);
2452
2453 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2454}
2455
2456/**
2457 * irdma_modify_qp_to_err - Modify a QP to error
2458 * @sc_qp: qp structure
2459 */
2460void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2461{
2462 struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2463 struct ib_qp_attr attr;
2464
5b1e985f 2465 if (qp->iwdev->rf->reset)
915cc7ac
MI		 2466 return;
2467 attr.qp_state = IB_QPS_ERR;
2468
2469 if (rdma_protocol_roce(qp->ibqp.device, 1))
2470 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2471 else
2472 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2473}
2474
2475void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2476{
2477 struct ib_event ibevent;
2478
2479 if (!iwqp->ibqp.event_handler)
2480 return;
2481
2482 switch (event) {
2483 case IRDMA_QP_EVENT_CATASTROPHIC:
2484 ibevent.event = IB_EVENT_QP_FATAL;
2485 break;
2486 case IRDMA_QP_EVENT_ACCESS_ERR:
2487 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2488 break;
2489 }
2490 ibevent.device = iwqp->ibqp.device;
2491 ibevent.element.qp = &iwqp->ibqp;
2492 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2493}
Linux 6.x block layer and io_uring tree(s)