projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/hns: Fixs hw access invalid dma memory error
[linux-2.6-block.git] / drivers / infiniband / hw / hns / hns_roce_hw_v2.c
CommitLineData
dd74282d
WHX		 1/*
2 * Copyright (c) 2016-2017 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/acpi.h>
34#include <linux/etherdevice.h>
35#include <linux/interrupt.h>
36#include <linux/kernel.h>
0b25c9cc 37#include <linux/types.h>
d4994d2f 38#include <net/addrconf.h>
610b8967 39#include <rdma/ib_addr.h>
a70c0739 40#include <rdma/ib_cache.h>
dd74282d 41#include <rdma/ib_umem.h>
bdeacabd 42#include <rdma/uverbs_ioctl.h>
dd74282d
WHX		 43
44#include "hnae3.h"
45#include "hns_roce_common.h"
46#include "hns_roce_device.h"
47#include "hns_roce_cmd.h"
48#include "hns_roce_hem.h"
a04ff739 49#include "hns_roce_hw_v2.h"
dd74282d 50
2d407888
WHX		 51static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
52 struct ib_sge *sg)
53{
54 dseg->lkey = cpu_to_le32(sg->lkey);
55 dseg->addr = cpu_to_le64(sg->addr);
56 dseg->len = cpu_to_le32(sg->length);
57}
58
68a997c5
YL		 59static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
60 struct hns_roce_wqe_frmr_seg *fseg,
61 const struct ib_reg_wr *wr)
62{
63 struct hns_roce_mr *mr = to_hr_mr(wr->mr);
64
65 /* use ib_access_flags */
66 roce_set_bit(rc_sq_wqe->byte_4,
67 V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
68 wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
69 roce_set_bit(rc_sq_wqe->byte_4,
70 V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
71 wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
72 roce_set_bit(rc_sq_wqe->byte_4,
73 V2_RC_FRMR_WQE_BYTE_4_RR_S,
74 wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
75 roce_set_bit(rc_sq_wqe->byte_4,
76 V2_RC_FRMR_WQE_BYTE_4_RW_S,
77 wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
78 roce_set_bit(rc_sq_wqe->byte_4,
79 V2_RC_FRMR_WQE_BYTE_4_LW_S,
80 wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
81
82 /* Data structure reuse may lead to confusion */
83 rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
84 rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
85
86 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
87 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
88 rc_sq_wqe->rkey = cpu_to_le32(wr->key);
89 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
90
91 fseg->pbl_size = cpu_to_le32(mr->pbl_size);
92 roce_set_field(fseg->mode_buf_pg_sz,
93 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
94 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
95 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
96 roce_set_bit(fseg->mode_buf_pg_sz,
97 V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
98}
99
384f8818
LO		 100static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,
101 const struct ib_atomic_wr *wr)
102{
103 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
104 aseg->fetchadd_swap_data = cpu_to_le64(wr->swap);
105 aseg->cmp_data = cpu_to_le64(wr->compare_add);
106 } else {
107 aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add);
108 aseg->cmp_data = 0;
109 }
110}
111
f696bf6d 112static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
0b25c9cc
WHX		 113 unsigned int *sge_ind)
114{
115 struct hns_roce_v2_wqe_data_seg *dseg;
116 struct ib_sge *sg;
117 int num_in_wqe = 0;
118 int extend_sge_num;
119 int fi_sge_num;
120 int se_sge_num;
121 int shift;
122 int i;
123
124 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
125 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
126 extend_sge_num = wr->num_sge - num_in_wqe;
127 sg = wr->sg_list + num_in_wqe;
128 shift = qp->hr_buf.page_shift;
129
130 /*
131 * Check whether wr->num_sge sges are in the same page. If not, we
132 * should calculate how many sges in the first page and the second
133 * page.
134 */
135 dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
136 fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
137 (uintptr_t)dseg) /
138 sizeof(struct hns_roce_v2_wqe_data_seg);
139 if (extend_sge_num > fi_sge_num) {
140 se_sge_num = extend_sge_num - fi_sge_num;
141 for (i = 0; i < fi_sge_num; i++) {
142 set_data_seg_v2(dseg++, sg + i);
143 (*sge_ind)++;
144 }
145 dseg = get_send_extend_sge(qp,
146 (*sge_ind) & (qp->sge.sge_cnt - 1));
147 for (i = 0; i < se_sge_num; i++) {
148 set_data_seg_v2(dseg++, sg + fi_sge_num + i);
149 (*sge_ind)++;
150 }
151 } else {
152 for (i = 0; i < extend_sge_num; i++) {
153 set_data_seg_v2(dseg++, sg + i);
154 (*sge_ind)++;
155 }
156 }
157}
158
f696bf6d 159static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
7bdee415 160 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
161 void *wqe, unsigned int *sge_ind,
d34ac5cd 162 const struct ib_send_wr **bad_wr)
7bdee415 163{
164 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
165 struct hns_roce_v2_wqe_data_seg *dseg = wqe;
166 struct hns_roce_qp *qp = to_hr_qp(ibqp);
167 int i;
168
169 if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
8b9b8d14 170 if (le32_to_cpu(rc_sq_wqe->msg_len) >
171 hr_dev->caps.max_sq_inline) {
7bdee415 172 *bad_wr = wr;
173 dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
174 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
175 return -EINVAL;
176 }
177
328d405b 178 if (wr->opcode == IB_WR_RDMA_READ) {
c80e0661 179 *bad_wr = wr;
328d405b 180 dev_err(hr_dev->dev, "Not support inline data!\n");
181 return -EINVAL;
182 }
183
7bdee415 184 for (i = 0; i < wr->num_sge; i++) {
185 memcpy(wqe, ((void *)wr->sg_list[i].addr),
186 wr->sg_list[i].length);
187 wqe += wr->sg_list[i].length;
188 }
189
190 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
191 1);
192 } else {
0b25c9cc 193 if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
7bdee415 194 for (i = 0; i < wr->num_sge; i++) {
195 if (likely(wr->sg_list[i].length)) {
196 set_data_seg_v2(dseg, wr->sg_list + i);
197 dseg++;
198 }
199 }
200 } else {
201 roce_set_field(rc_sq_wqe->byte_20,
202 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
203 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
204 (*sge_ind) & (qp->sge.sge_cnt - 1));
205
0b25c9cc 206 for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
7bdee415 207 if (likely(wr->sg_list[i].length)) {
208 set_data_seg_v2(dseg, wr->sg_list + i);
209 dseg++;
210 }
211 }
212
0b25c9cc 213 set_extend_sge(qp, wr, sge_ind);
7bdee415 214 }
215
216 roce_set_field(rc_sq_wqe->byte_16,
217 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
218 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
219 }
220
221 return 0;
222}
223
0425e3e6
YL		 224static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
225 const struct ib_qp_attr *attr,
226 int attr_mask, enum ib_qp_state cur_state,
227 enum ib_qp_state new_state);
228
d34ac5cd
BVA		 229static int hns_roce_v2_post_send(struct ib_qp *ibqp,
230 const struct ib_send_wr *wr,
231 const struct ib_send_wr **bad_wr)
2d407888
WHX		 232{
233 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
7bdee415 234 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
235 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
2d407888
WHX		 236 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
237 struct hns_roce_qp *qp = to_hr_qp(ibqp);
68a997c5 238 struct hns_roce_wqe_frmr_seg *fseg;
2d407888
WHX		 239 struct device *dev = hr_dev->dev;
240 struct hns_roce_v2_db sq_db;
0425e3e6 241 struct ib_qp_attr attr;
2d407888 242 unsigned int sge_ind = 0;
e8d18533 243 unsigned int owner_bit;
2d407888
WHX		 244 unsigned long flags;
245 unsigned int ind;
246 void *wqe = NULL;
7bdee415 247 bool loopback;
0425e3e6 248 int attr_mask;
55ba49cb 249 u32 tmp_len;
2d407888 250 int ret = 0;
b9c1ea40 251 u32 hr_op;
7bdee415 252 u8 *smac;
2d407888
WHX		 253 int nreq;
254 int i;
255
7bdee415 256 if (unlikely(ibqp->qp_type != IB_QPT_RC &&
257 ibqp->qp_type != IB_QPT_GSI &&
258 ibqp->qp_type != IB_QPT_UD)) {
2d407888 259 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
137ae320 260 *bad_wr = wr;
2d407888
WHX		 261 return -EOPNOTSUPP;
262 }
263
10bd2ade
YL		 264 if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
265 qp->state == IB_QPS_RTR)) {
2d407888
WHX		 266 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
267 *bad_wr = wr;
268 return -EINVAL;
269 }
270
271 spin_lock_irqsave(&qp->sq.lock, flags);
272 ind = qp->sq_next_wqe;
273 sge_ind = qp->next_sge;
274
275 for (nreq = 0; wr; ++nreq, wr = wr->next) {
276 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
277 ret = -ENOMEM;
278 *bad_wr = wr;
279 goto out;
280 }
281
282 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
283 dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
284 wr->num_sge, qp->sq.max_gs);
285 ret = -EINVAL;
286 *bad_wr = wr;
287 goto out;
288 }
289
290 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
291 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
292 wr->wr_id;
293
634f6390 294 owner_bit =
295 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
55ba49cb 296 tmp_len = 0;
2d407888 297
7bdee415 298 /* Corresponding to the QP type, wqe process separately */
299 if (ibqp->qp_type == IB_QPT_GSI) {
300 ud_sq_wqe = wqe;
301 memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
302
303 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
304 V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
305 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
306 V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
307 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
308 V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
309 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
310 V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
311 roce_set_field(ud_sq_wqe->byte_48,
312 V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
313 V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
314 ah->av.mac[4]);
315 roce_set_field(ud_sq_wqe->byte_48,
316 V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
317 V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
318 ah->av.mac[5]);
319
320 /* MAC loopback */
321 smac = (u8 *)hr_dev->dev_addr[qp->port];
322 loopback = ether_addr_equal_unaligned(ah->av.mac,
323 smac) ? 1 : 0;
324
325 roce_set_bit(ud_sq_wqe->byte_40,
326 V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
327
328 roce_set_field(ud_sq_wqe->byte_4,
329 V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
330 V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
331 HNS_ROCE_V2_WQE_OP_SEND);
2d407888 332
7bdee415 333 for (i = 0; i < wr->num_sge; i++)
8b9b8d14 334 tmp_len += wr->sg_list[i].length;
492b2bd0 335
8b9b8d14 336 ud_sq_wqe->msg_len =
337 cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
338
339 switch (wr->opcode) {
340 case IB_WR_SEND_WITH_IMM:
341 case IB_WR_RDMA_WRITE_WITH_IMM:
0c4a0e29
LO		 342 ud_sq_wqe->immtdata =
343 cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
8b9b8d14 344 break;
345 default:
346 ud_sq_wqe->immtdata = 0;
347 break;
348 }
651487c2 349
7bdee415 350 /* Set sig attr */
351 roce_set_bit(ud_sq_wqe->byte_4,
352 V2_UD_SEND_WQE_BYTE_4_CQE_S,
353 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
a49d761f 354
7bdee415 355 /* Set se attr */
356 roce_set_bit(ud_sq_wqe->byte_4,
357 V2_UD_SEND_WQE_BYTE_4_SE_S,
358 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
e8d18533 359
7bdee415 360 roce_set_bit(ud_sq_wqe->byte_4,
361 V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
362
363 roce_set_field(ud_sq_wqe->byte_16,
364 V2_UD_SEND_WQE_BYTE_16_PD_M,
365 V2_UD_SEND_WQE_BYTE_16_PD_S,
366 to_hr_pd(ibqp->pd)->pdn);
367
368 roce_set_field(ud_sq_wqe->byte_16,
369 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
370 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
371 wr->num_sge);
372
373 roce_set_field(ud_sq_wqe->byte_20,
374 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
375 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
376 sge_ind & (qp->sge.sge_cnt - 1));
377
378 roce_set_field(ud_sq_wqe->byte_24,
379 V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
380 V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
381 ud_sq_wqe->qkey =
8b9b8d14 382 cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
383 qp->qkey : ud_wr(wr)->remote_qkey);
7bdee415 384 roce_set_field(ud_sq_wqe->byte_32,
385 V2_UD_SEND_WQE_BYTE_32_DQPN_M,
386 V2_UD_SEND_WQE_BYTE_32_DQPN_S,
387 ud_wr(wr)->remote_qpn);
388
389 roce_set_field(ud_sq_wqe->byte_36,
390 V2_UD_SEND_WQE_BYTE_36_VLAN_M,
391 V2_UD_SEND_WQE_BYTE_36_VLAN_S,
8b9b8d14 392 le16_to_cpu(ah->av.vlan));
7bdee415 393 roce_set_field(ud_sq_wqe->byte_36,
394 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
395 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
396 ah->av.hop_limit);
397 roce_set_field(ud_sq_wqe->byte_36,
398 V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
399 V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
cdfa4ad5
LO		 400 ah->av.sl_tclass_flowlabel >>
401 HNS_ROCE_TCLASS_SHIFT);
7bdee415 402 roce_set_field(ud_sq_wqe->byte_40,
403 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
cdfa4ad5
LO		 404 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
405 ah->av.sl_tclass_flowlabel &
406 HNS_ROCE_FLOW_LABEL_MASK);
7bdee415 407 roce_set_field(ud_sq_wqe->byte_40,
408 V2_UD_SEND_WQE_BYTE_40_SL_M,
409 V2_UD_SEND_WQE_BYTE_40_SL_S,
8b9b8d14 410 le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
411 HNS_ROCE_SL_SHIFT);
7bdee415 412 roce_set_field(ud_sq_wqe->byte_40,
413 V2_UD_SEND_WQE_BYTE_40_PORTN_M,
414 V2_UD_SEND_WQE_BYTE_40_PORTN_S,
415 qp->port);
416
8320deb8
LO		 417 roce_set_bit(ud_sq_wqe->byte_40,
418 V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
419 ah->av.vlan_en ? 1 : 0);
7bdee415 420 roce_set_field(ud_sq_wqe->byte_48,
421 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
422 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
423 hns_get_gid_index(hr_dev, qp->phy_port,
424 ah->av.gid_index));
425
426 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
427 GID_LEN_V2);
428
0b25c9cc 429 set_extend_sge(qp, wr, &sge_ind);
7bdee415 430 ind++;
431 } else if (ibqp->qp_type == IB_QPT_RC) {
432 rc_sq_wqe = wqe;
433 memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
434 for (i = 0; i < wr->num_sge; i++)
8b9b8d14 435 tmp_len += wr->sg_list[i].length;
436
437 rc_sq_wqe->msg_len =
438 cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
7bdee415 439
8b9b8d14 440 switch (wr->opcode) {
441 case IB_WR_SEND_WITH_IMM:
442 case IB_WR_RDMA_WRITE_WITH_IMM:
0c4a0e29
LO		 443 rc_sq_wqe->immtdata =
444 cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
8b9b8d14 445 break;
446 case IB_WR_SEND_WITH_INV:
447 rc_sq_wqe->inv_key =
448 cpu_to_le32(wr->ex.invalidate_rkey);
449 break;
450 default:
451 rc_sq_wqe->immtdata = 0;
452 break;
453 }
7bdee415 454
455 roce_set_bit(rc_sq_wqe->byte_4,
456 V2_RC_SEND_WQE_BYTE_4_FENCE_S,
457 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
458
459 roce_set_bit(rc_sq_wqe->byte_4,
460 V2_RC_SEND_WQE_BYTE_4_SE_S,
461 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
462
463 roce_set_bit(rc_sq_wqe->byte_4,
464 V2_RC_SEND_WQE_BYTE_4_CQE_S,
465 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
466
467 roce_set_bit(rc_sq_wqe->byte_4,
468 V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
469
384f8818 470 wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
7bdee415 471 switch (wr->opcode) {
472 case IB_WR_RDMA_READ:
b9c1ea40 473 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;
7bdee415 474 rc_sq_wqe->rkey =
475 cpu_to_le32(rdma_wr(wr)->rkey);
476 rc_sq_wqe->va =
477 cpu_to_le64(rdma_wr(wr)->remote_addr);
478 break;
479 case IB_WR_RDMA_WRITE:
b9c1ea40 480 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;
7bdee415 481 rc_sq_wqe->rkey =
482 cpu_to_le32(rdma_wr(wr)->rkey);
483 rc_sq_wqe->va =
484 cpu_to_le64(rdma_wr(wr)->remote_addr);
485 break;
486 case IB_WR_RDMA_WRITE_WITH_IMM:
b9c1ea40 487 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;
7bdee415 488 rc_sq_wqe->rkey =
489 cpu_to_le32(rdma_wr(wr)->rkey);
490 rc_sq_wqe->va =
491 cpu_to_le64(rdma_wr(wr)->remote_addr);
492 break;
493 case IB_WR_SEND:
b9c1ea40 494 hr_op = HNS_ROCE_V2_WQE_OP_SEND;
7bdee415 495 break;
496 case IB_WR_SEND_WITH_INV:
b9c1ea40 497 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;
7bdee415 498 break;
499 case IB_WR_SEND_WITH_IMM:
b9c1ea40 500 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;
7bdee415 501 break;
502 case IB_WR_LOCAL_INV:
b9c1ea40 503 hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;
e93df010
LO		 504 roce_set_bit(rc_sq_wqe->byte_4,
505 V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
506 rc_sq_wqe->inv_key =
507 cpu_to_le32(wr->ex.invalidate_rkey);
7bdee415 508 break;
68a997c5
YL		 509 case IB_WR_REG_MR:
510 hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;
511 fseg = wqe;
512 set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));
513 break;
7bdee415 514 case IB_WR_ATOMIC_CMP_AND_SWP:
b9c1ea40 515 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;
384f8818
LO		 516 rc_sq_wqe->rkey =
517 cpu_to_le32(atomic_wr(wr)->rkey);
518 rc_sq_wqe->va =
d9581bf3 519 cpu_to_le64(atomic_wr(wr)->remote_addr);
7bdee415 520 break;
521 case IB_WR_ATOMIC_FETCH_AND_ADD:
b9c1ea40 522 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;
384f8818
LO		 523 rc_sq_wqe->rkey =
524 cpu_to_le32(atomic_wr(wr)->rkey);
525 rc_sq_wqe->va =
d9581bf3 526 cpu_to_le64(atomic_wr(wr)->remote_addr);
7bdee415 527 break;
528 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
b9c1ea40
LO		 529 hr_op =
530 HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;
7bdee415 531 break;
532 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
b9c1ea40
LO		 533 hr_op =
534 HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;
7bdee415 535 break;
536 default:
b9c1ea40 537 hr_op = HNS_ROCE_V2_WQE_OP_MASK;
7bdee415 538 break;
2d407888
WHX		 539 }
540
b9c1ea40
LO		 541 roce_set_field(rc_sq_wqe->byte_4,
542 V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
543 V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);
2d407888 544
d9581bf3
LO		 545 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
546 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
547 struct hns_roce_v2_wqe_data_seg *dseg;
548
549 dseg = wqe;
550 set_data_seg_v2(dseg, wr->sg_list);
551 wqe += sizeof(struct hns_roce_v2_wqe_data_seg);
552 set_atomic_seg(wqe, atomic_wr(wr));
553 roce_set_field(rc_sq_wqe->byte_16,
554 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
555 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
556 wr->num_sge);
68a997c5 557 } else if (wr->opcode != IB_WR_REG_MR) {
d9581bf3
LO		 558 ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,
559 wqe, &sge_ind, bad_wr);
560 if (ret)
561 goto out;
562 }
563
7bdee415 564 ind++;
2d407888 565 } else {
7bdee415 566 dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
567 spin_unlock_irqrestore(&qp->sq.lock, flags);
137ae320 568 *bad_wr = wr;
7bdee415 569 return -EOPNOTSUPP;
2d407888 570 }
2d407888
WHX		 571 }
572
573out:
574 if (likely(nreq)) {
575 qp->sq.head += nreq;
576 /* Memory barrier */
577 wmb();
578
579 sq_db.byte_4 = 0;
580 sq_db.parameter = 0;
581
582 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
583 V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
584 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
585 V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
cc3391cb 586 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
587 V2_DB_PARAMETER_IDX_S,
2d407888
WHX		 588 qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
589 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
590 V2_DB_PARAMETER_SL_S, qp->sl);
591
d3743fa9 592 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
2d407888
WHX		 593
594 qp->sq_next_wqe = ind;
595 qp->next_sge = sge_ind;
0425e3e6
YL		 596
597 if (qp->state == IB_QPS_ERR) {
598 attr_mask = IB_QP_STATE;
599 attr.qp_state = IB_QPS_ERR;
600
601 ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
602 qp->state, IB_QPS_ERR);
603 if (ret) {
604 spin_unlock_irqrestore(&qp->sq.lock, flags);
605 *bad_wr = wr;
606 return ret;
607 }
608 }
2d407888
WHX		 609 }
610
611 spin_unlock_irqrestore(&qp->sq.lock, flags);
612
613 return ret;
614}
615
d34ac5cd
BVA		 616static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
617 const struct ib_recv_wr *wr,
618 const struct ib_recv_wr **bad_wr)
2d407888
WHX		 619{
620 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
621 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
622 struct hns_roce_v2_wqe_data_seg *dseg;
0009c2db 623 struct hns_roce_rinl_sge *sge_list;
2d407888 624 struct device *dev = hr_dev->dev;
0425e3e6 625 struct ib_qp_attr attr;
2d407888
WHX		 626 unsigned long flags;
627 void *wqe = NULL;
0425e3e6 628 int attr_mask;
2d407888
WHX		 629 int ret = 0;
630 int nreq;
631 int ind;
632 int i;
633
634 spin_lock_irqsave(&hr_qp->rq.lock, flags);
635 ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
636
ced07769 637 if (hr_qp->state == IB_QPS_RESET) {
2d407888
WHX		 638 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
639 *bad_wr = wr;
640 return -EINVAL;
641 }
642
643 for (nreq = 0; wr; ++nreq, wr = wr->next) {
644 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
645 hr_qp->ibqp.recv_cq)) {
646 ret = -ENOMEM;
647 *bad_wr = wr;
648 goto out;
649 }
650
651 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
652 dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
653 wr->num_sge, hr_qp->rq.max_gs);
654 ret = -EINVAL;
655 *bad_wr = wr;
656 goto out;
657 }
658
659 wqe = get_recv_wqe(hr_qp, ind);
660 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
661 for (i = 0; i < wr->num_sge; i++) {
662 if (!wr->sg_list[i].length)
663 continue;
664 set_data_seg_v2(dseg, wr->sg_list + i);
665 dseg++;
666 }
667
668 if (i < hr_qp->rq.max_gs) {
778cc5a8 669 dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
670 dseg->addr = 0;
2d407888
WHX		 671 }
672
0009c2db 673 /* rq support inline data */
ecaaf1e2 674 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
675 sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
676 hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
677 (u32)wr->num_sge;
678 for (i = 0; i < wr->num_sge; i++) {
679 sge_list[i].addr =
680 (void *)(u64)wr->sg_list[i].addr;
681 sge_list[i].len = wr->sg_list[i].length;
682 }
0009c2db 683 }
684
2d407888
WHX		 685 hr_qp->rq.wrid[ind] = wr->wr_id;
686
687 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
688 }
689
690out:
691 if (likely(nreq)) {
692 hr_qp->rq.head += nreq;
693 /* Memory barrier */
694 wmb();
695
472bc0fb 696 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
0425e3e6
YL		 697
698 if (hr_qp->state == IB_QPS_ERR) {
699 attr_mask = IB_QP_STATE;
700 attr.qp_state = IB_QPS_ERR;
701
702 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
703 attr_mask, hr_qp->state,
704 IB_QPS_ERR);
705 if (ret) {
706 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
707 *bad_wr = wr;
708 return ret;
709 }
710 }
2d407888
WHX		 711 }
712 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
713
714 return ret;
715}
716
6a04aed6
WHX		 717static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
718 unsigned long instance_stage,
719 unsigned long reset_stage)
720{
721 /* When hardware reset has been completed once or more, we should stop
d3743fa9 722 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
6a04aed6
WHX		 723 * function, we should exit with error. If now at HNAE3_INIT_CLIENT
724 * stage of soft reset process, we should exit with error, and then
725 * HNAE3_INIT_CLIENT related process can rollback the operation like
726 * notifing hardware to free resources, HNAE3_INIT_CLIENT related
727 * process will exit with error to notify NIC driver to reschedule soft
728 * reset process once again.
729 */
730 hr_dev->is_reset = true;
d3743fa9 731 hr_dev->dis_db = true;
6a04aed6
WHX		 732
733 if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
734 instance_stage == HNS_ROCE_STATE_INIT)
735 return CMD_RST_PRC_EBUSY;
736
737 return CMD_RST_PRC_SUCCESS;
738}
739
740static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
741 unsigned long instance_stage,
742 unsigned long reset_stage)
743{
744 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
745 struct hnae3_handle *handle = priv->handle;
746 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
747
d3743fa9
WHX		 748 /* When hardware reset is detected, we should stop sending mailbox&cmq&
749 * doorbell to hardware. If now in .init_instance() function, we should
6a04aed6
WHX		 750 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
751 * process, we should exit with error, and then HNAE3_INIT_CLIENT
752 * related process can rollback the operation like notifing hardware to
753 * free resources, HNAE3_INIT_CLIENT related process will exit with
754 * error to notify NIC driver to reschedule soft reset process once
755 * again.
756 */
d3743fa9 757 hr_dev->dis_db = true;
6a04aed6
WHX		 758 if (!ops->get_hw_reset_stat(handle))
759 hr_dev->is_reset = true;
760
761 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
762 instance_stage == HNS_ROCE_STATE_INIT)
763 return CMD_RST_PRC_EBUSY;
764
765 return CMD_RST_PRC_SUCCESS;
766}
767
768static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
769{
770 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
771 struct hnae3_handle *handle = priv->handle;
772 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
773
774 /* When software reset is detected at .init_instance() function, we
d3743fa9
WHX		 775 * should stop sending mailbox&cmq&doorbell to hardware, and exit
776 * with error.
6a04aed6 777 */
d3743fa9 778 hr_dev->dis_db = true;
6a04aed6
WHX		 779 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
780 hr_dev->is_reset = true;
781
782 return CMD_RST_PRC_EBUSY;
783}
784
785static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
786{
787 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
788 struct hnae3_handle *handle = priv->handle;
789 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
790 unsigned long instance_stage; /* the current instance stage */
791 unsigned long reset_stage; /* the current reset stage */
792 unsigned long reset_cnt;
793 bool sw_resetting;
794 bool hw_resetting;
795
796 if (hr_dev->is_reset)
797 return CMD_RST_PRC_SUCCESS;
798
799 /* Get information about reset from NIC driver or RoCE driver itself,
800 * the meaning of the following variables from NIC driver are described
801 * as below:
802 * reset_cnt -- The count value of completed hardware reset.
803 * hw_resetting -- Whether hardware device is resetting now.
804 * sw_resetting -- Whether NIC's software reset process is running now.
805 */
806 instance_stage = handle->rinfo.instance_state;
807 reset_stage = handle->rinfo.reset_state;
808 reset_cnt = ops->ae_dev_reset_cnt(handle);
809 hw_resetting = ops->get_hw_reset_stat(handle);
810 sw_resetting = ops->ae_dev_resetting(handle);
811
812 if (reset_cnt != hr_dev->reset_cnt)
813 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
814 reset_stage);
815 else if (hw_resetting)
816 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
817 reset_stage);
818 else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
819 return hns_roce_v2_cmd_sw_resetting(hr_dev);
820
821 return 0;
822}
823
a04ff739
WHX		 824static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
825{
826 int ntu = ring->next_to_use;
827 int ntc = ring->next_to_clean;
828 int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
829
830 return ring->desc_num - used - 1;
831}
832
833static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
834 struct hns_roce_v2_cmq_ring *ring)
835{
836 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
837
838 ring->desc = kzalloc(size, GFP_KERNEL);
839 if (!ring->desc)
840 return -ENOMEM;
841
842 ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
843 DMA_BIDIRECTIONAL);
844 if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
845 ring->desc_dma_addr = 0;
846 kfree(ring->desc);
847 ring->desc = NULL;
848 return -ENOMEM;
849 }
850
851 return 0;
852}
853
854static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
855 struct hns_roce_v2_cmq_ring *ring)
856{
857 dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
858 ring->desc_num * sizeof(struct hns_roce_cmq_desc),
859 DMA_BIDIRECTIONAL);
90e7a4d5 860
861 ring->desc_dma_addr = 0;
a04ff739
WHX		 862 kfree(ring->desc);
863}
864
865static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
866{
867 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
868 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
869 &priv->cmq.csq : &priv->cmq.crq;
870
871 ring->flag = ring_type;
872 ring->next_to_clean = 0;
873 ring->next_to_use = 0;
874
875 return hns_roce_alloc_cmq_desc(hr_dev, ring);
876}
877
878static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
879{
880 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
881 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
882 &priv->cmq.csq : &priv->cmq.crq;
883 dma_addr_t dma = ring->desc_dma_addr;
884
885 if (ring_type == TYPE_CSQ) {
886 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
887 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
888 upper_32_bits(dma));
889 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
890 (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
891 HNS_ROCE_CMQ_ENABLE);
892 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
893 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
894 } else {
895 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
896 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
897 upper_32_bits(dma));
898 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
899 (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
900 HNS_ROCE_CMQ_ENABLE);
901 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
902 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
903 }
904}
905
906static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
907{
908 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
909 int ret;
910
911 /* Setup the queue entries for command queue */
426c4146
LO		 912 priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
913 priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
a04ff739
WHX		 914
915 /* Setup the lock for command queue */
916 spin_lock_init(&priv->cmq.csq.lock);
917 spin_lock_init(&priv->cmq.crq.lock);
918
919 /* Setup Tx write back timeout */
920 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
921
922 /* Init CSQ */
923 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
924 if (ret) {
925 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
926 return ret;
927 }
928
929 /* Init CRQ */
930 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
931 if (ret) {
932 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
933 goto err_crq;
934 }
935
936 /* Init CSQ REG */
937 hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
938
939 /* Init CRQ REG */
940 hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
941
942 return 0;
943
944err_crq:
945 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
946
947 return ret;
948}
949
950static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
951{
952 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
953
954 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
955 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
956}
957
281d0ccf
CIK		 958static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
959 enum hns_roce_opcode_type opcode,
960 bool is_read)
a04ff739
WHX		 961{
962 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
963 desc->opcode = cpu_to_le16(opcode);
964 desc->flag =
965 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
966 if (is_read)
967 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
968 else
969 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
970}
971
972static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
973{
974 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
975 u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
976
977 return head == priv->cmq.csq.next_to_use;
978}
979
980static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
981{
982 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
983 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
984 struct hns_roce_cmq_desc *desc;
985 u16 ntc = csq->next_to_clean;
986 u32 head;
987 int clean = 0;
988
989 desc = &csq->desc[ntc];
990 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
991 while (head != ntc) {
992 memset(desc, 0, sizeof(*desc));
993 ntc++;
994 if (ntc == csq->desc_num)
995 ntc = 0;
996 desc = &csq->desc[ntc];
997 clean++;
998 }
999 csq->next_to_clean = ntc;
1000
1001 return clean;
1002}
1003
6a04aed6
WHX		 1004static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1005 struct hns_roce_cmq_desc *desc, int num)
a04ff739
WHX		 1006{
1007 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1008 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1009 struct hns_roce_cmq_desc *desc_to_use;
1010 bool complete = false;
1011 u32 timeout = 0;
1012 int handle = 0;
1013 u16 desc_ret;
1014 int ret = 0;
1015 int ntc;
1016
1017 spin_lock_bh(&csq->lock);
1018
1019 if (num > hns_roce_cmq_space(csq)) {
1020 spin_unlock_bh(&csq->lock);
1021 return -EBUSY;
1022 }
1023
1024 /*
1025 * Record the location of desc in the cmq for this time
1026 * which will be use for hardware to write back
1027 */
1028 ntc = csq->next_to_use;
1029
1030 while (handle < num) {
1031 desc_to_use = &csq->desc[csq->next_to_use];
1032 *desc_to_use = desc[handle];
1033 dev_dbg(hr_dev->dev, "set cmq desc:\n");
1034 csq->next_to_use++;
1035 if (csq->next_to_use == csq->desc_num)
1036 csq->next_to_use = 0;
1037 handle++;
1038 }
1039
1040 /* Write to hardware */
1041 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
1042
1043 /*
1044 * If the command is sync, wait for the firmware to write back,
1045 * if multi descriptors to be sent, use the first one to check
1046 */
1047 if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
1048 do {
1049 if (hns_roce_cmq_csq_done(hr_dev))
1050 break;
988e175b 1051 udelay(1);
a04ff739
WHX		 1052 timeout++;
1053 } while (timeout < priv->cmq.tx_timeout);
1054 }
1055
1056 if (hns_roce_cmq_csq_done(hr_dev)) {
1057 complete = true;
1058 handle = 0;
1059 while (handle < num) {
1060 /* get the result of hardware write back */
1061 desc_to_use = &csq->desc[ntc];
1062 desc[handle] = *desc_to_use;
1063 dev_dbg(hr_dev->dev, "Get cmq desc:\n");
1064 desc_ret = desc[handle].retval;
1065 if (desc_ret == CMD_EXEC_SUCCESS)
1066 ret = 0;
1067 else
1068 ret = -EIO;
1069 priv->cmq.last_status = desc_ret;
1070 ntc++;
1071 handle++;
1072 if (ntc == csq->desc_num)
1073 ntc = 0;
1074 }
1075 }
1076
1077 if (!complete)
1078 ret = -EAGAIN;
1079
1080 /* clean the command send queue */
1081 handle = hns_roce_cmq_csq_clean(hr_dev);
1082 if (handle != num)
1083 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
1084 handle, num);
1085
1086 spin_unlock_bh(&csq->lock);
1087
1088 return ret;
1089}
1090
e95e52a1 1091static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
6a04aed6
WHX		 1092 struct hns_roce_cmq_desc *desc, int num)
1093{
1094 int retval;
1095 int ret;
1096
1097 ret = hns_roce_v2_rst_process_cmd(hr_dev);
1098 if (ret == CMD_RST_PRC_SUCCESS)
1099 return 0;
1100 if (ret == CMD_RST_PRC_EBUSY)
b417c087 1101 return -EBUSY;
6a04aed6
WHX		 1102
1103 ret = __hns_roce_cmq_send(hr_dev, desc, num);
1104 if (ret) {
1105 retval = hns_roce_v2_rst_process_cmd(hr_dev);
1106 if (retval == CMD_RST_PRC_SUCCESS)
1107 return 0;
1108 else if (retval == CMD_RST_PRC_EBUSY)
b417c087 1109 return -EBUSY;
6a04aed6
WHX		 1110 }
1111
1112 return ret;
1113}
1114
281d0ccf 1115static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
cfc85f3e
WHX		 1116{
1117 struct hns_roce_query_version *resp;
1118 struct hns_roce_cmq_desc desc;
1119 int ret;
1120
1121 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1122 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1123 if (ret)
1124 return ret;
1125
1126 resp = (struct hns_roce_query_version *)desc.data;
1127 hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
3a63c964
LO		 1128 hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1129
1130 return 0;
1131}
1132
89a6da3c
LC		 1133static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1134{
89a6da3c
LC		 1135 struct hns_roce_func_clear *resp;
1136 struct hns_roce_cmq_desc desc;
1137 unsigned long end;
1138 int ret;
1139
1140 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1141 resp = (struct hns_roce_func_clear *)desc.data;
1142
1143 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1144 if (ret) {
89a6da3c
LC		 1145 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1146 ret);
1147 return;
1148 }
1149
1150 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1151 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1152 while (end) {
1153 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1154 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1155
1156 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1157 true);
1158
1159 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1160 if (ret)
1161 continue;
1162
1163 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1164 hr_dev->is_reset = true;
1165 return;
1166 }
1167 }
1168
1169 dev_err(hr_dev->dev, "Func clear fail.\n");
1170}
1171
3a63c964
LO		 1172static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1173{
1174 struct hns_roce_query_fw_info *resp;
1175 struct hns_roce_cmq_desc desc;
1176 int ret;
1177
1178 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1179 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1180 if (ret)
1181 return ret;
1182
1183 resp = (struct hns_roce_query_fw_info *)desc.data;
1184 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
cfc85f3e
WHX		 1185
1186 return 0;
1187}
1188
1189static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1190{
1191 struct hns_roce_cfg_global_param *req;
1192 struct hns_roce_cmq_desc desc;
1193
1194 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1195 false);
1196
1197 req = (struct hns_roce_cfg_global_param *)desc.data;
1198 memset(req, 0, sizeof(*req));
1199 roce_set_field(req->time_cfg_udp_port,
1200 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1201 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1202 roce_set_field(req->time_cfg_udp_port,
1203 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1204 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
1205
1206 return hns_roce_cmq_send(hr_dev, &desc, 1);
1207}
1208
1209static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1210{
1211 struct hns_roce_cmq_desc desc[2];
6b63597d 1212 struct hns_roce_pf_res_a *req_a;
1213 struct hns_roce_pf_res_b *req_b;
cfc85f3e
WHX		 1214 int ret;
1215 int i;
1216
1217 for (i = 0; i < 2; i++) {
1218 hns_roce_cmq_setup_basic_desc(&desc[i],
1219 HNS_ROCE_OPC_QUERY_PF_RES, true);
1220
1221 if (i == 0)
1222 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1223 else
1224 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1225 }
1226
1227 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1228 if (ret)
1229 return ret;
1230
6b63597d 1231 req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1232 req_b = (struct hns_roce_pf_res_b *)desc[1].data;
cfc85f3e 1233
6b63597d 1234 hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
cfc85f3e
WHX		 1235 PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1236 PF_RES_DATA_1_PF_QPC_BT_NUM_S);
6b63597d 1237 hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
cfc85f3e
WHX		 1238 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1239 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
6b63597d 1240 hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
cfc85f3e
WHX		 1241 PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1242 PF_RES_DATA_3_PF_CQC_BT_NUM_S);
6b63597d 1243 hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
cfc85f3e
WHX		 1244 PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1245 PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1246
6b63597d 1247 hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1248 PF_RES_DATA_3_PF_SL_NUM_M,
1249 PF_RES_DATA_3_PF_SL_NUM_S);
6a157f7d
YL		 1250 hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
1251 PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
1252 PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
6b63597d 1253
cfc85f3e
WHX		 1254 return 0;
1255}
1256
0e40dc2f
YL		 1257static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
1258{
1259 struct hns_roce_pf_timer_res_a *req_a;
1260 struct hns_roce_cmq_desc desc[2];
1261 int ret, i;
1262
1263 for (i = 0; i < 2; i++) {
1264 hns_roce_cmq_setup_basic_desc(&desc[i],
1265 HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1266 true);
1267
1268 if (i == 0)
1269 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1270 else
1271 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1272 }
1273
1274 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1275 if (ret)
1276 return ret;
1277
1278 req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;
1279
1280 hr_dev->caps.qpc_timer_bt_num =
1281 roce_get_field(req_a->qpc_timer_bt_idx_num,
1282 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
1283 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
1284 hr_dev->caps.cqc_timer_bt_num =
1285 roce_get_field(req_a->cqc_timer_bt_idx_num,
1286 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
1287 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
1288
1289 return 0;
1290}
1291
0c1c3880
LO		 1292static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1293 int vf_id)
1294{
1295 struct hns_roce_cmq_desc desc;
1296 struct hns_roce_vf_switch *swt;
1297 int ret;
1298
1299 swt = (struct hns_roce_vf_switch *)desc.data;
1300 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1301 swt->rocee_sel |= cpu_to_le16(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1302 roce_set_field(swt->fun_id,
1303 VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1304 VF_SWITCH_DATA_FUN_ID_VF_ID_S,
1305 vf_id);
1306 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1307 if (ret)
1308 return ret;
1309 desc.flag =
1310 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1311 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1312 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
1313 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 1);
1314 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1315
1316 return hns_roce_cmq_send(hr_dev, &desc, 1);
1317}
1318
cfc85f3e
WHX		 1319static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1320{
1321 struct hns_roce_cmq_desc desc[2];
1322 struct hns_roce_vf_res_a *req_a;
1323 struct hns_roce_vf_res_b *req_b;
1324 int i;
1325
1326 req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1327 req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1328 memset(req_a, 0, sizeof(*req_a));
1329 memset(req_b, 0, sizeof(*req_b));
1330 for (i = 0; i < 2; i++) {
1331 hns_roce_cmq_setup_basic_desc(&desc[i],
1332 HNS_ROCE_OPC_ALLOC_VF_RES, false);
1333
1334 if (i == 0)
1335 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1336 else
1337 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1338
1339 if (i == 0) {
1340 roce_set_field(req_a->vf_qpc_bt_idx_num,
1341 VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1342 VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1343 roce_set_field(req_a->vf_qpc_bt_idx_num,
1344 VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1345 VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1346 HNS_ROCE_VF_QPC_BT_NUM);
1347
1348 roce_set_field(req_a->vf_srqc_bt_idx_num,
1349 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1350 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1351 roce_set_field(req_a->vf_srqc_bt_idx_num,
1352 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1353 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1354 HNS_ROCE_VF_SRQC_BT_NUM);
1355
1356 roce_set_field(req_a->vf_cqc_bt_idx_num,
1357 VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1358 VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1359 roce_set_field(req_a->vf_cqc_bt_idx_num,
1360 VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1361 VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1362 HNS_ROCE_VF_CQC_BT_NUM);
1363
1364 roce_set_field(req_a->vf_mpt_bt_idx_num,
1365 VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1366 VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1367 roce_set_field(req_a->vf_mpt_bt_idx_num,
1368 VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1369 VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1370 HNS_ROCE_VF_MPT_BT_NUM);
1371
1372 roce_set_field(req_a->vf_eqc_bt_idx_num,
1373 VF_RES_A_DATA_5_VF_EQC_IDX_M,
1374 VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1375 roce_set_field(req_a->vf_eqc_bt_idx_num,
1376 VF_RES_A_DATA_5_VF_EQC_NUM_M,
1377 VF_RES_A_DATA_5_VF_EQC_NUM_S,
1378 HNS_ROCE_VF_EQC_NUM);
1379 } else {
1380 roce_set_field(req_b->vf_smac_idx_num,
1381 VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1382 VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1383 roce_set_field(req_b->vf_smac_idx_num,
1384 VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1385 VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1386 HNS_ROCE_VF_SMAC_NUM);
1387
1388 roce_set_field(req_b->vf_sgid_idx_num,
1389 VF_RES_B_DATA_2_VF_SGID_IDX_M,
1390 VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1391 roce_set_field(req_b->vf_sgid_idx_num,
1392 VF_RES_B_DATA_2_VF_SGID_NUM_M,
1393 VF_RES_B_DATA_2_VF_SGID_NUM_S,
1394 HNS_ROCE_VF_SGID_NUM);
1395
1396 roce_set_field(req_b->vf_qid_idx_sl_num,
1397 VF_RES_B_DATA_3_VF_QID_IDX_M,
1398 VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1399 roce_set_field(req_b->vf_qid_idx_sl_num,
1400 VF_RES_B_DATA_3_VF_SL_NUM_M,
1401 VF_RES_B_DATA_3_VF_SL_NUM_S,
1402 HNS_ROCE_VF_SL_NUM);
6a157f7d
YL		 1403
1404 roce_set_field(req_b->vf_sccc_idx_num,
1405 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
1406 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
1407 roce_set_field(req_b->vf_sccc_idx_num,
1408 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
1409 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
1410 HNS_ROCE_VF_SCCC_BT_NUM);
cfc85f3e
WHX		 1411 }
1412 }
1413
1414 return hns_roce_cmq_send(hr_dev, desc, 2);
1415}
1416
a81fba28
WHX		 1417static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1418{
1419 u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1420 u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1421 u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1422 u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
6a157f7d 1423 u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
a81fba28
WHX		 1424 struct hns_roce_cfg_bt_attr *req;
1425 struct hns_roce_cmq_desc desc;
1426
1427 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1428 req = (struct hns_roce_cfg_bt_attr *)desc.data;
1429 memset(req, 0, sizeof(*req));
1430
1431 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1432 CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
5e6e78db 1433 hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1434 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1435 CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
5e6e78db 1436 hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1437 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1438 CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1439 qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1440
1441 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1442 CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
5e6e78db 1443 hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1444 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1445 CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
5e6e78db 1446 hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1447 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1448 CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1449 srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1450
1451 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1452 CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
5e6e78db 1453 hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1454 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1455 CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
5e6e78db 1456 hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1457 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1458 CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1459 cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1460
1461 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1462 CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
5e6e78db 1463 hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1464 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1465 CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
5e6e78db 1466 hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1467 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1468 CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1469 mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1470
6a157f7d
YL		 1471 roce_set_field(req->vf_sccc_cfg,
1472 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
1473 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
1474 hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1475 roce_set_field(req->vf_sccc_cfg,
1476 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
1477 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
1478 hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1479 roce_set_field(req->vf_sccc_cfg,
1480 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
1481 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
1482 sccc_hop_num ==
1483 HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
1484
a81fba28
WHX		 1485 return hns_roce_cmq_send(hr_dev, &desc, 1);
1486}
1487
cfc85f3e
WHX		 1488static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1489{
1490 struct hns_roce_caps *caps = &hr_dev->caps;
1491 int ret;
1492
1493 ret = hns_roce_cmq_query_hw_info(hr_dev);
3a63c964
LO		 1494 if (ret) {
1495 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
1496 ret);
1497 return ret;
1498 }
1499
1500 ret = hns_roce_query_fw_ver(hr_dev);
cfc85f3e
WHX		 1501 if (ret) {
1502 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1503 ret);
1504 return ret;
1505 }
1506
1507 ret = hns_roce_config_global_param(hr_dev);
1508 if (ret) {
1509 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1510 ret);
2349fdd4 1511 return ret;
cfc85f3e
WHX		 1512 }
1513
1514 /* Get pf resource owned by every pf */
1515 ret = hns_roce_query_pf_resource(hr_dev);
1516 if (ret) {
1517 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1518 ret);
1519 return ret;
1520 }
1521
0e40dc2f
YL		 1522 if (hr_dev->pci_dev->revision == 0x21) {
1523 ret = hns_roce_query_pf_timer_resource(hr_dev);
1524 if (ret) {
1525 dev_err(hr_dev->dev,
1526 "Query pf timer resource fail, ret = %d.\n",
1527 ret);
1528 return ret;
1529 }
1530 }
1531
cfc85f3e
WHX		 1532 ret = hns_roce_alloc_vf_resource(hr_dev);
1533 if (ret) {
1534 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1535 ret);
1536 return ret;
1537 }
1538
0c1c3880
LO		 1539 if (hr_dev->pci_dev->revision == 0x21) {
1540 ret = hns_roce_set_vf_switch_param(hr_dev, 0);
1541 if (ret) {
1542 dev_err(hr_dev->dev,
1543 "Set function switch param fail, ret = %d.\n",
1544 ret);
1545 return ret;
1546 }
1547 }
3a63c964
LO		 1548
1549 hr_dev->vendor_part_id = hr_dev->pci_dev->device;
1550 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
cfc85f3e
WHX		 1551
1552 caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
1553 caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
1554 caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
5c1f167a 1555 caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM;
704e0e61 1556 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
cfc85f3e 1557 caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
5c1f167a 1558 caps->max_srqwqes = HNS_ROCE_V2_MAX_SRQWQE_NUM;
cfc85f3e 1559 caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
05ad5482 1560 caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
cfc85f3e
WHX		 1561 caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1562 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
5c1f167a 1563 caps->max_srq_sg = HNS_ROCE_V2_MAX_SRQ_SGE_NUM;
cfc85f3e
WHX		 1564 caps->num_uars = HNS_ROCE_V2_UAR_NUM;
1565 caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
a5073d60
YL		 1566 caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
1567 caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
1568 caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
cfc85f3e
WHX		 1569 caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
1570 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
1571 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
5c1f167a
LO		 1572 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1573 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
cfc85f3e
WHX		 1574 caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
1575 caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1576 caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1577 caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1578 caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1579 caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1580 caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ;
1581 caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
e92f2c18 1582 caps->trrl_entry_sz = HNS_ROCE_V2_TRRL_ENTRY_SZ;
cfc85f3e 1583 caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
5c1f167a 1584 caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ;
cfc85f3e
WHX		 1585 caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1586 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
5c1f167a 1587 caps->idx_entry_sz = 4;
cfc85f3e
WHX		 1588 caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1589 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1590 caps->reserved_lkey = 0;
1591 caps->reserved_pds = 0;
1592 caps->reserved_mrws = 1;
1593 caps->reserved_uars = 0;
1594 caps->reserved_cqs = 0;
5c1f167a 1595 caps->reserved_srqs = 0;
06ef0ee4 1596 caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
cfc85f3e 1597
a25d13cb
SX		 1598 caps->qpc_ba_pg_sz = 0;
1599 caps->qpc_buf_pg_sz = 0;
1600 caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1601 caps->srqc_ba_pg_sz = 0;
1602 caps->srqc_buf_pg_sz = 0;
2b277dae 1603 caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
a25d13cb
SX		 1604 caps->cqc_ba_pg_sz = 0;
1605 caps->cqc_buf_pg_sz = 0;
1606 caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1607 caps->mpt_ba_pg_sz = 0;
1608 caps->mpt_buf_pg_sz = 0;
1609 caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
91fb4d83 1610 caps->pbl_ba_pg_sz = 2;
ff795f71
WHX		 1611 caps->pbl_buf_pg_sz = 0;
1612 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
6a93c77a
SX		 1613 caps->mtt_ba_pg_sz = 0;
1614 caps->mtt_buf_pg_sz = 0;
1615 caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
8d18ad83
LO		 1616 caps->wqe_sq_hop_num = 2;
1617 caps->wqe_sge_hop_num = 1;
1618 caps->wqe_rq_hop_num = 2;
f5662b4d 1619 caps->cqe_ba_pg_sz = 6;
6a93c77a
SX		 1620 caps->cqe_buf_pg_sz = 0;
1621 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
c7bcb134
LO		 1622 caps->srqwqe_ba_pg_sz = 0;
1623 caps->srqwqe_buf_pg_sz = 0;
1624 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
1625 caps->idx_ba_pg_sz = 0;
1626 caps->idx_buf_pg_sz = 0;
1627 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
a5073d60
YL		 1628 caps->eqe_ba_pg_sz = 0;
1629 caps->eqe_buf_pg_sz = 0;
1630 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
6b63597d 1631 caps->tsq_buf_pg_sz = 0;
29a1fe5d 1632 caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
a25d13cb 1633
023c1477 1634 caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR |
0009c2db 1635 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
e088a685 1636 HNS_ROCE_CAP_FLAG_RQ_INLINE |
0425e3e6
YL		 1637 HNS_ROCE_CAP_FLAG_RECORD_DB |
1638 HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
c7c28191
YL		 1639
1640 if (hr_dev->pci_dev->revision == 0x21)
68a997c5
YL		 1641 caps->flags |= HNS_ROCE_CAP_FLAG_MW |
1642 HNS_ROCE_CAP_FLAG_FRMR;
c7c28191 1643
cfc85f3e 1644 caps->pkey_table_len[0] = 1;
b5ff0f61 1645 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
a5073d60
YL		 1646 caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
1647 caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
cfc85f3e
WHX		 1648 caps->local_ca_ack_delay = 0;
1649 caps->max_mtu = IB_MTU_4096;
1650
d16da119
LO		 1651 caps->max_srqs = HNS_ROCE_V2_MAX_SRQ;
1652 caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
1653 caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
1654
6a157f7d 1655 if (hr_dev->pci_dev->revision == 0x21) {
d16da119 1656 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC |
aa84fa18
YL		 1657 HNS_ROCE_CAP_FLAG_SRQ |
1658 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1659
0e40dc2f
YL		 1660 caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1661 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1662 caps->qpc_timer_ba_pg_sz = 0;
1663 caps->qpc_timer_buf_pg_sz = 0;
1664 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1665 caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1666 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1667 caps->cqc_timer_ba_pg_sz = 0;
1668 caps->cqc_timer_buf_pg_sz = 0;
1669 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1670
6a157f7d
YL		 1671 caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;
1672 caps->sccc_ba_pg_sz = 0;
1673 caps->sccc_buf_pg_sz = 0;
1674 caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
1675 }
384f8818 1676
a81fba28
WHX		 1677 ret = hns_roce_v2_set_bt(hr_dev);
1678 if (ret)
1679 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1680 ret);
1681
1682 return ret;
cfc85f3e
WHX		 1683}
1684
6b63597d 1685static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
1686 enum hns_roce_link_table_type type)
1687{
1688 struct hns_roce_cmq_desc desc[2];
1689 struct hns_roce_cfg_llm_a *req_a =
1690 (struct hns_roce_cfg_llm_a *)desc[0].data;
1691 struct hns_roce_cfg_llm_b *req_b =
1692 (struct hns_roce_cfg_llm_b *)desc[1].data;
1693 struct hns_roce_v2_priv *priv = hr_dev->priv;
1694 struct hns_roce_link_table *link_tbl;
1695 struct hns_roce_link_table_entry *entry;
1696 enum hns_roce_opcode_type opcode;
1697 u32 page_num;
1698 int i;
1699
1700 switch (type) {
1701 case TSQ_LINK_TABLE:
1702 link_tbl = &priv->tsq;
1703 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
1704 break;
ded58ff9 1705 case TPQ_LINK_TABLE:
1706 link_tbl = &priv->tpq;
1707 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
1708 break;
6b63597d 1709 default:
1710 return -EINVAL;
1711 }
1712
1713 page_num = link_tbl->npages;
1714 entry = link_tbl->table.buf;
1715 memset(req_a, 0, sizeof(*req_a));
1716 memset(req_b, 0, sizeof(*req_b));
1717
1718 for (i = 0; i < 2; i++) {
1719 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
1720
1721 if (i == 0)
1722 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1723 else
1724 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1725
1726 if (i == 0) {
1727 req_a->base_addr_l = link_tbl->table.map & 0xffffffff;
1728 req_a->base_addr_h = (link_tbl->table.map >> 32) &
1729 0xffffffff;
1730 roce_set_field(req_a->depth_pgsz_init_en,
1731 CFG_LLM_QUE_DEPTH_M,
1732 CFG_LLM_QUE_DEPTH_S,
1733 link_tbl->npages);
1734 roce_set_field(req_a->depth_pgsz_init_en,
1735 CFG_LLM_QUE_PGSZ_M,
1736 CFG_LLM_QUE_PGSZ_S,
1737 link_tbl->pg_sz);
1738 req_a->head_ba_l = entry[0].blk_ba0;
1739 req_a->head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr;
1740 roce_set_field(req_a->head_ptr,
1741 CFG_LLM_HEAD_PTR_M,
1742 CFG_LLM_HEAD_PTR_S, 0);
1743 } else {
1744 req_b->tail_ba_l = entry[page_num - 1].blk_ba0;
1745 roce_set_field(req_b->tail_ba_h,
1746 CFG_LLM_TAIL_BA_H_M,
1747 CFG_LLM_TAIL_BA_H_S,
1748 entry[page_num - 1].blk_ba1_nxt_ptr &
1749 HNS_ROCE_LINK_TABLE_BA1_M);
1750 roce_set_field(req_b->tail_ptr,
1751 CFG_LLM_TAIL_PTR_M,
1752 CFG_LLM_TAIL_PTR_S,
1753 (entry[page_num - 2].blk_ba1_nxt_ptr &
1754 HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1755 HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1756 }
1757 }
1758 roce_set_field(req_a->depth_pgsz_init_en,
1759 CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
1760
1761 return hns_roce_cmq_send(hr_dev, desc, 2);
1762}
1763
1764static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
1765 enum hns_roce_link_table_type type)
1766{
1767 struct hns_roce_v2_priv *priv = hr_dev->priv;
1768 struct hns_roce_link_table *link_tbl;
1769 struct hns_roce_link_table_entry *entry;
1770 struct device *dev = hr_dev->dev;
1771 u32 buf_chk_sz;
1772 dma_addr_t t;
ded58ff9 1773 int func_num = 1;
6b63597d 1774 int pg_num_a;
1775 int pg_num_b;
1776 int pg_num;
1777 int size;
1778 int i;
1779
1780 switch (type) {
1781 case TSQ_LINK_TABLE:
1782 link_tbl = &priv->tsq;
1783 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
1784 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
1785 pg_num_b = hr_dev->caps.sl_num * 4 + 2;
1786 break;
ded58ff9 1787 case TPQ_LINK_TABLE:
1788 link_tbl = &priv->tpq;
1789 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
1790 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
1791 pg_num_b = 2 * 4 * func_num + 2;
1792 break;
6b63597d 1793 default:
1794 return -EINVAL;
1795 }
1796
1797 pg_num = max(pg_num_a, pg_num_b);
1798 size = pg_num * sizeof(struct hns_roce_link_table_entry);
1799
1800 link_tbl->table.buf = dma_alloc_coherent(dev, size,
1801 &link_tbl->table.map,
1802 GFP_KERNEL);
1803 if (!link_tbl->table.buf)
1804 goto out;
1805
1806 link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
1807 GFP_KERNEL);
1808 if (!link_tbl->pg_list)
1809 goto err_kcalloc_failed;
1810
1811 entry = link_tbl->table.buf;
1812 for (i = 0; i < pg_num; ++i) {
1813 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
1814 &t, GFP_KERNEL);
1815 if (!link_tbl->pg_list[i].buf)
1816 goto err_alloc_buf_failed;
1817
1818 link_tbl->pg_list[i].map = t;
6b63597d 1819
1820 entry[i].blk_ba0 = (t >> 12) & 0xffffffff;
1821 roce_set_field(entry[i].blk_ba1_nxt_ptr,
1822 HNS_ROCE_LINK_TABLE_BA1_M,
1823 HNS_ROCE_LINK_TABLE_BA1_S,
1824 t >> 44);
1825
1826 if (i < (pg_num - 1))
1827 roce_set_field(entry[i].blk_ba1_nxt_ptr,
1828 HNS_ROCE_LINK_TABLE_NXT_PTR_M,
1829 HNS_ROCE_LINK_TABLE_NXT_PTR_S,
1830 i + 1);
1831 }
1832 link_tbl->npages = pg_num;
1833 link_tbl->pg_sz = buf_chk_sz;
1834
1835 return hns_roce_config_link_table(hr_dev, type);
1836
1837err_alloc_buf_failed:
1838 for (i -= 1; i >= 0; i--)
1839 dma_free_coherent(dev, buf_chk_sz,
1840 link_tbl->pg_list[i].buf,
1841 link_tbl->pg_list[i].map);
1842 kfree(link_tbl->pg_list);
1843
1844err_kcalloc_failed:
1845 dma_free_coherent(dev, size, link_tbl->table.buf,
1846 link_tbl->table.map);
1847
1848out:
1849 return -ENOMEM;
1850}
1851
1852static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
1853 struct hns_roce_link_table *link_tbl)
1854{
1855 struct device *dev = hr_dev->dev;
1856 int size;
1857 int i;
1858
1859 size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
1860
1861 for (i = 0; i < link_tbl->npages; ++i)
1862 if (link_tbl->pg_list[i].buf)
1863 dma_free_coherent(dev, link_tbl->pg_sz,
1864 link_tbl->pg_list[i].buf,
1865 link_tbl->pg_list[i].map);
1866 kfree(link_tbl->pg_list);
1867
1868 dma_free_coherent(dev, size, link_tbl->table.buf,
1869 link_tbl->table.map);
1870}
1871
1872static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1873{
ded58ff9 1874 struct hns_roce_v2_priv *priv = hr_dev->priv;
0e40dc2f
YL		 1875 int qpc_count, cqc_count;
1876 int ret, i;
6b63597d 1877
1878 /* TSQ includes SQ doorbell and ack doorbell */
1879 ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
ded58ff9 1880 if (ret) {
6b63597d 1881 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
ded58ff9 1882 return ret;
1883 }
1884
1885 ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
1886 if (ret) {
1887 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
1888 goto err_tpq_init_failed;
1889 }
1890
0e40dc2f
YL		 1891 /* Alloc memory for QPC Timer buffer space chunk*/
1892 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
1893 qpc_count++) {
1894 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
1895 qpc_count);
1896 if (ret) {
1897 dev_err(hr_dev->dev, "QPC Timer get failed\n");
1898 goto err_qpc_timer_failed;
1899 }
1900 }
1901
1902 /* Alloc memory for CQC Timer buffer space chunk*/
1903 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
1904 cqc_count++) {
1905 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
1906 cqc_count);
1907 if (ret) {
1908 dev_err(hr_dev->dev, "CQC Timer get failed\n");
1909 goto err_cqc_timer_failed;
1910 }
1911 }
1912
ded58ff9 1913 return 0;
1914
0e40dc2f
YL		 1915err_cqc_timer_failed:
1916 for (i = 0; i < cqc_count; i++)
1917 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
1918
1919err_qpc_timer_failed:
1920 for (i = 0; i < qpc_count; i++)
1921 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
1922
1923 hns_roce_free_link_table(hr_dev, &priv->tpq);
1924
ded58ff9 1925err_tpq_init_failed:
1926 hns_roce_free_link_table(hr_dev, &priv->tsq);
6b63597d 1927
1928 return ret;
1929}
1930
1931static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
1932{
1933 struct hns_roce_v2_priv *priv = hr_dev->priv;
1934
89a6da3c
LC		 1935 if (hr_dev->pci_dev->revision == 0x21)
1936 hns_roce_function_clear(hr_dev);
1937
ded58ff9 1938 hns_roce_free_link_table(hr_dev, &priv->tpq);
6b63597d 1939 hns_roce_free_link_table(hr_dev, &priv->tsq);
1940}
1941
f747b689
LO		 1942static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
1943{
1944 struct hns_roce_cmq_desc desc;
1945 struct hns_roce_mbox_status *mb_st =
1946 (struct hns_roce_mbox_status *)desc.data;
1947 enum hns_roce_cmd_return_status status;
1948
1949 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
1950
1951 status = hns_roce_cmq_send(hr_dev, &desc, 1);
1952 if (status)
1953 return status;
1954
1955 return cpu_to_le32(mb_st->mb_status_hw_run);
1956}
1957
a680f2f3
WHX		 1958static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1959{
f747b689 1960 u32 status = hns_roce_query_mbox_status(hr_dev);
a680f2f3
WHX		 1961
1962 return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1963}
1964
1965static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1966{
f747b689 1967 u32 status = hns_roce_query_mbox_status(hr_dev);
a680f2f3
WHX		 1968
1969 return status & HNS_ROCE_HW_MB_STATUS_MASK;
1970}
1971
f747b689
LO		 1972static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
1973 u64 out_param, u32 in_modifier, u8 op_modifier,
1974 u16 op, u16 token, int event)
1975{
1976 struct hns_roce_cmq_desc desc;
1977 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
1978
1979 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
1980
1981 mb->in_param_l = cpu_to_le64(in_param);
1982 mb->in_param_h = cpu_to_le64(in_param) >> 32;
1983 mb->out_param_l = cpu_to_le64(out_param);
1984 mb->out_param_h = cpu_to_le64(out_param) >> 32;
1985 mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
1986 mb->token_event_en = cpu_to_le32(event << 16 | token);
1987
1988 return hns_roce_cmq_send(hr_dev, &desc, 1);
1989}
1990
a680f2f3
WHX		 1991static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1992 u64 out_param, u32 in_modifier, u8 op_modifier,
1993 u16 op, u16 token, int event)
1994{
1995 struct device *dev = hr_dev->dev;
a680f2f3 1996 unsigned long end;
f747b689 1997 int ret;
a680f2f3
WHX		 1998
1999 end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
2000 while (hns_roce_v2_cmd_pending(hr_dev)) {
2001 if (time_after(jiffies, end)) {
2002 dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
2003 (int)end);
2004 return -EAGAIN;
2005 }
2006 cond_resched();
2007 }
2008
f747b689
LO		 2009 ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2010 op_modifier, op, token, event);
2011 if (ret)
2012 dev_err(dev, "Post mailbox fail(%d)\n", ret);
a680f2f3 2013
f747b689 2014 return ret;
a680f2f3
WHX		 2015}
2016
2017static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2018 unsigned long timeout)
2019{
2020 struct device *dev = hr_dev->dev;
2021 unsigned long end = 0;
2022 u32 status;
2023
2024 end = msecs_to_jiffies(timeout) + jiffies;
2025 while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
2026 cond_resched();
2027
2028 if (hns_roce_v2_cmd_pending(hr_dev)) {
2029 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
2030 return -ETIMEDOUT;
2031 }
2032
2033 status = hns_roce_v2_cmd_complete(hr_dev);
2034 if (status != 0x1) {
6a04aed6
WHX		 2035 if (status == CMD_RST_PRC_EBUSY)
2036 return status;
2037
a680f2f3
WHX		 2038 dev_err(dev, "mailbox status 0x%x!\n", status);
2039 return -EBUSY;
2040 }
2041
2042 return 0;
2043}
2044
4db134a3 2045static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
2046 int gid_index, const union ib_gid *gid,
2047 enum hns_roce_sgid_type sgid_type)
2048{
2049 struct hns_roce_cmq_desc desc;
2050 struct hns_roce_cfg_sgid_tb *sgid_tb =
2051 (struct hns_roce_cfg_sgid_tb *)desc.data;
2052 u32 *p;
2053
2054 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2055
2056 roce_set_field(sgid_tb->table_idx_rsv,
2057 CFG_SGID_TB_TABLE_IDX_M,
2058 CFG_SGID_TB_TABLE_IDX_S, gid_index);
2059 roce_set_field(sgid_tb->vf_sgid_type_rsv,
2060 CFG_SGID_TB_VF_SGID_TYPE_M,
2061 CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2062
2063 p = (u32 *)&gid->raw[0];
2064 sgid_tb->vf_sgid_l = cpu_to_le32(*p);
2065
2066 p = (u32 *)&gid->raw[4];
2067 sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
2068
2069 p = (u32 *)&gid->raw[8];
2070 sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
2071
2072 p = (u32 *)&gid->raw[0xc];
2073 sgid_tb->vf_sgid_h = cpu_to_le32(*p);
2074
2075 return hns_roce_cmq_send(hr_dev, &desc, 1);
2076}
2077
b5ff0f61 2078static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
f4df9a7c 2079 int gid_index, const union ib_gid *gid,
b5ff0f61 2080 const struct ib_gid_attr *attr)
7afddafa 2081{
b5ff0f61 2082 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
4db134a3 2083 int ret;
7afddafa 2084
b5ff0f61
WHX		 2085 if (!gid || !attr)
2086 return -EINVAL;
2087
2088 if (attr->gid_type == IB_GID_TYPE_ROCE)
2089 sgid_type = GID_TYPE_FLAG_ROCE_V1;
2090
2091 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2092 if (ipv6_addr_v4mapped((void *)gid))
2093 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2094 else
2095 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2096 }
2097
4db134a3 2098 ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2099 if (ret)
2100 dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
b5ff0f61 2101
4db134a3 2102 return ret;
7afddafa
WHX		 2103}
2104
a74dc41d
WHX		 2105static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2106 u8 *addr)
7afddafa 2107{
e8e8b652 2108 struct hns_roce_cmq_desc desc;
2109 struct hns_roce_cfg_smac_tb *smac_tb =
2110 (struct hns_roce_cfg_smac_tb *)desc.data;
7afddafa
WHX		 2111 u16 reg_smac_h;
2112 u32 reg_smac_l;
e8e8b652 2113
2114 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
7afddafa
WHX		 2115
2116 reg_smac_l = *(u32 *)(&addr[0]);
e8e8b652 2117 reg_smac_h = *(u16 *)(&addr[4]);
7afddafa 2118
e8e8b652 2119 memset(smac_tb, 0, sizeof(*smac_tb));
2120 roce_set_field(smac_tb->tb_idx_rsv,
2121 CFG_SMAC_TB_IDX_M,
2122 CFG_SMAC_TB_IDX_S, phy_port);
2123 roce_set_field(smac_tb->vf_smac_h_rsv,
2124 CFG_SMAC_TB_VF_SMAC_H_M,
2125 CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
2126 smac_tb->vf_smac_l = reg_smac_l;
a74dc41d 2127
e8e8b652 2128 return hns_roce_cmq_send(hr_dev, &desc, 1);
7afddafa
WHX		 2129}
2130
ca088320
YL		 2131static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
2132 struct hns_roce_mr *mr)
3958cc56 2133{
3856ec55 2134 struct sg_dma_page_iter sg_iter;
db270c41 2135 u64 page_addr;
3958cc56 2136 u64 *pages;
3856ec55 2137 int i;
3958cc56 2138
ca088320
YL		 2139 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2140 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2141 roce_set_field(mpt_entry->byte_48_mode_ba,
2142 V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
2143 upper_32_bits(mr->pbl_ba >> 3));
2144
2145 pages = (u64 *)__get_free_page(GFP_KERNEL);
2146 if (!pages)
2147 return -ENOMEM;
2148
2149 i = 0;
3856ec55
SS		 2150 for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
2151 page_addr = sg_page_iter_dma_address(&sg_iter);
2152 pages[i] = page_addr >> 6;
2153
2154 /* Record the first 2 entry directly to MTPT table */
2155 if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
2156 goto found;
2157 i++;
ca088320
YL		 2158 }
2159found:
2160 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
2161 roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
2162 V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
2163
2164 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
2165 roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
2166 V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
2167 roce_set_field(mpt_entry->byte_64_buf_pa1,
2168 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2169 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2170 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2171
2172 free_page((unsigned long)pages);
2173
2174 return 0;
2175}
2176
2177static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
2178 unsigned long mtpt_idx)
2179{
2180 struct hns_roce_v2_mpt_entry *mpt_entry;
2181 int ret;
2182
3958cc56
WHX		 2183 mpt_entry = mb_buf;
2184 memset(mpt_entry, 0, sizeof(*mpt_entry));
2185
2186 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2187 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2188 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2189 V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
2190 HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
2191 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2192 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
5e6e78db
YL		 2193 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2194 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
3958cc56
WHX		 2195 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2196 V2_MPT_BYTE_4_PD_S, mr->pd);
3958cc56
WHX		 2197
2198 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
82342e49 2199 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
e93df010 2200 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
3958cc56
WHX		 2201 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
2202 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
384f8818
LO		 2203 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
2204 mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
3958cc56
WHX		 2205 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2206 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
2207 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2208 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
2209 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2210 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
3958cc56
WHX		 2211
2212 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
2213 mr->type == MR_TYPE_MR ? 0 : 1);
85e0274d 2214 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
2215 1);
3958cc56
WHX		 2216
2217 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2218 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2219 mpt_entry->lkey = cpu_to_le32(mr->key);
2220 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2221 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2222
2223 if (mr->type == MR_TYPE_DMA)
2224 return 0;
2225
ca088320 2226 ret = set_mtpt_pbl(mpt_entry, mr);
3958cc56 2227
ca088320 2228 return ret;
3958cc56
WHX		 2229}
2230
a2c80b7b
WHX		 2231static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
2232 struct hns_roce_mr *mr, int flags,
2233 u32 pdn, int mr_access_flags, u64 iova,
2234 u64 size, void *mb_buf)
2235{
2236 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
ca088320 2237 int ret = 0;
a2c80b7b 2238
ab22bf05
YL		 2239 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2240 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2241
a2c80b7b
WHX		 2242 if (flags & IB_MR_REREG_PD) {
2243 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2244 V2_MPT_BYTE_4_PD_S, pdn);
2245 mr->pd = pdn;
2246 }
2247
2248 if (flags & IB_MR_REREG_ACCESS) {
2249 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2250 V2_MPT_BYTE_8_BIND_EN_S,
2251 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
2252 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
ca088320
YL		 2253 V2_MPT_BYTE_8_ATOMIC_EN_S,
2254 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
a2c80b7b 2255 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
ca088320 2256 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
a2c80b7b 2257 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
ca088320 2258 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
a2c80b7b 2259 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
ca088320 2260 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
a2c80b7b
WHX		 2261 }
2262
2263 if (flags & IB_MR_REREG_TRANS) {
2264 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
2265 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
2266 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
2267 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
2268
a2c80b7b
WHX		 2269 mr->iova = iova;
2270 mr->size = size;
ca088320
YL		 2271
2272 ret = set_mtpt_pbl(mpt_entry, mr);
a2c80b7b
WHX		 2273 }
2274
ca088320 2275 return ret;
a2c80b7b
WHX		 2276}
2277
68a997c5
YL		 2278static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
2279{
2280 struct hns_roce_v2_mpt_entry *mpt_entry;
2281
2282 mpt_entry = mb_buf;
2283 memset(mpt_entry, 0, sizeof(*mpt_entry));
2284
2285 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2286 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2287 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2288 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
2289 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2290 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2291 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2292 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2293 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2294 V2_MPT_BYTE_4_PD_S, mr->pd);
2295
2296 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
2297 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2298 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2299
2300 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
2301 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2302 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
2303 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2304
2305 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2306
2307 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2308 roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
2309 V2_MPT_BYTE_48_PBL_BA_H_S,
2310 upper_32_bits(mr->pbl_ba >> 3));
2311
2312 roce_set_field(mpt_entry->byte_64_buf_pa1,
2313 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2314 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2315 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2316
2317 return 0;
2318}
2319
c7c28191
YL		 2320static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
2321{
2322 struct hns_roce_v2_mpt_entry *mpt_entry;
2323
2324 mpt_entry = mb_buf;
2325 memset(mpt_entry, 0, sizeof(*mpt_entry));
2326
2327 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2328 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2329 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2330 V2_MPT_BYTE_4_PD_S, mw->pdn);
2331 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2332 V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2333 V2_MPT_BYTE_4_PBL_HOP_NUM_S,
2334 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ?
2335 0 : mw->pbl_hop_num);
2336 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2337 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2338 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2339 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2340
2341 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2342 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2343
2344 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2345 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
2346 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2347 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
2348 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
2349
2350 roce_set_field(mpt_entry->byte_64_buf_pa1,
2351 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2352 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2353 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2354
2355 mpt_entry->lkey = cpu_to_le32(mw->rkey);
2356
2357 return 0;
2358}
2359
93aa2187
WHX		 2360static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2361{
2362 return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
2363 n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
2364}
2365
2366static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2367{
2368 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
2369
2370 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
2371 return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
2372 !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
2373}
2374
2375static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
2376{
2377 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2378}
2379
c7bcb134
LO		 2380static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
2381{
2382 return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
2383}
2384
2385static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
2386{
c7bcb134
LO		 2387 /* always called with interrupts disabled. */
2388 spin_lock(&srq->lock);
2389
97545b10 2390 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
c7bcb134
LO		 2391 srq->tail++;
2392
2393 spin_unlock(&srq->lock);
2394}
2395
93aa2187
WHX		 2396static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2397{
86188a88 2398 *hr_cq->set_ci_db = cons_index & 0xffffff;
93aa2187
WHX		 2399}
2400
926a01dc
WHX		 2401static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2402 struct hns_roce_srq *srq)
2403{
2404 struct hns_roce_v2_cqe *cqe, *dest;
2405 u32 prod_index;
2406 int nfreed = 0;
c7bcb134 2407 int wqe_index;
926a01dc
WHX		 2408 u8 owner_bit;
2409
2410 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
2411 ++prod_index) {
2412 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
2413 break;
2414 }
2415
2416 /*
2417 * Now backwards through the CQ, removing CQ entries
2418 * that match our QP by overwriting them with next entries.
2419 */
2420 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2421 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2422 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2423 V2_CQE_BYTE_16_LCL_QPN_S) &
2424 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
c7bcb134
LO		 2425 if (srq &&
2426 roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
2427 wqe_index = roce_get_field(cqe->byte_4,
2428 V2_CQE_BYTE_4_WQE_INDX_M,
2429 V2_CQE_BYTE_4_WQE_INDX_S);
2430 hns_roce_free_srq_wqe(srq, wqe_index);
2431 }
926a01dc
WHX		 2432 ++nfreed;
2433 } else if (nfreed) {
2434 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
2435 hr_cq->ib_cq.cqe);
2436 owner_bit = roce_get_bit(dest->byte_4,
2437 V2_CQE_BYTE_4_OWNER_S);
2438 memcpy(dest, cqe, sizeof(*cqe));
2439 roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
2440 owner_bit);
2441 }
2442 }
2443
2444 if (nfreed) {
2445 hr_cq->cons_index += nfreed;
2446 /*
2447 * Make sure update of buffer contents is done before
2448 * updating consumer index.
2449 */
2450 wmb();
2451 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2452 }
2453}
2454
2455static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2456 struct hns_roce_srq *srq)
2457{
2458 spin_lock_irq(&hr_cq->lock);
2459 __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
2460 spin_unlock_irq(&hr_cq->lock);
2461}
2462
93aa2187
WHX		 2463static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
2464 struct hns_roce_cq *hr_cq, void *mb_buf,
2465 u64 *mtts, dma_addr_t dma_handle, int nent,
2466 u32 vector)
2467{
2468 struct hns_roce_v2_cq_context *cq_context;
2469
2470 cq_context = mb_buf;
2471 memset(cq_context, 0, sizeof(*cq_context));
2472
2473 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
2474 V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
a5073d60
YL		 2475 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
2476 V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
93aa2187
WHX		 2477 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
2478 V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
2479 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
2480 V2_CQC_BYTE_4_CEQN_S, vector);
2481 cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
2482
2483 roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
2484 V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
2485
2486 cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
2487 cq_context->cqe_cur_blk_addr =
2488 cpu_to_le32(cq_context->cqe_cur_blk_addr);
2489
2490 roce_set_field(cq_context->byte_16_hop_addr,
2491 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
2492 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
2493 cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
2494 roce_set_field(cq_context->byte_16_hop_addr,
2495 V2_CQC_BYTE_16_CQE_HOP_NUM_M,
2496 V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
2497 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
2498
2499 cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
2500 roce_set_field(cq_context->byte_24_pgsz_addr,
2501 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
2502 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
2503 cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
2504 roce_set_field(cq_context->byte_24_pgsz_addr,
2505 V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
2506 V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
5e6e78db 2507 hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
93aa2187
WHX		 2508 roce_set_field(cq_context->byte_24_pgsz_addr,
2509 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
2510 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
5e6e78db 2511 hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
93aa2187
WHX		 2512
2513 cq_context->cqe_ba = (u32)(dma_handle >> 3);
2514
2515 roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
2516 V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
a5073d60 2517
9b44703d
YL		 2518 if (hr_cq->db_en)
2519 roce_set_bit(cq_context->byte_44_db_record,
2520 V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
2521
2522 roce_set_field(cq_context->byte_44_db_record,
2523 V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
2524 V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
2525 ((u32)hr_cq->db.dma) >> 1);
2526 cq_context->db_record_addr = hr_cq->db.dma >> 32;
2527
a5073d60
YL		 2528 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2529 V2_CQC_BYTE_56_CQ_MAX_CNT_M,
2530 V2_CQC_BYTE_56_CQ_MAX_CNT_S,
2531 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
2532 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2533 V2_CQC_BYTE_56_CQ_PERIOD_M,
2534 V2_CQC_BYTE_56_CQ_PERIOD_S,
2535 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
93aa2187
WHX		 2536}
2537
2538static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
2539 enum ib_cq_notify_flags flags)
2540{
d3743fa9 2541 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
93aa2187
WHX		 2542 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2543 u32 notification_flag;
2544 u32 doorbell[2];
2545
2546 doorbell[0] = 0;
2547 doorbell[1] = 0;
2548
2549 notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
2550 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
2551 /*
2552 * flags = 0; Notification Flag = 1, next
2553 * flags = 1; Notification Flag = 0, solocited
2554 */
2555 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
2556 hr_cq->cqn);
2557 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
2558 HNS_ROCE_V2_CQ_DB_NTR);
2559 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
2560 V2_CQ_DB_PARAMETER_CONS_IDX_S,
2561 hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2562 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
26beb85f 2563 V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
93aa2187
WHX		 2564 roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
2565 notification_flag);
2566
d3743fa9 2567 hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
93aa2187
WHX		 2568
2569 return 0;
2570}
2571
0009c2db 2572static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
2573 struct hns_roce_qp **cur_qp,
2574 struct ib_wc *wc)
2575{
2576 struct hns_roce_rinl_sge *sge_list;
2577 u32 wr_num, wr_cnt, sge_num;
2578 u32 sge_cnt, data_len, size;
2579 void *wqe_buf;
2580
2581 wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
2582 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
2583 wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
2584
2585 sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2586 sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2587 wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
2588 data_len = wc->byte_len;
2589
2590 for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
2591 size = min(sge_list[sge_cnt].len, data_len);
2592 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
2593
2594 data_len -= size;
2595 wqe_buf += size;
2596 }
2597
2598 if (data_len) {
2599 wc->status = IB_WC_LOC_LEN_ERR;
2600 return -EAGAIN;
2601 }
2602
2603 return 0;
2604}
2605
93aa2187
WHX		 2606static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2607 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2608{
c7bcb134 2609 struct hns_roce_srq *srq = NULL;
93aa2187
WHX		 2610 struct hns_roce_dev *hr_dev;
2611 struct hns_roce_v2_cqe *cqe;
2612 struct hns_roce_qp *hr_qp;
2613 struct hns_roce_wq *wq;
0425e3e6
YL		 2614 struct ib_qp_attr attr;
2615 int attr_mask;
93aa2187
WHX		 2616 int is_send;
2617 u16 wqe_ctr;
2618 u32 opcode;
2619 u32 status;
2620 int qpn;
0009c2db 2621 int ret;
93aa2187
WHX		 2622
2623 /* Find cqe according to consumer index */
2624 cqe = next_cqe_sw_v2(hr_cq);
2625 if (!cqe)
2626 return -EAGAIN;
2627
2628 ++hr_cq->cons_index;
2629 /* Memory barrier */
2630 rmb();
2631
2632 /* 0->SQ, 1->RQ */
2633 is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
2634
2635 qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2636 V2_CQE_BYTE_16_LCL_QPN_S);
2637
2638 if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2639 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2640 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2641 if (unlikely(!hr_qp)) {
2642 dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2643 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
2644 return -EINVAL;
2645 }
2646 *cur_qp = hr_qp;
2647 }
2648
2649 wc->qp = &(*cur_qp)->ibqp;
2650 wc->vendor_err = 0;
2651
c7bcb134
LO		 2652 if (is_send) {
2653 wq = &(*cur_qp)->sq;
2654 if ((*cur_qp)->sq_signal_bits) {
2655 /*
2656 * If sg_signal_bit is 1,
2657 * firstly tail pointer updated to wqe
2658 * which current cqe correspond to
2659 */
2660 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2661 V2_CQE_BYTE_4_WQE_INDX_M,
2662 V2_CQE_BYTE_4_WQE_INDX_S);
2663 wq->tail += (wqe_ctr - (u16)wq->tail) &
2664 (wq->wqe_cnt - 1);
2665 }
2666
2667 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2668 ++wq->tail;
2669 } else if ((*cur_qp)->ibqp.srq) {
2670 srq = to_hr_srq((*cur_qp)->ibqp.srq);
2671 wqe_ctr = le16_to_cpu(roce_get_field(cqe->byte_4,
2672 V2_CQE_BYTE_4_WQE_INDX_M,
2673 V2_CQE_BYTE_4_WQE_INDX_S));
2674 wc->wr_id = srq->wrid[wqe_ctr];
2675 hns_roce_free_srq_wqe(srq, wqe_ctr);
2676 } else {
2677 /* Update tail pointer, record wr_id */
2678 wq = &(*cur_qp)->rq;
2679 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2680 ++wq->tail;
2681 }
2682
93aa2187
WHX		 2683 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2684 V2_CQE_BYTE_4_STATUS_S);
2685 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2686 case HNS_ROCE_CQE_V2_SUCCESS:
2687 wc->status = IB_WC_SUCCESS;
2688 break;
2689 case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2690 wc->status = IB_WC_LOC_LEN_ERR;
2691 break;
2692 case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2693 wc->status = IB_WC_LOC_QP_OP_ERR;
2694 break;
2695 case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2696 wc->status = IB_WC_LOC_PROT_ERR;
2697 break;
2698 case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2699 wc->status = IB_WC_WR_FLUSH_ERR;
2700 break;
2701 case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2702 wc->status = IB_WC_MW_BIND_ERR;
2703 break;
2704 case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2705 wc->status = IB_WC_BAD_RESP_ERR;
2706 break;
2707 case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2708 wc->status = IB_WC_LOC_ACCESS_ERR;
2709 break;
2710 case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2711 wc->status = IB_WC_REM_INV_REQ_ERR;
2712 break;
2713 case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2714 wc->status = IB_WC_REM_ACCESS_ERR;
2715 break;
2716 case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2717 wc->status = IB_WC_REM_OP_ERR;
2718 break;
2719 case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2720 wc->status = IB_WC_RETRY_EXC_ERR;
2721 break;
2722 case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2723 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2724 break;
2725 case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2726 wc->status = IB_WC_REM_ABORT_ERR;
2727 break;
2728 default:
2729 wc->status = IB_WC_GENERAL_ERR;
2730 break;
2731 }
2732
0425e3e6
YL		 2733 /* flush cqe if wc status is error, excluding flush error */
2734 if ((wc->status != IB_WC_SUCCESS) &&
2735 (wc->status != IB_WC_WR_FLUSH_ERR)) {
2736 attr_mask = IB_QP_STATE;
2737 attr.qp_state = IB_QPS_ERR;
2738 return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2739 &attr, attr_mask,
2740 (*cur_qp)->state, IB_QPS_ERR);
2741 }
2742
2743 if (wc->status == IB_WC_WR_FLUSH_ERR)
93aa2187
WHX		 2744 return 0;
2745
2746 if (is_send) {
2747 wc->wc_flags = 0;
2748 /* SQ corresponding to CQE */
2749 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2750 V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2751 case HNS_ROCE_SQ_OPCODE_SEND:
2752 wc->opcode = IB_WC_SEND;
2753 break;
2754 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2755 wc->opcode = IB_WC_SEND;
2756 break;
2757 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2758 wc->opcode = IB_WC_SEND;
2759 wc->wc_flags |= IB_WC_WITH_IMM;
2760 break;
2761 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2762 wc->opcode = IB_WC_RDMA_READ;
2763 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2764 break;
2765 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2766 wc->opcode = IB_WC_RDMA_WRITE;
2767 break;
2768 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2769 wc->opcode = IB_WC_RDMA_WRITE;
2770 wc->wc_flags |= IB_WC_WITH_IMM;
2771 break;
2772 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2773 wc->opcode = IB_WC_LOCAL_INV;
2774 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2775 break;
2776 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2777 wc->opcode = IB_WC_COMP_SWAP;
2778 wc->byte_len = 8;
2779 break;
2780 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2781 wc->opcode = IB_WC_FETCH_ADD;
2782 wc->byte_len = 8;
2783 break;
2784 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2785 wc->opcode = IB_WC_MASKED_COMP_SWAP;
2786 wc->byte_len = 8;
2787 break;
2788 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2789 wc->opcode = IB_WC_MASKED_FETCH_ADD;
2790 wc->byte_len = 8;
2791 break;
2792 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2793 wc->opcode = IB_WC_REG_MR;
2794 break;
2795 case HNS_ROCE_SQ_OPCODE_BIND_MW:
2796 wc->opcode = IB_WC_REG_MR;
2797 break;
2798 default:
2799 wc->status = IB_WC_GENERAL_ERR;
2800 break;
2801 }
93aa2187
WHX		 2802 } else {
2803 /* RQ correspond to CQE */
2804 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2805
2806 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2807 V2_CQE_BYTE_4_OPCODE_S);
2808 switch (opcode & 0x1f) {
2809 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
2810 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2811 wc->wc_flags = IB_WC_WITH_IMM;
0c4a0e29
LO		 2812 wc->ex.imm_data =
2813 cpu_to_be32(le32_to_cpu(cqe->immtdata));
93aa2187
WHX		 2814 break;
2815 case HNS_ROCE_V2_OPCODE_SEND:
2816 wc->opcode = IB_WC_RECV;
2817 wc->wc_flags = 0;
2818 break;
2819 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
2820 wc->opcode = IB_WC_RECV;
2821 wc->wc_flags = IB_WC_WITH_IMM;
0c4a0e29
LO		 2822 wc->ex.imm_data =
2823 cpu_to_be32(le32_to_cpu(cqe->immtdata));
93aa2187
WHX		 2824 break;
2825 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
2826 wc->opcode = IB_WC_RECV;
2827 wc->wc_flags = IB_WC_WITH_INVALIDATE;
ccb8a29e 2828 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
93aa2187
WHX		 2829 break;
2830 default:
2831 wc->status = IB_WC_GENERAL_ERR;
2832 break;
2833 }
2834
0009c2db 2835 if ((wc->qp->qp_type == IB_QPT_RC ||
2836 wc->qp->qp_type == IB_QPT_UC) &&
2837 (opcode == HNS_ROCE_V2_OPCODE_SEND ||
2838 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
2839 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2840 (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2841 ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2842 if (ret)
2843 return -EAGAIN;
2844 }
2845
93aa2187
WHX		 2846 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2847 V2_CQE_BYTE_32_SL_S);
2848 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
2849 V2_CQE_BYTE_32_RMT_QPN_M,
2850 V2_CQE_BYTE_32_RMT_QPN_S);
15fc056f 2851 wc->slid = 0;
93aa2187
WHX		 2852 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
2853 V2_CQE_BYTE_32_GRH_S) ?
2854 IB_WC_GRH : 0);
6c1f08b3 2855 wc->port_num = roce_get_field(cqe->byte_32,
2856 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2857 wc->pkey_index = 0;
2eade675 2858 memcpy(wc->smac, cqe->smac, 4);
2859 wc->smac[4] = roce_get_field(cqe->byte_28,
2860 V2_CQE_BYTE_28_SMAC_4_M,
2861 V2_CQE_BYTE_28_SMAC_4_S);
2862 wc->smac[5] = roce_get_field(cqe->byte_28,
2863 V2_CQE_BYTE_28_SMAC_5_M,
2864 V2_CQE_BYTE_28_SMAC_5_S);
944e6409
LO		 2865 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
2866 wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
2867 V2_CQE_BYTE_28_VID_M,
2868 V2_CQE_BYTE_28_VID_S);
2869 } else {
2870 wc->vlan_id = 0xffff;
2871 }
2872
2eade675 2873 wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
2874 wc->network_hdr_type = roce_get_field(cqe->byte_28,
2875 V2_CQE_BYTE_28_PORT_TYPE_M,
2876 V2_CQE_BYTE_28_PORT_TYPE_S);
93aa2187
WHX		 2877 }
2878
2879 return 0;
2880}
2881
2882static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2883 struct ib_wc *wc)
2884{
2885 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2886 struct hns_roce_qp *cur_qp = NULL;
2887 unsigned long flags;
2888 int npolled;
2889
2890 spin_lock_irqsave(&hr_cq->lock, flags);
2891
2892 for (npolled = 0; npolled < num_entries; ++npolled) {
2893 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
2894 break;
2895 }
2896
2897 if (npolled) {
2898 /* Memory barrier */
2899 wmb();
2900 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2901 }
2902
2903 spin_unlock_irqrestore(&hr_cq->lock, flags);
2904
2905 return npolled;
2906}
2907
a81fba28
WHX		 2908static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
2909 struct hns_roce_hem_table *table, int obj,
2910 int step_idx)
2911{
2912 struct device *dev = hr_dev->dev;
2913 struct hns_roce_cmd_mailbox *mailbox;
2914 struct hns_roce_hem_iter iter;
2915 struct hns_roce_hem_mhop mhop;
2916 struct hns_roce_hem *hem;
2917 unsigned long mhop_obj = obj;
2918 int i, j, k;
2919 int ret = 0;
2920 u64 hem_idx = 0;
2921 u64 l1_idx = 0;
2922 u64 bt_ba = 0;
2923 u32 chunk_ba_num;
2924 u32 hop_num;
2925 u16 op = 0xff;
2926
2927 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2928 return 0;
2929
2930 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
2931 i = mhop.l0_idx;
2932 j = mhop.l1_idx;
2933 k = mhop.l2_idx;
2934 hop_num = mhop.hop_num;
2935 chunk_ba_num = mhop.bt_chunk_size / 8;
2936
2937 if (hop_num == 2) {
2938 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
2939 k;
2940 l1_idx = i * chunk_ba_num + j;
2941 } else if (hop_num == 1) {
2942 hem_idx = i * chunk_ba_num + j;
2943 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
2944 hem_idx = i;
2945 }
2946
2947 switch (table->type) {
2948 case HEM_TYPE_QPC:
2949 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2950 break;
2951 case HEM_TYPE_MTPT:
2952 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2953 break;
2954 case HEM_TYPE_CQC:
2955 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2956 break;
2957 case HEM_TYPE_SRQC:
2958 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2959 break;
6a157f7d
YL		 2960 case HEM_TYPE_SCCC:
2961 op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
2962 break;
0e40dc2f
YL		 2963 case HEM_TYPE_QPC_TIMER:
2964 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
2965 break;
2966 case HEM_TYPE_CQC_TIMER:
2967 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
2968 break;
a81fba28
WHX		 2969 default:
2970 dev_warn(dev, "Table %d not to be written by mailbox!\n",
2971 table->type);
2972 return 0;
2973 }
6a157f7d
YL		 2974
2975 if (table->type == HEM_TYPE_SCCC && step_idx)
2976 return 0;
2977
a81fba28
WHX		 2978 op += step_idx;
2979
2980 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2981 if (IS_ERR(mailbox))
2982 return PTR_ERR(mailbox);
2983
6ac16e40
YL		 2984 if (table->type == HEM_TYPE_SCCC)
2985 obj = mhop.l0_idx;
2986
a81fba28
WHX		 2987 if (check_whether_last_step(hop_num, step_idx)) {
2988 hem = table->hem[hem_idx];
2989 for (hns_roce_hem_first(hem, &iter);
2990 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2991 bt_ba = hns_roce_hem_addr(&iter);
2992
2993 /* configure the ba, tag, and op */
2994 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2995 obj, 0, op,
2996 HNS_ROCE_CMD_TIMEOUT_MSECS);
2997 }
2998 } else {
2999 if (step_idx == 0)
3000 bt_ba = table->bt_l0_dma_addr[i];
3001 else if (step_idx == 1 && hop_num == 2)
3002 bt_ba = table->bt_l1_dma_addr[l1_idx];
3003
3004 /* configure the ba, tag, and op */
3005 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3006 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3007 }
3008
3009 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3010 return ret;
3011}
3012
3013static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3014 struct hns_roce_hem_table *table, int obj,
3015 int step_idx)
3016{
3017 struct device *dev = hr_dev->dev;
3018 struct hns_roce_cmd_mailbox *mailbox;
3019 int ret = 0;
3020 u16 op = 0xff;
3021
3022 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3023 return 0;
3024
3025 switch (table->type) {
3026 case HEM_TYPE_QPC:
3027 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3028 break;
3029 case HEM_TYPE_MTPT:
3030 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3031 break;
3032 case HEM_TYPE_CQC:
3033 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3034 break;
6a157f7d 3035 case HEM_TYPE_SCCC:
0e40dc2f
YL		 3036 case HEM_TYPE_QPC_TIMER:
3037 case HEM_TYPE_CQC_TIMER:
6a157f7d 3038 break;
a81fba28
WHX		 3039 case HEM_TYPE_SRQC:
3040 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3041 break;
3042 default:
3043 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
3044 table->type);
3045 return 0;
3046 }
6a157f7d 3047
0e40dc2f
YL		 3048 if (table->type == HEM_TYPE_SCCC ||
3049 table->type == HEM_TYPE_QPC_TIMER ||
3050 table->type == HEM_TYPE_CQC_TIMER)
6a157f7d
YL		 3051 return 0;
3052
a81fba28
WHX		 3053 op += step_idx;
3054
3055 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3056 if (IS_ERR(mailbox))
3057 return PTR_ERR(mailbox);
3058
3059 /* configure the tag and op */
3060 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3061 HNS_ROCE_CMD_TIMEOUT_MSECS);
3062
3063 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3064 return ret;
3065}
3066
926a01dc 3067static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
926a01dc
WHX		 3068 enum ib_qp_state cur_state,
3069 enum ib_qp_state new_state,
3070 struct hns_roce_v2_qp_context *context,
3071 struct hns_roce_qp *hr_qp)
3072{
3073 struct hns_roce_cmd_mailbox *mailbox;
3074 int ret;
3075
3076 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3077 if (IS_ERR(mailbox))
3078 return PTR_ERR(mailbox);
3079
3080 memcpy(mailbox->buf, context, sizeof(*context) * 2);
3081
3082 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
3083 HNS_ROCE_CMD_MODIFY_QPC,
3084 HNS_ROCE_CMD_TIMEOUT_MSECS);
3085
3086 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3087
3088 return ret;
3089}
3090
ace1c541 3091static void set_access_flags(struct hns_roce_qp *hr_qp,
3092 struct hns_roce_v2_qp_context *context,
3093 struct hns_roce_v2_qp_context *qpc_mask,
3094 const struct ib_qp_attr *attr, int attr_mask)
3095{
3096 u8 dest_rd_atomic;
3097 u32 access_flags;
3098
c2799119 3099 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
ace1c541 3100 attr->max_dest_rd_atomic : hr_qp->resp_depth;
3101
c2799119 3102 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
ace1c541 3103 attr->qp_access_flags : hr_qp->atomic_rd_en;
3104
3105 if (!dest_rd_atomic)
3106 access_flags &= IB_ACCESS_REMOTE_WRITE;
3107
3108 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3109 !!(access_flags & IB_ACCESS_REMOTE_READ));
3110 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
3111
3112 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3113 !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3114 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
3115
3116 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3117 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3118 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
3119}
3120
926a01dc
WHX		 3121static void modify_qp_reset_to_init(struct ib_qp *ibqp,
3122 const struct ib_qp_attr *attr,
0fa95a9a 3123 int attr_mask,
926a01dc
WHX		 3124 struct hns_roce_v2_qp_context *context,
3125 struct hns_roce_v2_qp_context *qpc_mask)
3126{
ecaaf1e2 3127 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
926a01dc
WHX		 3128 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3129
3130 /*
3131 * In v2 engine, software pass context and context mask to hardware
3132 * when modifying qp. If software need modify some fields in context,
3133 * we should set all bits of the relevant fields in context mask to
3134 * 0 at the same time, else set them to 0x1.
3135 */
3136 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3137 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3138 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3139 V2_QPC_BYTE_4_TST_S, 0);
3140
0fa95a9a 3141 if (ibqp->qp_type == IB_QPT_GSI)
3142 roce_set_field(context->byte_4_sqpn_tst,
3143 V2_QPC_BYTE_4_SGE_SHIFT_M,
3144 V2_QPC_BYTE_4_SGE_SHIFT_S,
3145 ilog2((unsigned int)hr_qp->sge.sge_cnt));
3146 else
3147 roce_set_field(context->byte_4_sqpn_tst,
3148 V2_QPC_BYTE_4_SGE_SHIFT_M,
3149 V2_QPC_BYTE_4_SGE_SHIFT_S,
3150 hr_qp->sq.max_gs > 2 ?
3151 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3152
926a01dc
WHX		 3153 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3154 V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3155
3156 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3157 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3158 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3159 V2_QPC_BYTE_4_SQPN_S, 0);
3160
3161 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3162 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3163 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3164 V2_QPC_BYTE_16_PD_S, 0);
3165
3166 roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3167 V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
3168 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3169 V2_QPC_BYTE_20_RQWS_S, 0);
3170
3171 roce_set_field(context->byte_20_smac_sgid_idx,
3172 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3173 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3174 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3175 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3176
3177 roce_set_field(context->byte_20_smac_sgid_idx,
3178 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
c7bcb134
LO		 3179 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3180 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 :
926a01dc
WHX		 3181 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3182 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3183 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3184
3185 /* No VLAN need to set 0xFFF */
c8e46f8d
LO		 3186 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3187 V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
3188 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3189 V2_QPC_BYTE_24_VLAN_ID_S, 0);
926a01dc
WHX		 3190
3191 /*
3192 * Set some fields in context to zero, Because the default values
3193 * of all fields in context are zero, we need not set them to 0 again.
3194 * but we should set the relevant fields of context mask to 0.
3195 */
3196 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
3197 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
3198 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
3199 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
3200
2362ccee
LO		 3201 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,
3202 V2_QPC_BYTE_60_TEMPID_S, 0);
3203
3204 roce_set_field(qpc_mask->byte_60_qpst_tempid,
3205 V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,
3206 0);
3207 roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3208 V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);
3209 roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3210 V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);
926a01dc
WHX		 3211 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
3212 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
3213
e088a685
YL		 3214 if (hr_qp->rdb_en) {
3215 roce_set_bit(context->byte_68_rq_db,
3216 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
3217 roce_set_bit(qpc_mask->byte_68_rq_db,
3218 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
3219 }
3220
3221 roce_set_field(context->byte_68_rq_db,
3222 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3223 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
3224 ((u32)hr_qp->rdb.dma) >> 1);
3225 roce_set_field(qpc_mask->byte_68_rq_db,
3226 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3227 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
3228 context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
3229 qpc_mask->rq_db_record_addr = 0;
3230
ecaaf1e2 3231 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
3232 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
926a01dc
WHX		 3233 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
3234
3235 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3236 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3237 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3238 V2_QPC_BYTE_80_RX_CQN_S, 0);
3239 if (ibqp->srq) {
3240 roce_set_field(context->byte_76_srqn_op_en,
3241 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3242 to_hr_srq(ibqp->srq)->srqn);
3243 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3244 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3245 roce_set_bit(context->byte_76_srqn_op_en,
3246 V2_QPC_BYTE_76_SRQ_EN_S, 1);
3247 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3248 V2_QPC_BYTE_76_SRQ_EN_S, 0);
3249 }
3250
3251 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3252 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3253 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3254 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3255 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3256 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3257
3258 roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
3259 V2_QPC_BYTE_92_SRQ_INFO_S, 0);
3260
3261 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3262 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3263
3264 roce_set_field(qpc_mask->byte_104_rq_sge,
3265 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
3266 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
3267
3268 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3269 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3270 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3271 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3272 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3273 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3274 V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
3275
3276 qpc_mask->rq_rnr_timer = 0;
3277 qpc_mask->rx_msg_len = 0;
3278 qpc_mask->rx_rkey_pkt_info = 0;
3279 qpc_mask->rx_va = 0;
3280
3281 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3282 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3283 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3284 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3285
2362ccee
LO		 3286 roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,
3287 0);
926a01dc
WHX		 3288 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
3289 V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
3290 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
3291 V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
3292
3293 roce_set_field(qpc_mask->byte_144_raq,
3294 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
3295 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
926a01dc
WHX		 3296 roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
3297 V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
3298 roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
3299
3300 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
3301 V2_QPC_BYTE_148_RQ_MSN_S, 0);
3302 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
3303 V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
3304
3305 roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3306 V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3307 roce_set_field(qpc_mask->byte_152_raq,
3308 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
3309 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
3310
3311 roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
3312 V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
3313
3314 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3315 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3316 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3317 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3318 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
3319 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
3320
2362ccee
LO		 3321 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3322 V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);
3323 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3324 V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);
3325 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3326 V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);
926a01dc
WHX		 3327 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3328 V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
b5fddb7c 3329 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3330 V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
926a01dc
WHX		 3331 roce_set_field(qpc_mask->byte_168_irrl_idx,
3332 V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
3333 V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
3334
3335 roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3336 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
3337 roce_set_field(qpc_mask->byte_172_sq_psn,
3338 V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3339 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3340
3341 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
3342 0);
3343
68a997c5
YL		 3344 roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
3345 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);
3346
926a01dc
WHX		 3347 roce_set_field(qpc_mask->byte_176_msg_pktn,
3348 V2_QPC_BYTE_176_MSG_USE_PKTN_M,
3349 V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
3350 roce_set_field(qpc_mask->byte_176_msg_pktn,
3351 V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
3352 V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
3353
3354 roce_set_field(qpc_mask->byte_184_irrl_idx,
3355 V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
3356 V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
3357
3358 qpc_mask->cur_sge_offset = 0;
3359
3360 roce_set_field(qpc_mask->byte_192_ext_sge,
3361 V2_QPC_BYTE_192_CUR_SGE_IDX_M,
3362 V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
3363 roce_set_field(qpc_mask->byte_192_ext_sge,
3364 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
3365 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
3366
3367 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3368 V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3369
3370 roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
3371 V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
3372 roce_set_field(qpc_mask->byte_200_sq_max,
3373 V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
3374 V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
3375
3376 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
3377 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
3378
3379 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3380 V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3381
3382 qpc_mask->sq_timer = 0;
3383
3384 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3385 V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3386 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3387 roce_set_field(qpc_mask->byte_232_irrl_sge,
3388 V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3389 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3390
2362ccee
LO		 3391 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,
3392 0);
3393 roce_set_bit(qpc_mask->byte_232_irrl_sge,
3394 V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);
3395 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,
3396 0);
3397
926a01dc
WHX		 3398 qpc_mask->irrl_cur_sge_offset = 0;
3399
3400 roce_set_field(qpc_mask->byte_240_irrl_tail,
3401 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3402 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3403 roce_set_field(qpc_mask->byte_240_irrl_tail,
3404 V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
3405 V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
3406 roce_set_field(qpc_mask->byte_240_irrl_tail,
3407 V2_QPC_BYTE_240_RX_ACK_MSN_M,
3408 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3409
3410 roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
3411 V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3412 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
3413 0);
3414 roce_set_field(qpc_mask->byte_248_ack_psn,
3415 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3416 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3417 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
3418 0);
3419 roce_set_bit(qpc_mask->byte_248_ack_psn,
3420 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3421 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
3422 0);
3423
3424 hr_qp->access_flags = attr->qp_access_flags;
926a01dc
WHX		 3425 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3426 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3427 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3428 V2_QPC_BYTE_252_TX_CQN_S, 0);
3429
3430 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
3431 V2_QPC_BYTE_252_ERR_TYPE_S, 0);
3432
3433 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3434 V2_QPC_BYTE_256_RQ_CQE_IDX_M,
3435 V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
3436 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3437 V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
3438 V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
3439}
3440
3441static void modify_qp_init_to_init(struct ib_qp *ibqp,
3442 const struct ib_qp_attr *attr, int attr_mask,
3443 struct hns_roce_v2_qp_context *context,
3444 struct hns_roce_v2_qp_context *qpc_mask)
3445{
3446 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3447
3448 /*
3449 * In v2 engine, software pass context and context mask to hardware
3450 * when modifying qp. If software need modify some fields in context,
3451 * we should set all bits of the relevant fields in context mask to
3452 * 0 at the same time, else set them to 0x1.
3453 */
3454 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3455 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3456 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3457 V2_QPC_BYTE_4_TST_S, 0);
3458
0fa95a9a 3459 if (ibqp->qp_type == IB_QPT_GSI)
3460 roce_set_field(context->byte_4_sqpn_tst,
3461 V2_QPC_BYTE_4_SGE_SHIFT_M,
3462 V2_QPC_BYTE_4_SGE_SHIFT_S,
3463 ilog2((unsigned int)hr_qp->sge.sge_cnt));
3464 else
3465 roce_set_field(context->byte_4_sqpn_tst,
3466 V2_QPC_BYTE_4_SGE_SHIFT_M,
2a3d923f
LO		 3467 V2_QPC_BYTE_4_SGE_SHIFT_S,
3468 hr_qp->sq.max_gs >
3469 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
0fa95a9a 3470 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3471
926a01dc
WHX		 3472 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3473 V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3474
3475 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3476 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3477 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3478 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3479 0);
3480
3481 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3482 !!(attr->qp_access_flags &
3483 IB_ACCESS_REMOTE_WRITE));
3484 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3485 0);
3486
3487 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3488 !!(attr->qp_access_flags &
3489 IB_ACCESS_REMOTE_ATOMIC));
3490 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3491 0);
3492 } else {
3493 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3494 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3495 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3496 0);
3497
3498 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3499 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3500 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3501 0);
3502
3503 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3504 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3505 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3506 0);
3507 }
3508
3509 roce_set_field(context->byte_20_smac_sgid_idx,
3510 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3511 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3512 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3513 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3514
3515 roce_set_field(context->byte_20_smac_sgid_idx,
3516 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
c7bcb134
LO		 3517 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3518 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 :
926a01dc
WHX		 3519 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3520 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3521 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3522
3523 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3524 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3525 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3526 V2_QPC_BYTE_16_PD_S, 0);
3527
3528 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3529 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3530 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3531 V2_QPC_BYTE_80_RX_CQN_S, 0);
3532
3533 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
6d13b869 3534 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
926a01dc
WHX		 3535 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3536 V2_QPC_BYTE_252_TX_CQN_S, 0);
3537
3538 if (ibqp->srq) {
3539 roce_set_bit(context->byte_76_srqn_op_en,
3540 V2_QPC_BYTE_76_SRQ_EN_S, 1);
3541 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3542 V2_QPC_BYTE_76_SRQ_EN_S, 0);
3543 roce_set_field(context->byte_76_srqn_op_en,
3544 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3545 to_hr_srq(ibqp->srq)->srqn);
3546 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3547 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3548 }
3549
926a01dc
WHX		 3550 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3551 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3552 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3553 V2_QPC_BYTE_4_SQPN_S, 0);
3554
b6dd9b34 3555 if (attr_mask & IB_QP_DEST_QPN) {
3556 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3557 V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
3558 roce_set_field(qpc_mask->byte_56_dqpn_err,
3559 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3560 }
926a01dc
WHX		 3561}
3562
8d18ad83
LO		 3563static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
3564 struct hns_roce_qp *hr_qp, int mtt_cnt,
3565 u32 page_size)
3566{
3567 struct device *dev = hr_dev->dev;
3568
3569 if (hr_qp->rq.wqe_cnt < 1)
3570 return true;
3571
3572 if (mtt_cnt < 1) {
3573 dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n",
3574 hr_qp->qpn);
3575 return false;
3576 }
3577
3578 if (mtt_cnt < MTT_MIN_COUNT &&
3579 (hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
3580 dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n",
3581 hr_qp->qpn);
3582 return false;
3583 }
3584
3585 return true;
3586}
3587
926a01dc
WHX		 3588static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
3589 const struct ib_qp_attr *attr, int attr_mask,
3590 struct hns_roce_v2_qp_context *context,
3591 struct hns_roce_v2_qp_context *qpc_mask)
3592{
3593 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
3594 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3595 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3596 struct device *dev = hr_dev->dev;
8d18ad83 3597 u64 mtts[MTT_MIN_COUNT] = { 0 };
e92f2c18 3598 dma_addr_t dma_handle_3;
926a01dc 3599 dma_addr_t dma_handle_2;
8d18ad83 3600 u64 wqe_sge_ba;
926a01dc
WHX		 3601 u32 page_size;
3602 u8 port_num;
e92f2c18 3603 u64 *mtts_3;
926a01dc 3604 u64 *mtts_2;
8d18ad83 3605 int count;
926a01dc
WHX		 3606 u8 *dmac;
3607 u8 *smac;
3608 int port;
3609
3610 /* Search qp buf's mtts */
8d18ad83
LO		 3611 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3612 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3613 hr_qp->rq.offset / page_size, mtts,
3614 MTT_MIN_COUNT, &wqe_sge_ba);
3615 if (!ibqp->srq)
3616 if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
3617 return -EINVAL;
926a01dc
WHX		 3618
3619 /* Search IRRL's mtts */
3620 mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
3621 hr_qp->qpn, &dma_handle_2);
3622 if (!mtts_2) {
3623 dev_err(dev, "qp irrl_table find failed\n");
3624 return -EINVAL;
3625 }
3626
e92f2c18 3627 /* Search TRRL's mtts */
3628 mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3629 hr_qp->qpn, &dma_handle_3);
3630 if (!mtts_3) {
3631 dev_err(dev, "qp trrl_table find failed\n");
3632 return -EINVAL;
3633 }
3634
734f3863 3635 if (attr_mask & IB_QP_ALT_PATH) {
926a01dc
WHX		 3636 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
3637 return -EINVAL;
3638 }
3639
3640 dmac = (u8 *)attr->ah_attr.roce.dmac;
8d18ad83 3641 context->wqe_sge_ba = (u32)(wqe_sge_ba >> 3);
926a01dc
WHX		 3642 qpc_mask->wqe_sge_ba = 0;
3643
3644 /*
3645 * In v2 engine, software pass context and context mask to hardware
3646 * when modifying qp. If software need modify some fields in context,
3647 * we should set all bits of the relevant fields in context mask to
3648 * 0 at the same time, else set them to 0x1.
3649 */
3650 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
8d18ad83 3651 V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
926a01dc
WHX		 3652 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3653 V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3654
3655 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3656 V2_QPC_BYTE_12_SQ_HOP_NUM_S,
8d18ad83
LO		 3657 hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3658 0 : hr_dev->caps.wqe_sq_hop_num);
926a01dc
WHX		 3659 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3660 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3661
3662 roce_set_field(context->byte_20_smac_sgid_idx,
3663 V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3664 V2_QPC_BYTE_20_SGE_HOP_NUM_S,
8d18ad83
LO		 3665 ((ibqp->qp_type == IB_QPT_GSI) ||
3666 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3667 hr_dev->caps.wqe_sge_hop_num : 0);
926a01dc
WHX		 3668 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3669 V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3670 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3671
3672 roce_set_field(context->byte_20_smac_sgid_idx,
3673 V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3674 V2_QPC_BYTE_20_RQ_HOP_NUM_S,
8d18ad83
LO		 3675 hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3676 0 : hr_dev->caps.wqe_rq_hop_num);
926a01dc
WHX		 3677 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3678 V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3679 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3680
3681 roce_set_field(context->byte_16_buf_ba_pg_sz,
3682 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3683 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
8d18ad83 3684 hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
926a01dc
WHX		 3685 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3686 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3687 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3688
3689 roce_set_field(context->byte_16_buf_ba_pg_sz,
3690 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3691 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
5e6e78db 3692 hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
926a01dc
WHX		 3693 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3694 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3695 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3696
8d18ad83 3697 context->rq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 3698 qpc_mask->rq_cur_blk_addr = 0;
3699
3700 roce_set_field(context->byte_92_srq_info,
3701 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3702 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
8d18ad83 3703 mtts[0] >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 3704 roce_set_field(qpc_mask->byte_92_srq_info,
3705 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3706 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3707
8d18ad83 3708 context->rq_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 3709 qpc_mask->rq_nxt_blk_addr = 0;
3710
3711 roce_set_field(context->byte_104_rq_sge,
3712 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3713 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
8d18ad83 3714 mtts[1] >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 3715 roce_set_field(qpc_mask->byte_104_rq_sge,
3716 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3717 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3718
e92f2c18 3719 roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3720 V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
3721 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3722 V2_QPC_BYTE_132_TRRL_BA_S, 0);
3723 context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
3724 qpc_mask->trrl_ba = 0;
3725 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3726 V2_QPC_BYTE_140_TRRL_BA_S,
3727 (u32)(dma_handle_3 >> (32 + 16 + 4)));
3728 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3729 V2_QPC_BYTE_140_TRRL_BA_S, 0);
3730
d5514246 3731 context->irrl_ba = (u32)(dma_handle_2 >> 6);
926a01dc
WHX		 3732 qpc_mask->irrl_ba = 0;
3733 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3734 V2_QPC_BYTE_208_IRRL_BA_S,
d5514246 3735 dma_handle_2 >> (32 + 6));
926a01dc
WHX		 3736 roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3737 V2_QPC_BYTE_208_IRRL_BA_S, 0);
3738
3739 roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
3740 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
3741
3742 roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3743 hr_qp->sq_signal_bits);
3744 roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3745 0);
3746
3747 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3748
3749 smac = (u8 *)hr_dev->dev_addr[port];
3750 /* when dmac equals smac or loop_idc is 1, it should loopback */
3751 if (ether_addr_equal_unaligned(dmac, smac) ||
3752 hr_dev->loop_idc == 0x1) {
3753 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3754 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3755 }
3756
b6dd9b34 3757 if (attr_mask & IB_QP_DEST_QPN) {
3758 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3759 V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
3760 roce_set_field(qpc_mask->byte_56_dqpn_err,
3761 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3762 }
926a01dc
WHX		 3763
3764 /* Configure GID index */
3765 port_num = rdma_ah_get_port_num(&attr->ah_attr);
3766 roce_set_field(context->byte_20_smac_sgid_idx,
3767 V2_QPC_BYTE_20_SGID_IDX_M,
3768 V2_QPC_BYTE_20_SGID_IDX_S,
3769 hns_get_gid_index(hr_dev, port_num - 1,
3770 grh->sgid_index));
3771 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3772 V2_QPC_BYTE_20_SGID_IDX_M,
3773 V2_QPC_BYTE_20_SGID_IDX_S, 0);
2a3d923f 3774 memcpy(&(context->dmac), dmac, sizeof(u32));
926a01dc
WHX		 3775 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3776 V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
3777 qpc_mask->dmac = 0;
3778 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3779 V2_QPC_BYTE_52_DMAC_S, 0);
3780
2a3d923f 3781 /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
926a01dc
WHX		 3782 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3783 V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
3784 roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3785 V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3786
0fa95a9a 3787 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
3788 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3789 V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
6852af86 3790 else if (attr_mask & IB_QP_PATH_MTU)
0fa95a9a 3791 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3792 V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
3793
926a01dc
WHX		 3794 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3795 V2_QPC_BYTE_24_MTU_S, 0);
3796
926a01dc
WHX		 3797 roce_set_field(context->byte_84_rq_ci_pi,
3798 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3799 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3800 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3801 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3802 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3803
3804 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3805 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3806 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3807 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3808 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3809 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3810 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3811 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3812 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3813 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3814
3815 context->rq_rnr_timer = 0;
3816 qpc_mask->rq_rnr_timer = 0;
3817
926a01dc
WHX		 3818 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3819 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3820 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3821 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3822
2a3d923f 3823 /* rocee send 2^lp_sgen_ini segs every time */
926a01dc
WHX		 3824 roce_set_field(context->byte_168_irrl_idx,
3825 V2_QPC_BYTE_168_LP_SGEN_INI_M,
3826 V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
3827 roce_set_field(qpc_mask->byte_168_irrl_idx,
3828 V2_QPC_BYTE_168_LP_SGEN_INI_M,
3829 V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
3830
926a01dc
WHX		 3831 return 0;
3832}
3833
3834static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
3835 const struct ib_qp_attr *attr, int attr_mask,
3836 struct hns_roce_v2_qp_context *context,
3837 struct hns_roce_v2_qp_context *qpc_mask)
3838{
3839 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3840 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3841 struct device *dev = hr_dev->dev;
8d18ad83
LO		 3842 u64 sge_cur_blk = 0;
3843 u64 sq_cur_blk = 0;
befb63b4 3844 u32 page_size;
8d18ad83 3845 int count;
926a01dc
WHX		 3846
3847 /* Search qp buf's mtts */
8d18ad83
LO		 3848 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
3849 if (count < 1) {
3850 dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn);
926a01dc
WHX		 3851 return -EINVAL;
3852 }
3853
8d18ad83
LO		 3854 if (hr_qp->sge.offset) {
3855 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3856 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3857 hr_qp->sge.offset / page_size,
3858 &sge_cur_blk, 1, NULL);
3859 if (count < 1) {
3860 dev_err(dev, "qp(0x%lx) sge pa find failed\n",
3861 hr_qp->qpn);
3862 return -EINVAL;
3863 }
3864 }
3865
734f3863 3866 /* Not support alternate path and path migration */
3867 if ((attr_mask & IB_QP_ALT_PATH) ||
3868 (attr_mask & IB_QP_PATH_MIG_STATE)) {
926a01dc
WHX		 3869 dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3870 return -EINVAL;
3871 }
3872
3873 /*
3874 * In v2 engine, software pass context and context mask to hardware
3875 * when modifying qp. If software need modify some fields in context,
3876 * we should set all bits of the relevant fields in context mask to
3877 * 0 at the same time, else set them to 0x1.
3878 */
8d18ad83 3879 context->sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 3880 roce_set_field(context->byte_168_irrl_idx,
3881 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3882 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
8d18ad83 3883 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 3884 qpc_mask->sq_cur_blk_addr = 0;
3885 roce_set_field(qpc_mask->byte_168_irrl_idx,
3886 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3887 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3888
2a3d923f
LO		 3889 context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
3890 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
8d18ad83 3891 ((u32)(sge_cur_blk >>
2a3d923f 3892 PAGE_ADDR_SHIFT)) : 0;
befb63b4 3893 roce_set_field(context->byte_184_irrl_idx,
3894 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3895 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
2a3d923f
LO		 3896 ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
3897 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
8d18ad83 3898 (sge_cur_blk >>
befb63b4 3899 (32 + PAGE_ADDR_SHIFT)) : 0);
3900 qpc_mask->sq_cur_sge_blk_addr = 0;
3901 roce_set_field(qpc_mask->byte_184_irrl_idx,
3902 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3903 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3904
8d18ad83 3905 context->rx_sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 3906 roce_set_field(context->byte_232_irrl_sge,
3907 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3908 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
8d18ad83 3909 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 3910 qpc_mask->rx_sq_cur_blk_addr = 0;
3911 roce_set_field(qpc_mask->byte_232_irrl_sge,
3912 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3913 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
3914
3915 /*
3916 * Set some fields in context to zero, Because the default values
3917 * of all fields in context are zero, we need not set them to 0 again.
3918 * but we should set the relevant fields of context mask to 0.
3919 */
3920 roce_set_field(qpc_mask->byte_232_irrl_sge,
3921 V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3922 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3923
3924 roce_set_field(qpc_mask->byte_240_irrl_tail,
3925 V2_QPC_BYTE_240_RX_ACK_MSN_M,
3926 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3927
926a01dc
WHX		 3928 roce_set_field(qpc_mask->byte_248_ack_psn,
3929 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3930 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3931 roce_set_bit(qpc_mask->byte_248_ack_psn,
3932 V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
3933 roce_set_field(qpc_mask->byte_248_ack_psn,
3934 V2_QPC_BYTE_248_IRRL_PSN_M,
3935 V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3936
3937 roce_set_field(qpc_mask->byte_240_irrl_tail,
3938 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3939 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3940
926a01dc
WHX		 3941 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3942 V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3943 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3944
3945 roce_set_bit(qpc_mask->byte_248_ack_psn,
3946 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3947
3948 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3949 V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3950
926a01dc
WHX		 3951 roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3952 V2_QPC_BYTE_212_LSN_S, 0x100);
3953 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3954 V2_QPC_BYTE_212_LSN_S, 0);
3955
926a01dc
WHX		 3956 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3957 V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
926a01dc
WHX		 3958
3959 return 0;
3960}
3961
233673e4
LO		 3962static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,
3963 enum ib_qp_state new_state)
3964{
3965
3966 if ((cur_state != IB_QPS_RESET &&
3967 (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||
3968 ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &&
3969 (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||
3970 (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS))
3971 return true;
3972
3973 return false;
3974
3975}
3976
926a01dc
WHX		 3977static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
3978 const struct ib_qp_attr *attr,
3979 int attr_mask, enum ib_qp_state cur_state,
3980 enum ib_qp_state new_state)
3981{
3982 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3983 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3984 struct hns_roce_v2_qp_context *context;
3985 struct hns_roce_v2_qp_context *qpc_mask;
3986 struct device *dev = hr_dev->dev;
3987 int ret = -EINVAL;
3988
4e69cf1f 3989 context = kcalloc(2, sizeof(*context), GFP_ATOMIC);
926a01dc
WHX		 3990 if (!context)
3991 return -ENOMEM;
3992
3993 qpc_mask = context + 1;
3994 /*
3995 * In v2 engine, software pass context and context mask to hardware
3996 * when modifying qp. If software need modify some fields in context,
3997 * we should set all bits of the relevant fields in context mask to
3998 * 0 at the same time, else set them to 0x1.
3999 */
4000 memset(qpc_mask, 0xff, sizeof(*qpc_mask));
4001 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
9f507101 4002 memset(qpc_mask, 0, sizeof(*qpc_mask));
0fa95a9a 4003 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4004 qpc_mask);
926a01dc
WHX		 4005 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4006 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4007 qpc_mask);
4008 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4009 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4010 qpc_mask);
4011 if (ret)
4012 goto out;
4013 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4014 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4015 qpc_mask);
4016 if (ret)
4017 goto out;
233673e4 4018 } else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {
926a01dc
WHX		 4019 /* Nothing */
4020 ;
4021 } else {
4022 dev_err(dev, "Illegal state for QP!\n");
ac7cbf96 4023 ret = -EINVAL;
926a01dc
WHX		 4024 goto out;
4025 }
4026
0425e3e6
YL		 4027 /* When QP state is err, SQ and RQ WQE should be flushed */
4028 if (new_state == IB_QPS_ERR) {
4029 roce_set_field(context->byte_160_sq_ci_pi,
4030 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4031 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
4032 hr_qp->sq.head);
4033 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
4034 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4035 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
9c6ccc03
LO		 4036
4037 if (!ibqp->srq) {
4038 roce_set_field(context->byte_84_rq_ci_pi,
0425e3e6
YL		 4039 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4040 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
4041 hr_qp->rq.head);
9c6ccc03 4042 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
0425e3e6
YL		 4043 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4044 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
9c6ccc03 4045 }
0425e3e6
YL		 4046 }
4047
610b8967
LO		 4048 if (attr_mask & IB_QP_AV) {
4049 const struct ib_global_route *grh =
4050 rdma_ah_read_grh(&attr->ah_attr);
4051 const struct ib_gid_attr *gid_attr = NULL;
610b8967
LO		 4052 int is_roce_protocol;
4053 u16 vlan = 0xffff;
4054 u8 ib_port;
4055 u8 hr_port;
4056
4057 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num :
4058 hr_qp->port + 1;
4059 hr_port = ib_port - 1;
4060 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4061 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4062
4063 if (is_roce_protocol) {
4064 gid_attr = attr->ah_attr.grh.sgid_attr;
a70c0739
PP		 4065 ret = rdma_read_gid_l2_fields(gid_attr, &vlan, NULL);
4066 if (ret)
4067 goto out;
610b8967
LO		 4068 }
4069
a70c0739 4070 if (vlan < VLAN_CFI_MASK) {
caf3e406
LO		 4071 roce_set_bit(context->byte_76_srqn_op_en,
4072 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
4073 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4074 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
4075 roce_set_bit(context->byte_168_irrl_idx,
4076 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
4077 roce_set_bit(qpc_mask->byte_168_irrl_idx,
4078 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
4079 }
4080
c8e46f8d
LO		 4081 roce_set_field(context->byte_24_mtu_tc,
4082 V2_QPC_BYTE_24_VLAN_ID_M,
4083 V2_QPC_BYTE_24_VLAN_ID_S, vlan);
4084 roce_set_field(qpc_mask->byte_24_mtu_tc,
4085 V2_QPC_BYTE_24_VLAN_ID_M,
4086 V2_QPC_BYTE_24_VLAN_ID_S, 0);
4087
610b8967
LO		 4088 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4089 dev_err(hr_dev->dev,
4090 "sgid_index(%u) too large. max is %d\n",
4091 grh->sgid_index,
4092 hr_dev->caps.gid_table_len[hr_port]);
4093 ret = -EINVAL;
4094 goto out;
4095 }
4096
4097 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4098 dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
4099 ret = -EINVAL;
4100 goto out;
4101 }
4102
4103 roce_set_field(context->byte_52_udpspn_dmac,
4104 V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S,
4105 (gid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ?
4106 0 : 0x12b7);
4107
4108 roce_set_field(qpc_mask->byte_52_udpspn_dmac,
4109 V2_QPC_BYTE_52_UDPSPN_M,
4110 V2_QPC_BYTE_52_UDPSPN_S, 0);
4111
4112 roce_set_field(context->byte_20_smac_sgid_idx,
4113 V2_QPC_BYTE_20_SGID_IDX_M,
4114 V2_QPC_BYTE_20_SGID_IDX_S, grh->sgid_index);
4115
4116 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4117 V2_QPC_BYTE_20_SGID_IDX_M,
4118 V2_QPC_BYTE_20_SGID_IDX_S, 0);
4119
4120 roce_set_field(context->byte_24_mtu_tc,
4121 V2_QPC_BYTE_24_HOP_LIMIT_M,
4122 V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
4123 roce_set_field(qpc_mask->byte_24_mtu_tc,
4124 V2_QPC_BYTE_24_HOP_LIMIT_M,
4125 V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4126
157b52a0
LO		 4127 if (hr_dev->pci_dev->revision == 0x21 &&
4128 gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4129 roce_set_field(context->byte_24_mtu_tc,
4130 V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,
4131 grh->traffic_class >> 2);
4132 else
4133 roce_set_field(context->byte_24_mtu_tc,
4134 V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,
4135 grh->traffic_class);
610b8967
LO		 4136 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4137 V2_QPC_BYTE_24_TC_S, 0);
4138 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4139 V2_QPC_BYTE_28_FL_S, grh->flow_label);
4140 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4141 V2_QPC_BYTE_28_FL_S, 0);
4142 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4143 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4144 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4145 V2_QPC_BYTE_28_SL_S,
4146 rdma_ah_get_sl(&attr->ah_attr));
4147 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4148 V2_QPC_BYTE_28_SL_S, 0);
4149 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4150 }
4151
5b01b243
LO		 4152 if (attr_mask & IB_QP_TIMEOUT) {
4153 if (attr->timeout < 31) {
4154 roce_set_field(context->byte_28_at_fl,
4155 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4156 attr->timeout);
4157 roce_set_field(qpc_mask->byte_28_at_fl,
4158 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4159 0);
4160 } else {
4161 dev_warn(dev, "Local ACK timeout shall be 0 to 30.\n");
4162 }
4163 }
4164
4165 if (attr_mask & IB_QP_RETRY_CNT) {
4166 roce_set_field(context->byte_212_lsn,
4167 V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4168 V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
4169 attr->retry_cnt);
4170 roce_set_field(qpc_mask->byte_212_lsn,
4171 V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4172 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
4173
4174 roce_set_field(context->byte_212_lsn,
4175 V2_QPC_BYTE_212_RETRY_CNT_M,
4176 V2_QPC_BYTE_212_RETRY_CNT_S,
4177 attr->retry_cnt);
4178 roce_set_field(qpc_mask->byte_212_lsn,
4179 V2_QPC_BYTE_212_RETRY_CNT_M,
4180 V2_QPC_BYTE_212_RETRY_CNT_S, 0);
4181 }
4182
4183 if (attr_mask & IB_QP_RNR_RETRY) {
4184 roce_set_field(context->byte_244_rnr_rxack,
4185 V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4186 V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
4187 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4188 V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4189 V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
4190
4191 roce_set_field(context->byte_244_rnr_rxack,
4192 V2_QPC_BYTE_244_RNR_CNT_M,
4193 V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
4194 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4195 V2_QPC_BYTE_244_RNR_CNT_M,
4196 V2_QPC_BYTE_244_RNR_CNT_S, 0);
4197 }
4198
f04cc178
LO		 4199 if (attr_mask & IB_QP_SQ_PSN) {
4200 roce_set_field(context->byte_172_sq_psn,
4201 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4202 V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
4203 roce_set_field(qpc_mask->byte_172_sq_psn,
4204 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4205 V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
4206
4207 roce_set_field(context->byte_196_sq_psn,
4208 V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4209 V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
4210 roce_set_field(qpc_mask->byte_196_sq_psn,
4211 V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4212 V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
4213
4214 roce_set_field(context->byte_220_retry_psn_msn,
4215 V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4216 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
4217 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4218 V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4219 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
4220
4221 roce_set_field(context->byte_224_retry_msg,
4222 V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4223 V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
2a3d923f 4224 attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
f04cc178
LO		 4225 roce_set_field(qpc_mask->byte_224_retry_msg,
4226 V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4227 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
4228
4229 roce_set_field(context->byte_224_retry_msg,
4230 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4231 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
4232 attr->sq_psn);
4233 roce_set_field(qpc_mask->byte_224_retry_msg,
4234 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4235 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
4236
4237 roce_set_field(context->byte_244_rnr_rxack,
4238 V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4239 V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
4240 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4241 V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4242 V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
4243 }
4244
5b01b243
LO		 4245 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4246 attr->max_dest_rd_atomic) {
4247 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4248 V2_QPC_BYTE_140_RR_MAX_S,
4249 fls(attr->max_dest_rd_atomic - 1));
4250 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4251 V2_QPC_BYTE_140_RR_MAX_S, 0);
4252 }
4253
4254 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4255 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
4256 V2_QPC_BYTE_208_SR_MAX_S,
4257 fls(attr->max_rd_atomic - 1));
4258 roce_set_field(qpc_mask->byte_208_irrl,
4259 V2_QPC_BYTE_208_SR_MAX_M,
4260 V2_QPC_BYTE_208_SR_MAX_S, 0);
4261 }
4262
ace1c541 4263 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4264 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4265
5b01b243
LO		 4266 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4267 roce_set_field(context->byte_80_rnr_rx_cqn,
4268 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4269 V2_QPC_BYTE_80_MIN_RNR_TIME_S,
4270 attr->min_rnr_timer);
4271 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
4272 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4273 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
4274 }
4275
601f3e6d
LO		 4276 /* RC&UC required attr */
4277 if (attr_mask & IB_QP_RQ_PSN) {
4278 roce_set_field(context->byte_108_rx_reqepsn,
4279 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4280 V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
4281 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4282 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4283 V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
4284
4285 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
4286 V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
4287 roce_set_field(qpc_mask->byte_152_raq,
4288 V2_QPC_BYTE_152_RAQ_PSN_M,
4289 V2_QPC_BYTE_152_RAQ_PSN_S, 0);
4290 }
4291
5b01b243
LO		 4292 if (attr_mask & IB_QP_QKEY) {
4293 context->qkey_xrcd = attr->qkey;
4294 qpc_mask->qkey_xrcd = 0;
4295 hr_qp->qkey = attr->qkey;
4296 }
4297
c7bcb134
LO		 4298 roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
4299 ibqp->srq ? 1 : 0);
4300 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4301 V2_QPC_BYTE_108_INV_CREDIT_S, 0);
4302
926a01dc 4303 /* Every status migrate must change state */
2362ccee 4304 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
926a01dc 4305 V2_QPC_BYTE_60_QP_ST_S, new_state);
2362ccee 4306 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
926a01dc
WHX		 4307 V2_QPC_BYTE_60_QP_ST_S, 0);
4308
4309 /* SW pass context to HW */
8d18ad83 4310 ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state,
926a01dc
WHX		 4311 context, hr_qp);
4312 if (ret) {
4313 dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
4314 goto out;
4315 }
4316
4317 hr_qp->state = new_state;
4318
ace1c541 4319 if (attr_mask & IB_QP_ACCESS_FLAGS)
4320 hr_qp->atomic_rd_en = attr->qp_access_flags;
4321
926a01dc
WHX		 4322 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4323 hr_qp->resp_depth = attr->max_dest_rd_atomic;
4324 if (attr_mask & IB_QP_PORT) {
4325 hr_qp->port = attr->port_num - 1;
4326 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4327 }
4328
4329 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
4330 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
4331 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
4332 if (ibqp->send_cq != ibqp->recv_cq)
4333 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4334 hr_qp->qpn, NULL);
4335
4336 hr_qp->rq.head = 0;
4337 hr_qp->rq.tail = 0;
4338 hr_qp->sq.head = 0;
4339 hr_qp->sq.tail = 0;
4340 hr_qp->sq_next_wqe = 0;
4341 hr_qp->next_sge = 0;
e088a685
YL		 4342 if (hr_qp->rq.wqe_cnt)
4343 *hr_qp->rdb.db_record = 0;
926a01dc
WHX		 4344 }
4345
4346out:
4347 kfree(context);
4348 return ret;
4349}
4350
4351static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
4352{
4353 switch (state) {
4354 case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET;
4355 case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT;
4356 case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR;
4357 case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS;
4358 case HNS_ROCE_QP_ST_SQ_DRAINING:
4359 case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD;
4360 case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE;
4361 case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR;
4362 default: return -1;
4363 }
4364}
4365
4366static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
4367 struct hns_roce_qp *hr_qp,
4368 struct hns_roce_v2_qp_context *hr_context)
4369{
4370 struct hns_roce_cmd_mailbox *mailbox;
4371 int ret;
4372
4373 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4374 if (IS_ERR(mailbox))
4375 return PTR_ERR(mailbox);
4376
4377 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
4378 HNS_ROCE_CMD_QUERY_QPC,
4379 HNS_ROCE_CMD_TIMEOUT_MSECS);
4380 if (ret) {
4381 dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
4382 goto out;
4383 }
4384
4385 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
4386
4387out:
4388 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4389 return ret;
4390}
4391
4392static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4393 int qp_attr_mask,
4394 struct ib_qp_init_attr *qp_init_attr)
4395{
4396 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4397 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4398 struct hns_roce_v2_qp_context *context;
4399 struct device *dev = hr_dev->dev;
4400 int tmp_qp_state;
4401 int state;
4402 int ret;
4403
4404 context = kzalloc(sizeof(*context), GFP_KERNEL);
4405 if (!context)
4406 return -ENOMEM;
4407
4408 memset(qp_attr, 0, sizeof(*qp_attr));
4409 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4410
4411 mutex_lock(&hr_qp->mutex);
4412
4413 if (hr_qp->state == IB_QPS_RESET) {
4414 qp_attr->qp_state = IB_QPS_RESET;
63ea641f 4415 ret = 0;
926a01dc
WHX		 4416 goto done;
4417 }
4418
4419 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context);
4420 if (ret) {
4421 dev_err(dev, "query qpc error\n");
4422 ret = -EINVAL;
4423 goto out;
4424 }
4425
2362ccee 4426 state = roce_get_field(context->byte_60_qpst_tempid,
926a01dc
WHX		 4427 V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
4428 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
4429 if (tmp_qp_state == -1) {
4430 dev_err(dev, "Illegal ib_qp_state\n");
4431 ret = -EINVAL;
4432 goto out;
4433 }
4434 hr_qp->state = (u8)tmp_qp_state;
4435 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
4436 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc,
4437 V2_QPC_BYTE_24_MTU_M,
4438 V2_QPC_BYTE_24_MTU_S);
4439 qp_attr->path_mig_state = IB_MIG_ARMED;
2bf910d4 4440 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
926a01dc
WHX		 4441 if (hr_qp->ibqp.qp_type == IB_QPT_UD)
4442 qp_attr->qkey = V2_QKEY_VAL;
4443
4444 qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn,
4445 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4446 V2_QPC_BYTE_108_RX_REQ_EPSN_S);
4447 qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn,
4448 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4449 V2_QPC_BYTE_172_SQ_CUR_PSN_S);
4450 qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err,
4451 V2_QPC_BYTE_56_DQPN_M,
4452 V2_QPC_BYTE_56_DQPN_S);
4453 qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en,
2a3d923f
LO		 4454 V2_QPC_BYTE_76_RRE_S)) << V2_QP_RWE_S) |
4455 ((roce_get_bit(context->byte_76_srqn_op_en,
4456 V2_QPC_BYTE_76_RWE_S)) << V2_QP_RRE_S) |
4457 ((roce_get_bit(context->byte_76_srqn_op_en,
4458 V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
4459
926a01dc
WHX		 4460 if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
4461 hr_qp->ibqp.qp_type == IB_QPT_UC) {
4462 struct ib_global_route *grh =
4463 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
4464
4465 rdma_ah_set_sl(&qp_attr->ah_attr,
4466 roce_get_field(context->byte_28_at_fl,
4467 V2_QPC_BYTE_28_SL_M,
4468 V2_QPC_BYTE_28_SL_S));
4469 grh->flow_label = roce_get_field(context->byte_28_at_fl,
4470 V2_QPC_BYTE_28_FL_M,
4471 V2_QPC_BYTE_28_FL_S);
4472 grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx,
4473 V2_QPC_BYTE_20_SGID_IDX_M,
4474 V2_QPC_BYTE_20_SGID_IDX_S);
4475 grh->hop_limit = roce_get_field(context->byte_24_mtu_tc,
4476 V2_QPC_BYTE_24_HOP_LIMIT_M,
4477 V2_QPC_BYTE_24_HOP_LIMIT_S);
4478 grh->traffic_class = roce_get_field(context->byte_24_mtu_tc,
4479 V2_QPC_BYTE_24_TC_M,
4480 V2_QPC_BYTE_24_TC_S);
4481
4482 memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw));
4483 }
4484
4485 qp_attr->port_num = hr_qp->port + 1;
4486 qp_attr->sq_draining = 0;
4487 qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl,
4488 V2_QPC_BYTE_208_SR_MAX_M,
4489 V2_QPC_BYTE_208_SR_MAX_S);
4490 qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq,
4491 V2_QPC_BYTE_140_RR_MAX_M,
4492 V2_QPC_BYTE_140_RR_MAX_S);
4493 qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn,
4494 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4495 V2_QPC_BYTE_80_MIN_RNR_TIME_S);
4496 qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl,
4497 V2_QPC_BYTE_28_AT_M,
4498 V2_QPC_BYTE_28_AT_S);
4499 qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn,
4500 V2_QPC_BYTE_212_RETRY_CNT_M,
4501 V2_QPC_BYTE_212_RETRY_CNT_S);
4502 qp_attr->rnr_retry = context->rq_rnr_timer;
4503
4504done:
4505 qp_attr->cur_qp_state = qp_attr->qp_state;
4506 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
4507 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
4508
4509 if (!ibqp->uobject) {
4510 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
4511 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
4512 } else {
4513 qp_attr->cap.max_send_wr = 0;
4514 qp_attr->cap.max_send_sge = 0;
4515 }
4516
4517 qp_init_attr->cap = qp_attr->cap;
4518
4519out:
4520 mutex_unlock(&hr_qp->mutex);
4521 kfree(context);
4522 return ret;
4523}
4524
4525static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
4526 struct hns_roce_qp *hr_qp,
bdeacabd 4527 struct ib_udata *udata)
926a01dc
WHX		 4528{
4529 struct hns_roce_cq *send_cq, *recv_cq;
4530 struct device *dev = hr_dev->dev;
4531 int ret;
4532
4533 if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
4534 /* Modify qp to reset before destroying qp */
4535 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
4536 hr_qp->state, IB_QPS_RESET);
4537 if (ret) {
4538 dev_err(dev, "modify QP %06lx to ERR failed.\n",
4539 hr_qp->qpn);
4540 return ret;
4541 }
4542 }
4543
4544 send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
4545 recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
4546
4547 hns_roce_lock_cqs(send_cq, recv_cq);
4548
bdeacabd 4549 if (!udata) {
926a01dc
WHX		 4550 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
4551 to_hr_srq(hr_qp->ibqp.srq) : NULL);
4552 if (send_cq != recv_cq)
4553 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
4554 }
4555
4556 hns_roce_qp_remove(hr_dev, hr_qp);
4557
4558 hns_roce_unlock_cqs(send_cq, recv_cq);
4559
4560 hns_roce_qp_free(hr_dev, hr_qp);
4561
4562 /* Not special_QP, free their QPN */
4563 if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
4564 (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
4565 (hr_qp->ibqp.qp_type == IB_QPT_UD))
4566 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
4567
6fafe560 4568 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
926a01dc 4569
bdeacabd
SR		 4570 if (udata) {
4571 struct hns_roce_ucontext *context =
4572 rdma_udata_to_drv_context(
4573 udata,
4574 struct hns_roce_ucontext,
4575 ibucontext);
4576
0425e3e6 4577 if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
bdeacabd 4578 hns_roce_db_unmap_user(context, &hr_qp->sdb);
0425e3e6 4579
e088a685 4580 if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
bdeacabd 4581 hns_roce_db_unmap_user(context, &hr_qp->rdb);
926a01dc
WHX		 4582 } else {
4583 kfree(hr_qp->sq.wrid);
4584 kfree(hr_qp->rq.wrid);
4585 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
472bc0fb
YL		 4586 if (hr_qp->rq.wqe_cnt)
4587 hns_roce_free_db(hr_dev, &hr_qp->rdb);
926a01dc 4588 }
836a0fbb 4589 ib_umem_release(hr_qp->umem);
926a01dc 4590
c7bcb134
LO		 4591 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
4592 hr_qp->rq.wqe_cnt) {
0009c2db 4593 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4594 kfree(hr_qp->rq_inl_buf.wqe_list);
4595 }
4596
926a01dc
WHX		 4597 return 0;
4598}
4599
c4367a26 4600static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
926a01dc
WHX		 4601{
4602 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4603 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4604 int ret;
4605
bdeacabd 4606 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
926a01dc
WHX		 4607 if (ret) {
4608 dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret);
4609 return ret;
4610 }
4611
4612 if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
4613 kfree(hr_to_hr_sqp(hr_qp));
4614 else
4615 kfree(hr_qp);
4616
4617 return 0;
4618}
4619
aa84fa18
YL		 4620static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
4621 struct hns_roce_qp *hr_qp)
4622{
da91ddfd 4623 struct hns_roce_sccc_clr_done *resp;
aa84fa18
YL		 4624 struct hns_roce_sccc_clr *clr;
4625 struct hns_roce_cmq_desc desc;
4626 int ret, i;
4627
4628 mutex_lock(&hr_dev->qp_table.scc_mutex);
4629
4630 /* set scc ctx clear done flag */
4631 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
aa84fa18
YL		 4632 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4633 if (ret) {
4634 dev_err(hr_dev->dev, "Reset SCC ctx failed(%d)\n", ret);
4635 goto out;
4636 }
4637
4638 /* clear scc context */
4639 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
4640 clr = (struct hns_roce_sccc_clr *)desc.data;
4641 clr->qpn = cpu_to_le32(hr_qp->qpn);
4642 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4643 if (ret) {
4644 dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret);
4645 goto out;
4646 }
4647
4648 /* query scc context clear is done or not */
4649 resp = (struct hns_roce_sccc_clr_done *)desc.data;
4650 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
4651 hns_roce_cmq_setup_basic_desc(&desc,
4652 HNS_ROCE_OPC_QUERY_SCCC, true);
4653 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4654 if (ret) {
4655 dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret);
4656 goto out;
4657 }
4658
4659 if (resp->clr_done)
4660 goto out;
4661
4662 msleep(20);
4663 }
4664
4665 dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n");
4666 ret = -ETIMEDOUT;
4667
4668out:
4669 mutex_unlock(&hr_dev->qp_table.scc_mutex);
4670 return ret;
4671}
4672
b156269d 4673static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
4674{
4675 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
4676 struct hns_roce_v2_cq_context *cq_context;
4677 struct hns_roce_cq *hr_cq = to_hr_cq(cq);
4678 struct hns_roce_v2_cq_context *cqc_mask;
4679 struct hns_roce_cmd_mailbox *mailbox;
4680 int ret;
4681
4682 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4683 if (IS_ERR(mailbox))
4684 return PTR_ERR(mailbox);
4685
4686 cq_context = mailbox->buf;
4687 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
4688
4689 memset(cqc_mask, 0xff, sizeof(*cqc_mask));
4690
4691 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4692 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4693 cq_count);
4694 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4695 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4696 0);
4697 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4698 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4699 cq_period);
4700 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4701 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4702 0);
4703
4704 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
4705 HNS_ROCE_CMD_MODIFY_CQC,
4706 HNS_ROCE_CMD_TIMEOUT_MSECS);
4707 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4708 if (ret)
4709 dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
4710
4711 return ret;
4712}
4713
0425e3e6
YL		 4714static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
4715{
4716 struct hns_roce_qp *hr_qp;
4717 struct ib_qp_attr attr;
4718 int attr_mask;
4719 int ret;
4720
4721 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
4722 if (!hr_qp) {
4723 dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
4724 return;
4725 }
4726
4727 if (hr_qp->ibqp.uobject) {
4728 if (hr_qp->sdb_en == 1) {
4729 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
9c6ccc03
LO		 4730 if (hr_qp->rdb_en == 1)
4731 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
0425e3e6
YL		 4732 } else {
4733 dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
4734 return;
4735 }
4736 }
4737
4738 attr_mask = IB_QP_STATE;
4739 attr.qp_state = IB_QPS_ERR;
4740 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
4741 hr_qp->state, IB_QPS_ERR);
4742 if (ret)
4743 dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
4744 qpn);
4745}
4746
4747static void hns_roce_irq_work_handle(struct work_struct *work)
4748{
4749 struct hns_roce_work *irq_work =
4750 container_of(work, struct hns_roce_work, work);
b00a92c8 4751 struct device *dev = irq_work->hr_dev->dev;
0425e3e6 4752 u32 qpn = irq_work->qpn;
b00a92c8 4753 u32 cqn = irq_work->cqn;
0425e3e6
YL		 4754
4755 switch (irq_work->event_type) {
b00a92c8 4756 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4757 dev_info(dev, "Path migrated succeeded.\n");
4758 break;
4759 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4760 dev_warn(dev, "Path migration failed.\n");
4761 break;
4762 case HNS_ROCE_EVENT_TYPE_COMM_EST:
b00a92c8 4763 break;
4764 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4765 dev_warn(dev, "Send queue drained.\n");
4766 break;
0425e3e6 4767 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
e95c716c
YL		 4768 dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n",
4769 qpn, irq_work->sub_type);
b00a92c8 4770 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
b00a92c8 4771 break;
0425e3e6 4772 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
e95c716c
YL		 4773 dev_err(dev, "Invalid request local work queue 0x%x error.\n",
4774 qpn);
b00a92c8 4775 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4776 break;
0425e3e6 4777 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
e95c716c
YL		 4778 dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n",
4779 qpn, irq_work->sub_type);
0425e3e6 4780 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
b00a92c8 4781 break;
4782 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4783 dev_warn(dev, "SRQ limit reach.\n");
4784 break;
4785 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4786 dev_warn(dev, "SRQ last wqe reach.\n");
4787 break;
4788 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4789 dev_err(dev, "SRQ catas error.\n");
4790 break;
4791 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4792 dev_err(dev, "CQ 0x%x access err.\n", cqn);
4793 break;
4794 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4795 dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4796 break;
4797 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4798 dev_warn(dev, "DB overflow.\n");
4799 break;
4800 case HNS_ROCE_EVENT_TYPE_FLR:
4801 dev_warn(dev, "Function level reset.\n");
0425e3e6
YL		 4802 break;
4803 default:
4804 break;
4805 }
4806
4807 kfree(irq_work);
4808}
4809
4810static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
b00a92c8 4811 struct hns_roce_eq *eq,
4812 u32 qpn, u32 cqn)
0425e3e6
YL		 4813{
4814 struct hns_roce_work *irq_work;
4815
4816 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
4817 if (!irq_work)
4818 return;
4819
4820 INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4821 irq_work->hr_dev = hr_dev;
4822 irq_work->qpn = qpn;
b00a92c8 4823 irq_work->cqn = cqn;
0425e3e6
YL		 4824 irq_work->event_type = eq->event_type;
4825 irq_work->sub_type = eq->sub_type;
4826 queue_work(hr_dev->irq_workq, &(irq_work->work));
4827}
4828
a5073d60
YL		 4829static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4830{
d3743fa9 4831 struct hns_roce_dev *hr_dev = eq->hr_dev;
a5073d60
YL		 4832 u32 doorbell[2];
4833
4834 doorbell[0] = 0;
4835 doorbell[1] = 0;
4836
4837 if (eq->type_flag == HNS_ROCE_AEQ) {
4838 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4839 HNS_ROCE_V2_EQ_DB_CMD_S,
4840 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4841 HNS_ROCE_EQ_DB_CMD_AEQ :
4842 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
4843 } else {
4844 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
4845 HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
4846
4847 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4848 HNS_ROCE_V2_EQ_DB_CMD_S,
4849 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4850 HNS_ROCE_EQ_DB_CMD_CEQ :
4851 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
4852 }
4853
4854 roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
4855 HNS_ROCE_V2_EQ_DB_PARA_S,
4856 (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4857
d3743fa9 4858 hns_roce_write64(hr_dev, doorbell, eq->doorbell);
a5073d60
YL		 4859}
4860
a5073d60
YL		 4861static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
4862{
4863 u32 buf_chk_sz;
4864 unsigned long off;
4865
4866 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4867 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4868
4869 return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
4870 off % buf_chk_sz);
4871}
4872
4873static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
4874{
4875 u32 buf_chk_sz;
4876 unsigned long off;
4877
4878 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4879
4880 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4881
4882 if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4883 return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
4884 off % buf_chk_sz);
4885 else
4886 return (struct hns_roce_aeqe *)((u8 *)
4887 (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
4888}
4889
4890static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
4891{
4892 struct hns_roce_aeqe *aeqe;
4893
4894 if (!eq->hop_num)
4895 aeqe = get_aeqe_v2(eq, eq->cons_index);
4896 else
4897 aeqe = mhop_get_aeqe(eq, eq->cons_index);
4898
4899 return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
4900 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
4901}
4902
4903static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
4904 struct hns_roce_eq *eq)
4905{
4906 struct device *dev = hr_dev->dev;
4907 struct hns_roce_aeqe *aeqe;
4908 int aeqe_found = 0;
4909 int event_type;
0425e3e6 4910 int sub_type;
81fce629 4911 u32 srqn;
0425e3e6
YL		 4912 u32 qpn;
4913 u32 cqn;
a5073d60
YL		 4914
4915 while ((aeqe = next_aeqe_sw_v2(eq))) {
4044a3f4
YL		 4916
4917 /* Make sure we read AEQ entry after we have checked the
4918 * ownership bit
4919 */
4920 dma_rmb();
a5073d60
YL		 4921
4922 event_type = roce_get_field(aeqe->asyn,
4923 HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
4924 HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
0425e3e6
YL		 4925 sub_type = roce_get_field(aeqe->asyn,
4926 HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4927 HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4928 qpn = roce_get_field(aeqe->event.qp_event.qp,
4929 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4930 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4931 cqn = roce_get_field(aeqe->event.cq_event.cq,
4932 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4933 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
81fce629
LO		 4934 srqn = roce_get_field(aeqe->event.srq_event.srq,
4935 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4936 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
a5073d60
YL		 4937
4938 switch (event_type) {
4939 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
a5073d60 4940 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
a5073d60
YL		 4941 case HNS_ROCE_EVENT_TYPE_COMM_EST:
4942 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4943 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
81fce629 4944 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
a5073d60
YL		 4945 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4946 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
b00a92c8 4947 hns_roce_qp_event(hr_dev, qpn, event_type);
a5073d60
YL		 4948 break;
4949 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
a5073d60 4950 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
81fce629 4951 hns_roce_srq_event(hr_dev, srqn, event_type);
a5073d60
YL		 4952 break;
4953 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4954 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
b00a92c8 4955 hns_roce_cq_event(hr_dev, cqn, event_type);
a5073d60
YL		 4956 break;
4957 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
a5073d60
YL		 4958 break;
4959 case HNS_ROCE_EVENT_TYPE_MB:
4960 hns_roce_cmd_event(hr_dev,
4961 le16_to_cpu(aeqe->event.cmd.token),
4962 aeqe->event.cmd.status,
4963 le64_to_cpu(aeqe->event.cmd.out_param));
4964 break;
4965 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
a5073d60
YL		 4966 break;
4967 case HNS_ROCE_EVENT_TYPE_FLR:
a5073d60
YL		 4968 break;
4969 default:
4970 dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
4971 event_type, eq->eqn, eq->cons_index);
4972 break;
790b57f6 4973 }
a5073d60 4974
0425e3e6
YL		 4975 eq->event_type = event_type;
4976 eq->sub_type = sub_type;
a5073d60
YL		 4977 ++eq->cons_index;
4978 aeqe_found = 1;
4979
4980 if (eq->cons_index > (2 * eq->entries - 1)) {
4981 dev_warn(dev, "cons_index overflow, set back to 0.\n");
4982 eq->cons_index = 0;
4983 }
b00a92c8 4984 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
a5073d60
YL		 4985 }
4986
4987 set_eq_cons_index_v2(eq);
4988 return aeqe_found;
4989}
4990
4991static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
4992{
4993 u32 buf_chk_sz;
4994 unsigned long off;
4995
4996 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4997 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4998
4999 return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
5000 off % buf_chk_sz);
5001}
5002
5003static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
5004{
5005 u32 buf_chk_sz;
5006 unsigned long off;
5007
5008 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5009
5010 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5011
5012 if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
5013 return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
5014 off % buf_chk_sz);
5015 else
5016 return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
5017 buf_chk_sz]) + off % buf_chk_sz);
5018}
5019
5020static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5021{
5022 struct hns_roce_ceqe *ceqe;
5023
5024 if (!eq->hop_num)
5025 ceqe = get_ceqe_v2(eq, eq->cons_index);
5026 else
5027 ceqe = mhop_get_ceqe(eq, eq->cons_index);
5028
5029 return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5030 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5031}
5032
5033static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5034 struct hns_roce_eq *eq)
5035{
5036 struct device *dev = hr_dev->dev;
5037 struct hns_roce_ceqe *ceqe;
5038 int ceqe_found = 0;
5039 u32 cqn;
5040
5041 while ((ceqe = next_ceqe_sw_v2(eq))) {
5042
4044a3f4
YL		 5043 /* Make sure we read CEQ entry after we have checked the
5044 * ownership bit
5045 */
5046 dma_rmb();
5047
a5073d60
YL		 5048 cqn = roce_get_field(ceqe->comp,
5049 HNS_ROCE_V2_CEQE_COMP_CQN_M,
5050 HNS_ROCE_V2_CEQE_COMP_CQN_S);
5051
5052 hns_roce_cq_completion(hr_dev, cqn);
5053
5054 ++eq->cons_index;
5055 ceqe_found = 1;
5056
5057 if (eq->cons_index > (2 * eq->entries - 1)) {
5058 dev_warn(dev, "cons_index overflow, set back to 0.\n");
5059 eq->cons_index = 0;
5060 }
5061 }
5062
5063 set_eq_cons_index_v2(eq);
5064
5065 return ceqe_found;
5066}
5067
5068static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5069{
5070 struct hns_roce_eq *eq = eq_ptr;
5071 struct hns_roce_dev *hr_dev = eq->hr_dev;
5072 int int_work = 0;
5073
5074 if (eq->type_flag == HNS_ROCE_CEQ)
5075 /* Completion event interrupt */
5076 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5077 else
5078 /* Asychronous event interrupt */
5079 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5080
5081 return IRQ_RETVAL(int_work);
5082}
5083
5084static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5085{
5086 struct hns_roce_dev *hr_dev = dev_id;
5087 struct device *dev = hr_dev->dev;
5088 int int_work = 0;
5089 u32 int_st;
5090 u32 int_en;
5091
5092 /* Abnormal interrupt */
5093 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5094 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5095
5096 if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
2b9acb9a
XT		 5097 struct pci_dev *pdev = hr_dev->pci_dev;
5098 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5099 const struct hnae3_ae_ops *ops = ae_dev->ops;
5100
a5073d60
YL		 5101 dev_err(dev, "AEQ overflow!\n");
5102
5103 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);
5104 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5105
2b9acb9a
XT		 5106 /* Set reset level for reset_event() */
5107 if (ops->set_default_reset_request)
5108 ops->set_default_reset_request(ae_dev,
5109 HNAE3_FUNC_RESET);
5110 if (ops->reset_event)
5111 ops->reset_event(pdev, NULL);
5112
a5073d60
YL		 5113 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
5114 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5115
5116 int_work = 1;
5117 } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
5118 dev_err(dev, "BUS ERR!\n");
5119
5120 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);
5121 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5122
a5073d60
YL		 5123 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
5124 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5125
5126 int_work = 1;
5127 } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
5128 dev_err(dev, "OTHER ERR!\n");
5129
5130 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);
5131 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5132
a5073d60
YL		 5133 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
5134 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5135
5136 int_work = 1;
5137 } else
5138 dev_err(dev, "There is no abnormal irq found!\n");
5139
5140 return IRQ_RETVAL(int_work);
5141}
5142
5143static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5144 int eq_num, int enable_flag)
5145{
5146 int i;
5147
5148 if (enable_flag == EQ_ENABLE) {
5149 for (i = 0; i < eq_num; i++)
5150 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5151 i * EQ_REG_OFFSET,
5152 HNS_ROCE_V2_VF_EVENT_INT_EN_M);
5153
5154 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5155 HNS_ROCE_V2_VF_ABN_INT_EN_M);
5156 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5157 HNS_ROCE_V2_VF_ABN_INT_CFG_M);
5158 } else {
5159 for (i = 0; i < eq_num; i++)
5160 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5161 i * EQ_REG_OFFSET,
5162 HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
5163
5164 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5165 HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
5166 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5167 HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
5168 }
5169}
5170
5171static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5172{
5173 struct device *dev = hr_dev->dev;
5174 int ret;
5175
5176 if (eqn < hr_dev->caps.num_comp_vectors)
5177 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5178 0, HNS_ROCE_CMD_DESTROY_CEQC,
5179 HNS_ROCE_CMD_TIMEOUT_MSECS);
5180 else
5181 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5182 0, HNS_ROCE_CMD_DESTROY_AEQC,
5183 HNS_ROCE_CMD_TIMEOUT_MSECS);
5184 if (ret)
5185 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5186}
5187
5188static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
5189 struct hns_roce_eq *eq)
5190{
5191 struct device *dev = hr_dev->dev;
5192 u64 idx;
5193 u64 size;
5194 u32 buf_chk_sz;
5195 u32 bt_chk_sz;
5196 u32 mhop_num;
5197 int eqe_alloc;
a5073d60
YL		 5198 int i = 0;
5199 int j = 0;
5200
5201 mhop_num = hr_dev->caps.eqe_hop_num;
5202 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5203 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
a5073d60
YL		 5204
5205 /* hop_num = 0 */
5206 if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5207 dma_free_coherent(dev, (unsigned int)(eq->entries *
5208 eq->eqe_size), eq->bt_l0, eq->l0_dma);
5209 return;
5210 }
5211
5212 /* hop_num = 1 or hop = 2 */
5213 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5214 if (mhop_num == 1) {
5215 for (i = 0; i < eq->l0_last_num; i++) {
5216 if (i == eq->l0_last_num - 1) {
5217 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5218 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5219 dma_free_coherent(dev, size, eq->buf[i],
5220 eq->buf_dma[i]);
5221 break;
5222 }
5223 dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5224 eq->buf_dma[i]);
5225 }
5226 } else if (mhop_num == 2) {
5227 for (i = 0; i < eq->l0_last_num; i++) {
5228 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5229 eq->l1_dma[i]);
5230
2a3d923f
LO		 5231 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5232 idx = i * (bt_chk_sz / BA_BYTE_LEN) + j;
a5073d60
YL		 5233 if ((i == eq->l0_last_num - 1)
5234 && j == eq->l1_last_num - 1) {
5235 eqe_alloc = (buf_chk_sz / eq->eqe_size)
5236 * idx;
5237 size = (eq->entries - eqe_alloc)
5238 * eq->eqe_size;
5239 dma_free_coherent(dev, size,
5240 eq->buf[idx],
5241 eq->buf_dma[idx]);
5242 break;
5243 }
5244 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5245 eq->buf_dma[idx]);
5246 }
5247 }
5248 }
5249 kfree(eq->buf_dma);
5250 kfree(eq->buf);
5251 kfree(eq->l1_dma);
5252 kfree(eq->bt_l1);
5253 eq->buf_dma = NULL;
5254 eq->buf = NULL;
5255 eq->l1_dma = NULL;
5256 eq->bt_l1 = NULL;
5257}
5258
5259static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
5260 struct hns_roce_eq *eq)
5261{
5262 u32 buf_chk_sz;
5263
5264 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5265
5266 if (hr_dev->caps.eqe_hop_num) {
5267 hns_roce_mhop_free_eq(hr_dev, eq);
5268 return;
5269 }
5270
5271 if (eq->buf_list)
5272 dma_free_coherent(hr_dev->dev, buf_chk_sz,
5273 eq->buf_list->buf, eq->buf_list->map);
5274}
5275
5276static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
5277 struct hns_roce_eq *eq,
5278 void *mb_buf)
5279{
5280 struct hns_roce_eq_context *eqc;
5281
5282 eqc = mb_buf;
5283 memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5284
5285 /* init eqc */
5286 eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5287 eq->hop_num = hr_dev->caps.eqe_hop_num;
5288 eq->cons_index = 0;
5289 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5290 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5291 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5292 eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
5293 eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
5294 eq->shift = ilog2((unsigned int)eq->entries);
5295
5296 if (!eq->hop_num)
5297 eq->eqe_ba = eq->buf_list->map;
5298 else
5299 eq->eqe_ba = eq->l0_dma;
5300
5301 /* set eqc state */
5302 roce_set_field(eqc->byte_4,
5303 HNS_ROCE_EQC_EQ_ST_M,
5304 HNS_ROCE_EQC_EQ_ST_S,
5305 HNS_ROCE_V2_EQ_STATE_VALID);
5306
5307 /* set eqe hop num */
5308 roce_set_field(eqc->byte_4,
5309 HNS_ROCE_EQC_HOP_NUM_M,
5310 HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
5311
5312 /* set eqc over_ignore */
5313 roce_set_field(eqc->byte_4,
5314 HNS_ROCE_EQC_OVER_IGNORE_M,
5315 HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
5316
5317 /* set eqc coalesce */
5318 roce_set_field(eqc->byte_4,
5319 HNS_ROCE_EQC_COALESCE_M,
5320 HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
5321
5322 /* set eqc arm_state */
5323 roce_set_field(eqc->byte_4,
5324 HNS_ROCE_EQC_ARM_ST_M,
5325 HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
5326
5327 /* set eqn */
5328 roce_set_field(eqc->byte_4,
5329 HNS_ROCE_EQC_EQN_M,
5330 HNS_ROCE_EQC_EQN_S, eq->eqn);
5331
5332 /* set eqe_cnt */
5333 roce_set_field(eqc->byte_4,
5334 HNS_ROCE_EQC_EQE_CNT_M,
5335 HNS_ROCE_EQC_EQE_CNT_S,
5336 HNS_ROCE_EQ_INIT_EQE_CNT);
5337
5338 /* set eqe_ba_pg_sz */
5339 roce_set_field(eqc->byte_8,
5340 HNS_ROCE_EQC_BA_PG_SZ_M,
5e6e78db
YL		 5341 HNS_ROCE_EQC_BA_PG_SZ_S,
5342 eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
a5073d60
YL		 5343
5344 /* set eqe_buf_pg_sz */
5345 roce_set_field(eqc->byte_8,
5346 HNS_ROCE_EQC_BUF_PG_SZ_M,
5e6e78db
YL		 5347 HNS_ROCE_EQC_BUF_PG_SZ_S,
5348 eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
a5073d60
YL		 5349
5350 /* set eq_producer_idx */
5351 roce_set_field(eqc->byte_8,
5352 HNS_ROCE_EQC_PROD_INDX_M,
5353 HNS_ROCE_EQC_PROD_INDX_S,
5354 HNS_ROCE_EQ_INIT_PROD_IDX);
5355
5356 /* set eq_max_cnt */
5357 roce_set_field(eqc->byte_12,
5358 HNS_ROCE_EQC_MAX_CNT_M,
5359 HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
5360
5361 /* set eq_period */
5362 roce_set_field(eqc->byte_12,
5363 HNS_ROCE_EQC_PERIOD_M,
5364 HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
5365
5366 /* set eqe_report_timer */
5367 roce_set_field(eqc->eqe_report_timer,
5368 HNS_ROCE_EQC_REPORT_TIMER_M,
5369 HNS_ROCE_EQC_REPORT_TIMER_S,
5370 HNS_ROCE_EQ_INIT_REPORT_TIMER);
5371
5372 /* set eqe_ba [34:3] */
5373 roce_set_field(eqc->eqe_ba0,
5374 HNS_ROCE_EQC_EQE_BA_L_M,
5375 HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
5376
5377 /* set eqe_ba [64:35] */
5378 roce_set_field(eqc->eqe_ba1,
5379 HNS_ROCE_EQC_EQE_BA_H_M,
5380 HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
5381
5382 /* set eq shift */
5383 roce_set_field(eqc->byte_28,
5384 HNS_ROCE_EQC_SHIFT_M,
5385 HNS_ROCE_EQC_SHIFT_S, eq->shift);
5386
5387 /* set eq MSI_IDX */
5388 roce_set_field(eqc->byte_28,
5389 HNS_ROCE_EQC_MSI_INDX_M,
5390 HNS_ROCE_EQC_MSI_INDX_S,
5391 HNS_ROCE_EQ_INIT_MSI_IDX);
5392
5393 /* set cur_eqe_ba [27:12] */
5394 roce_set_field(eqc->byte_28,
5395 HNS_ROCE_EQC_CUR_EQE_BA_L_M,
5396 HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
5397
5398 /* set cur_eqe_ba [59:28] */
5399 roce_set_field(eqc->byte_32,
5400 HNS_ROCE_EQC_CUR_EQE_BA_M_M,
5401 HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
5402
5403 /* set cur_eqe_ba [63:60] */
5404 roce_set_field(eqc->byte_36,
5405 HNS_ROCE_EQC_CUR_EQE_BA_H_M,
5406 HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
5407
5408 /* set eq consumer idx */
5409 roce_set_field(eqc->byte_36,
5410 HNS_ROCE_EQC_CONS_INDX_M,
5411 HNS_ROCE_EQC_CONS_INDX_S,
5412 HNS_ROCE_EQ_INIT_CONS_IDX);
5413
5414 /* set nex_eqe_ba[43:12] */
5415 roce_set_field(eqc->nxt_eqe_ba0,
5416 HNS_ROCE_EQC_NXT_EQE_BA_L_M,
5417 HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
5418
5419 /* set nex_eqe_ba[63:44] */
5420 roce_set_field(eqc->nxt_eqe_ba1,
5421 HNS_ROCE_EQC_NXT_EQE_BA_H_M,
5422 HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
5423}
5424
5425static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
5426 struct hns_roce_eq *eq)
5427{
5428 struct device *dev = hr_dev->dev;
5429 int eq_alloc_done = 0;
5430 int eq_buf_cnt = 0;
5431 int eqe_alloc;
5432 u32 buf_chk_sz;
5433 u32 bt_chk_sz;
5434 u32 mhop_num;
5435 u64 size;
5436 u64 idx;
5437 int ba_num;
5438 int bt_num;
5439 int record_i;
5440 int record_j;
5441 int i = 0;
5442 int j = 0;
5443
5444 mhop_num = hr_dev->caps.eqe_hop_num;
5445 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5446 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5447
2a3d923f
LO		 5448 ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
5449 buf_chk_sz);
5450 bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN);
a5073d60
YL		 5451
5452 /* hop_num = 0 */
5453 if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5454 if (eq->entries > buf_chk_sz / eq->eqe_size) {
5455 dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
5456 eq->entries);
5457 return -EINVAL;
5458 }
5459 eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
5460 &(eq->l0_dma), GFP_KERNEL);
5461 if (!eq->bt_l0)
5462 return -ENOMEM;
5463
5464 eq->cur_eqe_ba = eq->l0_dma;
5465 eq->nxt_eqe_ba = 0;
5466
a5073d60
YL		 5467 return 0;
5468 }
5469
5470 eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
5471 if (!eq->buf_dma)
5472 return -ENOMEM;
5473 eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
5474 if (!eq->buf)
5475 goto err_kcalloc_buf;
5476
5477 if (mhop_num == 2) {
5478 eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
5479 if (!eq->l1_dma)
5480 goto err_kcalloc_l1_dma;
5481
5482 eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
5483 if (!eq->bt_l1)
5484 goto err_kcalloc_bt_l1;
5485 }
5486
5487 /* alloc L0 BT */
5488 eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
5489 if (!eq->bt_l0)
5490 goto err_dma_alloc_l0;
5491
5492 if (mhop_num == 1) {
2a3d923f 5493 if (ba_num > (bt_chk_sz / BA_BYTE_LEN))
a5073d60
YL		 5494 dev_err(dev, "ba_num %d is too large for 1 hop\n",
5495 ba_num);
5496
5497 /* alloc buf */
2a3d923f 5498 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
a5073d60
YL		 5499 if (eq_buf_cnt + 1 < ba_num) {
5500 size = buf_chk_sz;
5501 } else {
5502 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5503 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5504 }
750afb08 5505 eq->buf[i] = dma_alloc_coherent(dev, size,
a5073d60
YL		 5506 &(eq->buf_dma[i]),
5507 GFP_KERNEL);
5508 if (!eq->buf[i])
5509 goto err_dma_alloc_buf;
5510
a5073d60
YL		 5511 *(eq->bt_l0 + i) = eq->buf_dma[i];
5512
5513 eq_buf_cnt++;
5514 if (eq_buf_cnt >= ba_num)
5515 break;
5516 }
5517 eq->cur_eqe_ba = eq->buf_dma[0];
5518 eq->nxt_eqe_ba = eq->buf_dma[1];
5519
5520 } else if (mhop_num == 2) {
5521 /* alloc L1 BT and buf */
2a3d923f 5522 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
a5073d60
YL		 5523 eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
5524 &(eq->l1_dma[i]),
5525 GFP_KERNEL);
5526 if (!eq->bt_l1[i])
5527 goto err_dma_alloc_l1;
5528 *(eq->bt_l0 + i) = eq->l1_dma[i];
5529
2a3d923f
LO		 5530 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5531 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
a5073d60
YL		 5532 if (eq_buf_cnt + 1 < ba_num) {
5533 size = buf_chk_sz;
5534 } else {
5535 eqe_alloc = (buf_chk_sz / eq->eqe_size)
5536 * idx;
5537 size = (eq->entries - eqe_alloc)
5538 * eq->eqe_size;
5539 }
750afb08
LC		 5540 eq->buf[idx] = dma_alloc_coherent(dev, size,
5541 &(eq->buf_dma[idx]),
5542 GFP_KERNEL);
a5073d60
YL		 5543 if (!eq->buf[idx])
5544 goto err_dma_alloc_buf;
5545
a5073d60
YL		 5546 *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
5547
5548 eq_buf_cnt++;
5549 if (eq_buf_cnt >= ba_num) {
5550 eq_alloc_done = 1;
5551 break;
5552 }
5553 }
5554
5555 if (eq_alloc_done)
5556 break;
5557 }
5558 eq->cur_eqe_ba = eq->buf_dma[0];
5559 eq->nxt_eqe_ba = eq->buf_dma[1];
5560 }
5561
5562 eq->l0_last_num = i + 1;
5563 if (mhop_num == 2)
5564 eq->l1_last_num = j + 1;
5565
5566 return 0;
5567
5568err_dma_alloc_l1:
5569 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5570 eq->bt_l0 = NULL;
5571 eq->l0_dma = 0;
5572 for (i -= 1; i >= 0; i--) {
5573 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5574 eq->l1_dma[i]);
5575
2a3d923f
LO		 5576 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5577 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
a5073d60
YL		 5578 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5579 eq->buf_dma[idx]);
5580 }
5581 }
5582 goto err_dma_alloc_l0;
5583
5584err_dma_alloc_buf:
5585 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5586 eq->bt_l0 = NULL;
5587 eq->l0_dma = 0;
5588
5589 if (mhop_num == 1)
38759d61 5590 for (i -= 1; i >= 0; i--)
a5073d60
YL		 5591 dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5592 eq->buf_dma[i]);
5593 else if (mhop_num == 2) {
5594 record_i = i;
5595 record_j = j;
5596 for (; i >= 0; i--) {
5597 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5598 eq->l1_dma[i]);
5599
2a3d923f 5600 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
a5073d60
YL		 5601 if (i == record_i && j >= record_j)
5602 break;
5603
2a3d923f 5604 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
a5073d60
YL		 5605 dma_free_coherent(dev, buf_chk_sz,
5606 eq->buf[idx],
5607 eq->buf_dma[idx]);
5608 }
5609 }
5610 }
5611
5612err_dma_alloc_l0:
5613 kfree(eq->bt_l1);
5614 eq->bt_l1 = NULL;
5615
5616err_kcalloc_bt_l1:
5617 kfree(eq->l1_dma);
5618 eq->l1_dma = NULL;
5619
5620err_kcalloc_l1_dma:
5621 kfree(eq->buf);
5622 eq->buf = NULL;
5623
5624err_kcalloc_buf:
5625 kfree(eq->buf_dma);
5626 eq->buf_dma = NULL;
5627
5628 return -ENOMEM;
5629}
5630
5631static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5632 struct hns_roce_eq *eq,
5633 unsigned int eq_cmd)
5634{
5635 struct device *dev = hr_dev->dev;
5636 struct hns_roce_cmd_mailbox *mailbox;
5637 u32 buf_chk_sz = 0;
5638 int ret;
5639
5640 /* Allocate mailbox memory */
5641 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5642 if (IS_ERR(mailbox))
5643 return PTR_ERR(mailbox);
5644
5645 if (!hr_dev->caps.eqe_hop_num) {
5646 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5647
5648 eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
5649 GFP_KERNEL);
5650 if (!eq->buf_list) {
5651 ret = -ENOMEM;
5652 goto free_cmd_mbox;
5653 }
5654
750afb08 5655 eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
a5073d60
YL		 5656 &(eq->buf_list->map),
5657 GFP_KERNEL);
5658 if (!eq->buf_list->buf) {
5659 ret = -ENOMEM;
5660 goto err_alloc_buf;
5661 }
5662
a5073d60
YL		 5663 } else {
5664 ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
5665 if (ret) {
5666 ret = -ENOMEM;
5667 goto free_cmd_mbox;
5668 }
5669 }
5670
5671 hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
5672
5673 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5674 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5675 if (ret) {
ab178849 5676 dev_err(dev, "[mailbox cmd] create eqc failed.\n");
a5073d60
YL		 5677 goto err_cmd_mbox;
5678 }
5679
5680 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5681
5682 return 0;
5683
5684err_cmd_mbox:
5685 if (!hr_dev->caps.eqe_hop_num)
5686 dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5687 eq->buf_list->map);
5688 else {
5689 hns_roce_mhop_free_eq(hr_dev, eq);
5690 goto free_cmd_mbox;
5691 }
5692
5693err_alloc_buf:
5694 kfree(eq->buf_list);
5695
5696free_cmd_mbox:
5697 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5698
5699 return ret;
5700}
5701
5702static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
5703{
5704 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5705 struct device *dev = hr_dev->dev;
5706 struct hns_roce_eq *eq;
5707 unsigned int eq_cmd;
5708 int irq_num;
5709 int eq_num;
5710 int other_num;
5711 int comp_num;
5712 int aeq_num;
5713 int i, j, k;
5714 int ret;
5715
5716 other_num = hr_dev->caps.num_other_vectors;
5717 comp_num = hr_dev->caps.num_comp_vectors;
5718 aeq_num = hr_dev->caps.num_aeq_vectors;
5719
5720 eq_num = comp_num + aeq_num;
5721 irq_num = eq_num + other_num;
5722
5723 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
5724 if (!eq_table->eq)
5725 return -ENOMEM;
5726
5727 for (i = 0; i < irq_num; i++) {
5728 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5729 GFP_KERNEL);
5730 if (!hr_dev->irq_names[i]) {
5731 ret = -ENOMEM;
5732 goto err_failed_kzalloc;
5733 }
5734 }
5735
5736 /* create eq */
5737 for (j = 0; j < eq_num; j++) {
5738 eq = &eq_table->eq[j];
5739 eq->hr_dev = hr_dev;
5740 eq->eqn = j;
5741 if (j < comp_num) {
5742 /* CEQ */
5743 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5744 eq->type_flag = HNS_ROCE_CEQ;
5745 eq->entries = hr_dev->caps.ceqe_depth;
5746 eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5747 eq->irq = hr_dev->irq[j + other_num + aeq_num];
5748 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5749 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5750 } else {
5751 /* AEQ */
5752 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5753 eq->type_flag = HNS_ROCE_AEQ;
5754 eq->entries = hr_dev->caps.aeqe_depth;
5755 eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5756 eq->irq = hr_dev->irq[j - comp_num + other_num];
5757 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5758 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5759 }
5760
5761 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
5762 if (ret) {
5763 dev_err(dev, "eq create failed.\n");
5764 goto err_create_eq_fail;
5765 }
5766 }
5767
5768 /* enable irq */
5769 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5770
5771 /* irq contains: abnormal + AEQ + CEQ*/
5772 for (k = 0; k < irq_num; k++)
5773 if (k < other_num)
5774 snprintf((char *)hr_dev->irq_names[k],
5775 HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);
5776 else if (k < (other_num + aeq_num))
5777 snprintf((char *)hr_dev->irq_names[k],
5778 HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5779 k - other_num);
5780 else
5781 snprintf((char *)hr_dev->irq_names[k],
5782 HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5783 k - other_num - aeq_num);
5784
5785 for (k = 0; k < irq_num; k++) {
5786 if (k < other_num)
5787 ret = request_irq(hr_dev->irq[k],
5788 hns_roce_v2_msix_interrupt_abn,
5789 0, hr_dev->irq_names[k], hr_dev);
5790
5791 else if (k < (other_num + comp_num))
5792 ret = request_irq(eq_table->eq[k - other_num].irq,
5793 hns_roce_v2_msix_interrupt_eq,
5794 0, hr_dev->irq_names[k + aeq_num],
5795 &eq_table->eq[k - other_num]);
5796 else
5797 ret = request_irq(eq_table->eq[k - other_num].irq,
5798 hns_roce_v2_msix_interrupt_eq,
5799 0, hr_dev->irq_names[k - comp_num],
5800 &eq_table->eq[k - other_num]);
5801 if (ret) {
5802 dev_err(dev, "Request irq error!\n");
5803 goto err_request_irq_fail;
5804 }
5805 }
5806
0425e3e6
YL		 5807 hr_dev->irq_workq =
5808 create_singlethread_workqueue("hns_roce_irq_workqueue");
5809 if (!hr_dev->irq_workq) {
5810 dev_err(dev, "Create irq workqueue failed!\n");
f1a31542 5811 ret = -ENOMEM;
0425e3e6
YL		 5812 goto err_request_irq_fail;
5813 }
5814
a5073d60
YL		 5815 return 0;
5816
5817err_request_irq_fail:
5818 for (k -= 1; k >= 0; k--)
5819 if (k < other_num)
5820 free_irq(hr_dev->irq[k], hr_dev);
5821 else
5822 free_irq(eq_table->eq[k - other_num].irq,
5823 &eq_table->eq[k - other_num]);
5824
5825err_create_eq_fail:
5826 for (j -= 1; j >= 0; j--)
5827 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]);
5828
5829err_failed_kzalloc:
5830 for (i -= 1; i >= 0; i--)
5831 kfree(hr_dev->irq_names[i]);
5832 kfree(eq_table->eq);
5833
5834 return ret;
5835}
5836
5837static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5838{
5839 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5840 int irq_num;
5841 int eq_num;
5842 int i;
5843
5844 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5845 irq_num = eq_num + hr_dev->caps.num_other_vectors;
5846
5847 /* Disable irq */
5848 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5849
5850 for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5851 free_irq(hr_dev->irq[i], hr_dev);
5852
5853 for (i = 0; i < eq_num; i++) {
5854 hns_roce_v2_destroy_eqc(hr_dev, i);
5855
5856 free_irq(eq_table->eq[i].irq, &eq_table->eq[i]);
5857
5858 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
5859 }
5860
5861 for (i = 0; i < irq_num; i++)
5862 kfree(hr_dev->irq_names[i]);
5863
5864 kfree(eq_table->eq);
0425e3e6
YL		 5865
5866 flush_workqueue(hr_dev->irq_workq);
5867 destroy_workqueue(hr_dev->irq_workq);
a5073d60
YL		 5868}
5869
c7bcb134
LO		 5870static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
5871 struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
5872 u32 cqn, void *mb_buf, u64 *mtts_wqe,
5873 u64 *mtts_idx, dma_addr_t dma_handle_wqe,
5874 dma_addr_t dma_handle_idx)
5875{
5876 struct hns_roce_srq_context *srq_context;
5877
5878 srq_context = mb_buf;
5879 memset(srq_context, 0, sizeof(*srq_context));
5880
5881 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
5882 SRQC_BYTE_4_SRQ_ST_S, 1);
5883
5884 roce_set_field(srq_context->byte_4_srqn_srqst,
5885 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
5886 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
5887 (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
5888 hr_dev->caps.srqwqe_hop_num));
5889 roce_set_field(srq_context->byte_4_srqn_srqst,
5890 SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
5891 ilog2(srq->max));
5892
5893 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
5894 SRQC_BYTE_4_SRQN_S, srq->srqn);
5895
5896 roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5897 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5898
5899 roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
5900 SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
5901
5902 srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
5903
5904 roce_set_field(srq_context->byte_24_wqe_bt_ba,
5905 SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
5906 SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
5907 cpu_to_le32(dma_handle_wqe >> 35));
5908
5909 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
5910 SRQC_BYTE_28_PD_S, pdn);
5911 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
5912 SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
5913 fls(srq->max_gs - 1));
5914
5915 srq_context->idx_bt_ba = (u32)(dma_handle_idx >> 3);
5916 srq_context->idx_bt_ba = cpu_to_le32(srq_context->idx_bt_ba);
5917 roce_set_field(srq_context->rsv_idx_bt_ba,
5918 SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
5919 SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
5920 cpu_to_le32(dma_handle_idx >> 35));
5921
5922 srq_context->idx_cur_blk_addr = (u32)(mtts_idx[0] >> PAGE_ADDR_SHIFT);
5923 srq_context->idx_cur_blk_addr =
5924 cpu_to_le32(srq_context->idx_cur_blk_addr);
5925 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5926 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
5927 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
5928 cpu_to_le32((mtts_idx[0]) >> (32 + PAGE_ADDR_SHIFT)));
5929 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5930 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
5931 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
5932 hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
5933 hr_dev->caps.idx_hop_num);
5934
5935 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5936 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
5937 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
5938 hr_dev->caps.idx_ba_pg_sz);
5939 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5940 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
5941 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
5942 hr_dev->caps.idx_buf_pg_sz);
5943
5944 srq_context->idx_nxt_blk_addr = (u32)(mtts_idx[1] >> PAGE_ADDR_SHIFT);
5945 srq_context->idx_nxt_blk_addr =
5946 cpu_to_le32(srq_context->idx_nxt_blk_addr);
5947 roce_set_field(srq_context->rsv_idxnxtblkaddr,
5948 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
5949 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
5950 cpu_to_le32((mtts_idx[1]) >> (32 + PAGE_ADDR_SHIFT)));
5951 roce_set_field(srq_context->byte_56_xrc_cqn,
5952 SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
5953 cqn);
5954 roce_set_field(srq_context->byte_56_xrc_cqn,
5955 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
5956 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
5957 hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
5958 roce_set_field(srq_context->byte_56_xrc_cqn,
5959 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
5960 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
5961 hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
5962
5963 roce_set_bit(srq_context->db_record_addr_record_en,
5964 SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
5965}
5966
5967static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5968 struct ib_srq_attr *srq_attr,
5969 enum ib_srq_attr_mask srq_attr_mask,
5970 struct ib_udata *udata)
5971{
5972 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5973 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5974 struct hns_roce_srq_context *srq_context;
5975 struct hns_roce_srq_context *srqc_mask;
5976 struct hns_roce_cmd_mailbox *mailbox;
5977 int ret;
5978
5979 if (srq_attr_mask & IB_SRQ_LIMIT) {
5980 if (srq_attr->srq_limit >= srq->max)
5981 return -EINVAL;
5982
5983 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5984 if (IS_ERR(mailbox))
5985 return PTR_ERR(mailbox);
5986
5987 srq_context = mailbox->buf;
5988 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
5989
5990 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
5991
5992 roce_set_field(srq_context->byte_8_limit_wl,
5993 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5994 SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
5995 roce_set_field(srqc_mask->byte_8_limit_wl,
5996 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5997 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5998
5999 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
6000 HNS_ROCE_CMD_MODIFY_SRQC,
6001 HNS_ROCE_CMD_TIMEOUT_MSECS);
6002 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6003 if (ret) {
6004 dev_err(hr_dev->dev,
6005 "MODIFY SRQ Failed to cmd mailbox.\n");
6006 return ret;
6007 }
6008 }
6009
6010 return 0;
6011}
6012
c3c668e7 6013static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
c7bcb134
LO		 6014{
6015 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6016 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6017 struct hns_roce_srq_context *srq_context;
6018 struct hns_roce_cmd_mailbox *mailbox;
6019 int limit_wl;
6020 int ret;
6021
6022 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6023 if (IS_ERR(mailbox))
6024 return PTR_ERR(mailbox);
6025
6026 srq_context = mailbox->buf;
6027 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
6028 HNS_ROCE_CMD_QUERY_SRQC,
6029 HNS_ROCE_CMD_TIMEOUT_MSECS);
6030 if (ret) {
6031 dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
6032 goto out;
6033 }
6034
6035 limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
6036 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6037 SRQC_BYTE_8_SRQ_LIMIT_WL_S);
6038
6039 attr->srq_limit = limit_wl;
6040 attr->max_wr = srq->max - 1;
6041 attr->max_sge = srq->max_gs;
6042
6043 memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
6044
6045out:
6046 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6047 return ret;
6048}
6049
97545b10
LO		 6050static int find_empty_entry(struct hns_roce_idx_que *idx_que,
6051 unsigned long size)
c7bcb134 6052{
97545b10 6053 int wqe_idx;
c7bcb134 6054
97545b10
LO		 6055 if (unlikely(bitmap_full(idx_que->bitmap, size)))
6056 return -ENOSPC;
6057
6058 wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
6059
6060 bitmap_set(idx_que->bitmap, wqe_idx, 1);
c7bcb134 6061
97545b10 6062 return wqe_idx;
c7bcb134
LO		 6063}
6064
6065static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
6066 int cur_idx, int wqe_idx)
6067{
6068 unsigned int *addr;
6069
6070 addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
6071 cur_idx * idx_que->entry_sz);
6072 *addr = wqe_idx;
6073}
6074
6075static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
6076 const struct ib_recv_wr *wr,
6077 const struct ib_recv_wr **bad_wr)
6078{
d3743fa9 6079 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
c7bcb134
LO		 6080 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6081 struct hns_roce_v2_wqe_data_seg *dseg;
6082 struct hns_roce_v2_db srq_db;
6083 unsigned long flags;
6084 int ret = 0;
6085 int wqe_idx;
6086 void *wqe;
6087 int nreq;
6088 int ind;
6089 int i;
6090
6091 spin_lock_irqsave(&srq->lock, flags);
6092
6093 ind = srq->head & (srq->max - 1);
6094
6095 for (nreq = 0; wr; ++nreq, wr = wr->next) {
6096 if (unlikely(wr->num_sge > srq->max_gs)) {
6097 ret = -EINVAL;
6098 *bad_wr = wr;
6099 break;
6100 }
6101
6102 if (unlikely(srq->head == srq->tail)) {
6103 ret = -ENOMEM;
6104 *bad_wr = wr;
6105 break;
6106 }
6107
97545b10
LO		 6108 wqe_idx = find_empty_entry(&srq->idx_que, srq->max);
6109 if (wqe_idx < 0) {
6110 ret = -ENOMEM;
6111 *bad_wr = wr;
6112 break;
6113 }
6114
c7bcb134
LO		 6115 fill_idx_queue(&srq->idx_que, ind, wqe_idx);
6116 wqe = get_srq_wqe(srq, wqe_idx);
6117 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
6118
6119 for (i = 0; i < wr->num_sge; ++i) {
6120 dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
6121 dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
6122 dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
6123 }
6124
6125 if (i < srq->max_gs) {
4f18904c
LO		 6126 dseg[i].len = 0;
6127 dseg[i].lkey = cpu_to_le32(0x100);
6128 dseg[i].addr = 0;
c7bcb134
LO		 6129 }
6130
6131 srq->wrid[wqe_idx] = wr->wr_id;
6132 ind = (ind + 1) & (srq->max - 1);
6133 }
6134
6135 if (likely(nreq)) {
6136 srq->head += nreq;
6137
6138 /*
6139 * Make sure that descriptors are written before
6140 * doorbell record.
6141 */
6142 wmb();
6143
2a3d923f
LO		 6144 srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
6145 (srq->srqn & V2_DB_BYTE_4_TAG_M);
c7bcb134
LO		 6146 srq_db.parameter = srq->head;
6147
d3743fa9 6148 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
c7bcb134
LO		 6149
6150 }
6151
6152 spin_unlock_irqrestore(&srq->lock, flags);
6153
6154 return ret;
6155}
6156
e1c9a0dc
LO		 6157static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6158 .query_cqc_info = hns_roce_v2_query_cqc_info,
6159};
6160
7f645a58
KH		 6161static const struct ib_device_ops hns_roce_v2_dev_ops = {
6162 .destroy_qp = hns_roce_v2_destroy_qp,
6163 .modify_cq = hns_roce_v2_modify_cq,
6164 .poll_cq = hns_roce_v2_poll_cq,
6165 .post_recv = hns_roce_v2_post_recv,
6166 .post_send = hns_roce_v2_post_send,
6167 .query_qp = hns_roce_v2_query_qp,
6168 .req_notify_cq = hns_roce_v2_req_notify_cq,
6169};
6170
6171static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6172 .modify_srq = hns_roce_v2_modify_srq,
6173 .post_srq_recv = hns_roce_v2_post_srq_recv,
6174 .query_srq = hns_roce_v2_query_srq,
6175};
6176
a04ff739
WHX		 6177static const struct hns_roce_hw hns_roce_hw_v2 = {
6178 .cmq_init = hns_roce_v2_cmq_init,
6179 .cmq_exit = hns_roce_v2_cmq_exit,
cfc85f3e 6180 .hw_profile = hns_roce_v2_profile,
6b63597d 6181 .hw_init = hns_roce_v2_init,
6182 .hw_exit = hns_roce_v2_exit,
a680f2f3
WHX		 6183 .post_mbox = hns_roce_v2_post_mbox,
6184 .chk_mbox = hns_roce_v2_chk_mbox,
6a04aed6 6185 .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
7afddafa
WHX		 6186 .set_gid = hns_roce_v2_set_gid,
6187 .set_mac = hns_roce_v2_set_mac,
3958cc56 6188 .write_mtpt = hns_roce_v2_write_mtpt,
a2c80b7b 6189 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
68a997c5 6190 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
c7c28191 6191 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
93aa2187 6192 .write_cqc = hns_roce_v2_write_cqc,
a81fba28
WHX		 6193 .set_hem = hns_roce_v2_set_hem,
6194 .clear_hem = hns_roce_v2_clear_hem,
926a01dc
WHX		 6195 .modify_qp = hns_roce_v2_modify_qp,
6196 .query_qp = hns_roce_v2_query_qp,
6197 .destroy_qp = hns_roce_v2_destroy_qp,
aa84fa18 6198 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
b156269d 6199 .modify_cq = hns_roce_v2_modify_cq,
2d407888
WHX		 6200 .post_send = hns_roce_v2_post_send,
6201 .post_recv = hns_roce_v2_post_recv,
93aa2187
WHX		 6202 .req_notify_cq = hns_roce_v2_req_notify_cq,
6203 .poll_cq = hns_roce_v2_poll_cq,
a5073d60
YL		 6204 .init_eq = hns_roce_v2_init_eq_table,
6205 .cleanup_eq = hns_roce_v2_cleanup_eq_table,
c7bcb134
LO		 6206 .write_srqc = hns_roce_v2_write_srqc,
6207 .modify_srq = hns_roce_v2_modify_srq,
6208 .query_srq = hns_roce_v2_query_srq,
6209 .post_srq_recv = hns_roce_v2_post_srq_recv,
7f645a58
KH		 6210 .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6211 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
a04ff739 6212};
dd74282d
WHX		 6213
6214static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6215 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6216 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
aaa31567
LO		 6217 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6218 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
dd74282d
WHX		 6219 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6220 /* required last entry */
6221 {0, }
6222};
6223
f97a62c3 6224MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6225
dd74282d
WHX		 6226static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6227 struct hnae3_handle *handle)
6228{
d061effc 6229 struct hns_roce_v2_priv *priv = hr_dev->priv;
a5073d60 6230 int i;
dd74282d 6231
dd74282d 6232 hr_dev->hw = &hns_roce_hw_v2;
e1c9a0dc 6233 hr_dev->dfx = &hns_roce_dfx_hw_v2;
2d407888
WHX		 6234 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6235 hr_dev->odb_offset = hr_dev->sdb_offset;
dd74282d
WHX		 6236
6237 /* Get info from NIC driver. */
6238 hr_dev->reg_base = handle->rinfo.roce_io_base;
6239 hr_dev->caps.num_ports = 1;
6240 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6241 hr_dev->iboe.phy_port[0] = 0;
6242
d4994d2f 6243 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6244 hr_dev->iboe.netdevs[0]->dev_addr);
6245
a5073d60
YL		 6246 for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
6247 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6248 i + handle->rinfo.base_vector);
6249
dd74282d 6250 /* cmd issue mode: 0 is poll, 1 is event */
a5073d60 6251 hr_dev->cmd_mod = 1;
dd74282d
WHX		 6252 hr_dev->loop_idc = 0;
6253
d061effc
WHX		 6254 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6255 priv->handle = handle;
6256
dd74282d
WHX		 6257 return 0;
6258}
6259
d061effc 6260static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
dd74282d
WHX		 6261{
6262 struct hns_roce_dev *hr_dev;
6263 int ret;
6264
459cc69f 6265 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
dd74282d
WHX		 6266 if (!hr_dev)
6267 return -ENOMEM;
6268
a04ff739
WHX		 6269 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6270 if (!hr_dev->priv) {
6271 ret = -ENOMEM;
6272 goto error_failed_kzalloc;
6273 }
6274
dd74282d
WHX		 6275 hr_dev->pci_dev = handle->pdev;
6276 hr_dev->dev = &handle->pdev->dev;
dd74282d
WHX		 6277
6278 ret = hns_roce_hw_v2_get_cfg(hr_dev, handle);
6279 if (ret) {
6280 dev_err(hr_dev->dev, "Get Configuration failed!\n");
6281 goto error_failed_get_cfg;
6282 }
6283
6284 ret = hns_roce_init(hr_dev);
6285 if (ret) {
6286 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6287 goto error_failed_get_cfg;
6288 }
6289
d061effc
WHX		 6290 handle->priv = hr_dev;
6291
dd74282d
WHX		 6292 return 0;
6293
6294error_failed_get_cfg:
a04ff739
WHX		 6295 kfree(hr_dev->priv);
6296
6297error_failed_kzalloc:
dd74282d
WHX		 6298 ib_dealloc_device(&hr_dev->ib_dev);
6299
6300 return ret;
6301}
6302
d061effc 6303static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
dd74282d
WHX		 6304 bool reset)
6305{
6306 struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
6307
cb7a94c9
WHX		 6308 if (!hr_dev)
6309 return;
6310
d061effc 6311 handle->priv = NULL;
dd74282d 6312 hns_roce_exit(hr_dev);
a04ff739 6313 kfree(hr_dev->priv);
dd74282d
WHX		 6314 ib_dealloc_device(&hr_dev->ib_dev);
6315}
6316
d061effc
WHX		 6317static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6318{
6319 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
07c2339a 6320 const struct pci_device_id *id;
d061effc
WHX		 6321 struct device *dev = &handle->pdev->dev;
6322 int ret;
6323
6324 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6325
6326 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6327 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6328 goto reset_chk_err;
6329 }
6330
07c2339a
LO		 6331 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6332 if (!id)
6333 return 0;
6334
d061effc
WHX		 6335 ret = __hns_roce_hw_v2_init_instance(handle);
6336 if (ret) {
6337 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6338 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6339 if (ops->ae_dev_resetting(handle) ||
6340 ops->get_hw_reset_stat(handle))
6341 goto reset_chk_err;
6342 else
6343 return ret;
6344 }
6345
6346 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6347
6348
6349 return 0;
6350
6351reset_chk_err:
6352 dev_err(dev, "Device is busy in resetting state.\n"
6353 "please retry later.\n");
6354
6355 return -EBUSY;
6356}
6357
6358static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6359 bool reset)
6360{
6361 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6362 return;
6363
6364 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6365
6366 __hns_roce_hw_v2_uninit_instance(handle, reset);
6367
6368 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6369}
cb7a94c9
WHX		 6370static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6371{
d061effc 6372 struct hns_roce_dev *hr_dev;
cb7a94c9
WHX		 6373 struct ib_event event;
6374
d061effc
WHX		 6375 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6376 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6377 return 0;
cb7a94c9
WHX		 6378 }
6379
d061effc
WHX		 6380 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6381 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6382
6383 hr_dev = (struct hns_roce_dev *)handle->priv;
6384 if (!hr_dev)
6385 return 0;
6386
726be12f 6387 hr_dev->is_reset = true;
cb7a94c9 6388 hr_dev->active = false;
d3743fa9 6389 hr_dev->dis_db = true;
cb7a94c9
WHX		 6390
6391 event.event = IB_EVENT_DEVICE_FATAL;
6392 event.device = &hr_dev->ib_dev;
6393 event.element.port_num = 1;
6394 ib_dispatch_event(&event);
6395
6396 return 0;
6397}
6398
6399static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6400{
d061effc 6401 struct device *dev = &handle->pdev->dev;
cb7a94c9
WHX		 6402 int ret;
6403
d061effc
WHX		 6404 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6405 &handle->rinfo.state)) {
6406 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6407 return 0;
6408 }
6409
6410 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6411
6412 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6413 ret = __hns_roce_hw_v2_init_instance(handle);
cb7a94c9
WHX		 6414 if (ret) {
6415 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6416 * callback function, RoCE Engine reinitialize. If RoCE reinit
6417 * failed, we should inform NIC driver.
6418 */
6419 handle->priv = NULL;
d061effc
WHX		 6420 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6421 } else {
6422 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6423 dev_info(dev, "Reset done, RoCE client reinit finished.\n");
cb7a94c9
WHX		 6424 }
6425
6426 return ret;
6427}
6428
6429static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6430{
d061effc
WHX		 6431 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6432 return 0;
6433
6434 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6435 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
cb7a94c9 6436 msleep(100);
d061effc
WHX		 6437 __hns_roce_hw_v2_uninit_instance(handle, false);
6438
cb7a94c9
WHX		 6439 return 0;
6440}
6441
6442static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6443 enum hnae3_reset_notify_type type)
6444{
6445 int ret = 0;
6446
6447 switch (type) {
6448 case HNAE3_DOWN_CLIENT:
6449 ret = hns_roce_hw_v2_reset_notify_down(handle);
6450 break;
6451 case HNAE3_INIT_CLIENT:
6452 ret = hns_roce_hw_v2_reset_notify_init(handle);
6453 break;
6454 case HNAE3_UNINIT_CLIENT:
6455 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6456 break;
6457 default:
6458 break;
6459 }
6460
6461 return ret;
6462}
6463
dd74282d
WHX		 6464static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6465 .init_instance = hns_roce_hw_v2_init_instance,
6466 .uninit_instance = hns_roce_hw_v2_uninit_instance,
cb7a94c9 6467 .reset_notify = hns_roce_hw_v2_reset_notify,
dd74282d
WHX		 6468};
6469
6470static struct hnae3_client hns_roce_hw_v2_client = {
6471 .name = "hns_roce_hw_v2",
6472 .type = HNAE3_CLIENT_ROCE,
6473 .ops = &hns_roce_hw_v2_ops,
6474};
6475
6476static int __init hns_roce_hw_v2_init(void)
6477{
6478 return hnae3_register_client(&hns_roce_hw_v2_client);
6479}
6480
6481static void __exit hns_roce_hw_v2_exit(void)
6482{
6483 hnae3_unregister_client(&hns_roce_hw_v2_client);
6484}
6485
6486module_init(hns_roce_hw_v2_init);
6487module_exit(hns_roce_hw_v2_exit);
6488
6489MODULE_LICENSE("Dual BSD/GPL");
6490MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
6491MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
6492MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
6493MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");
Linux 6.x block layer and io_uring tree(s)