projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/hns: Cleanups of magic numbers
[linux-2.6-block.git] / drivers / infiniband / hw / hns / hns_roce_hw_v2.c
CommitLineData
dd74282d
WHX		 1/*
2 * Copyright (c) 2016-2017 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/acpi.h>
34#include <linux/etherdevice.h>
35#include <linux/interrupt.h>
36#include <linux/kernel.h>
0b25c9cc 37#include <linux/types.h>
d4994d2f 38#include <net/addrconf.h>
610b8967 39#include <rdma/ib_addr.h>
a70c0739 40#include <rdma/ib_cache.h>
dd74282d 41#include <rdma/ib_umem.h>
bdeacabd 42#include <rdma/uverbs_ioctl.h>
dd74282d
WHX		 43
44#include "hnae3.h"
45#include "hns_roce_common.h"
46#include "hns_roce_device.h"
47#include "hns_roce_cmd.h"
48#include "hns_roce_hem.h"
a04ff739 49#include "hns_roce_hw_v2.h"
dd74282d 50
2d407888
WHX		 51static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
52 struct ib_sge *sg)
53{
54 dseg->lkey = cpu_to_le32(sg->lkey);
55 dseg->addr = cpu_to_le64(sg->addr);
56 dseg->len = cpu_to_le32(sg->length);
57}
58
68a997c5
YL		 59static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
60 struct hns_roce_wqe_frmr_seg *fseg,
61 const struct ib_reg_wr *wr)
62{
63 struct hns_roce_mr *mr = to_hr_mr(wr->mr);
64
65 /* use ib_access_flags */
60262b10 66 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
68a997c5 67 wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
60262b10 68 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
68a997c5 69 wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
60262b10 70 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RR_S,
68a997c5 71 wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
60262b10 72 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_RW_S,
68a997c5 73 wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
60262b10 74 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_LW_S,
68a997c5
YL		 75 wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
76
77 /* Data structure reuse may lead to confusion */
78 rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
79 rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
80
81 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
82 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
83 rc_sq_wqe->rkey = cpu_to_le32(wr->key);
84 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
85
86 fseg->pbl_size = cpu_to_le32(mr->pbl_size);
87 roce_set_field(fseg->mode_buf_pg_sz,
88 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
89 V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
90 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
91 roce_set_bit(fseg->mode_buf_pg_sz,
92 V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
93}
94
384f8818
LO		 95static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,
96 const struct ib_atomic_wr *wr)
97{
98 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
99 aseg->fetchadd_swap_data = cpu_to_le64(wr->swap);
100 aseg->cmp_data = cpu_to_le64(wr->compare_add);
101 } else {
102 aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add);
103 aseg->cmp_data = 0;
104 }
105}
106
f696bf6d 107static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
468d020e 108 unsigned int *sge_ind, int valid_num_sge)
0b25c9cc
WHX		 109{
110 struct hns_roce_v2_wqe_data_seg *dseg;
111 struct ib_sge *sg;
112 int num_in_wqe = 0;
113 int extend_sge_num;
114 int fi_sge_num;
115 int se_sge_num;
116 int shift;
117 int i;
118
119 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
120 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
468d020e 121 extend_sge_num = valid_num_sge - num_in_wqe;
0b25c9cc
WHX		 122 sg = wr->sg_list + num_in_wqe;
123 shift = qp->hr_buf.page_shift;
124
125 /*
126 * Check whether wr->num_sge sges are in the same page. If not, we
127 * should calculate how many sges in the first page and the second
128 * page.
129 */
130 dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
131 fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
132 (uintptr_t)dseg) /
133 sizeof(struct hns_roce_v2_wqe_data_seg);
134 if (extend_sge_num > fi_sge_num) {
135 se_sge_num = extend_sge_num - fi_sge_num;
136 for (i = 0; i < fi_sge_num; i++) {
137 set_data_seg_v2(dseg++, sg + i);
138 (*sge_ind)++;
139 }
140 dseg = get_send_extend_sge(qp,
141 (*sge_ind) & (qp->sge.sge_cnt - 1));
142 for (i = 0; i < se_sge_num; i++) {
143 set_data_seg_v2(dseg++, sg + fi_sge_num + i);
144 (*sge_ind)++;
145 }
146 } else {
147 for (i = 0; i < extend_sge_num; i++) {
148 set_data_seg_v2(dseg++, sg + i);
149 (*sge_ind)++;
150 }
151 }
152}
153
f696bf6d 154static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
7bdee415 155 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
156 void *wqe, unsigned int *sge_ind,
468d020e 157 int valid_num_sge,
d34ac5cd 158 const struct ib_send_wr **bad_wr)
7bdee415 159{
160 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
161 struct hns_roce_v2_wqe_data_seg *dseg = wqe;
162 struct hns_roce_qp *qp = to_hr_qp(ibqp);
468d020e 163 int j = 0;
7bdee415 164 int i;
165
468d020e 166 if (wr->send_flags & IB_SEND_INLINE && valid_num_sge) {
8b9b8d14 167 if (le32_to_cpu(rc_sq_wqe->msg_len) >
168 hr_dev->caps.max_sq_inline) {
7bdee415 169 *bad_wr = wr;
170 dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
171 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
172 return -EINVAL;
173 }
174
328d405b 175 if (wr->opcode == IB_WR_RDMA_READ) {
c80e0661 176 *bad_wr = wr;
328d405b 177 dev_err(hr_dev->dev, "Not support inline data!\n");
178 return -EINVAL;
179 }
180
7bdee415 181 for (i = 0; i < wr->num_sge; i++) {
182 memcpy(wqe, ((void *)wr->sg_list[i].addr),
183 wr->sg_list[i].length);
184 wqe += wr->sg_list[i].length;
185 }
186
187 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
188 1);
189 } else {
468d020e 190 if (valid_num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
7bdee415 191 for (i = 0; i < wr->num_sge; i++) {
192 if (likely(wr->sg_list[i].length)) {
193 set_data_seg_v2(dseg, wr->sg_list + i);
194 dseg++;
195 }
196 }
197 } else {
198 roce_set_field(rc_sq_wqe->byte_20,
199 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
200 V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
201 (*sge_ind) & (qp->sge.sge_cnt - 1));
202
468d020e
LO		 203 for (i = 0; i < wr->num_sge &&
204 j < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
7bdee415 205 if (likely(wr->sg_list[i].length)) {
206 set_data_seg_v2(dseg, wr->sg_list + i);
207 dseg++;
468d020e 208 j++;
7bdee415 209 }
210 }
211
468d020e 212 set_extend_sge(qp, wr, sge_ind, valid_num_sge);
7bdee415 213 }
214
215 roce_set_field(rc_sq_wqe->byte_16,
216 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
468d020e 217 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, valid_num_sge);
7bdee415 218 }
219
220 return 0;
221}
222
0425e3e6
YL		 223static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
224 const struct ib_qp_attr *attr,
225 int attr_mask, enum ib_qp_state cur_state,
226 enum ib_qp_state new_state);
227
626903e9
XW		 228static int check_send_valid(struct hns_roce_dev *hr_dev,
229 struct hns_roce_qp *hr_qp)
230{
231 struct ib_qp *ibqp = &hr_qp->ibqp;
232 struct device *dev = hr_dev->dev;
233
234 if (unlikely(ibqp->qp_type != IB_QPT_RC &&
235 ibqp->qp_type != IB_QPT_GSI &&
236 ibqp->qp_type != IB_QPT_UD)) {
237 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
238 return -EOPNOTSUPP;
239 } else if (unlikely(hr_qp->state == IB_QPS_RESET ||
240 hr_qp->state == IB_QPS_INIT ||
241 hr_qp->state == IB_QPS_RTR)) {
242 dev_err(dev, "Post WQE fail, QP state %d!\n", hr_qp->state);
243 return -EINVAL;
244 } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {
245 dev_err(dev, "Post WQE fail, dev state %d!\n", hr_dev->state);
246 return -EIO;
247 }
248
249 return 0;
250}
251
d34ac5cd
BVA		 252static int hns_roce_v2_post_send(struct ib_qp *ibqp,
253 const struct ib_send_wr *wr,
254 const struct ib_send_wr **bad_wr)
2d407888
WHX		 255{
256 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
7bdee415 257 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
258 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
2d407888
WHX		 259 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
260 struct hns_roce_qp *qp = to_hr_qp(ibqp);
68a997c5 261 struct hns_roce_wqe_frmr_seg *fseg;
2d407888
WHX		 262 struct device *dev = hr_dev->dev;
263 struct hns_roce_v2_db sq_db;
0425e3e6 264 struct ib_qp_attr attr;
e8d18533 265 unsigned int owner_bit;
47688202
YL		 266 unsigned int sge_idx;
267 unsigned int wqe_idx;
2d407888 268 unsigned long flags;
468d020e 269 int valid_num_sge;
2d407888 270 void *wqe = NULL;
7bdee415 271 bool loopback;
0425e3e6 272 int attr_mask;
55ba49cb 273 u32 tmp_len;
b9c1ea40 274 u32 hr_op;
7bdee415 275 u8 *smac;
2d407888 276 int nreq;
626903e9 277 int ret;
2d407888
WHX		 278 int i;
279
626903e9 280 spin_lock_irqsave(&qp->sq.lock, flags);
2d407888 281
626903e9
XW		 282 ret = check_send_valid(hr_dev, qp);
283 if (ret) {
2d407888 284 *bad_wr = wr;
626903e9
XW		 285 nreq = 0;
286 goto out;
2d407888
WHX		 287 }
288
47688202 289 sge_idx = qp->next_sge;
2d407888
WHX		 290
291 for (nreq = 0; wr; ++nreq, wr = wr->next) {
292 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
293 ret = -ENOMEM;
294 *bad_wr = wr;
295 goto out;
296 }
297
47688202
YL		 298 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
299
2d407888
WHX		 300 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
301 dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
302 wr->num_sge, qp->sq.max_gs);
303 ret = -EINVAL;
304 *bad_wr = wr;
305 goto out;
306 }
307
47688202
YL		 308 wqe = get_send_wqe(qp, wqe_idx);
309 qp->sq.wrid[wqe_idx] = wr->wr_id;
634f6390 310 owner_bit =
311 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
468d020e 312 valid_num_sge = 0;
55ba49cb 313 tmp_len = 0;
2d407888 314
468d020e
LO		 315 for (i = 0; i < wr->num_sge; i++) {
316 if (likely(wr->sg_list[i].length)) {
317 tmp_len += wr->sg_list[i].length;
318 valid_num_sge++;
319 }
320 }
321
7bdee415 322 /* Corresponding to the QP type, wqe process separately */
323 if (ibqp->qp_type == IB_QPT_GSI) {
324 ud_sq_wqe = wqe;
325 memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
326
327 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
328 V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
329 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
330 V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
331 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
332 V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
333 roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
334 V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
335 roce_set_field(ud_sq_wqe->byte_48,
336 V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
337 V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
338 ah->av.mac[4]);
339 roce_set_field(ud_sq_wqe->byte_48,
340 V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
341 V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
342 ah->av.mac[5]);
343
344 /* MAC loopback */
345 smac = (u8 *)hr_dev->dev_addr[qp->port];
346 loopback = ether_addr_equal_unaligned(ah->av.mac,
347 smac) ? 1 : 0;
348
349 roce_set_bit(ud_sq_wqe->byte_40,
350 V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
351
352 roce_set_field(ud_sq_wqe->byte_4,
353 V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
354 V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
355 HNS_ROCE_V2_WQE_OP_SEND);
2d407888 356
8b9b8d14 357 ud_sq_wqe->msg_len =
358 cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
359
360 switch (wr->opcode) {
361 case IB_WR_SEND_WITH_IMM:
362 case IB_WR_RDMA_WRITE_WITH_IMM:
0c4a0e29
LO		 363 ud_sq_wqe->immtdata =
364 cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
8b9b8d14 365 break;
366 default:
367 ud_sq_wqe->immtdata = 0;
368 break;
369 }
651487c2 370
7bdee415 371 /* Set sig attr */
372 roce_set_bit(ud_sq_wqe->byte_4,
373 V2_UD_SEND_WQE_BYTE_4_CQE_S,
374 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
a49d761f 375
7bdee415 376 /* Set se attr */
377 roce_set_bit(ud_sq_wqe->byte_4,
378 V2_UD_SEND_WQE_BYTE_4_SE_S,
379 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
e8d18533 380
7bdee415 381 roce_set_bit(ud_sq_wqe->byte_4,
382 V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
383
384 roce_set_field(ud_sq_wqe->byte_16,
385 V2_UD_SEND_WQE_BYTE_16_PD_M,
386 V2_UD_SEND_WQE_BYTE_16_PD_S,
387 to_hr_pd(ibqp->pd)->pdn);
388
389 roce_set_field(ud_sq_wqe->byte_16,
390 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
391 V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
468d020e 392 valid_num_sge);
7bdee415 393
394 roce_set_field(ud_sq_wqe->byte_20,
395 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
396 V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
47688202 397 sge_idx & (qp->sge.sge_cnt - 1));
7bdee415 398
399 roce_set_field(ud_sq_wqe->byte_24,
400 V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
401 V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
402 ud_sq_wqe->qkey =
8b9b8d14 403 cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
404 qp->qkey : ud_wr(wr)->remote_qkey);
7bdee415 405 roce_set_field(ud_sq_wqe->byte_32,
406 V2_UD_SEND_WQE_BYTE_32_DQPN_M,
407 V2_UD_SEND_WQE_BYTE_32_DQPN_S,
408 ud_wr(wr)->remote_qpn);
409
410 roce_set_field(ud_sq_wqe->byte_36,
411 V2_UD_SEND_WQE_BYTE_36_VLAN_M,
412 V2_UD_SEND_WQE_BYTE_36_VLAN_S,
32883228 413 ah->av.vlan_id);
7bdee415 414 roce_set_field(ud_sq_wqe->byte_36,
415 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
416 V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
417 ah->av.hop_limit);
418 roce_set_field(ud_sq_wqe->byte_36,
419 V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
420 V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
82e620d9 421 ah->av.tclass);
7bdee415 422 roce_set_field(ud_sq_wqe->byte_40,
423 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
cdfa4ad5 424 V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
82e620d9 425 ah->av.flowlabel);
7bdee415 426 roce_set_field(ud_sq_wqe->byte_40,
427 V2_UD_SEND_WQE_BYTE_40_SL_M,
428 V2_UD_SEND_WQE_BYTE_40_SL_S,
82e620d9 429 ah->av.sl);
7bdee415 430 roce_set_field(ud_sq_wqe->byte_40,
431 V2_UD_SEND_WQE_BYTE_40_PORTN_M,
432 V2_UD_SEND_WQE_BYTE_40_PORTN_S,
433 qp->port);
434
8320deb8
LO		 435 roce_set_bit(ud_sq_wqe->byte_40,
436 V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
437 ah->av.vlan_en ? 1 : 0);
7bdee415 438 roce_set_field(ud_sq_wqe->byte_48,
439 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
440 V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
441 hns_get_gid_index(hr_dev, qp->phy_port,
442 ah->av.gid_index));
443
444 memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
445 GID_LEN_V2);
446
468d020e 447 set_extend_sge(qp, wr, &sge_idx, valid_num_sge);
7bdee415 448 } else if (ibqp->qp_type == IB_QPT_RC) {
449 rc_sq_wqe = wqe;
450 memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
8b9b8d14 451
452 rc_sq_wqe->msg_len =
453 cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
7bdee415 454
8b9b8d14 455 switch (wr->opcode) {
456 case IB_WR_SEND_WITH_IMM:
457 case IB_WR_RDMA_WRITE_WITH_IMM:
0c4a0e29
LO		 458 rc_sq_wqe->immtdata =
459 cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
8b9b8d14 460 break;
461 case IB_WR_SEND_WITH_INV:
462 rc_sq_wqe->inv_key =
463 cpu_to_le32(wr->ex.invalidate_rkey);
464 break;
465 default:
466 rc_sq_wqe->immtdata = 0;
467 break;
468 }
7bdee415 469
470 roce_set_bit(rc_sq_wqe->byte_4,
471 V2_RC_SEND_WQE_BYTE_4_FENCE_S,
472 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
473
474 roce_set_bit(rc_sq_wqe->byte_4,
475 V2_RC_SEND_WQE_BYTE_4_SE_S,
476 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
477
478 roce_set_bit(rc_sq_wqe->byte_4,
479 V2_RC_SEND_WQE_BYTE_4_CQE_S,
480 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
481
482 roce_set_bit(rc_sq_wqe->byte_4,
483 V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
484
384f8818 485 wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
7bdee415 486 switch (wr->opcode) {
487 case IB_WR_RDMA_READ:
b9c1ea40 488 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;
7bdee415 489 rc_sq_wqe->rkey =
490 cpu_to_le32(rdma_wr(wr)->rkey);
491 rc_sq_wqe->va =
492 cpu_to_le64(rdma_wr(wr)->remote_addr);
493 break;
494 case IB_WR_RDMA_WRITE:
b9c1ea40 495 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;
7bdee415 496 rc_sq_wqe->rkey =
497 cpu_to_le32(rdma_wr(wr)->rkey);
498 rc_sq_wqe->va =
499 cpu_to_le64(rdma_wr(wr)->remote_addr);
500 break;
501 case IB_WR_RDMA_WRITE_WITH_IMM:
b9c1ea40 502 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;
7bdee415 503 rc_sq_wqe->rkey =
504 cpu_to_le32(rdma_wr(wr)->rkey);
505 rc_sq_wqe->va =
506 cpu_to_le64(rdma_wr(wr)->remote_addr);
507 break;
508 case IB_WR_SEND:
b9c1ea40 509 hr_op = HNS_ROCE_V2_WQE_OP_SEND;
7bdee415 510 break;
511 case IB_WR_SEND_WITH_INV:
b9c1ea40 512 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;
7bdee415 513 break;
514 case IB_WR_SEND_WITH_IMM:
b9c1ea40 515 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;
7bdee415 516 break;
517 case IB_WR_LOCAL_INV:
b9c1ea40 518 hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;
e93df010
LO		 519 roce_set_bit(rc_sq_wqe->byte_4,
520 V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
521 rc_sq_wqe->inv_key =
522 cpu_to_le32(wr->ex.invalidate_rkey);
7bdee415 523 break;
68a997c5
YL		 524 case IB_WR_REG_MR:
525 hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;
526 fseg = wqe;
527 set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));
528 break;
7bdee415 529 case IB_WR_ATOMIC_CMP_AND_SWP:
b9c1ea40 530 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;
384f8818
LO		 531 rc_sq_wqe->rkey =
532 cpu_to_le32(atomic_wr(wr)->rkey);
533 rc_sq_wqe->va =
d9581bf3 534 cpu_to_le64(atomic_wr(wr)->remote_addr);
7bdee415 535 break;
536 case IB_WR_ATOMIC_FETCH_AND_ADD:
b9c1ea40 537 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;
384f8818
LO		 538 rc_sq_wqe->rkey =
539 cpu_to_le32(atomic_wr(wr)->rkey);
540 rc_sq_wqe->va =
d9581bf3 541 cpu_to_le64(atomic_wr(wr)->remote_addr);
7bdee415 542 break;
543 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
b9c1ea40
LO		 544 hr_op =
545 HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;
7bdee415 546 break;
547 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
b9c1ea40
LO		 548 hr_op =
549 HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;
7bdee415 550 break;
551 default:
b9c1ea40 552 hr_op = HNS_ROCE_V2_WQE_OP_MASK;
7bdee415 553 break;
2d407888
WHX		 554 }
555
b9c1ea40
LO		 556 roce_set_field(rc_sq_wqe->byte_4,
557 V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
558 V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);
2d407888 559
d9581bf3
LO		 560 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
561 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
562 struct hns_roce_v2_wqe_data_seg *dseg;
563
564 dseg = wqe;
565 set_data_seg_v2(dseg, wr->sg_list);
566 wqe += sizeof(struct hns_roce_v2_wqe_data_seg);
567 set_atomic_seg(wqe, atomic_wr(wr));
568 roce_set_field(rc_sq_wqe->byte_16,
569 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
570 V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
468d020e 571 valid_num_sge);
68a997c5 572 } else if (wr->opcode != IB_WR_REG_MR) {
d9581bf3 573 ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,
468d020e
LO		 574 wqe, &sge_idx,
575 valid_num_sge, bad_wr);
d9581bf3
LO		 576 if (ret)
577 goto out;
578 }
2d407888 579 } else {
7bdee415 580 dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
581 spin_unlock_irqrestore(&qp->sq.lock, flags);
137ae320 582 *bad_wr = wr;
7bdee415 583 return -EOPNOTSUPP;
2d407888 584 }
2d407888
WHX		 585 }
586
587out:
588 if (likely(nreq)) {
589 qp->sq.head += nreq;
590 /* Memory barrier */
591 wmb();
592
593 sq_db.byte_4 = 0;
594 sq_db.parameter = 0;
595
596 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
597 V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
598 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
599 V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
cc3391cb 600 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
601 V2_DB_PARAMETER_IDX_S,
2d407888
WHX		 602 qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
603 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
604 V2_DB_PARAMETER_SL_S, qp->sl);
605
d3743fa9 606 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
2d407888 607
47688202 608 qp->next_sge = sge_idx;
0425e3e6
YL		 609
610 if (qp->state == IB_QPS_ERR) {
611 attr_mask = IB_QP_STATE;
612 attr.qp_state = IB_QPS_ERR;
613
614 ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
615 qp->state, IB_QPS_ERR);
616 if (ret) {
617 spin_unlock_irqrestore(&qp->sq.lock, flags);
618 *bad_wr = wr;
619 return ret;
620 }
621 }
2d407888
WHX		 622 }
623
624 spin_unlock_irqrestore(&qp->sq.lock, flags);
625
626 return ret;
627}
628
626903e9
XW		 629static int check_recv_valid(struct hns_roce_dev *hr_dev,
630 struct hns_roce_qp *hr_qp)
631{
632 if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
633 return -EIO;
634 else if (hr_qp->state == IB_QPS_RESET)
635 return -EINVAL;
636
637 return 0;
638}
639
d34ac5cd
BVA		 640static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
641 const struct ib_recv_wr *wr,
642 const struct ib_recv_wr **bad_wr)
2d407888
WHX		 643{
644 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
645 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
646 struct hns_roce_v2_wqe_data_seg *dseg;
0009c2db 647 struct hns_roce_rinl_sge *sge_list;
2d407888 648 struct device *dev = hr_dev->dev;
0425e3e6 649 struct ib_qp_attr attr;
2d407888
WHX		 650 unsigned long flags;
651 void *wqe = NULL;
0425e3e6 652 int attr_mask;
47688202 653 u32 wqe_idx;
2d407888 654 int nreq;
626903e9 655 int ret;
2d407888
WHX		 656 int i;
657
658 spin_lock_irqsave(&hr_qp->rq.lock, flags);
2d407888 659
626903e9
XW		 660 ret = check_recv_valid(hr_dev, hr_qp);
661 if (ret) {
2d407888 662 *bad_wr = wr;
626903e9
XW		 663 nreq = 0;
664 goto out;
2d407888
WHX		 665 }
666
667 for (nreq = 0; wr; ++nreq, wr = wr->next) {
668 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
669 hr_qp->ibqp.recv_cq)) {
670 ret = -ENOMEM;
671 *bad_wr = wr;
672 goto out;
673 }
674
47688202
YL		 675 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
676
2d407888
WHX		 677 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
678 dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
679 wr->num_sge, hr_qp->rq.max_gs);
680 ret = -EINVAL;
681 *bad_wr = wr;
682 goto out;
683 }
684
47688202 685 wqe = get_recv_wqe(hr_qp, wqe_idx);
2d407888
WHX		 686 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
687 for (i = 0; i < wr->num_sge; i++) {
688 if (!wr->sg_list[i].length)
689 continue;
690 set_data_seg_v2(dseg, wr->sg_list + i);
691 dseg++;
692 }
693
694 if (i < hr_qp->rq.max_gs) {
778cc5a8 695 dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
696 dseg->addr = 0;
2d407888
WHX		 697 }
698
0009c2db 699 /* rq support inline data */
ecaaf1e2 700 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
47688202
YL		 701 sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
702 hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt =
ecaaf1e2 703 (u32)wr->num_sge;
704 for (i = 0; i < wr->num_sge; i++) {
705 sge_list[i].addr =
706 (void *)(u64)wr->sg_list[i].addr;
707 sge_list[i].len = wr->sg_list[i].length;
708 }
0009c2db 709 }
710
47688202 711 hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
2d407888
WHX		 712 }
713
714out:
715 if (likely(nreq)) {
716 hr_qp->rq.head += nreq;
717 /* Memory barrier */
718 wmb();
719
472bc0fb 720 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
0425e3e6
YL		 721
722 if (hr_qp->state == IB_QPS_ERR) {
723 attr_mask = IB_QP_STATE;
724 attr.qp_state = IB_QPS_ERR;
725
726 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
727 attr_mask, hr_qp->state,
728 IB_QPS_ERR);
729 if (ret) {
730 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
731 *bad_wr = wr;
732 return ret;
733 }
734 }
2d407888
WHX		 735 }
736 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
737
738 return ret;
739}
740
6a04aed6
WHX		 741static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
742 unsigned long instance_stage,
743 unsigned long reset_stage)
744{
745 /* When hardware reset has been completed once or more, we should stop
d3743fa9 746 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
6a04aed6
WHX		 747 * function, we should exit with error. If now at HNAE3_INIT_CLIENT
748 * stage of soft reset process, we should exit with error, and then
749 * HNAE3_INIT_CLIENT related process can rollback the operation like
750 * notifing hardware to free resources, HNAE3_INIT_CLIENT related
751 * process will exit with error to notify NIC driver to reschedule soft
752 * reset process once again.
753 */
754 hr_dev->is_reset = true;
d3743fa9 755 hr_dev->dis_db = true;
6a04aed6
WHX		 756
757 if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
758 instance_stage == HNS_ROCE_STATE_INIT)
759 return CMD_RST_PRC_EBUSY;
760
761 return CMD_RST_PRC_SUCCESS;
762}
763
764static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
765 unsigned long instance_stage,
766 unsigned long reset_stage)
767{
768 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
769 struct hnae3_handle *handle = priv->handle;
770 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
771
d3743fa9
WHX		 772 /* When hardware reset is detected, we should stop sending mailbox&cmq&
773 * doorbell to hardware. If now in .init_instance() function, we should
6a04aed6
WHX		 774 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
775 * process, we should exit with error, and then HNAE3_INIT_CLIENT
776 * related process can rollback the operation like notifing hardware to
777 * free resources, HNAE3_INIT_CLIENT related process will exit with
778 * error to notify NIC driver to reschedule soft reset process once
779 * again.
780 */
d3743fa9 781 hr_dev->dis_db = true;
6a04aed6
WHX		 782 if (!ops->get_hw_reset_stat(handle))
783 hr_dev->is_reset = true;
784
785 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
786 instance_stage == HNS_ROCE_STATE_INIT)
787 return CMD_RST_PRC_EBUSY;
788
789 return CMD_RST_PRC_SUCCESS;
790}
791
792static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
793{
794 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
795 struct hnae3_handle *handle = priv->handle;
796 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
797
798 /* When software reset is detected at .init_instance() function, we
d3743fa9
WHX		 799 * should stop sending mailbox&cmq&doorbell to hardware, and exit
800 * with error.
6a04aed6 801 */
d3743fa9 802 hr_dev->dis_db = true;
6a04aed6
WHX		 803 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
804 hr_dev->is_reset = true;
805
806 return CMD_RST_PRC_EBUSY;
807}
808
809static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
810{
811 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
812 struct hnae3_handle *handle = priv->handle;
813 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
814 unsigned long instance_stage; /* the current instance stage */
815 unsigned long reset_stage; /* the current reset stage */
816 unsigned long reset_cnt;
817 bool sw_resetting;
818 bool hw_resetting;
819
820 if (hr_dev->is_reset)
821 return CMD_RST_PRC_SUCCESS;
822
823 /* Get information about reset from NIC driver or RoCE driver itself,
824 * the meaning of the following variables from NIC driver are described
825 * as below:
826 * reset_cnt -- The count value of completed hardware reset.
827 * hw_resetting -- Whether hardware device is resetting now.
828 * sw_resetting -- Whether NIC's software reset process is running now.
829 */
830 instance_stage = handle->rinfo.instance_state;
831 reset_stage = handle->rinfo.reset_state;
832 reset_cnt = ops->ae_dev_reset_cnt(handle);
833 hw_resetting = ops->get_hw_reset_stat(handle);
834 sw_resetting = ops->ae_dev_resetting(handle);
835
836 if (reset_cnt != hr_dev->reset_cnt)
837 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
838 reset_stage);
839 else if (hw_resetting)
840 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
841 reset_stage);
842 else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
843 return hns_roce_v2_cmd_sw_resetting(hr_dev);
844
845 return 0;
846}
847
a04ff739
WHX		 848static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
849{
850 int ntu = ring->next_to_use;
851 int ntc = ring->next_to_clean;
852 int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
853
854 return ring->desc_num - used - 1;
855}
856
857static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
858 struct hns_roce_v2_cmq_ring *ring)
859{
860 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
861
862 ring->desc = kzalloc(size, GFP_KERNEL);
863 if (!ring->desc)
864 return -ENOMEM;
865
866 ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
867 DMA_BIDIRECTIONAL);
868 if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
869 ring->desc_dma_addr = 0;
870 kfree(ring->desc);
871 ring->desc = NULL;
872 return -ENOMEM;
873 }
874
875 return 0;
876}
877
878static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
879 struct hns_roce_v2_cmq_ring *ring)
880{
881 dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
882 ring->desc_num * sizeof(struct hns_roce_cmq_desc),
883 DMA_BIDIRECTIONAL);
90e7a4d5 884
885 ring->desc_dma_addr = 0;
a04ff739
WHX		 886 kfree(ring->desc);
887}
888
889static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
890{
891 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
892 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
893 &priv->cmq.csq : &priv->cmq.crq;
894
895 ring->flag = ring_type;
896 ring->next_to_clean = 0;
897 ring->next_to_use = 0;
898
899 return hns_roce_alloc_cmq_desc(hr_dev, ring);
900}
901
902static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
903{
904 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
905 struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
906 &priv->cmq.csq : &priv->cmq.crq;
907 dma_addr_t dma = ring->desc_dma_addr;
908
909 if (ring_type == TYPE_CSQ) {
910 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
911 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
912 upper_32_bits(dma));
913 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
2288b3b3 914 ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
a04ff739
WHX		 915 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
916 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
917 } else {
918 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
919 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
920 upper_32_bits(dma));
921 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
2288b3b3 922 ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
a04ff739
WHX		 923 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
924 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
925 }
926}
927
928static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
929{
930 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
931 int ret;
932
933 /* Setup the queue entries for command queue */
426c4146
LO		 934 priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
935 priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
a04ff739
WHX		 936
937 /* Setup the lock for command queue */
938 spin_lock_init(&priv->cmq.csq.lock);
939 spin_lock_init(&priv->cmq.crq.lock);
940
941 /* Setup Tx write back timeout */
942 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
943
944 /* Init CSQ */
945 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
946 if (ret) {
947 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
948 return ret;
949 }
950
951 /* Init CRQ */
952 ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
953 if (ret) {
954 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
955 goto err_crq;
956 }
957
958 /* Init CSQ REG */
959 hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
960
961 /* Init CRQ REG */
962 hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
963
964 return 0;
965
966err_crq:
967 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
968
969 return ret;
970}
971
972static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
973{
974 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
975
976 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
977 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
978}
979
281d0ccf
CIK		 980static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
981 enum hns_roce_opcode_type opcode,
982 bool is_read)
a04ff739
WHX		 983{
984 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
985 desc->opcode = cpu_to_le16(opcode);
986 desc->flag =
987 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
988 if (is_read)
989 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
990 else
991 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
992}
993
994static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
995{
996 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
997 u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
998
999 return head == priv->cmq.csq.next_to_use;
1000}
1001
1002static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
1003{
1004 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1005 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1006 struct hns_roce_cmq_desc *desc;
1007 u16 ntc = csq->next_to_clean;
1008 u32 head;
1009 int clean = 0;
1010
1011 desc = &csq->desc[ntc];
1012 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
1013 while (head != ntc) {
1014 memset(desc, 0, sizeof(*desc));
1015 ntc++;
1016 if (ntc == csq->desc_num)
1017 ntc = 0;
1018 desc = &csq->desc[ntc];
1019 clean++;
1020 }
1021 csq->next_to_clean = ntc;
1022
1023 return clean;
1024}
1025
6a04aed6
WHX		 1026static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1027 struct hns_roce_cmq_desc *desc, int num)
a04ff739
WHX		 1028{
1029 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1030 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1031 struct hns_roce_cmq_desc *desc_to_use;
1032 bool complete = false;
1033 u32 timeout = 0;
1034 int handle = 0;
1035 u16 desc_ret;
1036 int ret = 0;
1037 int ntc;
1038
1039 spin_lock_bh(&csq->lock);
1040
1041 if (num > hns_roce_cmq_space(csq)) {
1042 spin_unlock_bh(&csq->lock);
1043 return -EBUSY;
1044 }
1045
1046 /*
1047 * Record the location of desc in the cmq for this time
1048 * which will be use for hardware to write back
1049 */
1050 ntc = csq->next_to_use;
1051
1052 while (handle < num) {
1053 desc_to_use = &csq->desc[csq->next_to_use];
1054 *desc_to_use = desc[handle];
1055 dev_dbg(hr_dev->dev, "set cmq desc:\n");
1056 csq->next_to_use++;
1057 if (csq->next_to_use == csq->desc_num)
1058 csq->next_to_use = 0;
1059 handle++;
1060 }
1061
1062 /* Write to hardware */
1063 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
1064
1065 /*
1066 * If the command is sync, wait for the firmware to write back,
1067 * if multi descriptors to be sent, use the first one to check
1068 */
bfe86035 1069 if (le16_to_cpu(desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
a04ff739
WHX		 1070 do {
1071 if (hns_roce_cmq_csq_done(hr_dev))
1072 break;
988e175b 1073 udelay(1);
a04ff739
WHX		 1074 timeout++;
1075 } while (timeout < priv->cmq.tx_timeout);
1076 }
1077
1078 if (hns_roce_cmq_csq_done(hr_dev)) {
1079 complete = true;
1080 handle = 0;
1081 while (handle < num) {
1082 /* get the result of hardware write back */
1083 desc_to_use = &csq->desc[ntc];
1084 desc[handle] = *desc_to_use;
1085 dev_dbg(hr_dev->dev, "Get cmq desc:\n");
bfe86035 1086 desc_ret = le16_to_cpu(desc[handle].retval);
a04ff739
WHX		 1087 if (desc_ret == CMD_EXEC_SUCCESS)
1088 ret = 0;
1089 else
1090 ret = -EIO;
1091 priv->cmq.last_status = desc_ret;
1092 ntc++;
1093 handle++;
1094 if (ntc == csq->desc_num)
1095 ntc = 0;
1096 }
1097 }
1098
1099 if (!complete)
1100 ret = -EAGAIN;
1101
1102 /* clean the command send queue */
1103 handle = hns_roce_cmq_csq_clean(hr_dev);
1104 if (handle != num)
1105 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
1106 handle, num);
1107
1108 spin_unlock_bh(&csq->lock);
1109
1110 return ret;
1111}
1112
e95e52a1 1113static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
6a04aed6
WHX		 1114 struct hns_roce_cmq_desc *desc, int num)
1115{
1116 int retval;
1117 int ret;
1118
1119 ret = hns_roce_v2_rst_process_cmd(hr_dev);
1120 if (ret == CMD_RST_PRC_SUCCESS)
1121 return 0;
1122 if (ret == CMD_RST_PRC_EBUSY)
b417c087 1123 return -EBUSY;
6a04aed6
WHX		 1124
1125 ret = __hns_roce_cmq_send(hr_dev, desc, num);
1126 if (ret) {
1127 retval = hns_roce_v2_rst_process_cmd(hr_dev);
1128 if (retval == CMD_RST_PRC_SUCCESS)
1129 return 0;
1130 else if (retval == CMD_RST_PRC_EBUSY)
b417c087 1131 return -EBUSY;
6a04aed6
WHX		 1132 }
1133
1134 return ret;
1135}
1136
281d0ccf 1137static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
cfc85f3e
WHX		 1138{
1139 struct hns_roce_query_version *resp;
1140 struct hns_roce_cmq_desc desc;
1141 int ret;
1142
1143 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1144 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1145 if (ret)
1146 return ret;
1147
1148 resp = (struct hns_roce_query_version *)desc.data;
bfe86035 1149 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
3a63c964
LO		 1150 hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1151
1152 return 0;
1153}
1154
e075da5e
LC		 1155static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
1156{
1157 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1158 struct hnae3_handle *handle = priv->handle;
1159 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1160 unsigned long reset_cnt;
1161 bool sw_resetting;
1162 bool hw_resetting;
1163
1164 reset_cnt = ops->ae_dev_reset_cnt(handle);
1165 hw_resetting = ops->get_hw_reset_stat(handle);
1166 sw_resetting = ops->ae_dev_resetting(handle);
1167
1168 if (reset_cnt != hr_dev->reset_cnt || hw_resetting || sw_resetting)
1169 return true;
1170
1171 return false;
1172}
1173
1174static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
1175 int flag)
1176{
1177 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1178 struct hnae3_handle *handle = priv->handle;
1179 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1180 unsigned long instance_stage;
1181 unsigned long reset_cnt;
1182 unsigned long end;
1183 bool sw_resetting;
1184 bool hw_resetting;
1185
1186 instance_stage = handle->rinfo.instance_state;
1187 reset_cnt = ops->ae_dev_reset_cnt(handle);
1188 hw_resetting = ops->get_hw_reset_stat(handle);
1189 sw_resetting = ops->ae_dev_resetting(handle);
1190
1191 if (reset_cnt != hr_dev->reset_cnt) {
1192 hr_dev->dis_db = true;
1193 hr_dev->is_reset = true;
1194 dev_info(hr_dev->dev, "Func clear success after reset.\n");
1195 } else if (hw_resetting) {
1196 hr_dev->dis_db = true;
1197
1198 dev_warn(hr_dev->dev,
1199 "Func clear is pending, device in resetting state.\n");
1200 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1201 while (end) {
1202 if (!ops->get_hw_reset_stat(handle)) {
1203 hr_dev->is_reset = true;
1204 dev_info(hr_dev->dev,
1205 "Func clear success after reset.\n");
1206 return;
1207 }
1208 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1209 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1210 }
1211
1212 dev_warn(hr_dev->dev, "Func clear failed.\n");
1213 } else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) {
1214 hr_dev->dis_db = true;
1215
1216 dev_warn(hr_dev->dev,
1217 "Func clear is pending, device in resetting state.\n");
1218 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1219 while (end) {
1220 if (ops->ae_dev_reset_cnt(handle) !=
1221 hr_dev->reset_cnt) {
1222 hr_dev->is_reset = true;
1223 dev_info(hr_dev->dev,
1224 "Func clear success after sw reset\n");
1225 return;
1226 }
1227 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1228 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1229 }
1230
1231 dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n");
1232 } else {
1233 if (retval && !flag)
1234 dev_warn(hr_dev->dev,
1235 "Func clear read failed, ret = %d.\n", retval);
1236
1237 dev_warn(hr_dev->dev, "Func clear failed.\n");
1238 }
1239}
89a6da3c
LC		 1240static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1241{
e075da5e 1242 bool fclr_write_fail_flag = false;
89a6da3c
LC		 1243 struct hns_roce_func_clear *resp;
1244 struct hns_roce_cmq_desc desc;
1245 unsigned long end;
e075da5e
LC		 1246 int ret = 0;
1247
1248 if (hns_roce_func_clr_chk_rst(hr_dev))
1249 goto out;
89a6da3c
LC		 1250
1251 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1252 resp = (struct hns_roce_func_clear *)desc.data;
1253
1254 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1255 if (ret) {
e075da5e 1256 fclr_write_fail_flag = true;
89a6da3c
LC		 1257 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1258 ret);
e075da5e 1259 goto out;
89a6da3c
LC		 1260 }
1261
1262 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1263 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1264 while (end) {
e075da5e
LC		 1265 if (hns_roce_func_clr_chk_rst(hr_dev))
1266 goto out;
89a6da3c
LC		 1267 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1268 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1269
1270 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1271 true);
1272
1273 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1274 if (ret)
1275 continue;
1276
1277 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1278 hr_dev->is_reset = true;
1279 return;
1280 }
1281 }
1282
e075da5e 1283out:
e075da5e 1284 hns_roce_func_clr_rst_prc(hr_dev, ret, fclr_write_fail_flag);
89a6da3c
LC		 1285}
1286
3a63c964
LO		 1287static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1288{
1289 struct hns_roce_query_fw_info *resp;
1290 struct hns_roce_cmq_desc desc;
1291 int ret;
1292
1293 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1294 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1295 if (ret)
1296 return ret;
1297
1298 resp = (struct hns_roce_query_fw_info *)desc.data;
1299 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
cfc85f3e
WHX		 1300
1301 return 0;
1302}
1303
1304static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1305{
1306 struct hns_roce_cfg_global_param *req;
1307 struct hns_roce_cmq_desc desc;
1308
1309 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1310 false);
1311
1312 req = (struct hns_roce_cfg_global_param *)desc.data;
1313 memset(req, 0, sizeof(*req));
1314 roce_set_field(req->time_cfg_udp_port,
1315 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1316 CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1317 roce_set_field(req->time_cfg_udp_port,
1318 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1319 CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
1320
1321 return hns_roce_cmq_send(hr_dev, &desc, 1);
1322}
1323
1324static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1325{
1326 struct hns_roce_cmq_desc desc[2];
6b63597d 1327 struct hns_roce_pf_res_a *req_a;
1328 struct hns_roce_pf_res_b *req_b;
cfc85f3e
WHX		 1329 int ret;
1330 int i;
1331
1332 for (i = 0; i < 2; i++) {
1333 hns_roce_cmq_setup_basic_desc(&desc[i],
1334 HNS_ROCE_OPC_QUERY_PF_RES, true);
1335
1336 if (i == 0)
1337 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1338 else
1339 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1340 }
1341
1342 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1343 if (ret)
1344 return ret;
1345
6b63597d 1346 req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1347 req_b = (struct hns_roce_pf_res_b *)desc[1].data;
cfc85f3e 1348
6b63597d 1349 hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
cfc85f3e
WHX		 1350 PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1351 PF_RES_DATA_1_PF_QPC_BT_NUM_S);
6b63597d 1352 hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
cfc85f3e
WHX		 1353 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1354 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
6b63597d 1355 hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
cfc85f3e
WHX		 1356 PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1357 PF_RES_DATA_3_PF_CQC_BT_NUM_S);
6b63597d 1358 hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
cfc85f3e
WHX		 1359 PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1360 PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1361
6b63597d 1362 hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1363 PF_RES_DATA_3_PF_SL_NUM_M,
1364 PF_RES_DATA_3_PF_SL_NUM_S);
6a157f7d
YL		 1365 hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
1366 PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
1367 PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
6b63597d 1368
cfc85f3e
WHX		 1369 return 0;
1370}
1371
0e40dc2f
YL		 1372static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
1373{
1374 struct hns_roce_pf_timer_res_a *req_a;
1375 struct hns_roce_cmq_desc desc[2];
1376 int ret, i;
1377
1378 for (i = 0; i < 2; i++) {
1379 hns_roce_cmq_setup_basic_desc(&desc[i],
1380 HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1381 true);
1382
1383 if (i == 0)
1384 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1385 else
1386 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1387 }
1388
1389 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1390 if (ret)
1391 return ret;
1392
1393 req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;
1394
1395 hr_dev->caps.qpc_timer_bt_num =
1396 roce_get_field(req_a->qpc_timer_bt_idx_num,
1397 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
1398 PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
1399 hr_dev->caps.cqc_timer_bt_num =
1400 roce_get_field(req_a->cqc_timer_bt_idx_num,
1401 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
1402 PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
1403
1404 return 0;
1405}
1406
60262b10 1407static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
0c1c3880
LO		 1408{
1409 struct hns_roce_cmq_desc desc;
1410 struct hns_roce_vf_switch *swt;
1411 int ret;
1412
1413 swt = (struct hns_roce_vf_switch *)desc.data;
1414 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
bfe86035 1415 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
60262b10
LO		 1416 roce_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1417 VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id);
0c1c3880
LO		 1418 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1419 if (ret)
1420 return ret;
60262b10 1421
0c1c3880
LO		 1422 desc.flag =
1423 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1424 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1425 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
d967e262 1426 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0);
0c1c3880
LO		 1427 roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1428
1429 return hns_roce_cmq_send(hr_dev, &desc, 1);
1430}
1431
cfc85f3e
WHX		 1432static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1433{
1434 struct hns_roce_cmq_desc desc[2];
1435 struct hns_roce_vf_res_a *req_a;
1436 struct hns_roce_vf_res_b *req_b;
1437 int i;
1438
1439 req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1440 req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1441 memset(req_a, 0, sizeof(*req_a));
1442 memset(req_b, 0, sizeof(*req_b));
1443 for (i = 0; i < 2; i++) {
1444 hns_roce_cmq_setup_basic_desc(&desc[i],
1445 HNS_ROCE_OPC_ALLOC_VF_RES, false);
1446
1447 if (i == 0)
1448 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1449 else
1450 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1451
1452 if (i == 0) {
1453 roce_set_field(req_a->vf_qpc_bt_idx_num,
1454 VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1455 VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1456 roce_set_field(req_a->vf_qpc_bt_idx_num,
1457 VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1458 VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1459 HNS_ROCE_VF_QPC_BT_NUM);
1460
1461 roce_set_field(req_a->vf_srqc_bt_idx_num,
1462 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1463 VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1464 roce_set_field(req_a->vf_srqc_bt_idx_num,
1465 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1466 VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1467 HNS_ROCE_VF_SRQC_BT_NUM);
1468
1469 roce_set_field(req_a->vf_cqc_bt_idx_num,
1470 VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1471 VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1472 roce_set_field(req_a->vf_cqc_bt_idx_num,
1473 VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1474 VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1475 HNS_ROCE_VF_CQC_BT_NUM);
1476
1477 roce_set_field(req_a->vf_mpt_bt_idx_num,
1478 VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1479 VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1480 roce_set_field(req_a->vf_mpt_bt_idx_num,
1481 VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1482 VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1483 HNS_ROCE_VF_MPT_BT_NUM);
1484
1485 roce_set_field(req_a->vf_eqc_bt_idx_num,
1486 VF_RES_A_DATA_5_VF_EQC_IDX_M,
1487 VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1488 roce_set_field(req_a->vf_eqc_bt_idx_num,
1489 VF_RES_A_DATA_5_VF_EQC_NUM_M,
1490 VF_RES_A_DATA_5_VF_EQC_NUM_S,
1491 HNS_ROCE_VF_EQC_NUM);
1492 } else {
1493 roce_set_field(req_b->vf_smac_idx_num,
1494 VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1495 VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1496 roce_set_field(req_b->vf_smac_idx_num,
1497 VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1498 VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1499 HNS_ROCE_VF_SMAC_NUM);
1500
1501 roce_set_field(req_b->vf_sgid_idx_num,
1502 VF_RES_B_DATA_2_VF_SGID_IDX_M,
1503 VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1504 roce_set_field(req_b->vf_sgid_idx_num,
1505 VF_RES_B_DATA_2_VF_SGID_NUM_M,
1506 VF_RES_B_DATA_2_VF_SGID_NUM_S,
1507 HNS_ROCE_VF_SGID_NUM);
1508
1509 roce_set_field(req_b->vf_qid_idx_sl_num,
1510 VF_RES_B_DATA_3_VF_QID_IDX_M,
1511 VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1512 roce_set_field(req_b->vf_qid_idx_sl_num,
1513 VF_RES_B_DATA_3_VF_SL_NUM_M,
1514 VF_RES_B_DATA_3_VF_SL_NUM_S,
1515 HNS_ROCE_VF_SL_NUM);
6a157f7d
YL		 1516
1517 roce_set_field(req_b->vf_sccc_idx_num,
1518 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
1519 VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
1520 roce_set_field(req_b->vf_sccc_idx_num,
1521 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
1522 VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
1523 HNS_ROCE_VF_SCCC_BT_NUM);
cfc85f3e
WHX		 1524 }
1525 }
1526
1527 return hns_roce_cmq_send(hr_dev, desc, 2);
1528}
1529
a81fba28
WHX		 1530static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1531{
1532 u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1533 u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1534 u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1535 u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
6a157f7d 1536 u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
a81fba28
WHX		 1537 struct hns_roce_cfg_bt_attr *req;
1538 struct hns_roce_cmq_desc desc;
1539
1540 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1541 req = (struct hns_roce_cfg_bt_attr *)desc.data;
1542 memset(req, 0, sizeof(*req));
1543
1544 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1545 CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
5e6e78db 1546 hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1547 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1548 CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
5e6e78db 1549 hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1550 roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1551 CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1552 qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1553
1554 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1555 CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
5e6e78db 1556 hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1557 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1558 CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
5e6e78db 1559 hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1560 roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1561 CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1562 srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1563
1564 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1565 CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
5e6e78db 1566 hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1567 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1568 CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
5e6e78db 1569 hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1570 roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1571 CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1572 cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1573
1574 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1575 CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
5e6e78db 1576 hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1577 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1578 CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
5e6e78db 1579 hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
a81fba28
WHX		 1580 roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1581 CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1582 mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1583
6a157f7d
YL		 1584 roce_set_field(req->vf_sccc_cfg,
1585 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
1586 CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
1587 hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1588 roce_set_field(req->vf_sccc_cfg,
1589 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
1590 CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
1591 hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1592 roce_set_field(req->vf_sccc_cfg,
1593 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
1594 CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
1595 sccc_hop_num ==
1596 HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
1597
a81fba28
WHX		 1598 return hns_roce_cmq_send(hr_dev, &desc, 1);
1599}
1600
ba6bb7e9
LO		 1601static void set_default_caps(struct hns_roce_dev *hr_dev)
1602{
1603 struct hns_roce_caps *caps = &hr_dev->caps;
1604
1605 caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM;
1606 caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM;
1607 caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM;
1608 caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM;
1609 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM;
1610 caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM;
1611 caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1612 caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1613 caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1614 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE;
1615 caps->num_uars = HNS_ROCE_V2_UAR_NUM;
1616 caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM;
1617 caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM;
1618 caps->num_comp_vectors = HNS_ROCE_V2_COMP_VEC_NUM;
1619 caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1620 caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM;
1621 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
1622 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
1623 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1624 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
1625 caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM;
1626 caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1627 caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1628 caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1629 caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1630 caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1631 caps->qpc_entry_sz = HNS_ROCE_V2_QPC_ENTRY_SZ;
1632 caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ;
7db82697 1633 caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ;
ba6bb7e9
LO		 1634 caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ;
1635 caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ;
1636 caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1637 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
1638 caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ;
1639 caps->cq_entry_sz = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1640 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1641 caps->reserved_lkey = 0;
1642 caps->reserved_pds = 0;
1643 caps->reserved_mrws = 1;
1644 caps->reserved_uars = 0;
1645 caps->reserved_cqs = 0;
1646 caps->reserved_srqs = 0;
1647 caps->reserved_qps = HNS_ROCE_V2_RSV_QPS;
1648
1649 caps->qpc_ba_pg_sz = 0;
1650 caps->qpc_buf_pg_sz = 0;
1651 caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1652 caps->srqc_ba_pg_sz = 0;
1653 caps->srqc_buf_pg_sz = 0;
1654 caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1655 caps->cqc_ba_pg_sz = 0;
1656 caps->cqc_buf_pg_sz = 0;
1657 caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1658 caps->mpt_ba_pg_sz = 0;
1659 caps->mpt_buf_pg_sz = 0;
1660 caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM;
1661 caps->mtt_ba_pg_sz = 0;
1662 caps->mtt_buf_pg_sz = 0;
1663 caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM;
1664 caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM;
1665 caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM;
1666 caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM;
1667 caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
1668 caps->cqe_buf_pg_sz = 0;
1669 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM;
1670 caps->srqwqe_ba_pg_sz = 0;
1671 caps->srqwqe_buf_pg_sz = 0;
1672 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM;
1673 caps->idx_ba_pg_sz = 0;
1674 caps->idx_buf_pg_sz = 0;
1675 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM;
1676 caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1677
1678 caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR |
1679 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1680 HNS_ROCE_CAP_FLAG_RQ_INLINE |
1681 HNS_ROCE_CAP_FLAG_RECORD_DB |
1682 HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1683
1684 caps->pkey_table_len[0] = 1;
1685 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1686 caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM;
1687 caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM;
1688 caps->local_ca_ack_delay = 0;
1689 caps->max_mtu = IB_MTU_4096;
1690
1691 caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR;
1692 caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE;
1693
1694 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B) {
1695 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW |
1696 HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR |
1697 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1698
1699 caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1700 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1701 caps->qpc_timer_ba_pg_sz = 0;
1702 caps->qpc_timer_buf_pg_sz = 0;
1703 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1704 caps->num_cqc_timer = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1705 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1706 caps->cqc_timer_ba_pg_sz = 0;
1707 caps->cqc_timer_buf_pg_sz = 0;
1708 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1709
1710 caps->sccc_entry_sz = HNS_ROCE_V2_SCCC_ENTRY_SZ;
1711 caps->sccc_ba_pg_sz = 0;
1712 caps->sccc_buf_pg_sz = 0;
1713 caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM;
1714 }
1715}
1716
1717static void calc_pg_sz(int obj_num, int obj_size, int hop_num, int ctx_bt_num,
1718 int *buf_page_size, int *bt_page_size, u32 hem_type)
1719{
1720 u64 obj_per_chunk;
1721 int bt_chunk_size = 1 << PAGE_SHIFT;
1722 int buf_chunk_size = 1 << PAGE_SHIFT;
1723 int obj_per_chunk_default = buf_chunk_size / obj_size;
1724
1725 *buf_page_size = 0;
1726 *bt_page_size = 0;
1727
1728 switch (hop_num) {
1729 case 3:
1730 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1731 (bt_chunk_size / BA_BYTE_LEN) *
1732 (bt_chunk_size / BA_BYTE_LEN) *
1733 obj_per_chunk_default;
1734 break;
1735 case 2:
1736 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1737 (bt_chunk_size / BA_BYTE_LEN) *
1738 obj_per_chunk_default;
1739 break;
1740 case 1:
1741 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1742 obj_per_chunk_default;
1743 break;
1744 case HNS_ROCE_HOP_NUM_0:
1745 obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
1746 break;
1747 default:
1748 pr_err("Table %d not support hop_num = %d!\n", hem_type,
1749 hop_num);
1750 return;
1751 }
1752
1753 if (hem_type >= HEM_TYPE_MTT)
1754 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
1755 else
1756 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
1757}
1758
1759static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
1760{
1761 struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
1762 struct hns_roce_caps *caps = &hr_dev->caps;
1763 struct hns_roce_query_pf_caps_a *resp_a;
1764 struct hns_roce_query_pf_caps_b *resp_b;
1765 struct hns_roce_query_pf_caps_c *resp_c;
1766 struct hns_roce_query_pf_caps_d *resp_d;
1767 struct hns_roce_query_pf_caps_e *resp_e;
1768 int ctx_hop_num;
1769 int pbl_hop_num;
1770 int ret;
1771 int i;
1772
1773 for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
1774 hns_roce_cmq_setup_basic_desc(&desc[i],
1775 HNS_ROCE_OPC_QUERY_PF_CAPS_NUM,
1776 true);
1777 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
1778 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1779 else
1780 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1781 }
1782
1783 ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
1784 if (ret)
1785 return ret;
1786
1787 resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
1788 resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
1789 resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
1790 resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
1791 resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
1792
1793 caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
1794 caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
1795 caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
1796 caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
1797 caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg);
1798 caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer);
1799 caps->num_cqc_timer = le16_to_cpu(resp_a->num_cqc_timer);
1800 caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
1801 caps->num_aeq_vectors = resp_a->num_aeq_vectors;
1802 caps->num_other_vectors = resp_a->num_other_vectors;
1803 caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
1804 caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
1805 caps->max_srq_desc_sz = resp_a->max_srq_desc_sz;
1806 caps->cq_entry_sz = resp_a->cq_entry_sz;
1807
1808 caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
1809 caps->irrl_entry_sz = resp_b->irrl_entry_sz;
1810 caps->trrl_entry_sz = resp_b->trrl_entry_sz;
1811 caps->cqc_entry_sz = resp_b->cqc_entry_sz;
1812 caps->srqc_entry_sz = resp_b->srqc_entry_sz;
1813 caps->idx_entry_sz = resp_b->idx_entry_sz;
1814 caps->sccc_entry_sz = resp_b->scc_ctx_entry_sz;
1815 caps->max_mtu = resp_b->max_mtu;
1816 caps->qpc_entry_sz = le16_to_cpu(resp_b->qpc_entry_sz);
1817 caps->min_cqes = resp_b->min_cqes;
1818 caps->min_wqes = resp_b->min_wqes;
1819 caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
1820 caps->pkey_table_len[0] = resp_b->pkey_table_len;
1821 caps->phy_num_uars = resp_b->phy_num_uars;
1822 ctx_hop_num = resp_b->ctx_hop_num;
1823 pbl_hop_num = resp_b->pbl_hop_num;
1824
1825 caps->num_pds = 1 << roce_get_field(resp_c->cap_flags_num_pds,
1826 V2_QUERY_PF_CAPS_C_NUM_PDS_M,
1827 V2_QUERY_PF_CAPS_C_NUM_PDS_S);
1828 caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
1829 V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
1830 V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
1831 caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
1832 V2_QUERY_PF_CAPS_C_NUM_CQS_M,
1833 V2_QUERY_PF_CAPS_C_NUM_CQS_S);
1834 caps->gid_table_len[0] = roce_get_field(resp_c->max_gid_num_cqs,
1835 V2_QUERY_PF_CAPS_C_MAX_GID_M,
1836 V2_QUERY_PF_CAPS_C_MAX_GID_S);
1837 caps->max_cqes = 1 << roce_get_field(resp_c->cq_depth,
1838 V2_QUERY_PF_CAPS_C_CQ_DEPTH_M,
1839 V2_QUERY_PF_CAPS_C_CQ_DEPTH_S);
1840 caps->num_mtpts = 1 << roce_get_field(resp_c->num_mrws,
1841 V2_QUERY_PF_CAPS_C_NUM_MRWS_M,
1842 V2_QUERY_PF_CAPS_C_NUM_MRWS_S);
1843 caps->num_qps = 1 << roce_get_field(resp_c->ord_num_qps,
1844 V2_QUERY_PF_CAPS_C_NUM_QPS_M,
1845 V2_QUERY_PF_CAPS_C_NUM_QPS_S);
1846 caps->max_qp_init_rdma = roce_get_field(resp_c->ord_num_qps,
1847 V2_QUERY_PF_CAPS_C_MAX_ORD_M,
1848 V2_QUERY_PF_CAPS_C_MAX_ORD_S);
1849 caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
1850 caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
1851 caps->num_srqs = 1 << roce_get_field(resp_d->wq_hop_num_max_srqs,
1852 V2_QUERY_PF_CAPS_D_NUM_SRQS_M,
1853 V2_QUERY_PF_CAPS_D_NUM_SRQS_S);
1854 caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
1855 caps->ceqe_depth = 1 << roce_get_field(resp_d->num_ceqs_ceq_depth,
1856 V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M,
1857 V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S);
1858 caps->num_comp_vectors = roce_get_field(resp_d->num_ceqs_ceq_depth,
1859 V2_QUERY_PF_CAPS_D_NUM_CEQS_M,
1860 V2_QUERY_PF_CAPS_D_NUM_CEQS_S);
1861 caps->aeqe_depth = 1 << roce_get_field(resp_d->arm_st_aeq_depth,
1862 V2_QUERY_PF_CAPS_D_AEQ_DEPTH_M,
1863 V2_QUERY_PF_CAPS_D_AEQ_DEPTH_S);
1864 caps->default_aeq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
1865 V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_M,
1866 V2_QUERY_PF_CAPS_D_AEQ_ARM_ST_S);
1867 caps->default_ceq_arm_st = roce_get_field(resp_d->arm_st_aeq_depth,
1868 V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_M,
1869 V2_QUERY_PF_CAPS_D_CEQ_ARM_ST_S);
1870 caps->reserved_pds = roce_get_field(resp_d->num_uars_rsv_pds,
1871 V2_QUERY_PF_CAPS_D_RSV_PDS_M,
1872 V2_QUERY_PF_CAPS_D_RSV_PDS_S);
1873 caps->num_uars = 1 << roce_get_field(resp_d->num_uars_rsv_pds,
1874 V2_QUERY_PF_CAPS_D_NUM_UARS_M,
1875 V2_QUERY_PF_CAPS_D_NUM_UARS_S);
1876 caps->reserved_qps = roce_get_field(resp_d->rsv_uars_rsv_qps,
1877 V2_QUERY_PF_CAPS_D_RSV_QPS_M,
1878 V2_QUERY_PF_CAPS_D_RSV_QPS_S);
1879 caps->reserved_uars = roce_get_field(resp_d->rsv_uars_rsv_qps,
1880 V2_QUERY_PF_CAPS_D_RSV_UARS_M,
1881 V2_QUERY_PF_CAPS_D_RSV_UARS_S);
1882 caps->reserved_mrws = roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
1883 V2_QUERY_PF_CAPS_E_RSV_MRWS_M,
1884 V2_QUERY_PF_CAPS_E_RSV_MRWS_S);
1885 caps->chunk_sz = 1 << roce_get_field(resp_e->chunk_size_shift_rsv_mrws,
1886 V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M,
1887 V2_QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S);
1888 caps->reserved_cqs = roce_get_field(resp_e->rsv_cqs,
1889 V2_QUERY_PF_CAPS_E_RSV_CQS_M,
1890 V2_QUERY_PF_CAPS_E_RSV_CQS_S);
1891 caps->reserved_srqs = roce_get_field(resp_e->rsv_srqs,
1892 V2_QUERY_PF_CAPS_E_RSV_SRQS_M,
1893 V2_QUERY_PF_CAPS_E_RSV_SRQS_S);
1894 caps->reserved_lkey = roce_get_field(resp_e->rsv_lkey,
1895 V2_QUERY_PF_CAPS_E_RSV_LKEYS_M,
1896 V2_QUERY_PF_CAPS_E_RSV_LKEYS_S);
1897 caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
1898 caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
1899 caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
1900 caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
1901
1902 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1903 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1904 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
1905 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
1906 caps->mtt_ba_pg_sz = 0;
1907 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
1908 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1909 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
1910
1911 caps->qpc_hop_num = ctx_hop_num;
1912 caps->srqc_hop_num = ctx_hop_num;
1913 caps->cqc_hop_num = ctx_hop_num;
1914 caps->mpt_hop_num = ctx_hop_num;
1915 caps->mtt_hop_num = pbl_hop_num;
1916 caps->cqe_hop_num = pbl_hop_num;
1917 caps->srqwqe_hop_num = pbl_hop_num;
1918 caps->idx_hop_num = pbl_hop_num;
1919 caps->wqe_sq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
1920 V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M,
1921 V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S);
1922 caps->wqe_sge_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
1923 V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M,
1924 V2_QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S);
1925 caps->wqe_rq_hop_num = roce_get_field(resp_d->wq_hop_num_max_srqs,
1926 V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M,
1927 V2_QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S);
1928
1929 calc_pg_sz(caps->num_qps, caps->qpc_entry_sz, caps->qpc_hop_num,
1930 caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
1931 HEM_TYPE_QPC);
1932 calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
1933 caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
1934 HEM_TYPE_MTPT);
1935 calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
1936 caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
1937 HEM_TYPE_CQC);
1938 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, caps->srqc_hop_num,
1939 caps->srqc_bt_num, &caps->srqc_buf_pg_sz,
1940 &caps->srqc_ba_pg_sz, HEM_TYPE_SRQC);
1941
1942 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B) {
1943 caps->sccc_hop_num = ctx_hop_num;
1944 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1945 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
1946
1947 calc_pg_sz(caps->num_qps, caps->sccc_entry_sz,
1948 caps->sccc_hop_num, caps->sccc_bt_num,
1949 &caps->sccc_buf_pg_sz, &caps->sccc_ba_pg_sz,
1950 HEM_TYPE_SCCC);
1951 calc_pg_sz(caps->num_cqc_timer, caps->cqc_timer_entry_sz,
1952 caps->cqc_timer_hop_num, caps->cqc_timer_bt_num,
1953 &caps->cqc_timer_buf_pg_sz,
1954 &caps->cqc_timer_ba_pg_sz, HEM_TYPE_CQC_TIMER);
1955 }
1956
1957 calc_pg_sz(caps->num_cqe_segs, caps->mtt_entry_sz, caps->cqe_hop_num,
1958 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
1959 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
1960 caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
1961 &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
1962 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, caps->idx_hop_num,
1963 1, &caps->idx_buf_pg_sz, &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
1964
1965 return 0;
1966}
1967
cfc85f3e
WHX		 1968static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1969{
1970 struct hns_roce_caps *caps = &hr_dev->caps;
1971 int ret;
1972
1973 ret = hns_roce_cmq_query_hw_info(hr_dev);
3a63c964
LO		 1974 if (ret) {
1975 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
1976 ret);
1977 return ret;
1978 }
1979
1980 ret = hns_roce_query_fw_ver(hr_dev);
cfc85f3e
WHX		 1981 if (ret) {
1982 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1983 ret);
1984 return ret;
1985 }
1986
1987 ret = hns_roce_config_global_param(hr_dev);
1988 if (ret) {
1989 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1990 ret);
2349fdd4 1991 return ret;
cfc85f3e
WHX		 1992 }
1993
1994 /* Get pf resource owned by every pf */
1995 ret = hns_roce_query_pf_resource(hr_dev);
1996 if (ret) {
1997 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1998 ret);
1999 return ret;
2000 }
2001
b14c95be 2002 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B) {
0e40dc2f
YL		 2003 ret = hns_roce_query_pf_timer_resource(hr_dev);
2004 if (ret) {
2005 dev_err(hr_dev->dev,
2006 "Query pf timer resource fail, ret = %d.\n",
2007 ret);
2008 return ret;
2009 }
2010 }
2011
cfc85f3e
WHX		 2012 ret = hns_roce_alloc_vf_resource(hr_dev);
2013 if (ret) {
2014 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
2015 ret);
2016 return ret;
2017 }
2018
b14c95be 2019 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B) {
0c1c3880
LO		 2020 ret = hns_roce_set_vf_switch_param(hr_dev, 0);
2021 if (ret) {
2022 dev_err(hr_dev->dev,
2023 "Set function switch param fail, ret = %d.\n",
2024 ret);
2025 return ret;
2026 }
2027 }
3a63c964
LO		 2028
2029 hr_dev->vendor_part_id = hr_dev->pci_dev->device;
2030 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
cfc85f3e 2031
cfc85f3e
WHX		 2032 caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS;
2033 caps->num_cqe_segs = HNS_ROCE_V2_MAX_CQE_SEGS;
5c1f167a
LO		 2034 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
2035 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
cfc85f3e 2036
80a78570 2037 caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
ff795f71
WHX		 2038 caps->pbl_buf_pg_sz = 0;
2039 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
a5073d60
YL		 2040 caps->eqe_ba_pg_sz = 0;
2041 caps->eqe_buf_pg_sz = 0;
2042 caps->eqe_hop_num = HNS_ROCE_EQE_HOP_NUM;
6b63597d 2043 caps->tsq_buf_pg_sz = 0;
aa84fa18 2044
80a78570
LO		 2045 ret = hns_roce_query_pf_caps(hr_dev);
2046 if (ret)
2047 set_default_caps(hr_dev);
384f8818 2048
a81fba28
WHX		 2049 ret = hns_roce_v2_set_bt(hr_dev);
2050 if (ret)
2051 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
2052 ret);
2053
2054 return ret;
cfc85f3e
WHX		 2055}
2056
6b63597d 2057static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
2058 enum hns_roce_link_table_type type)
2059{
2060 struct hns_roce_cmq_desc desc[2];
2061 struct hns_roce_cfg_llm_a *req_a =
2062 (struct hns_roce_cfg_llm_a *)desc[0].data;
2063 struct hns_roce_cfg_llm_b *req_b =
2064 (struct hns_roce_cfg_llm_b *)desc[1].data;
2065 struct hns_roce_v2_priv *priv = hr_dev->priv;
2066 struct hns_roce_link_table *link_tbl;
2067 struct hns_roce_link_table_entry *entry;
2068 enum hns_roce_opcode_type opcode;
2069 u32 page_num;
2070 int i;
2071
2072 switch (type) {
2073 case TSQ_LINK_TABLE:
2074 link_tbl = &priv->tsq;
2075 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
2076 break;
ded58ff9 2077 case TPQ_LINK_TABLE:
2078 link_tbl = &priv->tpq;
2079 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
2080 break;
6b63597d 2081 default:
2082 return -EINVAL;
2083 }
2084
2085 page_num = link_tbl->npages;
2086 entry = link_tbl->table.buf;
2087 memset(req_a, 0, sizeof(*req_a));
2088 memset(req_b, 0, sizeof(*req_b));
2089
2090 for (i = 0; i < 2; i++) {
2091 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
2092
2093 if (i == 0)
2094 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2095 else
2096 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2097
2098 if (i == 0) {
bfe86035
LC		 2099 req_a->base_addr_l =
2100 cpu_to_le32(link_tbl->table.map & 0xffffffff);
2101 req_a->base_addr_h =
2102 cpu_to_le32(link_tbl->table.map >> 32);
6b63597d 2103 roce_set_field(req_a->depth_pgsz_init_en,
60262b10 2104 CFG_LLM_QUE_DEPTH_M, CFG_LLM_QUE_DEPTH_S,
6b63597d 2105 link_tbl->npages);
2106 roce_set_field(req_a->depth_pgsz_init_en,
60262b10 2107 CFG_LLM_QUE_PGSZ_M, CFG_LLM_QUE_PGSZ_S,
6b63597d 2108 link_tbl->pg_sz);
bfe86035
LC		 2109 req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
2110 req_a->head_ba_h_nxtptr =
2111 cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
60262b10 2112 roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M,
6b63597d 2113 CFG_LLM_HEAD_PTR_S, 0);
2114 } else {
bfe86035
LC		 2115 req_b->tail_ba_l =
2116 cpu_to_le32(entry[page_num - 1].blk_ba0);
60262b10 2117 roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
6b63597d 2118 CFG_LLM_TAIL_BA_H_S,
2119 entry[page_num - 1].blk_ba1_nxt_ptr &
60262b10
LO		 2120 HNS_ROCE_LINK_TABLE_BA1_M);
2121 roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M,
6b63597d 2122 CFG_LLM_TAIL_PTR_S,
2123 (entry[page_num - 2].blk_ba1_nxt_ptr &
60262b10
LO		 2124 HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
2125 HNS_ROCE_LINK_TABLE_NXT_PTR_S);
6b63597d 2126 }
2127 }
60262b10
LO		 2128 roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M,
2129 CFG_LLM_INIT_EN_S, 1);
6b63597d 2130
2131 return hns_roce_cmq_send(hr_dev, desc, 2);
2132}
2133
2134static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
2135 enum hns_roce_link_table_type type)
2136{
2137 struct hns_roce_v2_priv *priv = hr_dev->priv;
2138 struct hns_roce_link_table *link_tbl;
2139 struct hns_roce_link_table_entry *entry;
2140 struct device *dev = hr_dev->dev;
2141 u32 buf_chk_sz;
2142 dma_addr_t t;
ded58ff9 2143 int func_num = 1;
6b63597d 2144 int pg_num_a;
2145 int pg_num_b;
2146 int pg_num;
2147 int size;
2148 int i;
2149
2150 switch (type) {
2151 case TSQ_LINK_TABLE:
2152 link_tbl = &priv->tsq;
2153 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
2154 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
2155 pg_num_b = hr_dev->caps.sl_num * 4 + 2;
2156 break;
ded58ff9 2157 case TPQ_LINK_TABLE:
2158 link_tbl = &priv->tpq;
2159 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
2160 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
2161 pg_num_b = 2 * 4 * func_num + 2;
2162 break;
6b63597d 2163 default:
2164 return -EINVAL;
2165 }
2166
2167 pg_num = max(pg_num_a, pg_num_b);
2168 size = pg_num * sizeof(struct hns_roce_link_table_entry);
2169
2170 link_tbl->table.buf = dma_alloc_coherent(dev, size,
2171 &link_tbl->table.map,
2172 GFP_KERNEL);
2173 if (!link_tbl->table.buf)
2174 goto out;
2175
2176 link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
2177 GFP_KERNEL);
2178 if (!link_tbl->pg_list)
2179 goto err_kcalloc_failed;
2180
2181 entry = link_tbl->table.buf;
2182 for (i = 0; i < pg_num; ++i) {
2183 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
2184 &t, GFP_KERNEL);
2185 if (!link_tbl->pg_list[i].buf)
2186 goto err_alloc_buf_failed;
2187
2188 link_tbl->pg_list[i].map = t;
6b63597d 2189
bfe86035
LC		 2190 entry[i].blk_ba0 = (u32)(t >> 12);
2191 entry[i].blk_ba1_nxt_ptr = (u32)(t >> 44);
6b63597d 2192
2193 if (i < (pg_num - 1))
bfe86035
LC		 2194 entry[i].blk_ba1_nxt_ptr |=
2195 (i + 1) << HNS_ROCE_LINK_TABLE_NXT_PTR_S;
2196
6b63597d 2197 }
2198 link_tbl->npages = pg_num;
2199 link_tbl->pg_sz = buf_chk_sz;
2200
2201 return hns_roce_config_link_table(hr_dev, type);
2202
2203err_alloc_buf_failed:
2204 for (i -= 1; i >= 0; i--)
2205 dma_free_coherent(dev, buf_chk_sz,
2206 link_tbl->pg_list[i].buf,
2207 link_tbl->pg_list[i].map);
2208 kfree(link_tbl->pg_list);
2209
2210err_kcalloc_failed:
2211 dma_free_coherent(dev, size, link_tbl->table.buf,
2212 link_tbl->table.map);
2213
2214out:
2215 return -ENOMEM;
2216}
2217
2218static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
2219 struct hns_roce_link_table *link_tbl)
2220{
2221 struct device *dev = hr_dev->dev;
2222 int size;
2223 int i;
2224
2225 size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
2226
2227 for (i = 0; i < link_tbl->npages; ++i)
2228 if (link_tbl->pg_list[i].buf)
2229 dma_free_coherent(dev, link_tbl->pg_sz,
2230 link_tbl->pg_list[i].buf,
2231 link_tbl->pg_list[i].map);
2232 kfree(link_tbl->pg_list);
2233
2234 dma_free_coherent(dev, size, link_tbl->table.buf,
2235 link_tbl->table.map);
2236}
2237
2238static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
2239{
ded58ff9 2240 struct hns_roce_v2_priv *priv = hr_dev->priv;
0e40dc2f
YL		 2241 int qpc_count, cqc_count;
2242 int ret, i;
6b63597d 2243
2244 /* TSQ includes SQ doorbell and ack doorbell */
2245 ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
ded58ff9 2246 if (ret) {
6b63597d 2247 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
ded58ff9 2248 return ret;
2249 }
2250
2251 ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
2252 if (ret) {
2253 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
2254 goto err_tpq_init_failed;
2255 }
2256
6def7de6 2257 /* Alloc memory for QPC Timer buffer space chunk */
0e40dc2f
YL		 2258 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
2259 qpc_count++) {
2260 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
2261 qpc_count);
2262 if (ret) {
2263 dev_err(hr_dev->dev, "QPC Timer get failed\n");
2264 goto err_qpc_timer_failed;
2265 }
2266 }
2267
6def7de6 2268 /* Alloc memory for CQC Timer buffer space chunk */
0e40dc2f
YL		 2269 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
2270 cqc_count++) {
2271 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
2272 cqc_count);
2273 if (ret) {
2274 dev_err(hr_dev->dev, "CQC Timer get failed\n");
2275 goto err_cqc_timer_failed;
2276 }
2277 }
2278
ded58ff9 2279 return 0;
2280
0e40dc2f
YL		 2281err_cqc_timer_failed:
2282 for (i = 0; i < cqc_count; i++)
2283 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2284
2285err_qpc_timer_failed:
2286 for (i = 0; i < qpc_count; i++)
2287 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2288
2289 hns_roce_free_link_table(hr_dev, &priv->tpq);
2290
ded58ff9 2291err_tpq_init_failed:
2292 hns_roce_free_link_table(hr_dev, &priv->tsq);
6b63597d 2293
2294 return ret;
2295}
2296
2297static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
2298{
2299 struct hns_roce_v2_priv *priv = hr_dev->priv;
2300
b14c95be 2301 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B)
89a6da3c
LC		 2302 hns_roce_function_clear(hr_dev);
2303
ded58ff9 2304 hns_roce_free_link_table(hr_dev, &priv->tpq);
6b63597d 2305 hns_roce_free_link_table(hr_dev, &priv->tsq);
2306}
2307
f747b689
LO		 2308static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
2309{
2310 struct hns_roce_cmq_desc desc;
2311 struct hns_roce_mbox_status *mb_st =
2312 (struct hns_roce_mbox_status *)desc.data;
2313 enum hns_roce_cmd_return_status status;
2314
2315 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
2316
2317 status = hns_roce_cmq_send(hr_dev, &desc, 1);
2318 if (status)
2319 return status;
2320
bfe86035 2321 return le32_to_cpu(mb_st->mb_status_hw_run);
f747b689
LO		 2322}
2323
a680f2f3
WHX		 2324static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
2325{
f747b689 2326 u32 status = hns_roce_query_mbox_status(hr_dev);
a680f2f3
WHX		 2327
2328 return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
2329}
2330
2331static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
2332{
f747b689 2333 u32 status = hns_roce_query_mbox_status(hr_dev);
a680f2f3
WHX		 2334
2335 return status & HNS_ROCE_HW_MB_STATUS_MASK;
2336}
2337
f747b689
LO		 2338static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
2339 u64 out_param, u32 in_modifier, u8 op_modifier,
2340 u16 op, u16 token, int event)
2341{
2342 struct hns_roce_cmq_desc desc;
2343 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
2344
2345 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
2346
bfe86035
LC		 2347 mb->in_param_l = cpu_to_le32(in_param);
2348 mb->in_param_h = cpu_to_le32(in_param >> 32);
2349 mb->out_param_l = cpu_to_le32(out_param);
2350 mb->out_param_h = cpu_to_le32(out_param >> 32);
f747b689
LO		 2351 mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
2352 mb->token_event_en = cpu_to_le32(event << 16 | token);
2353
2354 return hns_roce_cmq_send(hr_dev, &desc, 1);
2355}
2356
a680f2f3
WHX		 2357static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
2358 u64 out_param, u32 in_modifier, u8 op_modifier,
2359 u16 op, u16 token, int event)
2360{
2361 struct device *dev = hr_dev->dev;
a680f2f3 2362 unsigned long end;
f747b689 2363 int ret;
a680f2f3
WHX		 2364
2365 end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
2366 while (hns_roce_v2_cmd_pending(hr_dev)) {
2367 if (time_after(jiffies, end)) {
2368 dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
2369 (int)end);
2370 return -EAGAIN;
2371 }
2372 cond_resched();
2373 }
2374
f747b689
LO		 2375 ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2376 op_modifier, op, token, event);
2377 if (ret)
2378 dev_err(dev, "Post mailbox fail(%d)\n", ret);
a680f2f3 2379
f747b689 2380 return ret;
a680f2f3
WHX		 2381}
2382
2383static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2384 unsigned long timeout)
2385{
2386 struct device *dev = hr_dev->dev;
617cf24f 2387 unsigned long end;
a680f2f3
WHX		 2388 u32 status;
2389
2390 end = msecs_to_jiffies(timeout) + jiffies;
2391 while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
2392 cond_resched();
2393
2394 if (hns_roce_v2_cmd_pending(hr_dev)) {
2395 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
2396 return -ETIMEDOUT;
2397 }
2398
2399 status = hns_roce_v2_cmd_complete(hr_dev);
2400 if (status != 0x1) {
6a04aed6
WHX		 2401 if (status == CMD_RST_PRC_EBUSY)
2402 return status;
2403
a680f2f3
WHX		 2404 dev_err(dev, "mailbox status 0x%x!\n", status);
2405 return -EBUSY;
2406 }
2407
2408 return 0;
2409}
2410
4db134a3 2411static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
2412 int gid_index, const union ib_gid *gid,
2413 enum hns_roce_sgid_type sgid_type)
2414{
2415 struct hns_roce_cmq_desc desc;
2416 struct hns_roce_cfg_sgid_tb *sgid_tb =
2417 (struct hns_roce_cfg_sgid_tb *)desc.data;
2418 u32 *p;
2419
2420 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2421
60262b10 2422 roce_set_field(sgid_tb->table_idx_rsv, CFG_SGID_TB_TABLE_IDX_M,
4db134a3 2423 CFG_SGID_TB_TABLE_IDX_S, gid_index);
60262b10 2424 roce_set_field(sgid_tb->vf_sgid_type_rsv, CFG_SGID_TB_VF_SGID_TYPE_M,
4db134a3 2425 CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2426
2427 p = (u32 *)&gid->raw[0];
2428 sgid_tb->vf_sgid_l = cpu_to_le32(*p);
2429
2430 p = (u32 *)&gid->raw[4];
2431 sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
2432
2433 p = (u32 *)&gid->raw[8];
2434 sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
2435
2436 p = (u32 *)&gid->raw[0xc];
2437 sgid_tb->vf_sgid_h = cpu_to_le32(*p);
2438
2439 return hns_roce_cmq_send(hr_dev, &desc, 1);
2440}
2441
b5ff0f61 2442static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
f4df9a7c 2443 int gid_index, const union ib_gid *gid,
b5ff0f61 2444 const struct ib_gid_attr *attr)
7afddafa 2445{
b5ff0f61 2446 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
4db134a3 2447 int ret;
7afddafa 2448
b5ff0f61
WHX		 2449 if (!gid || !attr)
2450 return -EINVAL;
2451
2452 if (attr->gid_type == IB_GID_TYPE_ROCE)
2453 sgid_type = GID_TYPE_FLAG_ROCE_V1;
2454
2455 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2456 if (ipv6_addr_v4mapped((void *)gid))
2457 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2458 else
2459 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2460 }
2461
4db134a3 2462 ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2463 if (ret)
2464 dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
b5ff0f61 2465
4db134a3 2466 return ret;
7afddafa
WHX		 2467}
2468
a74dc41d
WHX		 2469static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2470 u8 *addr)
7afddafa 2471{
e8e8b652 2472 struct hns_roce_cmq_desc desc;
2473 struct hns_roce_cfg_smac_tb *smac_tb =
2474 (struct hns_roce_cfg_smac_tb *)desc.data;
7afddafa
WHX		 2475 u16 reg_smac_h;
2476 u32 reg_smac_l;
e8e8b652 2477
2478 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
7afddafa
WHX		 2479
2480 reg_smac_l = *(u32 *)(&addr[0]);
e8e8b652 2481 reg_smac_h = *(u16 *)(&addr[4]);
7afddafa 2482
e8e8b652 2483 memset(smac_tb, 0, sizeof(*smac_tb));
2484 roce_set_field(smac_tb->tb_idx_rsv,
2485 CFG_SMAC_TB_IDX_M,
2486 CFG_SMAC_TB_IDX_S, phy_port);
2487 roce_set_field(smac_tb->vf_smac_h_rsv,
2488 CFG_SMAC_TB_VF_SMAC_H_M,
2489 CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
bfe86035 2490 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
a74dc41d 2491
e8e8b652 2492 return hns_roce_cmq_send(hr_dev, &desc, 1);
7afddafa
WHX		 2493}
2494
ca088320
YL		 2495static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
2496 struct hns_roce_mr *mr)
3958cc56 2497{
3856ec55 2498 struct sg_dma_page_iter sg_iter;
db270c41 2499 u64 page_addr;
3958cc56 2500 u64 *pages;
3856ec55 2501 int i;
3958cc56 2502
ca088320
YL		 2503 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2504 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2505 roce_set_field(mpt_entry->byte_48_mode_ba,
2506 V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
2507 upper_32_bits(mr->pbl_ba >> 3));
2508
2509 pages = (u64 *)__get_free_page(GFP_KERNEL);
2510 if (!pages)
2511 return -ENOMEM;
2512
2513 i = 0;
3856ec55
SS		 2514 for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
2515 page_addr = sg_page_iter_dma_address(&sg_iter);
2516 pages[i] = page_addr >> 6;
2517
2518 /* Record the first 2 entry directly to MTPT table */
2519 if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
2520 goto found;
2521 i++;
ca088320
YL		 2522 }
2523found:
2524 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
2525 roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
2526 V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
2527
2528 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
2529 roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
2530 V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
2531 roce_set_field(mpt_entry->byte_64_buf_pa1,
2532 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2533 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2534 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2535
2536 free_page((unsigned long)pages);
2537
2538 return 0;
2539}
2540
2541static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
2542 unsigned long mtpt_idx)
2543{
2544 struct hns_roce_v2_mpt_entry *mpt_entry;
2545 int ret;
2546
3958cc56
WHX		 2547 mpt_entry = mb_buf;
2548 memset(mpt_entry, 0, sizeof(*mpt_entry));
2549
2550 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2551 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2552 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2553 V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
2554 HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
2555 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2556 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
5e6e78db
YL		 2557 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2558 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
3958cc56
WHX		 2559 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2560 V2_MPT_BYTE_4_PD_S, mr->pd);
3958cc56
WHX		 2561
2562 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
82342e49 2563 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
e93df010 2564 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
3958cc56
WHX		 2565 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
2566 (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
384f8818
LO		 2567 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
2568 mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
3958cc56
WHX		 2569 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2570 (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
2571 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2572 (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
2573 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2574 (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
3958cc56
WHX		 2575
2576 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
2577 mr->type == MR_TYPE_MR ? 0 : 1);
85e0274d 2578 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
2579 1);
3958cc56
WHX		 2580
2581 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2582 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2583 mpt_entry->lkey = cpu_to_le32(mr->key);
2584 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2585 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2586
2587 if (mr->type == MR_TYPE_DMA)
2588 return 0;
2589
ca088320 2590 ret = set_mtpt_pbl(mpt_entry, mr);
3958cc56 2591
ca088320 2592 return ret;
3958cc56
WHX		 2593}
2594
a2c80b7b
WHX		 2595static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
2596 struct hns_roce_mr *mr, int flags,
2597 u32 pdn, int mr_access_flags, u64 iova,
2598 u64 size, void *mb_buf)
2599{
2600 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
ca088320 2601 int ret = 0;
a2c80b7b 2602
ab22bf05
YL		 2603 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2604 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2605
a2c80b7b
WHX		 2606 if (flags & IB_MR_REREG_PD) {
2607 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2608 V2_MPT_BYTE_4_PD_S, pdn);
2609 mr->pd = pdn;
2610 }
2611
2612 if (flags & IB_MR_REREG_ACCESS) {
2613 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2614 V2_MPT_BYTE_8_BIND_EN_S,
2615 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
2616 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
ca088320
YL		 2617 V2_MPT_BYTE_8_ATOMIC_EN_S,
2618 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
a2c80b7b 2619 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
ca088320 2620 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
a2c80b7b 2621 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
ca088320 2622 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
a2c80b7b 2623 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
ca088320 2624 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
a2c80b7b
WHX		 2625 }
2626
2627 if (flags & IB_MR_REREG_TRANS) {
2628 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
2629 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
2630 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
2631 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
2632
a2c80b7b
WHX		 2633 mr->iova = iova;
2634 mr->size = size;
ca088320
YL		 2635
2636 ret = set_mtpt_pbl(mpt_entry, mr);
a2c80b7b
WHX		 2637 }
2638
ca088320 2639 return ret;
a2c80b7b
WHX		 2640}
2641
68a997c5
YL		 2642static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
2643{
2644 struct hns_roce_v2_mpt_entry *mpt_entry;
2645
2646 mpt_entry = mb_buf;
2647 memset(mpt_entry, 0, sizeof(*mpt_entry));
2648
2649 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2650 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2651 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2652 V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
2653 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2654 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2655 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2656 mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2657 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2658 V2_MPT_BYTE_4_PD_S, mr->pd);
2659
2660 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
2661 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2662 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2663
2664 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
2665 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2666 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
2667 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2668
2669 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2670
2671 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2672 roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
2673 V2_MPT_BYTE_48_PBL_BA_H_S,
2674 upper_32_bits(mr->pbl_ba >> 3));
2675
2676 roce_set_field(mpt_entry->byte_64_buf_pa1,
2677 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2678 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2679 mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2680
2681 return 0;
2682}
2683
c7c28191
YL		 2684static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
2685{
2686 struct hns_roce_v2_mpt_entry *mpt_entry;
2687
2688 mpt_entry = mb_buf;
2689 memset(mpt_entry, 0, sizeof(*mpt_entry));
2690
2691 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2692 V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2693 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2694 V2_MPT_BYTE_4_PD_S, mw->pdn);
60262b10 2695 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
c7c28191 2696 V2_MPT_BYTE_4_PBL_HOP_NUM_S,
60262b10
LO		 2697 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
2698 mw->pbl_hop_num);
c7c28191
YL		 2699 roce_set_field(mpt_entry->byte_4_pd_hop_st,
2700 V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2701 V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2702 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2703
2704 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2705 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2706
2707 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2708 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
2709 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2710 roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
2711 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
2712
2713 roce_set_field(mpt_entry->byte_64_buf_pa1,
2714 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2715 V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2716 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2717
2718 mpt_entry->lkey = cpu_to_le32(mw->rkey);
2719
2720 return 0;
2721}
2722
93aa2187
WHX		 2723static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2724{
18a96d25 2725 return hns_roce_buf_offset(&hr_cq->buf, n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
93aa2187
WHX		 2726}
2727
2728static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2729{
2730 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
2731
2732 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
2733 return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
e2b2744a 2734 !!(n & hr_cq->cq_depth)) ? cqe : NULL;
93aa2187
WHX		 2735}
2736
2737static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
2738{
2739 return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2740}
2741
c7bcb134
LO		 2742static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
2743{
2744 return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
2745}
2746
2747static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
2748{
c7bcb134
LO		 2749 /* always called with interrupts disabled. */
2750 spin_lock(&srq->lock);
2751
97545b10 2752 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
c7bcb134
LO		 2753 srq->tail++;
2754
2755 spin_unlock(&srq->lock);
2756}
2757
93aa2187
WHX		 2758static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2759{
b14c95be 2760 *hr_cq->set_ci_db = cons_index & V2_CQ_DB_PARAMETER_CONS_IDX_M;
93aa2187
WHX		 2761}
2762
926a01dc
WHX		 2763static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2764 struct hns_roce_srq *srq)
2765{
2766 struct hns_roce_v2_cqe *cqe, *dest;
2767 u32 prod_index;
2768 int nfreed = 0;
c7bcb134 2769 int wqe_index;
926a01dc
WHX		 2770 u8 owner_bit;
2771
2772 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
2773 ++prod_index) {
d7e5ca88 2774 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
926a01dc
WHX		 2775 break;
2776 }
2777
2778 /*
2779 * Now backwards through the CQ, removing CQ entries
2780 * that match our QP by overwriting them with next entries.
2781 */
2782 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2783 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2784 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2785 V2_CQE_BYTE_16_LCL_QPN_S) &
2786 HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
c7bcb134
LO		 2787 if (srq &&
2788 roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
2789 wqe_index = roce_get_field(cqe->byte_4,
2790 V2_CQE_BYTE_4_WQE_INDX_M,
2791 V2_CQE_BYTE_4_WQE_INDX_S);
2792 hns_roce_free_srq_wqe(srq, wqe_index);
2793 }
926a01dc
WHX		 2794 ++nfreed;
2795 } else if (nfreed) {
2796 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
2797 hr_cq->ib_cq.cqe);
2798 owner_bit = roce_get_bit(dest->byte_4,
2799 V2_CQE_BYTE_4_OWNER_S);
2800 memcpy(dest, cqe, sizeof(*cqe));
2801 roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
2802 owner_bit);
2803 }
2804 }
2805
2806 if (nfreed) {
2807 hr_cq->cons_index += nfreed;
2808 /*
2809 * Make sure update of buffer contents is done before
2810 * updating consumer index.
2811 */
2812 wmb();
2813 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2814 }
2815}
2816
2817static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2818 struct hns_roce_srq *srq)
2819{
2820 spin_lock_irq(&hr_cq->lock);
2821 __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
2822 spin_unlock_irq(&hr_cq->lock);
2823}
2824
93aa2187
WHX		 2825static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
2826 struct hns_roce_cq *hr_cq, void *mb_buf,
e2b2744a 2827 u64 *mtts, dma_addr_t dma_handle)
93aa2187
WHX		 2828{
2829 struct hns_roce_v2_cq_context *cq_context;
2830
2831 cq_context = mb_buf;
2832 memset(cq_context, 0, sizeof(*cq_context));
2833
2834 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
2835 V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
a5073d60
YL		 2836 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
2837 V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
93aa2187 2838 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
60262b10 2839 V2_CQC_BYTE_4_SHIFT_S, ilog2(hr_cq->cq_depth));
93aa2187 2840 roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
e2b2744a 2841 V2_CQC_BYTE_4_CEQN_S, hr_cq->vector);
93aa2187
WHX		 2842
2843 roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
2844 V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
2845
bfe86035 2846 cq_context->cqe_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
93aa2187
WHX		 2847
2848 roce_set_field(cq_context->byte_16_hop_addr,
2849 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
2850 V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
bfe86035 2851 mtts[0] >> (32 + PAGE_ADDR_SHIFT));
93aa2187
WHX		 2852 roce_set_field(cq_context->byte_16_hop_addr,
2853 V2_CQC_BYTE_16_CQE_HOP_NUM_M,
2854 V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
2855 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
2856
bfe86035 2857 cq_context->cqe_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
93aa2187
WHX		 2858 roce_set_field(cq_context->byte_24_pgsz_addr,
2859 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
2860 V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
bfe86035 2861 mtts[1] >> (32 + PAGE_ADDR_SHIFT));
93aa2187
WHX		 2862 roce_set_field(cq_context->byte_24_pgsz_addr,
2863 V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
2864 V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
5e6e78db 2865 hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
93aa2187
WHX		 2866 roce_set_field(cq_context->byte_24_pgsz_addr,
2867 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
2868 V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
5e6e78db 2869 hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
93aa2187 2870
bfe86035 2871 cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
93aa2187
WHX		 2872
2873 roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
2874 V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
a5073d60 2875
9b44703d
YL		 2876 if (hr_cq->db_en)
2877 roce_set_bit(cq_context->byte_44_db_record,
2878 V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
2879
2880 roce_set_field(cq_context->byte_44_db_record,
2881 V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
2882 V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
2883 ((u32)hr_cq->db.dma) >> 1);
bfe86035 2884 cq_context->db_record_addr = cpu_to_le32(hr_cq->db.dma >> 32);
9b44703d 2885
a5073d60
YL		 2886 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2887 V2_CQC_BYTE_56_CQ_MAX_CNT_M,
2888 V2_CQC_BYTE_56_CQ_MAX_CNT_S,
2889 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
2890 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2891 V2_CQC_BYTE_56_CQ_PERIOD_M,
2892 V2_CQC_BYTE_56_CQ_PERIOD_S,
2893 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
93aa2187
WHX		 2894}
2895
2896static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
2897 enum ib_cq_notify_flags flags)
2898{
d3743fa9 2899 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
93aa2187
WHX		 2900 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2901 u32 notification_flag;
bfe86035 2902 __le32 doorbell[2];
93aa2187
WHX		 2903
2904 doorbell[0] = 0;
2905 doorbell[1] = 0;
2906
2907 notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
2908 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
2909 /*
2910 * flags = 0; Notification Flag = 1, next
2911 * flags = 1; Notification Flag = 0, solocited
2912 */
2913 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
2914 hr_cq->cqn);
2915 roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
2916 HNS_ROCE_V2_CQ_DB_NTR);
2917 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
2918 V2_CQ_DB_PARAMETER_CONS_IDX_S,
2919 hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2920 roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
26beb85f 2921 V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
93aa2187
WHX		 2922 roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
2923 notification_flag);
2924
d3743fa9 2925 hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
93aa2187
WHX		 2926
2927 return 0;
2928}
2929
0009c2db 2930static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
2931 struct hns_roce_qp **cur_qp,
2932 struct ib_wc *wc)
2933{
2934 struct hns_roce_rinl_sge *sge_list;
2935 u32 wr_num, wr_cnt, sge_num;
2936 u32 sge_cnt, data_len, size;
2937 void *wqe_buf;
2938
2939 wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
2940 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
2941 wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
2942
2943 sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2944 sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2945 wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
2946 data_len = wc->byte_len;
2947
2948 for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
2949 size = min(sge_list[sge_cnt].len, data_len);
2950 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
2951
2952 data_len -= size;
2953 wqe_buf += size;
2954 }
2955
2956 if (data_len) {
2957 wc->status = IB_WC_LOC_LEN_ERR;
2958 return -EAGAIN;
2959 }
2960
2961 return 0;
2962}
2963
626903e9
XW		 2964static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
2965 int num_entries, struct ib_wc *wc)
2966{
2967 unsigned int left;
2968 int npolled = 0;
2969
2970 left = wq->head - wq->tail;
2971 if (left == 0)
2972 return 0;
2973
2974 left = min_t(unsigned int, (unsigned int)num_entries, left);
2975 while (npolled < left) {
2976 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2977 wc->status = IB_WC_WR_FLUSH_ERR;
2978 wc->vendor_err = 0;
2979 wc->qp = &hr_qp->ibqp;
2980
2981 wq->tail++;
2982 wc++;
2983 npolled++;
2984 }
2985
2986 return npolled;
2987}
2988
2989static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
2990 struct ib_wc *wc)
2991{
2992 struct hns_roce_qp *hr_qp;
2993 int npolled = 0;
2994
2995 list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
2996 npolled += sw_comp(hr_qp, &hr_qp->sq,
2997 num_entries - npolled, wc + npolled);
2998 if (npolled >= num_entries)
2999 goto out;
3000 }
3001
3002 list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
3003 npolled += sw_comp(hr_qp, &hr_qp->rq,
3004 num_entries - npolled, wc + npolled);
3005 if (npolled >= num_entries)
3006 goto out;
3007 }
3008
3009out:
3010 return npolled;
3011}
3012
93aa2187
WHX		 3013static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
3014 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
3015{
c7bcb134 3016 struct hns_roce_srq *srq = NULL;
93aa2187
WHX		 3017 struct hns_roce_dev *hr_dev;
3018 struct hns_roce_v2_cqe *cqe;
3019 struct hns_roce_qp *hr_qp;
3020 struct hns_roce_wq *wq;
0425e3e6
YL		 3021 struct ib_qp_attr attr;
3022 int attr_mask;
93aa2187
WHX		 3023 int is_send;
3024 u16 wqe_ctr;
3025 u32 opcode;
3026 u32 status;
3027 int qpn;
0009c2db 3028 int ret;
93aa2187
WHX		 3029
3030 /* Find cqe according to consumer index */
3031 cqe = next_cqe_sw_v2(hr_cq);
3032 if (!cqe)
3033 return -EAGAIN;
3034
3035 ++hr_cq->cons_index;
3036 /* Memory barrier */
3037 rmb();
3038
3039 /* 0->SQ, 1->RQ */
3040 is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
3041
3042 qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
3043 V2_CQE_BYTE_16_LCL_QPN_S);
3044
3045 if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
3046 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3047 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3048 if (unlikely(!hr_qp)) {
3049 dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
3050 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
3051 return -EINVAL;
3052 }
3053 *cur_qp = hr_qp;
3054 }
3055
3056 wc->qp = &(*cur_qp)->ibqp;
3057 wc->vendor_err = 0;
3058
c7bcb134
LO		 3059 if (is_send) {
3060 wq = &(*cur_qp)->sq;
3061 if ((*cur_qp)->sq_signal_bits) {
3062 /*
3063 * If sg_signal_bit is 1,
3064 * firstly tail pointer updated to wqe
3065 * which current cqe correspond to
3066 */
3067 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
3068 V2_CQE_BYTE_4_WQE_INDX_M,
3069 V2_CQE_BYTE_4_WQE_INDX_S);
3070 wq->tail += (wqe_ctr - (u16)wq->tail) &
3071 (wq->wqe_cnt - 1);
3072 }
3073
3074 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3075 ++wq->tail;
3076 } else if ((*cur_qp)->ibqp.srq) {
3077 srq = to_hr_srq((*cur_qp)->ibqp.srq);
bfe86035
LC		 3078 wqe_ctr = (u16)roce_get_field(cqe->byte_4,
3079 V2_CQE_BYTE_4_WQE_INDX_M,
3080 V2_CQE_BYTE_4_WQE_INDX_S);
c7bcb134
LO		 3081 wc->wr_id = srq->wrid[wqe_ctr];
3082 hns_roce_free_srq_wqe(srq, wqe_ctr);
3083 } else {
3084 /* Update tail pointer, record wr_id */
3085 wq = &(*cur_qp)->rq;
3086 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3087 ++wq->tail;
3088 }
3089
93aa2187
WHX		 3090 status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
3091 V2_CQE_BYTE_4_STATUS_S);
3092 switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
3093 case HNS_ROCE_CQE_V2_SUCCESS:
3094 wc->status = IB_WC_SUCCESS;
3095 break;
3096 case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
3097 wc->status = IB_WC_LOC_LEN_ERR;
3098 break;
3099 case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
3100 wc->status = IB_WC_LOC_QP_OP_ERR;
3101 break;
3102 case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
3103 wc->status = IB_WC_LOC_PROT_ERR;
3104 break;
3105 case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
3106 wc->status = IB_WC_WR_FLUSH_ERR;
3107 break;
3108 case HNS_ROCE_CQE_V2_MW_BIND_ERR:
3109 wc->status = IB_WC_MW_BIND_ERR;
3110 break;
3111 case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
3112 wc->status = IB_WC_BAD_RESP_ERR;
3113 break;
3114 case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
3115 wc->status = IB_WC_LOC_ACCESS_ERR;
3116 break;
3117 case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
3118 wc->status = IB_WC_REM_INV_REQ_ERR;
3119 break;
3120 case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
3121 wc->status = IB_WC_REM_ACCESS_ERR;
3122 break;
3123 case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
3124 wc->status = IB_WC_REM_OP_ERR;
3125 break;
3126 case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
3127 wc->status = IB_WC_RETRY_EXC_ERR;
3128 break;
3129 case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
3130 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
3131 break;
3132 case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
3133 wc->status = IB_WC_REM_ABORT_ERR;
3134 break;
3135 default:
3136 wc->status = IB_WC_GENERAL_ERR;
3137 break;
3138 }
3139
0425e3e6
YL		 3140 /* flush cqe if wc status is error, excluding flush error */
3141 if ((wc->status != IB_WC_SUCCESS) &&
3142 (wc->status != IB_WC_WR_FLUSH_ERR)) {
3143 attr_mask = IB_QP_STATE;
3144 attr.qp_state = IB_QPS_ERR;
3145 return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
3146 &attr, attr_mask,
3147 (*cur_qp)->state, IB_QPS_ERR);
3148 }
3149
3150 if (wc->status == IB_WC_WR_FLUSH_ERR)
93aa2187
WHX		 3151 return 0;
3152
3153 if (is_send) {
3154 wc->wc_flags = 0;
3155 /* SQ corresponding to CQE */
3156 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
3157 V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
3158 case HNS_ROCE_SQ_OPCODE_SEND:
3159 wc->opcode = IB_WC_SEND;
3160 break;
3161 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
3162 wc->opcode = IB_WC_SEND;
3163 break;
3164 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
3165 wc->opcode = IB_WC_SEND;
3166 wc->wc_flags |= IB_WC_WITH_IMM;
3167 break;
3168 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
3169 wc->opcode = IB_WC_RDMA_READ;
3170 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3171 break;
3172 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
3173 wc->opcode = IB_WC_RDMA_WRITE;
3174 break;
3175 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
3176 wc->opcode = IB_WC_RDMA_WRITE;
3177 wc->wc_flags |= IB_WC_WITH_IMM;
3178 break;
3179 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
3180 wc->opcode = IB_WC_LOCAL_INV;
3181 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3182 break;
3183 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
3184 wc->opcode = IB_WC_COMP_SWAP;
3185 wc->byte_len = 8;
3186 break;
3187 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
3188 wc->opcode = IB_WC_FETCH_ADD;
3189 wc->byte_len = 8;
3190 break;
3191 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
3192 wc->opcode = IB_WC_MASKED_COMP_SWAP;
3193 wc->byte_len = 8;
3194 break;
3195 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
3196 wc->opcode = IB_WC_MASKED_FETCH_ADD;
3197 wc->byte_len = 8;
3198 break;
3199 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
3200 wc->opcode = IB_WC_REG_MR;
3201 break;
3202 case HNS_ROCE_SQ_OPCODE_BIND_MW:
3203 wc->opcode = IB_WC_REG_MR;
3204 break;
3205 default:
3206 wc->status = IB_WC_GENERAL_ERR;
3207 break;
3208 }
93aa2187
WHX		 3209 } else {
3210 /* RQ correspond to CQE */
3211 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3212
3213 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
3214 V2_CQE_BYTE_4_OPCODE_S);
3215 switch (opcode & 0x1f) {
3216 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
3217 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3218 wc->wc_flags = IB_WC_WITH_IMM;
0c4a0e29
LO		 3219 wc->ex.imm_data =
3220 cpu_to_be32(le32_to_cpu(cqe->immtdata));
93aa2187
WHX		 3221 break;
3222 case HNS_ROCE_V2_OPCODE_SEND:
3223 wc->opcode = IB_WC_RECV;
3224 wc->wc_flags = 0;
3225 break;
3226 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
3227 wc->opcode = IB_WC_RECV;
3228 wc->wc_flags = IB_WC_WITH_IMM;
0c4a0e29
LO		 3229 wc->ex.imm_data =
3230 cpu_to_be32(le32_to_cpu(cqe->immtdata));
93aa2187
WHX		 3231 break;
3232 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
3233 wc->opcode = IB_WC_RECV;
3234 wc->wc_flags = IB_WC_WITH_INVALIDATE;
ccb8a29e 3235 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
93aa2187
WHX		 3236 break;
3237 default:
3238 wc->status = IB_WC_GENERAL_ERR;
3239 break;
3240 }
3241
0009c2db 3242 if ((wc->qp->qp_type == IB_QPT_RC ||
3243 wc->qp->qp_type == IB_QPT_UC) &&
3244 (opcode == HNS_ROCE_V2_OPCODE_SEND ||
3245 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
3246 opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
3247 (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
3248 ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
3249 if (ret)
3250 return -EAGAIN;
3251 }
3252
93aa2187
WHX		 3253 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
3254 V2_CQE_BYTE_32_SL_S);
3255 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
3256 V2_CQE_BYTE_32_RMT_QPN_M,
3257 V2_CQE_BYTE_32_RMT_QPN_S);
15fc056f 3258 wc->slid = 0;
93aa2187
WHX		 3259 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
3260 V2_CQE_BYTE_32_GRH_S) ?
3261 IB_WC_GRH : 0);
6c1f08b3 3262 wc->port_num = roce_get_field(cqe->byte_32,
3263 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
3264 wc->pkey_index = 0;
2eade675 3265 memcpy(wc->smac, cqe->smac, 4);
3266 wc->smac[4] = roce_get_field(cqe->byte_28,
3267 V2_CQE_BYTE_28_SMAC_4_M,
3268 V2_CQE_BYTE_28_SMAC_4_S);
3269 wc->smac[5] = roce_get_field(cqe->byte_28,
3270 V2_CQE_BYTE_28_SMAC_5_M,
3271 V2_CQE_BYTE_28_SMAC_5_S);
0e1aa6f0 3272 wc->wc_flags |= IB_WC_WITH_SMAC;
944e6409
LO		 3273 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
3274 wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
3275 V2_CQE_BYTE_28_VID_M,
3276 V2_CQE_BYTE_28_VID_S);
0e1aa6f0 3277 wc->wc_flags |= IB_WC_WITH_VLAN;
944e6409
LO		 3278 } else {
3279 wc->vlan_id = 0xffff;
3280 }
3281
2eade675 3282 wc->network_hdr_type = roce_get_field(cqe->byte_28,
3283 V2_CQE_BYTE_28_PORT_TYPE_M,
3284 V2_CQE_BYTE_28_PORT_TYPE_S);
93aa2187
WHX		 3285 }
3286
3287 return 0;
3288}
3289
3290static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
3291 struct ib_wc *wc)
3292{
626903e9 3293 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
93aa2187
WHX		 3294 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3295 struct hns_roce_qp *cur_qp = NULL;
3296 unsigned long flags;
3297 int npolled;
3298
3299 spin_lock_irqsave(&hr_cq->lock, flags);
3300
626903e9
XW		 3301 /*
3302 * When the device starts to reset, the state is RST_DOWN. At this time,
3303 * there may still be some valid CQEs in the hardware that are not
3304 * polled. Therefore, it is not allowed to switch to the software mode
3305 * immediately. When the state changes to UNINIT, CQE no longer exists
3306 * in the hardware, and then switch to software mode.
3307 */
3308 if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
3309 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
3310 goto out;
3311 }
3312
93aa2187
WHX		 3313 for (npolled = 0; npolled < num_entries; ++npolled) {
3314 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
3315 break;
3316 }
3317
3318 if (npolled) {
3319 /* Memory barrier */
3320 wmb();
3321 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
3322 }
3323
626903e9 3324out:
93aa2187
WHX		 3325 spin_unlock_irqrestore(&hr_cq->lock, flags);
3326
3327 return npolled;
3328}
3329
260c3b34
YL		 3330static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
3331 int step_idx)
3332{
3333 int op;
3334
3335 if (type == HEM_TYPE_SCCC && step_idx)
3336 return -EINVAL;
3337
3338 switch (type) {
3339 case HEM_TYPE_QPC:
3340 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
3341 break;
3342 case HEM_TYPE_MTPT:
3343 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
3344 break;
3345 case HEM_TYPE_CQC:
3346 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
3347 break;
3348 case HEM_TYPE_SRQC:
3349 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
3350 break;
3351 case HEM_TYPE_SCCC:
3352 op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
3353 break;
3354 case HEM_TYPE_QPC_TIMER:
3355 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
3356 break;
3357 case HEM_TYPE_CQC_TIMER:
3358 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
3359 break;
3360 default:
3361 dev_warn(hr_dev->dev,
3362 "Table %d not to be written by mailbox!\n", type);
3363 return -EINVAL;
3364 }
3365
3366 return op + step_idx;
3367}
3368
a81fba28
WHX		 3369static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
3370 struct hns_roce_hem_table *table, int obj,
3371 int step_idx)
3372{
a81fba28
WHX		 3373 struct hns_roce_cmd_mailbox *mailbox;
3374 struct hns_roce_hem_iter iter;
3375 struct hns_roce_hem_mhop mhop;
3376 struct hns_roce_hem *hem;
3377 unsigned long mhop_obj = obj;
3378 int i, j, k;
3379 int ret = 0;
3380 u64 hem_idx = 0;
3381 u64 l1_idx = 0;
3382 u64 bt_ba = 0;
3383 u32 chunk_ba_num;
3384 u32 hop_num;
260c3b34 3385 int op;
a81fba28
WHX		 3386
3387 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3388 return 0;
3389
3390 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
3391 i = mhop.l0_idx;
3392 j = mhop.l1_idx;
3393 k = mhop.l2_idx;
3394 hop_num = mhop.hop_num;
3395 chunk_ba_num = mhop.bt_chunk_size / 8;
3396
3397 if (hop_num == 2) {
3398 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
3399 k;
3400 l1_idx = i * chunk_ba_num + j;
3401 } else if (hop_num == 1) {
3402 hem_idx = i * chunk_ba_num + j;
3403 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
3404 hem_idx = i;
3405 }
3406
260c3b34
YL		 3407 op = get_op_for_set_hem(hr_dev, table->type, step_idx);
3408 if (op == -EINVAL)
a81fba28 3409 return 0;
a81fba28
WHX		 3410
3411 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3412 if (IS_ERR(mailbox))
3413 return PTR_ERR(mailbox);
3414
6ac16e40
YL		 3415 if (table->type == HEM_TYPE_SCCC)
3416 obj = mhop.l0_idx;
3417
a81fba28
WHX		 3418 if (check_whether_last_step(hop_num, step_idx)) {
3419 hem = table->hem[hem_idx];
3420 for (hns_roce_hem_first(hem, &iter);
3421 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
3422 bt_ba = hns_roce_hem_addr(&iter);
3423
3424 /* configure the ba, tag, and op */
3425 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
3426 obj, 0, op,
3427 HNS_ROCE_CMD_TIMEOUT_MSECS);
3428 }
3429 } else {
3430 if (step_idx == 0)
3431 bt_ba = table->bt_l0_dma_addr[i];
3432 else if (step_idx == 1 && hop_num == 2)
3433 bt_ba = table->bt_l1_dma_addr[l1_idx];
3434
3435 /* configure the ba, tag, and op */
3436 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3437 0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3438 }
3439
3440 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3441 return ret;
3442}
3443
3444static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3445 struct hns_roce_hem_table *table, int obj,
3446 int step_idx)
3447{
3448 struct device *dev = hr_dev->dev;
3449 struct hns_roce_cmd_mailbox *mailbox;
617cf24f 3450 int ret;
a81fba28
WHX		 3451 u16 op = 0xff;
3452
3453 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3454 return 0;
3455
3456 switch (table->type) {
3457 case HEM_TYPE_QPC:
3458 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3459 break;
3460 case HEM_TYPE_MTPT:
3461 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3462 break;
3463 case HEM_TYPE_CQC:
3464 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3465 break;
6a157f7d 3466 case HEM_TYPE_SCCC:
0e40dc2f
YL		 3467 case HEM_TYPE_QPC_TIMER:
3468 case HEM_TYPE_CQC_TIMER:
6a157f7d 3469 break;
a81fba28
WHX		 3470 case HEM_TYPE_SRQC:
3471 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3472 break;
3473 default:
3474 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
3475 table->type);
3476 return 0;
3477 }
6a157f7d 3478
0e40dc2f
YL		 3479 if (table->type == HEM_TYPE_SCCC ||
3480 table->type == HEM_TYPE_QPC_TIMER ||
3481 table->type == HEM_TYPE_CQC_TIMER)
6a157f7d
YL		 3482 return 0;
3483
a81fba28
WHX		 3484 op += step_idx;
3485
3486 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3487 if (IS_ERR(mailbox))
3488 return PTR_ERR(mailbox);
3489
3490 /* configure the tag and op */
3491 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3492 HNS_ROCE_CMD_TIMEOUT_MSECS);
3493
3494 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3495 return ret;
3496}
3497
926a01dc 3498static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
926a01dc
WHX		 3499 struct hns_roce_v2_qp_context *context,
3500 struct hns_roce_qp *hr_qp)
3501{
3502 struct hns_roce_cmd_mailbox *mailbox;
3503 int ret;
3504
3505 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3506 if (IS_ERR(mailbox))
3507 return PTR_ERR(mailbox);
3508
3509 memcpy(mailbox->buf, context, sizeof(*context) * 2);
3510
3511 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
3512 HNS_ROCE_CMD_MODIFY_QPC,
3513 HNS_ROCE_CMD_TIMEOUT_MSECS);
3514
3515 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3516
3517 return ret;
3518}
3519
ace1c541 3520static void set_access_flags(struct hns_roce_qp *hr_qp,
3521 struct hns_roce_v2_qp_context *context,
3522 struct hns_roce_v2_qp_context *qpc_mask,
3523 const struct ib_qp_attr *attr, int attr_mask)
3524{
3525 u8 dest_rd_atomic;
3526 u32 access_flags;
3527
c2799119 3528 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
ace1c541 3529 attr->max_dest_rd_atomic : hr_qp->resp_depth;
3530
c2799119 3531 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
ace1c541 3532 attr->qp_access_flags : hr_qp->atomic_rd_en;
3533
3534 if (!dest_rd_atomic)
3535 access_flags &= IB_ACCESS_REMOTE_WRITE;
3536
3537 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3538 !!(access_flags & IB_ACCESS_REMOTE_READ));
3539 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
3540
3541 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3542 !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3543 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
3544
3545 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3546 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3547 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
7db82697
JZ		 3548 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S,
3549 !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3550 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_EXT_ATE_S, 0);
ace1c541 3551}
3552
99441ab5
XW		 3553static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
3554 struct hns_roce_v2_qp_context *context,
3555 struct hns_roce_v2_qp_context *qpc_mask)
3556{
3557 if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
3558 roce_set_field(context->byte_4_sqpn_tst,
3559 V2_QPC_BYTE_4_SGE_SHIFT_M,
3560 V2_QPC_BYTE_4_SGE_SHIFT_S,
3561 ilog2((unsigned int)hr_qp->sge.sge_cnt));
3562 else
3563 roce_set_field(context->byte_4_sqpn_tst,
3564 V2_QPC_BYTE_4_SGE_SHIFT_M,
3565 V2_QPC_BYTE_4_SGE_SHIFT_S,
3566 hr_qp->sq.max_gs >
3567 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
3568 ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3569
3570 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3571 V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3572
3573 roce_set_field(context->byte_20_smac_sgid_idx,
3574 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3575 ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3576 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3577 V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3578
3579 roce_set_field(context->byte_20_smac_sgid_idx,
3580 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3581 (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3582 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT ||
3583 hr_qp->ibqp.srq) ? 0 :
3584 ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3585
3586 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3587 V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3588}
3589
926a01dc
WHX		 3590static void modify_qp_reset_to_init(struct ib_qp *ibqp,
3591 const struct ib_qp_attr *attr,
0fa95a9a 3592 int attr_mask,
926a01dc
WHX		 3593 struct hns_roce_v2_qp_context *context,
3594 struct hns_roce_v2_qp_context *qpc_mask)
3595{
ecaaf1e2 3596 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
926a01dc
WHX		 3597 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3598
3599 /*
3600 * In v2 engine, software pass context and context mask to hardware
3601 * when modifying qp. If software need modify some fields in context,
3602 * we should set all bits of the relevant fields in context mask to
3603 * 0 at the same time, else set them to 0x1.
3604 */
3605 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3606 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3607 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3608 V2_QPC_BYTE_4_TST_S, 0);
3609
926a01dc
WHX		 3610 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3611 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3612 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3613 V2_QPC_BYTE_4_SQPN_S, 0);
3614
3615 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3616 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3617 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3618 V2_QPC_BYTE_16_PD_S, 0);
3619
3620 roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3621 V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
3622 roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3623 V2_QPC_BYTE_20_RQWS_S, 0);
3624
99441ab5 3625 set_qpc_wqe_cnt(hr_qp, context, qpc_mask);
926a01dc
WHX		 3626
3627 /* No VLAN need to set 0xFFF */
c8e46f8d
LO		 3628 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3629 V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
3630 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3631 V2_QPC_BYTE_24_VLAN_ID_S, 0);
926a01dc
WHX		 3632
3633 /*
3634 * Set some fields in context to zero, Because the default values
3635 * of all fields in context are zero, we need not set them to 0 again.
3636 * but we should set the relevant fields of context mask to 0.
3637 */
3638 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
3639 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
3640 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
3641 roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
3642
2362ccee
LO		 3643 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,
3644 V2_QPC_BYTE_60_TEMPID_S, 0);
3645
3646 roce_set_field(qpc_mask->byte_60_qpst_tempid,
3647 V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,
3648 0);
3649 roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3650 V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);
3651 roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3652 V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);
926a01dc
WHX		 3653 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
3654 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
3655
e088a685
YL		 3656 if (hr_qp->rdb_en) {
3657 roce_set_bit(context->byte_68_rq_db,
3658 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
3659 roce_set_bit(qpc_mask->byte_68_rq_db,
3660 V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
3661 }
3662
3663 roce_set_field(context->byte_68_rq_db,
3664 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3665 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
3666 ((u32)hr_qp->rdb.dma) >> 1);
3667 roce_set_field(qpc_mask->byte_68_rq_db,
3668 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3669 V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
bfe86035 3670 context->rq_db_record_addr = cpu_to_le32(hr_qp->rdb.dma >> 32);
e088a685
YL		 3671 qpc_mask->rq_db_record_addr = 0;
3672
ecaaf1e2 3673 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
3674 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
926a01dc
WHX		 3675 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
3676
3677 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3678 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3679 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3680 V2_QPC_BYTE_80_RX_CQN_S, 0);
3681 if (ibqp->srq) {
3682 roce_set_field(context->byte_76_srqn_op_en,
3683 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3684 to_hr_srq(ibqp->srq)->srqn);
3685 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3686 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3687 roce_set_bit(context->byte_76_srqn_op_en,
3688 V2_QPC_BYTE_76_SRQ_EN_S, 1);
3689 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3690 V2_QPC_BYTE_76_SRQ_EN_S, 0);
3691 }
3692
3693 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3694 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3695 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3696 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3697 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3698 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3699
3700 roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
3701 V2_QPC_BYTE_92_SRQ_INFO_S, 0);
3702
3703 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3704 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3705
3706 roce_set_field(qpc_mask->byte_104_rq_sge,
3707 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
3708 V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
3709
3710 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3711 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3712 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3713 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3714 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3715 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3716 V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
3717
3718 qpc_mask->rq_rnr_timer = 0;
3719 qpc_mask->rx_msg_len = 0;
3720 qpc_mask->rx_rkey_pkt_info = 0;
3721 qpc_mask->rx_va = 0;
3722
3723 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3724 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3725 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3726 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3727
2362ccee
LO		 3728 roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,
3729 0);
926a01dc
WHX		 3730 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
3731 V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
3732 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
3733 V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
3734
3735 roce_set_field(qpc_mask->byte_144_raq,
3736 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
3737 V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
926a01dc
WHX		 3738 roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
3739 V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
3740 roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
3741
3742 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
3743 V2_QPC_BYTE_148_RQ_MSN_S, 0);
3744 roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
3745 V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
3746
3747 roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3748 V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3749 roce_set_field(qpc_mask->byte_152_raq,
3750 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
3751 V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
3752
3753 roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
3754 V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
3755
3756 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3757 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3758 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3759 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3760 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
3761 V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
3762
2362ccee
LO		 3763 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3764 V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);
3765 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3766 V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);
3767 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3768 V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);
926a01dc
WHX		 3769 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3770 V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
b5fddb7c 3771 roce_set_bit(qpc_mask->byte_168_irrl_idx,
3772 V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
926a01dc
WHX		 3773 roce_set_field(qpc_mask->byte_168_irrl_idx,
3774 V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
3775 V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
3776
3777 roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3778 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
3779 roce_set_field(qpc_mask->byte_172_sq_psn,
3780 V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3781 V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3782
3783 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
3784 0);
3785
68a997c5
YL		 3786 roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
3787 roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);
3788
926a01dc
WHX		 3789 roce_set_field(qpc_mask->byte_176_msg_pktn,
3790 V2_QPC_BYTE_176_MSG_USE_PKTN_M,
3791 V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
3792 roce_set_field(qpc_mask->byte_176_msg_pktn,
3793 V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
3794 V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
3795
3796 roce_set_field(qpc_mask->byte_184_irrl_idx,
3797 V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
3798 V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
3799
3800 qpc_mask->cur_sge_offset = 0;
3801
3802 roce_set_field(qpc_mask->byte_192_ext_sge,
3803 V2_QPC_BYTE_192_CUR_SGE_IDX_M,
3804 V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
3805 roce_set_field(qpc_mask->byte_192_ext_sge,
3806 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
3807 V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
3808
3809 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3810 V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3811
3812 roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
3813 V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
3814 roce_set_field(qpc_mask->byte_200_sq_max,
3815 V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
3816 V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
3817
3818 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
3819 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
3820
3821 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3822 V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3823
3824 qpc_mask->sq_timer = 0;
3825
3826 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3827 V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3828 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3829 roce_set_field(qpc_mask->byte_232_irrl_sge,
3830 V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3831 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3832
2362ccee
LO		 3833 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,
3834 0);
3835 roce_set_bit(qpc_mask->byte_232_irrl_sge,
3836 V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);
3837 roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,
3838 0);
3839
926a01dc
WHX		 3840 qpc_mask->irrl_cur_sge_offset = 0;
3841
3842 roce_set_field(qpc_mask->byte_240_irrl_tail,
3843 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3844 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3845 roce_set_field(qpc_mask->byte_240_irrl_tail,
3846 V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
3847 V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
3848 roce_set_field(qpc_mask->byte_240_irrl_tail,
3849 V2_QPC_BYTE_240_RX_ACK_MSN_M,
3850 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3851
3852 roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
3853 V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3854 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
3855 0);
3856 roce_set_field(qpc_mask->byte_248_ack_psn,
3857 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3858 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3859 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
3860 0);
3861 roce_set_bit(qpc_mask->byte_248_ack_psn,
3862 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3863 roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
3864 0);
3865
3866 hr_qp->access_flags = attr->qp_access_flags;
926a01dc
WHX		 3867 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3868 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3869 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3870 V2_QPC_BYTE_252_TX_CQN_S, 0);
3871
3872 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
3873 V2_QPC_BYTE_252_ERR_TYPE_S, 0);
3874
3875 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3876 V2_QPC_BYTE_256_RQ_CQE_IDX_M,
3877 V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
3878 roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3879 V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
3880 V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
3881}
3882
3883static void modify_qp_init_to_init(struct ib_qp *ibqp,
3884 const struct ib_qp_attr *attr, int attr_mask,
3885 struct hns_roce_v2_qp_context *context,
3886 struct hns_roce_v2_qp_context *qpc_mask)
3887{
3888 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3889
3890 /*
3891 * In v2 engine, software pass context and context mask to hardware
3892 * when modifying qp. If software need modify some fields in context,
3893 * we should set all bits of the relevant fields in context mask to
3894 * 0 at the same time, else set them to 0x1.
3895 */
3896 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3897 V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3898 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3899 V2_QPC_BYTE_4_TST_S, 0);
3900
926a01dc
WHX		 3901 if (attr_mask & IB_QP_ACCESS_FLAGS) {
3902 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3903 !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3904 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3905 0);
3906
3907 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3908 !!(attr->qp_access_flags &
3909 IB_ACCESS_REMOTE_WRITE));
3910 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3911 0);
3912
3913 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3914 !!(attr->qp_access_flags &
3915 IB_ACCESS_REMOTE_ATOMIC));
3916 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3917 0);
7db82697
JZ		 3918 roce_set_bit(context->byte_76_srqn_op_en,
3919 V2_QPC_BYTE_76_EXT_ATE_S,
3920 !!(attr->qp_access_flags &
3921 IB_ACCESS_REMOTE_ATOMIC));
3922 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3923 V2_QPC_BYTE_76_EXT_ATE_S, 0);
926a01dc
WHX		 3924 } else {
3925 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3926 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3927 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3928 0);
3929
3930 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3931 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3932 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3933 0);
3934
3935 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3936 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3937 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3938 0);
7db82697
JZ		 3939 roce_set_bit(context->byte_76_srqn_op_en,
3940 V2_QPC_BYTE_76_EXT_ATE_S,
3941 !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3942 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3943 V2_QPC_BYTE_76_EXT_ATE_S, 0);
926a01dc
WHX		 3944 }
3945
926a01dc
WHX		 3946 roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3947 V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3948 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3949 V2_QPC_BYTE_16_PD_S, 0);
3950
3951 roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3952 V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3953 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3954 V2_QPC_BYTE_80_RX_CQN_S, 0);
3955
3956 roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
6d13b869 3957 V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
926a01dc
WHX		 3958 roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3959 V2_QPC_BYTE_252_TX_CQN_S, 0);
3960
3961 if (ibqp->srq) {
3962 roce_set_bit(context->byte_76_srqn_op_en,
3963 V2_QPC_BYTE_76_SRQ_EN_S, 1);
3964 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3965 V2_QPC_BYTE_76_SRQ_EN_S, 0);
3966 roce_set_field(context->byte_76_srqn_op_en,
3967 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3968 to_hr_srq(ibqp->srq)->srqn);
3969 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3970 V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3971 }
3972
926a01dc
WHX		 3973 roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3974 V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3975 roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3976 V2_QPC_BYTE_4_SQPN_S, 0);
3977
b6dd9b34 3978 if (attr_mask & IB_QP_DEST_QPN) {
3979 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3980 V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
3981 roce_set_field(qpc_mask->byte_56_dqpn_err,
3982 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3983 }
926a01dc
WHX		 3984}
3985
8d18ad83
LO		 3986static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
3987 struct hns_roce_qp *hr_qp, int mtt_cnt,
3988 u32 page_size)
3989{
3990 struct device *dev = hr_dev->dev;
3991
3992 if (hr_qp->rq.wqe_cnt < 1)
3993 return true;
3994
3995 if (mtt_cnt < 1) {
3996 dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n",
3997 hr_qp->qpn);
3998 return false;
3999 }
4000
4001 if (mtt_cnt < MTT_MIN_COUNT &&
4002 (hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
4003 dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n",
4004 hr_qp->qpn);
4005 return false;
4006 }
4007
4008 return true;
4009}
4010
926a01dc
WHX		 4011static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
4012 const struct ib_qp_attr *attr, int attr_mask,
4013 struct hns_roce_v2_qp_context *context,
4014 struct hns_roce_v2_qp_context *qpc_mask)
4015{
4016 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4017 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4018 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4019 struct device *dev = hr_dev->dev;
8d18ad83 4020 u64 mtts[MTT_MIN_COUNT] = { 0 };
e92f2c18 4021 dma_addr_t dma_handle_3;
926a01dc 4022 dma_addr_t dma_handle_2;
8d18ad83 4023 u64 wqe_sge_ba;
926a01dc
WHX		 4024 u32 page_size;
4025 u8 port_num;
e92f2c18 4026 u64 *mtts_3;
926a01dc 4027 u64 *mtts_2;
8d18ad83 4028 int count;
926a01dc
WHX		 4029 u8 *dmac;
4030 u8 *smac;
4031 int port;
4032
4033 /* Search qp buf's mtts */
8d18ad83
LO		 4034 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
4035 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
4036 hr_qp->rq.offset / page_size, mtts,
4037 MTT_MIN_COUNT, &wqe_sge_ba);
4038 if (!ibqp->srq)
4039 if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
4040 return -EINVAL;
926a01dc
WHX		 4041
4042 /* Search IRRL's mtts */
4043 mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
4044 hr_qp->qpn, &dma_handle_2);
4045 if (!mtts_2) {
4046 dev_err(dev, "qp irrl_table find failed\n");
4047 return -EINVAL;
4048 }
4049
e92f2c18 4050 /* Search TRRL's mtts */
4051 mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
4052 hr_qp->qpn, &dma_handle_3);
4053 if (!mtts_3) {
4054 dev_err(dev, "qp trrl_table find failed\n");
4055 return -EINVAL;
4056 }
4057
734f3863 4058 if (attr_mask & IB_QP_ALT_PATH) {
926a01dc
WHX		 4059 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
4060 return -EINVAL;
4061 }
4062
4063 dmac = (u8 *)attr->ah_attr.roce.dmac;
bfe86035 4064 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
926a01dc
WHX		 4065 qpc_mask->wqe_sge_ba = 0;
4066
4067 /*
4068 * In v2 engine, software pass context and context mask to hardware
4069 * when modifying qp. If software need modify some fields in context,
4070 * we should set all bits of the relevant fields in context mask to
4071 * 0 at the same time, else set them to 0x1.
4072 */
4073 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
8d18ad83 4074 V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
926a01dc
WHX		 4075 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
4076 V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
4077
4078 roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
4079 V2_QPC_BYTE_12_SQ_HOP_NUM_S,
8d18ad83
LO		 4080 hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
4081 0 : hr_dev->caps.wqe_sq_hop_num);
926a01dc
WHX		 4082 roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
4083 V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
4084
4085 roce_set_field(context->byte_20_smac_sgid_idx,
4086 V2_QPC_BYTE_20_SGE_HOP_NUM_M,
4087 V2_QPC_BYTE_20_SGE_HOP_NUM_S,
8d18ad83
LO		 4088 ((ibqp->qp_type == IB_QPT_GSI) ||
4089 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
4090 hr_dev->caps.wqe_sge_hop_num : 0);
926a01dc
WHX		 4091 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4092 V2_QPC_BYTE_20_SGE_HOP_NUM_M,
4093 V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
4094
4095 roce_set_field(context->byte_20_smac_sgid_idx,
4096 V2_QPC_BYTE_20_RQ_HOP_NUM_M,
4097 V2_QPC_BYTE_20_RQ_HOP_NUM_S,
8d18ad83
LO		 4098 hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
4099 0 : hr_dev->caps.wqe_rq_hop_num);
926a01dc
WHX		 4100 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4101 V2_QPC_BYTE_20_RQ_HOP_NUM_M,
4102 V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
4103
4104 roce_set_field(context->byte_16_buf_ba_pg_sz,
4105 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
4106 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
8d18ad83 4107 hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
926a01dc
WHX		 4108 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
4109 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
4110 V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
4111
4112 roce_set_field(context->byte_16_buf_ba_pg_sz,
4113 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
4114 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
5e6e78db 4115 hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
926a01dc
WHX		 4116 roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
4117 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
4118 V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
4119
bfe86035 4120 context->rq_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 4121 qpc_mask->rq_cur_blk_addr = 0;
4122
4123 roce_set_field(context->byte_92_srq_info,
4124 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
4125 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
8d18ad83 4126 mtts[0] >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 4127 roce_set_field(qpc_mask->byte_92_srq_info,
4128 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
4129 V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
4130
bfe86035 4131 context->rq_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 4132 qpc_mask->rq_nxt_blk_addr = 0;
4133
4134 roce_set_field(context->byte_104_rq_sge,
4135 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
4136 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
8d18ad83 4137 mtts[1] >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 4138 roce_set_field(qpc_mask->byte_104_rq_sge,
4139 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
4140 V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
4141
e92f2c18 4142 roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
4143 V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
4144 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
4145 V2_QPC_BYTE_132_TRRL_BA_S, 0);
bfe86035 4146 context->trrl_ba = cpu_to_le32(dma_handle_3 >> (16 + 4));
e92f2c18 4147 qpc_mask->trrl_ba = 0;
4148 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
4149 V2_QPC_BYTE_140_TRRL_BA_S,
4150 (u32)(dma_handle_3 >> (32 + 16 + 4)));
4151 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
4152 V2_QPC_BYTE_140_TRRL_BA_S, 0);
4153
bfe86035 4154 context->irrl_ba = cpu_to_le32(dma_handle_2 >> 6);
926a01dc
WHX		 4155 qpc_mask->irrl_ba = 0;
4156 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
4157 V2_QPC_BYTE_208_IRRL_BA_S,
d5514246 4158 dma_handle_2 >> (32 + 6));
926a01dc
WHX		 4159 roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
4160 V2_QPC_BYTE_208_IRRL_BA_S, 0);
4161
4162 roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
4163 roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
4164
4165 roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
4166 hr_qp->sq_signal_bits);
4167 roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
4168 0);
4169
4170 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
4171
4172 smac = (u8 *)hr_dev->dev_addr[port];
4173 /* when dmac equals smac or loop_idc is 1, it should loopback */
4174 if (ether_addr_equal_unaligned(dmac, smac) ||
4175 hr_dev->loop_idc == 0x1) {
4176 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
4177 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
4178 }
4179
b6dd9b34 4180 if (attr_mask & IB_QP_DEST_QPN) {
4181 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
4182 V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
4183 roce_set_field(qpc_mask->byte_56_dqpn_err,
4184 V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
4185 }
926a01dc
WHX		 4186
4187 /* Configure GID index */
4188 port_num = rdma_ah_get_port_num(&attr->ah_attr);
4189 roce_set_field(context->byte_20_smac_sgid_idx,
60262b10 4190 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
926a01dc
WHX		 4191 hns_get_gid_index(hr_dev, port_num - 1,
4192 grh->sgid_index));
4193 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
60262b10 4194 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
2a3d923f 4195 memcpy(&(context->dmac), dmac, sizeof(u32));
926a01dc
WHX		 4196 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
4197 V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
4198 qpc_mask->dmac = 0;
4199 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
4200 V2_QPC_BYTE_52_DMAC_S, 0);
4201
2a3d923f 4202 /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
926a01dc
WHX		 4203 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
4204 V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
4205 roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
4206 V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
4207
0fa95a9a 4208 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
4209 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
4210 V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
6852af86 4211 else if (attr_mask & IB_QP_PATH_MTU)
0fa95a9a 4212 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
4213 V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
4214
926a01dc
WHX		 4215 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
4216 V2_QPC_BYTE_24_MTU_S, 0);
4217
926a01dc
WHX		 4218 roce_set_field(context->byte_84_rq_ci_pi,
4219 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4220 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
4221 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4222 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4223 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4224
4225 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4226 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
4227 V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
4228 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4229 V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
4230 roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
4231 V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
4232 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4233 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
4234 V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
4235
4236 context->rq_rnr_timer = 0;
4237 qpc_mask->rq_rnr_timer = 0;
4238
926a01dc
WHX		 4239 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
4240 V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
4241 roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
4242 V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
4243
2a3d923f 4244 /* rocee send 2^lp_sgen_ini segs every time */
926a01dc
WHX		 4245 roce_set_field(context->byte_168_irrl_idx,
4246 V2_QPC_BYTE_168_LP_SGEN_INI_M,
4247 V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
4248 roce_set_field(qpc_mask->byte_168_irrl_idx,
4249 V2_QPC_BYTE_168_LP_SGEN_INI_M,
4250 V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
4251
926a01dc
WHX		 4252 return 0;
4253}
4254
4255static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
4256 const struct ib_qp_attr *attr, int attr_mask,
4257 struct hns_roce_v2_qp_context *context,
4258 struct hns_roce_v2_qp_context *qpc_mask)
4259{
4260 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4261 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4262 struct device *dev = hr_dev->dev;
8d18ad83
LO		 4263 u64 sge_cur_blk = 0;
4264 u64 sq_cur_blk = 0;
befb63b4 4265 u32 page_size;
8d18ad83 4266 int count;
926a01dc
WHX		 4267
4268 /* Search qp buf's mtts */
8d18ad83
LO		 4269 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
4270 if (count < 1) {
4271 dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn);
926a01dc
WHX		 4272 return -EINVAL;
4273 }
4274
8d18ad83
LO		 4275 if (hr_qp->sge.offset) {
4276 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
4277 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
4278 hr_qp->sge.offset / page_size,
4279 &sge_cur_blk, 1, NULL);
4280 if (count < 1) {
4281 dev_err(dev, "qp(0x%lx) sge pa find failed\n",
4282 hr_qp->qpn);
4283 return -EINVAL;
4284 }
4285 }
4286
734f3863 4287 /* Not support alternate path and path migration */
4288 if ((attr_mask & IB_QP_ALT_PATH) ||
4289 (attr_mask & IB_QP_PATH_MIG_STATE)) {
926a01dc
WHX		 4290 dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
4291 return -EINVAL;
4292 }
4293
4294 /*
4295 * In v2 engine, software pass context and context mask to hardware
4296 * when modifying qp. If software need modify some fields in context,
4297 * we should set all bits of the relevant fields in context mask to
4298 * 0 at the same time, else set them to 0x1.
4299 */
bfe86035 4300 context->sq_cur_blk_addr = cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 4301 roce_set_field(context->byte_168_irrl_idx,
4302 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
4303 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
8d18ad83 4304 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 4305 qpc_mask->sq_cur_blk_addr = 0;
4306 roce_set_field(qpc_mask->byte_168_irrl_idx,
4307 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
4308 V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
4309
2a3d923f
LO		 4310 context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
4311 hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
bfe86035
LC		 4312 cpu_to_le32(sge_cur_blk >>
4313 PAGE_ADDR_SHIFT) : 0;
befb63b4 4314 roce_set_field(context->byte_184_irrl_idx,
4315 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
4316 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
2a3d923f
LO		 4317 ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
4318 HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
8d18ad83 4319 (sge_cur_blk >>
befb63b4 4320 (32 + PAGE_ADDR_SHIFT)) : 0);
4321 qpc_mask->sq_cur_sge_blk_addr = 0;
4322 roce_set_field(qpc_mask->byte_184_irrl_idx,
4323 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
4324 V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
4325
bfe86035
LC		 4326 context->rx_sq_cur_blk_addr =
4327 cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
926a01dc
WHX		 4328 roce_set_field(context->byte_232_irrl_sge,
4329 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
4330 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
8d18ad83 4331 sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
926a01dc
WHX		 4332 qpc_mask->rx_sq_cur_blk_addr = 0;
4333 roce_set_field(qpc_mask->byte_232_irrl_sge,
4334 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
4335 V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
4336
4337 /*
4338 * Set some fields in context to zero, Because the default values
4339 * of all fields in context are zero, we need not set them to 0 again.
4340 * but we should set the relevant fields of context mask to 0.
4341 */
4342 roce_set_field(qpc_mask->byte_232_irrl_sge,
4343 V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
4344 V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
4345
4346 roce_set_field(qpc_mask->byte_240_irrl_tail,
4347 V2_QPC_BYTE_240_RX_ACK_MSN_M,
4348 V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
4349
926a01dc
WHX		 4350 roce_set_field(qpc_mask->byte_248_ack_psn,
4351 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
4352 V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
4353 roce_set_bit(qpc_mask->byte_248_ack_psn,
4354 V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
4355 roce_set_field(qpc_mask->byte_248_ack_psn,
4356 V2_QPC_BYTE_248_IRRL_PSN_M,
4357 V2_QPC_BYTE_248_IRRL_PSN_S, 0);
4358
4359 roce_set_field(qpc_mask->byte_240_irrl_tail,
4360 V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
4361 V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
4362
926a01dc
WHX		 4363 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4364 V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
4365 V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
4366
4367 roce_set_bit(qpc_mask->byte_248_ack_psn,
4368 V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
4369
4370 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
4371 V2_QPC_BYTE_212_CHECK_FLG_S, 0);
4372
926a01dc
WHX		 4373 roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4374 V2_QPC_BYTE_212_LSN_S, 0x100);
4375 roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
4376 V2_QPC_BYTE_212_LSN_S, 0);
4377
926a01dc
WHX		 4378 roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
4379 V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
926a01dc
WHX		 4380
4381 return 0;
4382}
4383
233673e4
LO		 4384static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,
4385 enum ib_qp_state new_state)
4386{
4387
4388 if ((cur_state != IB_QPS_RESET &&
4389 (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||
4390 ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &&
4391 (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||
4392 (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS))
4393 return true;
4394
4395 return false;
4396
4397}
4398
606bf89e
LO		 4399static int hns_roce_v2_set_path(struct ib_qp *ibqp,
4400 const struct ib_qp_attr *attr,
4401 int attr_mask,
4402 struct hns_roce_v2_qp_context *context,
4403 struct hns_roce_v2_qp_context *qpc_mask)
926a01dc 4404{
606bf89e 4405 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
926a01dc
WHX		 4406 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4407 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
606bf89e
LO		 4408 const struct ib_gid_attr *gid_attr = NULL;
4409 int is_roce_protocol;
32883228 4410 u16 vlan_id = 0xffff;
606bf89e 4411 bool is_udp = false;
606bf89e
LO		 4412 u8 ib_port;
4413 u8 hr_port;
4414 int ret;
926a01dc 4415
606bf89e
LO		 4416 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
4417 hr_port = ib_port - 1;
4418 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4419 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4420
4421 if (is_roce_protocol) {
4422 gid_attr = attr->ah_attr.grh.sgid_attr;
32883228 4423 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
606bf89e
LO		 4424 if (ret)
4425 return ret;
4426
4427 if (gid_attr)
4428 is_udp = (gid_attr->gid_type ==
4429 IB_GID_TYPE_ROCE_UDP_ENCAP);
4430 }
4431
32883228 4432 if (vlan_id < VLAN_N_VID) {
606bf89e
LO		 4433 roce_set_bit(context->byte_76_srqn_op_en,
4434 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
4435 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4436 V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
4437 roce_set_bit(context->byte_168_irrl_idx,
4438 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
4439 roce_set_bit(qpc_mask->byte_168_irrl_idx,
4440 V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
4441 }
4442
4443 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
32883228 4444 V2_QPC_BYTE_24_VLAN_ID_S, vlan_id);
606bf89e
LO		 4445 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
4446 V2_QPC_BYTE_24_VLAN_ID_S, 0);
4447
4448 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4449 dev_err(hr_dev->dev, "sgid_index(%u) too large. max is %d\n",
4450 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
4451 return -EINVAL;
4452 }
4453
4454 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4455 dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
4456 return -EINVAL;
4457 }
4458
4459 roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4460 V2_QPC_BYTE_52_UDPSPN_S,
4461 is_udp ? 0x12b7 : 0);
4462
4463 roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
4464 V2_QPC_BYTE_52_UDPSPN_S, 0);
4465
4466 roce_set_field(context->byte_20_smac_sgid_idx,
4467 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S,
4468 grh->sgid_index);
4469
4470 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4471 V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
4472
4473 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4474 V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
4475 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
4476 V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4477
b14c95be 4478 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_B && is_udp)
606bf89e
LO		 4479 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4480 V2_QPC_BYTE_24_TC_S, grh->traffic_class >> 2);
4481 else
4482 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4483 V2_QPC_BYTE_24_TC_S, grh->traffic_class);
4484 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4485 V2_QPC_BYTE_24_TC_S, 0);
4486 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4487 V2_QPC_BYTE_28_FL_S, grh->flow_label);
4488 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4489 V2_QPC_BYTE_28_FL_S, 0);
4490 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4491 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4492 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4493 V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr));
4494 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4495 V2_QPC_BYTE_28_SL_S, 0);
4496 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4497
4498 return 0;
4499}
4500
4501static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
4502 const struct ib_qp_attr *attr,
4503 int attr_mask,
4504 enum ib_qp_state cur_state,
4505 enum ib_qp_state new_state,
4506 struct hns_roce_v2_qp_context *context,
4507 struct hns_roce_v2_qp_context *qpc_mask)
4508{
4509 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4510 int ret = 0;
926a01dc 4511
926a01dc 4512 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
9f507101 4513 memset(qpc_mask, 0, sizeof(*qpc_mask));
0fa95a9a 4514 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4515 qpc_mask);
926a01dc
WHX		 4516 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4517 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4518 qpc_mask);
4519 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4520 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4521 qpc_mask);
4522 if (ret)
4523 goto out;
4524 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4525 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4526 qpc_mask);
4527 if (ret)
4528 goto out;
233673e4 4529 } else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {
926a01dc
WHX		 4530 /* Nothing */
4531 ;
4532 } else {
606bf89e 4533 dev_err(hr_dev->dev, "Illegal state for QP!\n");
ac7cbf96 4534 ret = -EINVAL;
926a01dc
WHX		 4535 goto out;
4536 }
4537
606bf89e
LO		 4538out:
4539 return ret;
4540}
9c6ccc03 4541
606bf89e
LO		 4542static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
4543 const struct ib_qp_attr *attr,
4544 int attr_mask,
4545 struct hns_roce_v2_qp_context *context,
4546 struct hns_roce_v2_qp_context *qpc_mask)
4547{
4548 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4549 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4550 int ret = 0;
0425e3e6 4551
610b8967 4552 if (attr_mask & IB_QP_AV) {
606bf89e
LO		 4553 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
4554 qpc_mask);
4555 if (ret)
4556 return ret;
610b8967
LO		 4557 }
4558
5b01b243
LO		 4559 if (attr_mask & IB_QP_TIMEOUT) {
4560 if (attr->timeout < 31) {
4561 roce_set_field(context->byte_28_at_fl,
4562 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4563 attr->timeout);
4564 roce_set_field(qpc_mask->byte_28_at_fl,
4565 V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4566 0);
4567 } else {
606bf89e
LO		 4568 dev_warn(hr_dev->dev,
4569 "Local ACK timeout shall be 0 to 30.\n");
5b01b243
LO		 4570 }
4571 }
4572
4573 if (attr_mask & IB_QP_RETRY_CNT) {
4574 roce_set_field(context->byte_212_lsn,
4575 V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4576 V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
4577 attr->retry_cnt);
4578 roce_set_field(qpc_mask->byte_212_lsn,
4579 V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4580 V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
4581
4582 roce_set_field(context->byte_212_lsn,
4583 V2_QPC_BYTE_212_RETRY_CNT_M,
60262b10 4584 V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
5b01b243
LO		 4585 roce_set_field(qpc_mask->byte_212_lsn,
4586 V2_QPC_BYTE_212_RETRY_CNT_M,
4587 V2_QPC_BYTE_212_RETRY_CNT_S, 0);
4588 }
4589
4590 if (attr_mask & IB_QP_RNR_RETRY) {
4591 roce_set_field(context->byte_244_rnr_rxack,
4592 V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4593 V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
4594 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4595 V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4596 V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
4597
4598 roce_set_field(context->byte_244_rnr_rxack,
4599 V2_QPC_BYTE_244_RNR_CNT_M,
4600 V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
4601 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4602 V2_QPC_BYTE_244_RNR_CNT_M,
4603 V2_QPC_BYTE_244_RNR_CNT_S, 0);
4604 }
4605
606bf89e 4606 /* RC&UC&UD required attr */
f04cc178
LO		 4607 if (attr_mask & IB_QP_SQ_PSN) {
4608 roce_set_field(context->byte_172_sq_psn,
4609 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4610 V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
4611 roce_set_field(qpc_mask->byte_172_sq_psn,
4612 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4613 V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
4614
4615 roce_set_field(context->byte_196_sq_psn,
4616 V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4617 V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
4618 roce_set_field(qpc_mask->byte_196_sq_psn,
4619 V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4620 V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
4621
4622 roce_set_field(context->byte_220_retry_psn_msn,
4623 V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4624 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
4625 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4626 V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4627 V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
4628
4629 roce_set_field(context->byte_224_retry_msg,
4630 V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4631 V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
2a3d923f 4632 attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
f04cc178
LO		 4633 roce_set_field(qpc_mask->byte_224_retry_msg,
4634 V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4635 V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
4636
4637 roce_set_field(context->byte_224_retry_msg,
4638 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4639 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
4640 attr->sq_psn);
4641 roce_set_field(qpc_mask->byte_224_retry_msg,
4642 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4643 V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
4644
4645 roce_set_field(context->byte_244_rnr_rxack,
4646 V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4647 V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
4648 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4649 V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4650 V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
4651 }
4652
5b01b243
LO		 4653 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4654 attr->max_dest_rd_atomic) {
4655 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4656 V2_QPC_BYTE_140_RR_MAX_S,
4657 fls(attr->max_dest_rd_atomic - 1));
4658 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4659 V2_QPC_BYTE_140_RR_MAX_S, 0);
4660 }
4661
4662 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4663 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
4664 V2_QPC_BYTE_208_SR_MAX_S,
4665 fls(attr->max_rd_atomic - 1));
4666 roce_set_field(qpc_mask->byte_208_irrl,
4667 V2_QPC_BYTE_208_SR_MAX_M,
4668 V2_QPC_BYTE_208_SR_MAX_S, 0);
4669 }
4670
ace1c541 4671 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4672 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4673
5b01b243
LO		 4674 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4675 roce_set_field(context->byte_80_rnr_rx_cqn,
4676 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4677 V2_QPC_BYTE_80_MIN_RNR_TIME_S,
4678 attr->min_rnr_timer);
4679 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
4680 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4681 V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
4682 }
4683
601f3e6d
LO		 4684 /* RC&UC required attr */
4685 if (attr_mask & IB_QP_RQ_PSN) {
4686 roce_set_field(context->byte_108_rx_reqepsn,
4687 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4688 V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
4689 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4690 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4691 V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
4692
4693 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
4694 V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
4695 roce_set_field(qpc_mask->byte_152_raq,
4696 V2_QPC_BYTE_152_RAQ_PSN_M,
4697 V2_QPC_BYTE_152_RAQ_PSN_S, 0);
4698 }
4699
5b01b243 4700 if (attr_mask & IB_QP_QKEY) {
bfe86035 4701 context->qkey_xrcd = cpu_to_le32(attr->qkey);
5b01b243
LO		 4702 qpc_mask->qkey_xrcd = 0;
4703 hr_qp->qkey = attr->qkey;
4704 }
4705
606bf89e
LO		 4706 return ret;
4707}
4708
4709static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
4710 const struct ib_qp_attr *attr,
4711 int attr_mask)
4712{
4713 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4714 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4715
4716 if (attr_mask & IB_QP_ACCESS_FLAGS)
4717 hr_qp->atomic_rd_en = attr->qp_access_flags;
4718
4719 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4720 hr_qp->resp_depth = attr->max_dest_rd_atomic;
4721 if (attr_mask & IB_QP_PORT) {
4722 hr_qp->port = attr->port_num - 1;
4723 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4724 }
4725}
4726
4727static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
4728 const struct ib_qp_attr *attr,
4729 int attr_mask, enum ib_qp_state cur_state,
4730 enum ib_qp_state new_state)
4731{
4732 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4733 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4b42d05d
LC		 4734 struct hns_roce_v2_qp_context ctx[2];
4735 struct hns_roce_v2_qp_context *context = ctx;
4736 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
606bf89e 4737 struct device *dev = hr_dev->dev;
b5c229dc 4738 int ret;
606bf89e 4739
606bf89e
LO		 4740 /*
4741 * In v2 engine, software pass context and context mask to hardware
4742 * when modifying qp. If software need modify some fields in context,
4743 * we should set all bits of the relevant fields in context mask to
4744 * 0 at the same time, else set them to 0x1.
4745 */
4b42d05d 4746 memset(context, 0, sizeof(*context));
606bf89e
LO		 4747 memset(qpc_mask, 0xff, sizeof(*qpc_mask));
4748 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
4749 new_state, context, qpc_mask);
4750 if (ret)
4751 goto out;
4752
4753 /* When QP state is err, SQ and RQ WQE should be flushed */
4754 if (new_state == IB_QPS_ERR) {
4755 roce_set_field(context->byte_160_sq_ci_pi,
4756 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4757 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
4758 hr_qp->sq.head);
4759 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
4760 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4761 V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
4762
4763 if (!ibqp->srq) {
4764 roce_set_field(context->byte_84_rq_ci_pi,
4765 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4766 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
4767 hr_qp->rq.head);
4768 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4769 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4770 V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4771 }
4772 }
4773
4774 /* Configure the optional fields */
4775 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
4776 qpc_mask);
4777 if (ret)
4778 goto out;
4779
c7bcb134
LO		 4780 roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
4781 ibqp->srq ? 1 : 0);
4782 roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4783 V2_QPC_BYTE_108_INV_CREDIT_S, 0);
4784
926a01dc 4785 /* Every status migrate must change state */
2362ccee 4786 roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
926a01dc 4787 V2_QPC_BYTE_60_QP_ST_S, new_state);
2362ccee 4788 roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
926a01dc
WHX		 4789 V2_QPC_BYTE_60_QP_ST_S, 0);
4790
4791 /* SW pass context to HW */
032b0574 4792 ret = hns_roce_v2_qp_modify(hr_dev, ctx, hr_qp);
926a01dc
WHX		 4793 if (ret) {
4794 dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
4795 goto out;
4796 }
4797
4798 hr_qp->state = new_state;
4799
606bf89e 4800 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
926a01dc
WHX		 4801
4802 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
4803 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
4804 ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
4805 if (ibqp->send_cq != ibqp->recv_cq)
4806 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4807 hr_qp->qpn, NULL);
4808
4809 hr_qp->rq.head = 0;
4810 hr_qp->rq.tail = 0;
4811 hr_qp->sq.head = 0;
4812 hr_qp->sq.tail = 0;
926a01dc 4813 hr_qp->next_sge = 0;
e088a685
YL		 4814 if (hr_qp->rq.wqe_cnt)
4815 *hr_qp->rdb.db_record = 0;
926a01dc
WHX		 4816 }
4817
4818out:
926a01dc
WHX		 4819 return ret;
4820}
4821
4822static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
4823{
4824 switch (state) {
4825 case HNS_ROCE_QP_ST_RST: return IB_QPS_RESET;
4826 case HNS_ROCE_QP_ST_INIT: return IB_QPS_INIT;
4827 case HNS_ROCE_QP_ST_RTR: return IB_QPS_RTR;
4828 case HNS_ROCE_QP_ST_RTS: return IB_QPS_RTS;
4829 case HNS_ROCE_QP_ST_SQ_DRAINING:
4830 case HNS_ROCE_QP_ST_SQD: return IB_QPS_SQD;
4831 case HNS_ROCE_QP_ST_SQER: return IB_QPS_SQE;
4832 case HNS_ROCE_QP_ST_ERR: return IB_QPS_ERR;
4833 default: return -1;
4834 }
4835}
4836
4837static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
4838 struct hns_roce_qp *hr_qp,
4839 struct hns_roce_v2_qp_context *hr_context)
4840{
4841 struct hns_roce_cmd_mailbox *mailbox;
4842 int ret;
4843
4844 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4845 if (IS_ERR(mailbox))
4846 return PTR_ERR(mailbox);
4847
4848 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
4849 HNS_ROCE_CMD_QUERY_QPC,
4850 HNS_ROCE_CMD_TIMEOUT_MSECS);
4851 if (ret) {
4852 dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
4853 goto out;
4854 }
4855
4856 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
4857
4858out:
4859 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4860 return ret;
4861}
4862
4863static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4864 int qp_attr_mask,
4865 struct ib_qp_init_attr *qp_init_attr)
4866{
4867 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4868 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4b42d05d 4869 struct hns_roce_v2_qp_context context = {};
926a01dc
WHX		 4870 struct device *dev = hr_dev->dev;
4871 int tmp_qp_state;
4872 int state;
4873 int ret;
4874
926a01dc
WHX		 4875 memset(qp_attr, 0, sizeof(*qp_attr));
4876 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4877
4878 mutex_lock(&hr_qp->mutex);
4879
4880 if (hr_qp->state == IB_QPS_RESET) {
4881 qp_attr->qp_state = IB_QPS_RESET;
63ea641f 4882 ret = 0;
926a01dc
WHX		 4883 goto done;
4884 }
4885
4b42d05d 4886 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
926a01dc
WHX		 4887 if (ret) {
4888 dev_err(dev, "query qpc error\n");
4889 ret = -EINVAL;
4890 goto out;
4891 }
4892
4b42d05d 4893 state = roce_get_field(context.byte_60_qpst_tempid,
926a01dc
WHX		 4894 V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
4895 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
4896 if (tmp_qp_state == -1) {
4897 dev_err(dev, "Illegal ib_qp_state\n");
4898 ret = -EINVAL;
4899 goto out;
4900 }
4901 hr_qp->state = (u8)tmp_qp_state;
4902 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
4b42d05d 4903 qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context.byte_24_mtu_tc,
926a01dc
WHX		 4904 V2_QPC_BYTE_24_MTU_M,
4905 V2_QPC_BYTE_24_MTU_S);
4906 qp_attr->path_mig_state = IB_MIG_ARMED;
2bf910d4 4907 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
926a01dc
WHX		 4908 if (hr_qp->ibqp.qp_type == IB_QPT_UD)
4909 qp_attr->qkey = V2_QKEY_VAL;
4910
4b42d05d 4911 qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
926a01dc
WHX		 4912 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4913 V2_QPC_BYTE_108_RX_REQ_EPSN_S);
4b42d05d 4914 qp_attr->sq_psn = (u32)roce_get_field(context.byte_172_sq_psn,
926a01dc
WHX		 4915 V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4916 V2_QPC_BYTE_172_SQ_CUR_PSN_S);
4b42d05d 4917 qp_attr->dest_qp_num = (u8)roce_get_field(context.byte_56_dqpn_err,
926a01dc
WHX		 4918 V2_QPC_BYTE_56_DQPN_M,
4919 V2_QPC_BYTE_56_DQPN_S);
4b42d05d 4920 qp_attr->qp_access_flags = ((roce_get_bit(context.byte_76_srqn_op_en,
98c09b8c 4921 V2_QPC_BYTE_76_RRE_S)) << V2_QP_RRE_S) |
4b42d05d 4922 ((roce_get_bit(context.byte_76_srqn_op_en,
98c09b8c 4923 V2_QPC_BYTE_76_RWE_S)) << V2_QP_RWE_S) |
4b42d05d 4924 ((roce_get_bit(context.byte_76_srqn_op_en,
2a3d923f
LO		 4925 V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
4926
926a01dc
WHX		 4927 if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
4928 hr_qp->ibqp.qp_type == IB_QPT_UC) {
4929 struct ib_global_route *grh =
4930 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
4931
4932 rdma_ah_set_sl(&qp_attr->ah_attr,
4b42d05d 4933 roce_get_field(context.byte_28_at_fl,
926a01dc
WHX		 4934 V2_QPC_BYTE_28_SL_M,
4935 V2_QPC_BYTE_28_SL_S));
4b42d05d 4936 grh->flow_label = roce_get_field(context.byte_28_at_fl,
926a01dc
WHX		 4937 V2_QPC_BYTE_28_FL_M,
4938 V2_QPC_BYTE_28_FL_S);
4b42d05d 4939 grh->sgid_index = roce_get_field(context.byte_20_smac_sgid_idx,
926a01dc
WHX		 4940 V2_QPC_BYTE_20_SGID_IDX_M,
4941 V2_QPC_BYTE_20_SGID_IDX_S);
4b42d05d 4942 grh->hop_limit = roce_get_field(context.byte_24_mtu_tc,
926a01dc
WHX		 4943 V2_QPC_BYTE_24_HOP_LIMIT_M,
4944 V2_QPC_BYTE_24_HOP_LIMIT_S);
4b42d05d 4945 grh->traffic_class = roce_get_field(context.byte_24_mtu_tc,
926a01dc
WHX		 4946 V2_QPC_BYTE_24_TC_M,
4947 V2_QPC_BYTE_24_TC_S);
4948
4b42d05d 4949 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
926a01dc
WHX		 4950 }
4951
4952 qp_attr->port_num = hr_qp->port + 1;
4953 qp_attr->sq_draining = 0;
4b42d05d 4954 qp_attr->max_rd_atomic = 1 << roce_get_field(context.byte_208_irrl,
926a01dc
WHX		 4955 V2_QPC_BYTE_208_SR_MAX_M,
4956 V2_QPC_BYTE_208_SR_MAX_S);
4b42d05d 4957 qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context.byte_140_raq,
926a01dc
WHX		 4958 V2_QPC_BYTE_140_RR_MAX_M,
4959 V2_QPC_BYTE_140_RR_MAX_S);
4b42d05d 4960 qp_attr->min_rnr_timer = (u8)roce_get_field(context.byte_80_rnr_rx_cqn,
926a01dc
WHX		 4961 V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4962 V2_QPC_BYTE_80_MIN_RNR_TIME_S);
4b42d05d 4963 qp_attr->timeout = (u8)roce_get_field(context.byte_28_at_fl,
926a01dc
WHX		 4964 V2_QPC_BYTE_28_AT_M,
4965 V2_QPC_BYTE_28_AT_S);
4b42d05d 4966 qp_attr->retry_cnt = roce_get_field(context.byte_212_lsn,
926a01dc
WHX		 4967 V2_QPC_BYTE_212_RETRY_CNT_M,
4968 V2_QPC_BYTE_212_RETRY_CNT_S);
bfe86035 4969 qp_attr->rnr_retry = le32_to_cpu(context.rq_rnr_timer);
926a01dc
WHX		 4970
4971done:
4972 qp_attr->cur_qp_state = qp_attr->qp_state;
4973 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
4974 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
4975
4976 if (!ibqp->uobject) {
4977 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
4978 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
4979 } else {
4980 qp_attr->cap.max_send_wr = 0;
4981 qp_attr->cap.max_send_sge = 0;
4982 }
4983
4984 qp_init_attr->cap = qp_attr->cap;
4985
4986out:
4987 mutex_unlock(&hr_qp->mutex);
926a01dc
WHX		 4988 return ret;
4989}
4990
4991static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
4992 struct hns_roce_qp *hr_qp,
bdeacabd 4993 struct ib_udata *udata)
926a01dc
WHX		 4994{
4995 struct hns_roce_cq *send_cq, *recv_cq;
db50077b 4996 struct ib_device *ibdev = &hr_dev->ib_dev;
626903e9 4997 unsigned long flags;
d302c6e3 4998 int ret = 0;
926a01dc
WHX		 4999
5000 if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
5001 /* Modify qp to reset before destroying qp */
5002 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
5003 hr_qp->state, IB_QPS_RESET);
d302c6e3 5004 if (ret)
db50077b 5005 ibdev_err(ibdev, "modify QP to Reset failed.\n");
926a01dc
WHX		 5006 }
5007
626903e9
XW		 5008 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
5009 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
926a01dc 5010
626903e9 5011 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
926a01dc
WHX		 5012 hns_roce_lock_cqs(send_cq, recv_cq);
5013
626903e9
XW		 5014 list_del(&hr_qp->node);
5015 list_del(&hr_qp->sq_node);
5016 list_del(&hr_qp->rq_node);
5017
bdeacabd 5018 if (!udata) {
626903e9
XW		 5019 if (recv_cq)
5020 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
5021 (hr_qp->ibqp.srq ?
5022 to_hr_srq(hr_qp->ibqp.srq) :
5023 NULL));
5024
5025 if (send_cq && send_cq != recv_cq)
926a01dc 5026 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
626903e9 5027
926a01dc
WHX		 5028 }
5029
5030 hns_roce_qp_remove(hr_dev, hr_qp);
5031
5032 hns_roce_unlock_cqs(send_cq, recv_cq);
626903e9 5033 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
926a01dc
WHX		 5034
5035 hns_roce_qp_free(hr_dev, hr_qp);
5036
5037 /* Not special_QP, free their QPN */
5038 if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
5039 (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
5040 (hr_qp->ibqp.qp_type == IB_QPT_UD))
5041 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
5042
6fafe560 5043 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
926a01dc 5044
bdeacabd
SR		 5045 if (udata) {
5046 struct hns_roce_ucontext *context =
5047 rdma_udata_to_drv_context(
5048 udata,
5049 struct hns_roce_ucontext,
5050 ibucontext);
5051
0425e3e6 5052 if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
bdeacabd 5053 hns_roce_db_unmap_user(context, &hr_qp->sdb);
0425e3e6 5054
e088a685 5055 if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
bdeacabd 5056 hns_roce_db_unmap_user(context, &hr_qp->rdb);
926a01dc
WHX		 5057 } else {
5058 kfree(hr_qp->sq.wrid);
5059 kfree(hr_qp->rq.wrid);
5060 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
472bc0fb
YL		 5061 if (hr_qp->rq.wqe_cnt)
5062 hns_roce_free_db(hr_dev, &hr_qp->rdb);
926a01dc 5063 }
836a0fbb 5064 ib_umem_release(hr_qp->umem);
926a01dc 5065
c7bcb134
LO		 5066 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
5067 hr_qp->rq.wqe_cnt) {
0009c2db 5068 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
5069 kfree(hr_qp->rq_inl_buf.wqe_list);
5070 }
5071
d302c6e3 5072 return ret;
926a01dc
WHX		 5073}
5074
c4367a26 5075static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
926a01dc
WHX		 5076{
5077 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5078 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5079 int ret;
5080
bdeacabd 5081 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
d302c6e3 5082 if (ret)
db50077b
LO		 5083 ibdev_err(&hr_dev->ib_dev, "Destroy qp 0x%06lx failed(%d)\n",
5084 hr_qp->qpn, ret);
926a01dc 5085
16a11e0b 5086 kfree(hr_qp);
926a01dc
WHX		 5087
5088 return 0;
5089}
5090
aa84fa18
YL		 5091static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
5092 struct hns_roce_qp *hr_qp)
5093{
da91ddfd 5094 struct hns_roce_sccc_clr_done *resp;
aa84fa18
YL		 5095 struct hns_roce_sccc_clr *clr;
5096 struct hns_roce_cmq_desc desc;
5097 int ret, i;
5098
5099 mutex_lock(&hr_dev->qp_table.scc_mutex);
5100
5101 /* set scc ctx clear done flag */
5102 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
aa84fa18
YL		 5103 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5104 if (ret) {
5105 dev_err(hr_dev->dev, "Reset SCC ctx failed(%d)\n", ret);
5106 goto out;
5107 }
5108
5109 /* clear scc context */
5110 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
5111 clr = (struct hns_roce_sccc_clr *)desc.data;
5112 clr->qpn = cpu_to_le32(hr_qp->qpn);
5113 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5114 if (ret) {
5115 dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret);
5116 goto out;
5117 }
5118
5119 /* query scc context clear is done or not */
5120 resp = (struct hns_roce_sccc_clr_done *)desc.data;
5121 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
5122 hns_roce_cmq_setup_basic_desc(&desc,
5123 HNS_ROCE_OPC_QUERY_SCCC, true);
5124 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5125 if (ret) {
5126 dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret);
5127 goto out;
5128 }
5129
5130 if (resp->clr_done)
5131 goto out;
5132
5133 msleep(20);
5134 }
5135
5136 dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n");
5137 ret = -ETIMEDOUT;
5138
5139out:
5140 mutex_unlock(&hr_dev->qp_table.scc_mutex);
5141 return ret;
5142}
5143
b156269d 5144static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
5145{
5146 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
5147 struct hns_roce_v2_cq_context *cq_context;
5148 struct hns_roce_cq *hr_cq = to_hr_cq(cq);
5149 struct hns_roce_v2_cq_context *cqc_mask;
5150 struct hns_roce_cmd_mailbox *mailbox;
5151 int ret;
5152
5153 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5154 if (IS_ERR(mailbox))
5155 return PTR_ERR(mailbox);
5156
5157 cq_context = mailbox->buf;
5158 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
5159
5160 memset(cqc_mask, 0xff, sizeof(*cqc_mask));
5161
5162 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
5163 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
5164 cq_count);
5165 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
5166 V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
5167 0);
5168 roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
5169 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
5170 cq_period);
5171 roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
5172 V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
5173 0);
5174
5175 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
5176 HNS_ROCE_CMD_MODIFY_CQC,
5177 HNS_ROCE_CMD_TIMEOUT_MSECS);
5178 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5179 if (ret)
5180 dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
5181
5182 return ret;
5183}
5184
0425e3e6
YL		 5185static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
5186{
5187 struct hns_roce_qp *hr_qp;
5188 struct ib_qp_attr attr;
5189 int attr_mask;
5190 int ret;
5191
5192 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
5193 if (!hr_qp) {
5194 dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
5195 return;
5196 }
5197
5198 if (hr_qp->ibqp.uobject) {
5199 if (hr_qp->sdb_en == 1) {
5200 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
9c6ccc03
LO		 5201 if (hr_qp->rdb_en == 1)
5202 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
0425e3e6
YL		 5203 } else {
5204 dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
5205 return;
5206 }
5207 }
5208
5209 attr_mask = IB_QP_STATE;
5210 attr.qp_state = IB_QPS_ERR;
5211 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
5212 hr_qp->state, IB_QPS_ERR);
5213 if (ret)
5214 dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
5215 qpn);
5216}
5217
5218static void hns_roce_irq_work_handle(struct work_struct *work)
5219{
5220 struct hns_roce_work *irq_work =
5221 container_of(work, struct hns_roce_work, work);
b00a92c8 5222 struct device *dev = irq_work->hr_dev->dev;
0425e3e6 5223 u32 qpn = irq_work->qpn;
b00a92c8 5224 u32 cqn = irq_work->cqn;
0425e3e6
YL		 5225
5226 switch (irq_work->event_type) {
b00a92c8 5227 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
5228 dev_info(dev, "Path migrated succeeded.\n");
5229 break;
5230 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
5231 dev_warn(dev, "Path migration failed.\n");
5232 break;
5233 case HNS_ROCE_EVENT_TYPE_COMM_EST:
b00a92c8 5234 break;
5235 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5236 dev_warn(dev, "Send queue drained.\n");
5237 break;
0425e3e6 5238 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
e95c716c
YL		 5239 dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n",
5240 qpn, irq_work->sub_type);
b00a92c8 5241 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
b00a92c8 5242 break;
0425e3e6 5243 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
e95c716c
YL		 5244 dev_err(dev, "Invalid request local work queue 0x%x error.\n",
5245 qpn);
b00a92c8 5246 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
5247 break;
0425e3e6 5248 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
e95c716c
YL		 5249 dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n",
5250 qpn, irq_work->sub_type);
0425e3e6 5251 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
b00a92c8 5252 break;
5253 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
5254 dev_warn(dev, "SRQ limit reach.\n");
5255 break;
5256 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
5257 dev_warn(dev, "SRQ last wqe reach.\n");
5258 break;
5259 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
5260 dev_err(dev, "SRQ catas error.\n");
5261 break;
5262 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5263 dev_err(dev, "CQ 0x%x access err.\n", cqn);
5264 break;
5265 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
5266 dev_warn(dev, "CQ 0x%x overflow\n", cqn);
5267 break;
5268 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
5269 dev_warn(dev, "DB overflow.\n");
5270 break;
5271 case HNS_ROCE_EVENT_TYPE_FLR:
5272 dev_warn(dev, "Function level reset.\n");
0425e3e6
YL		 5273 break;
5274 default:
5275 break;
5276 }
5277
5278 kfree(irq_work);
5279}
5280
5281static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
b00a92c8 5282 struct hns_roce_eq *eq,
5283 u32 qpn, u32 cqn)
0425e3e6
YL		 5284{
5285 struct hns_roce_work *irq_work;
5286
5287 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
5288 if (!irq_work)
5289 return;
5290
5291 INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
5292 irq_work->hr_dev = hr_dev;
5293 irq_work->qpn = qpn;
b00a92c8 5294 irq_work->cqn = cqn;
0425e3e6
YL		 5295 irq_work->event_type = eq->event_type;
5296 irq_work->sub_type = eq->sub_type;
5297 queue_work(hr_dev->irq_workq, &(irq_work->work));
5298}
5299
a5073d60
YL		 5300static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
5301{
d3743fa9 5302 struct hns_roce_dev *hr_dev = eq->hr_dev;
880f133c 5303 __le32 doorbell[2] = {};
a5073d60
YL		 5304
5305 if (eq->type_flag == HNS_ROCE_AEQ) {
5306 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
5307 HNS_ROCE_V2_EQ_DB_CMD_S,
5308 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
5309 HNS_ROCE_EQ_DB_CMD_AEQ :
5310 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
5311 } else {
5312 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
5313 HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
5314
5315 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
5316 HNS_ROCE_V2_EQ_DB_CMD_S,
5317 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
5318 HNS_ROCE_EQ_DB_CMD_CEQ :
5319 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
5320 }
5321
5322 roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
5323 HNS_ROCE_V2_EQ_DB_PARA_S,
5324 (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
5325
d3743fa9 5326 hns_roce_write64(hr_dev, doorbell, eq->doorbell);
a5073d60
YL		 5327}
5328
a5073d60
YL		 5329static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
5330{
5331 u32 buf_chk_sz;
5332 unsigned long off;
5333
5334 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5335 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
5336
5337 return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
5338 off % buf_chk_sz);
5339}
5340
5341static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
5342{
5343 u32 buf_chk_sz;
5344 unsigned long off;
5345
5346 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5347
5348 off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
5349
5350 if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
5351 return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
5352 off % buf_chk_sz);
5353 else
5354 return (struct hns_roce_aeqe *)((u8 *)
5355 (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
5356}
5357
5358static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
5359{
5360 struct hns_roce_aeqe *aeqe;
5361
5362 if (!eq->hop_num)
5363 aeqe = get_aeqe_v2(eq, eq->cons_index);
5364 else
5365 aeqe = mhop_get_aeqe(eq, eq->cons_index);
5366
5367 return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
5368 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
5369}
5370
5371static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
5372 struct hns_roce_eq *eq)
5373{
5374 struct device *dev = hr_dev->dev;
e7f40440 5375 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
a5073d60
YL		 5376 int aeqe_found = 0;
5377 int event_type;
0425e3e6 5378 int sub_type;
81fce629 5379 u32 srqn;
0425e3e6
YL		 5380 u32 qpn;
5381 u32 cqn;
a5073d60 5382
e7f40440 5383 while (aeqe) {
4044a3f4
YL		 5384 /* Make sure we read AEQ entry after we have checked the
5385 * ownership bit
5386 */
5387 dma_rmb();
a5073d60
YL		 5388
5389 event_type = roce_get_field(aeqe->asyn,
5390 HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
5391 HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
0425e3e6
YL		 5392 sub_type = roce_get_field(aeqe->asyn,
5393 HNS_ROCE_V2_AEQE_SUB_TYPE_M,
5394 HNS_ROCE_V2_AEQE_SUB_TYPE_S);
5395 qpn = roce_get_field(aeqe->event.qp_event.qp,
5396 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5397 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
5398 cqn = roce_get_field(aeqe->event.cq_event.cq,
5399 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5400 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
81fce629
LO		 5401 srqn = roce_get_field(aeqe->event.srq_event.srq,
5402 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
5403 HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
a5073d60
YL		 5404
5405 switch (event_type) {
5406 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
a5073d60 5407 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
a5073d60
YL		 5408 case HNS_ROCE_EVENT_TYPE_COMM_EST:
5409 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
5410 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
81fce629 5411 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
a5073d60
YL		 5412 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
5413 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
b00a92c8 5414 hns_roce_qp_event(hr_dev, qpn, event_type);
a5073d60
YL		 5415 break;
5416 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
a5073d60 5417 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
81fce629 5418 hns_roce_srq_event(hr_dev, srqn, event_type);
a5073d60
YL		 5419 break;
5420 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
5421 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
b00a92c8 5422 hns_roce_cq_event(hr_dev, cqn, event_type);
a5073d60
YL		 5423 break;
5424 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
a5073d60
YL		 5425 break;
5426 case HNS_ROCE_EVENT_TYPE_MB:
5427 hns_roce_cmd_event(hr_dev,
5428 le16_to_cpu(aeqe->event.cmd.token),
5429 aeqe->event.cmd.status,
5430 le64_to_cpu(aeqe->event.cmd.out_param));
5431 break;
5432 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
a5073d60
YL		 5433 break;
5434 case HNS_ROCE_EVENT_TYPE_FLR:
a5073d60
YL		 5435 break;
5436 default:
5437 dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
5438 event_type, eq->eqn, eq->cons_index);
5439 break;
790b57f6 5440 }
a5073d60 5441
0425e3e6
YL		 5442 eq->event_type = event_type;
5443 eq->sub_type = sub_type;
a5073d60
YL		 5444 ++eq->cons_index;
5445 aeqe_found = 1;
5446
249f2f92 5447 if (eq->cons_index > (2 * eq->entries - 1))
a5073d60 5448 eq->cons_index = 0;
249f2f92 5449
b00a92c8 5450 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
e7f40440
LC		 5451
5452 aeqe = next_aeqe_sw_v2(eq);
a5073d60
YL		 5453 }
5454
5455 set_eq_cons_index_v2(eq);
5456 return aeqe_found;
5457}
5458
5459static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
5460{
5461 u32 buf_chk_sz;
5462 unsigned long off;
5463
5464 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5465 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5466
5467 return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
5468 off % buf_chk_sz);
5469}
5470
5471static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
5472{
5473 u32 buf_chk_sz;
5474 unsigned long off;
5475
5476 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5477
5478 off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5479
5480 if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
5481 return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
5482 off % buf_chk_sz);
5483 else
5484 return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
5485 buf_chk_sz]) + off % buf_chk_sz);
5486}
5487
5488static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5489{
5490 struct hns_roce_ceqe *ceqe;
5491
5492 if (!eq->hop_num)
5493 ceqe = get_ceqe_v2(eq, eq->cons_index);
5494 else
5495 ceqe = mhop_get_ceqe(eq, eq->cons_index);
5496
5497 return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5498 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5499}
5500
5501static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5502 struct hns_roce_eq *eq)
5503{
5504 struct device *dev = hr_dev->dev;
e7f40440 5505 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
a5073d60
YL		 5506 int ceqe_found = 0;
5507 u32 cqn;
5508
e7f40440 5509 while (ceqe) {
4044a3f4
YL		 5510 /* Make sure we read CEQ entry after we have checked the
5511 * ownership bit
5512 */
5513 dma_rmb();
5514
60262b10 5515 cqn = roce_get_field(ceqe->comp, HNS_ROCE_V2_CEQE_COMP_CQN_M,
a5073d60
YL		 5516 HNS_ROCE_V2_CEQE_COMP_CQN_S);
5517
5518 hns_roce_cq_completion(hr_dev, cqn);
5519
5520 ++eq->cons_index;
5521 ceqe_found = 1;
5522
90c559b1 5523 if (eq->cons_index > (EQ_DEPTH_COEFF * eq->entries - 1)) {
a5073d60
YL		 5524 dev_warn(dev, "cons_index overflow, set back to 0.\n");
5525 eq->cons_index = 0;
5526 }
e7f40440
LC		 5527
5528 ceqe = next_ceqe_sw_v2(eq);
a5073d60
YL		 5529 }
5530
5531 set_eq_cons_index_v2(eq);
5532
5533 return ceqe_found;
5534}
5535
5536static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5537{
5538 struct hns_roce_eq *eq = eq_ptr;
5539 struct hns_roce_dev *hr_dev = eq->hr_dev;
5540 int int_work = 0;
5541
5542 if (eq->type_flag == HNS_ROCE_CEQ)
5543 /* Completion event interrupt */
5544 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5545 else
5546 /* Asychronous event interrupt */
5547 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5548
5549 return IRQ_RETVAL(int_work);
5550}
5551
5552static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5553{
5554 struct hns_roce_dev *hr_dev = dev_id;
5555 struct device *dev = hr_dev->dev;
5556 int int_work = 0;
5557 u32 int_st;
5558 u32 int_en;
5559
5560 /* Abnormal interrupt */
5561 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5562 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5563
bfe86035 5564 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
2b9acb9a
XT		 5565 struct pci_dev *pdev = hr_dev->pci_dev;
5566 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5567 const struct hnae3_ae_ops *ops = ae_dev->ops;
5568
a5073d60
YL		 5569 dev_err(dev, "AEQ overflow!\n");
5570
bfe86035 5571 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S;
a5073d60
YL		 5572 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5573
2b9acb9a
XT		 5574 /* Set reset level for reset_event() */
5575 if (ops->set_default_reset_request)
5576 ops->set_default_reset_request(ae_dev,
5577 HNAE3_FUNC_RESET);
5578 if (ops->reset_event)
5579 ops->reset_event(pdev, NULL);
5580
bfe86035 5581 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
a5073d60
YL		 5582 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5583
5584 int_work = 1;
bfe86035 5585 } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
a5073d60
YL		 5586 dev_err(dev, "BUS ERR!\n");
5587
bfe86035 5588 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S;
a5073d60
YL		 5589 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5590
bfe86035 5591 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
a5073d60
YL		 5592 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5593
5594 int_work = 1;
bfe86035 5595 } else if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
a5073d60
YL		 5596 dev_err(dev, "OTHER ERR!\n");
5597
bfe86035 5598 int_st |= 1 << HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S;
a5073d60
YL		 5599 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5600
bfe86035 5601 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
a5073d60
YL		 5602 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5603
5604 int_work = 1;
5605 } else
5606 dev_err(dev, "There is no abnormal irq found!\n");
5607
5608 return IRQ_RETVAL(int_work);
5609}
5610
5611static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5612 int eq_num, int enable_flag)
5613{
5614 int i;
5615
5616 if (enable_flag == EQ_ENABLE) {
5617 for (i = 0; i < eq_num; i++)
5618 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5619 i * EQ_REG_OFFSET,
5620 HNS_ROCE_V2_VF_EVENT_INT_EN_M);
5621
5622 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5623 HNS_ROCE_V2_VF_ABN_INT_EN_M);
5624 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5625 HNS_ROCE_V2_VF_ABN_INT_CFG_M);
5626 } else {
5627 for (i = 0; i < eq_num; i++)
5628 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5629 i * EQ_REG_OFFSET,
5630 HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
5631
5632 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5633 HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
5634 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5635 HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
5636 }
5637}
5638
5639static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5640{
5641 struct device *dev = hr_dev->dev;
5642 int ret;
5643
5644 if (eqn < hr_dev->caps.num_comp_vectors)
5645 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5646 0, HNS_ROCE_CMD_DESTROY_CEQC,
5647 HNS_ROCE_CMD_TIMEOUT_MSECS);
5648 else
5649 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5650 0, HNS_ROCE_CMD_DESTROY_AEQC,
5651 HNS_ROCE_CMD_TIMEOUT_MSECS);
5652 if (ret)
5653 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5654}
5655
5656static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
5657 struct hns_roce_eq *eq)
5658{
5659 struct device *dev = hr_dev->dev;
5660 u64 idx;
5661 u64 size;
5662 u32 buf_chk_sz;
5663 u32 bt_chk_sz;
5664 u32 mhop_num;
5665 int eqe_alloc;
a5073d60
YL		 5666 int i = 0;
5667 int j = 0;
5668
5669 mhop_num = hr_dev->caps.eqe_hop_num;
5670 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5671 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
a5073d60 5672
a5073d60
YL		 5673 if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5674 dma_free_coherent(dev, (unsigned int)(eq->entries *
5675 eq->eqe_size), eq->bt_l0, eq->l0_dma);
5676 return;
5677 }
5678
a5073d60
YL		 5679 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5680 if (mhop_num == 1) {
5681 for (i = 0; i < eq->l0_last_num; i++) {
5682 if (i == eq->l0_last_num - 1) {
5683 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5684 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5685 dma_free_coherent(dev, size, eq->buf[i],
5686 eq->buf_dma[i]);
5687 break;
5688 }
5689 dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5690 eq->buf_dma[i]);
5691 }
5692 } else if (mhop_num == 2) {
5693 for (i = 0; i < eq->l0_last_num; i++) {
5694 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5695 eq->l1_dma[i]);
5696
2a3d923f
LO		 5697 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5698 idx = i * (bt_chk_sz / BA_BYTE_LEN) + j;
a5073d60
YL		 5699 if ((i == eq->l0_last_num - 1)
5700 && j == eq->l1_last_num - 1) {
5701 eqe_alloc = (buf_chk_sz / eq->eqe_size)
5702 * idx;
5703 size = (eq->entries - eqe_alloc)
5704 * eq->eqe_size;
5705 dma_free_coherent(dev, size,
5706 eq->buf[idx],
5707 eq->buf_dma[idx]);
5708 break;
5709 }
5710 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5711 eq->buf_dma[idx]);
5712 }
5713 }
5714 }
5715 kfree(eq->buf_dma);
5716 kfree(eq->buf);
5717 kfree(eq->l1_dma);
5718 kfree(eq->bt_l1);
5719 eq->buf_dma = NULL;
5720 eq->buf = NULL;
5721 eq->l1_dma = NULL;
5722 eq->bt_l1 = NULL;
5723}
5724
5725static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
5726 struct hns_roce_eq *eq)
5727{
5728 u32 buf_chk_sz;
5729
5730 buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5731
5732 if (hr_dev->caps.eqe_hop_num) {
5733 hns_roce_mhop_free_eq(hr_dev, eq);
5734 return;
5735 }
5736
b681a052
LO		 5737 dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf,
5738 eq->buf_list->map);
5739 kfree(eq->buf_list);
a5073d60
YL		 5740}
5741
5742static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
5743 struct hns_roce_eq *eq,
5744 void *mb_buf)
5745{
5746 struct hns_roce_eq_context *eqc;
5747
5748 eqc = mb_buf;
5749 memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5750
5751 /* init eqc */
5752 eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5753 eq->hop_num = hr_dev->caps.eqe_hop_num;
5754 eq->cons_index = 0;
5755 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5756 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5757 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5758 eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
5759 eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
5760 eq->shift = ilog2((unsigned int)eq->entries);
5761
5762 if (!eq->hop_num)
5763 eq->eqe_ba = eq->buf_list->map;
5764 else
5765 eq->eqe_ba = eq->l0_dma;
5766
5767 /* set eqc state */
60262b10 5768 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQ_ST_M, HNS_ROCE_EQC_EQ_ST_S,
a5073d60
YL		 5769 HNS_ROCE_V2_EQ_STATE_VALID);
5770
5771 /* set eqe hop num */
60262b10 5772 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_HOP_NUM_M,
a5073d60
YL		 5773 HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
5774
5775 /* set eqc over_ignore */
60262b10 5776 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_OVER_IGNORE_M,
a5073d60
YL		 5777 HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
5778
5779 /* set eqc coalesce */
60262b10 5780 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_COALESCE_M,
a5073d60
YL		 5781 HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
5782
5783 /* set eqc arm_state */
60262b10 5784 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_ARM_ST_M,
a5073d60
YL		 5785 HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
5786
5787 /* set eqn */
60262b10
LO		 5788 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQN_M, HNS_ROCE_EQC_EQN_S,
5789 eq->eqn);
a5073d60
YL		 5790
5791 /* set eqe_cnt */
60262b10
LO		 5792 roce_set_field(eqc->byte_4, HNS_ROCE_EQC_EQE_CNT_M,
5793 HNS_ROCE_EQC_EQE_CNT_S, HNS_ROCE_EQ_INIT_EQE_CNT);
a5073d60
YL		 5794
5795 /* set eqe_ba_pg_sz */
60262b10 5796 roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M,
5e6e78db
YL		 5797 HNS_ROCE_EQC_BA_PG_SZ_S,
5798 eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
a5073d60
YL		 5799
5800 /* set eqe_buf_pg_sz */
60262b10 5801 roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M,
5e6e78db
YL		 5802 HNS_ROCE_EQC_BUF_PG_SZ_S,
5803 eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
a5073d60
YL		 5804
5805 /* set eq_producer_idx */
60262b10
LO		 5806 roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M,
5807 HNS_ROCE_EQC_PROD_INDX_S, HNS_ROCE_EQ_INIT_PROD_IDX);
a5073d60
YL		 5808
5809 /* set eq_max_cnt */
60262b10 5810 roce_set_field(eqc->byte_12, HNS_ROCE_EQC_MAX_CNT_M,
a5073d60
YL		 5811 HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
5812
5813 /* set eq_period */
60262b10 5814 roce_set_field(eqc->byte_12, HNS_ROCE_EQC_PERIOD_M,
a5073d60
YL		 5815 HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
5816
5817 /* set eqe_report_timer */
60262b10 5818 roce_set_field(eqc->eqe_report_timer, HNS_ROCE_EQC_REPORT_TIMER_M,
a5073d60
YL		 5819 HNS_ROCE_EQC_REPORT_TIMER_S,
5820 HNS_ROCE_EQ_INIT_REPORT_TIMER);
5821
5822 /* set eqe_ba [34:3] */
60262b10 5823 roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M,
a5073d60
YL		 5824 HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
5825
5826 /* set eqe_ba [64:35] */
60262b10 5827 roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M,
a5073d60
YL		 5828 HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
5829
5830 /* set eq shift */
60262b10
LO		 5831 roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S,
5832 eq->shift);
a5073d60
YL		 5833
5834 /* set eq MSI_IDX */
60262b10
LO		 5835 roce_set_field(eqc->byte_28, HNS_ROCE_EQC_MSI_INDX_M,
5836 HNS_ROCE_EQC_MSI_INDX_S, HNS_ROCE_EQ_INIT_MSI_IDX);
a5073d60
YL		 5837
5838 /* set cur_eqe_ba [27:12] */
60262b10 5839 roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M,
a5073d60
YL		 5840 HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
5841
5842 /* set cur_eqe_ba [59:28] */
60262b10 5843 roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M,
a5073d60
YL		 5844 HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
5845
5846 /* set cur_eqe_ba [63:60] */
60262b10 5847 roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M,
a5073d60
YL		 5848 HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
5849
5850 /* set eq consumer idx */
60262b10
LO		 5851 roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M,
5852 HNS_ROCE_EQC_CONS_INDX_S, HNS_ROCE_EQ_INIT_CONS_IDX);
a5073d60
YL		 5853
5854 /* set nex_eqe_ba[43:12] */
60262b10 5855 roce_set_field(eqc->nxt_eqe_ba0, HNS_ROCE_EQC_NXT_EQE_BA_L_M,
a5073d60
YL		 5856 HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
5857
5858 /* set nex_eqe_ba[63:44] */
60262b10 5859 roce_set_field(eqc->nxt_eqe_ba1, HNS_ROCE_EQC_NXT_EQE_BA_H_M,
a5073d60
YL		 5860 HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
5861}
5862
5863static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
5864 struct hns_roce_eq *eq)
5865{
5866 struct device *dev = hr_dev->dev;
5867 int eq_alloc_done = 0;
5868 int eq_buf_cnt = 0;
5869 int eqe_alloc;
5870 u32 buf_chk_sz;
5871 u32 bt_chk_sz;
5872 u32 mhop_num;
5873 u64 size;
5874 u64 idx;
5875 int ba_num;
5876 int bt_num;
5877 int record_i;
5878 int record_j;
5879 int i = 0;
5880 int j = 0;
5881
5882 mhop_num = hr_dev->caps.eqe_hop_num;
5883 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5884 bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5885
2a3d923f
LO		 5886 ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
5887 buf_chk_sz);
5888 bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN);
a5073d60 5889
a5073d60
YL		 5890 if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5891 if (eq->entries > buf_chk_sz / eq->eqe_size) {
5892 dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
5893 eq->entries);
5894 return -EINVAL;
5895 }
5896 eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
5897 &(eq->l0_dma), GFP_KERNEL);
5898 if (!eq->bt_l0)
5899 return -ENOMEM;
5900
5901 eq->cur_eqe_ba = eq->l0_dma;
5902 eq->nxt_eqe_ba = 0;
5903
a5073d60
YL		 5904 return 0;
5905 }
5906
5907 eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
5908 if (!eq->buf_dma)
5909 return -ENOMEM;
5910 eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
5911 if (!eq->buf)
5912 goto err_kcalloc_buf;
5913
5914 if (mhop_num == 2) {
5915 eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
5916 if (!eq->l1_dma)
5917 goto err_kcalloc_l1_dma;
5918
5919 eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
5920 if (!eq->bt_l1)
5921 goto err_kcalloc_bt_l1;
5922 }
5923
5924 /* alloc L0 BT */
5925 eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
5926 if (!eq->bt_l0)
5927 goto err_dma_alloc_l0;
5928
5929 if (mhop_num == 1) {
2a3d923f 5930 if (ba_num > (bt_chk_sz / BA_BYTE_LEN))
a5073d60
YL		 5931 dev_err(dev, "ba_num %d is too large for 1 hop\n",
5932 ba_num);
5933
5934 /* alloc buf */
2a3d923f 5935 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
a5073d60
YL		 5936 if (eq_buf_cnt + 1 < ba_num) {
5937 size = buf_chk_sz;
5938 } else {
5939 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5940 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5941 }
750afb08 5942 eq->buf[i] = dma_alloc_coherent(dev, size,
a5073d60
YL		 5943 &(eq->buf_dma[i]),
5944 GFP_KERNEL);
5945 if (!eq->buf[i])
5946 goto err_dma_alloc_buf;
5947
a5073d60
YL		 5948 *(eq->bt_l0 + i) = eq->buf_dma[i];
5949
5950 eq_buf_cnt++;
5951 if (eq_buf_cnt >= ba_num)
5952 break;
5953 }
5954 eq->cur_eqe_ba = eq->buf_dma[0];
bf8c02f9
XW		 5955 if (ba_num > 1)
5956 eq->nxt_eqe_ba = eq->buf_dma[1];
a5073d60
YL		 5957
5958 } else if (mhop_num == 2) {
5959 /* alloc L1 BT and buf */
2a3d923f 5960 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
a5073d60
YL		 5961 eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
5962 &(eq->l1_dma[i]),
5963 GFP_KERNEL);
5964 if (!eq->bt_l1[i])
5965 goto err_dma_alloc_l1;
5966 *(eq->bt_l0 + i) = eq->l1_dma[i];
5967
2a3d923f
LO		 5968 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5969 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
a5073d60
YL		 5970 if (eq_buf_cnt + 1 < ba_num) {
5971 size = buf_chk_sz;
5972 } else {
5973 eqe_alloc = (buf_chk_sz / eq->eqe_size)
5974 * idx;
5975 size = (eq->entries - eqe_alloc)
5976 * eq->eqe_size;
5977 }
750afb08
LC		 5978 eq->buf[idx] = dma_alloc_coherent(dev, size,
5979 &(eq->buf_dma[idx]),
5980 GFP_KERNEL);
a5073d60
YL		 5981 if (!eq->buf[idx])
5982 goto err_dma_alloc_buf;
5983
a5073d60
YL		 5984 *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
5985
5986 eq_buf_cnt++;
5987 if (eq_buf_cnt >= ba_num) {
5988 eq_alloc_done = 1;
5989 break;
5990 }
5991 }
5992
5993 if (eq_alloc_done)
5994 break;
5995 }
5996 eq->cur_eqe_ba = eq->buf_dma[0];
bf8c02f9
XW		 5997 if (ba_num > 1)
5998 eq->nxt_eqe_ba = eq->buf_dma[1];
a5073d60
YL		 5999 }
6000
6001 eq->l0_last_num = i + 1;
6002 if (mhop_num == 2)
6003 eq->l1_last_num = j + 1;
6004
6005 return 0;
6006
6007err_dma_alloc_l1:
6008 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
6009 eq->bt_l0 = NULL;
6010 eq->l0_dma = 0;
6011 for (i -= 1; i >= 0; i--) {
6012 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
6013 eq->l1_dma[i]);
6014
2a3d923f
LO		 6015 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
6016 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
a5073d60
YL		 6017 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
6018 eq->buf_dma[idx]);
6019 }
6020 }
6021 goto err_dma_alloc_l0;
6022
6023err_dma_alloc_buf:
6024 dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
6025 eq->bt_l0 = NULL;
6026 eq->l0_dma = 0;
6027
6028 if (mhop_num == 1)
38759d61 6029 for (i -= 1; i >= 0; i--)
a5073d60
YL		 6030 dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
6031 eq->buf_dma[i]);
6032 else if (mhop_num == 2) {
6033 record_i = i;
6034 record_j = j;
6035 for (; i >= 0; i--) {
6036 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
6037 eq->l1_dma[i]);
6038
2a3d923f 6039 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
a5073d60
YL		 6040 if (i == record_i && j >= record_j)
6041 break;
6042
2a3d923f 6043 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
a5073d60
YL		 6044 dma_free_coherent(dev, buf_chk_sz,
6045 eq->buf[idx],
6046 eq->buf_dma[idx]);
6047 }
6048 }
6049 }
6050
6051err_dma_alloc_l0:
6052 kfree(eq->bt_l1);
6053 eq->bt_l1 = NULL;
6054
6055err_kcalloc_bt_l1:
6056 kfree(eq->l1_dma);
6057 eq->l1_dma = NULL;
6058
6059err_kcalloc_l1_dma:
6060 kfree(eq->buf);
6061 eq->buf = NULL;
6062
6063err_kcalloc_buf:
6064 kfree(eq->buf_dma);
6065 eq->buf_dma = NULL;
6066
6067 return -ENOMEM;
6068}
6069
6070static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
6071 struct hns_roce_eq *eq,
6072 unsigned int eq_cmd)
6073{
6074 struct device *dev = hr_dev->dev;
6075 struct hns_roce_cmd_mailbox *mailbox;
6076 u32 buf_chk_sz = 0;
6077 int ret;
6078
6079 /* Allocate mailbox memory */
6080 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6081 if (IS_ERR(mailbox))
6082 return PTR_ERR(mailbox);
6083
6084 if (!hr_dev->caps.eqe_hop_num) {
6085 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
6086
6087 eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
6088 GFP_KERNEL);
6089 if (!eq->buf_list) {
6090 ret = -ENOMEM;
6091 goto free_cmd_mbox;
6092 }
6093
750afb08 6094 eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
a5073d60
YL		 6095 &(eq->buf_list->map),
6096 GFP_KERNEL);
6097 if (!eq->buf_list->buf) {
6098 ret = -ENOMEM;
6099 goto err_alloc_buf;
6100 }
6101
a5073d60
YL		 6102 } else {
6103 ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
6104 if (ret) {
6105 ret = -ENOMEM;
6106 goto free_cmd_mbox;
6107 }
6108 }
6109
6110 hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
6111
6112 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
6113 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
6114 if (ret) {
ab178849 6115 dev_err(dev, "[mailbox cmd] create eqc failed.\n");
a5073d60
YL		 6116 goto err_cmd_mbox;
6117 }
6118
6119 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6120
6121 return 0;
6122
6123err_cmd_mbox:
6124 if (!hr_dev->caps.eqe_hop_num)
6125 dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
6126 eq->buf_list->map);
6127 else {
6128 hns_roce_mhop_free_eq(hr_dev, eq);
6129 goto free_cmd_mbox;
6130 }
6131
6132err_alloc_buf:
6133 kfree(eq->buf_list);
6134
6135free_cmd_mbox:
6136 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6137
6138 return ret;
6139}
6140
33db6f94
YL		 6141static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
6142 int comp_num, int aeq_num, int other_num)
6143{
6144 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6145 int i, j;
6146 int ret;
6147
6148 for (i = 0; i < irq_num; i++) {
6149 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
6150 GFP_KERNEL);
6151 if (!hr_dev->irq_names[i]) {
6152 ret = -ENOMEM;
6153 goto err_kzalloc_failed;
6154 }
6155 }
6156
6def7de6 6157 /* irq contains: abnormal + AEQ + CEQ */
bebdb83f 6158 for (j = 0; j < other_num; j++)
60262b10
LO		 6159 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6160 "hns-abn-%d", j);
bebdb83f
LC		 6161
6162 for (j = other_num; j < (other_num + aeq_num); j++)
60262b10
LO		 6163 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6164 "hns-aeq-%d", j - other_num);
bebdb83f
LC		 6165
6166 for (j = (other_num + aeq_num); j < irq_num; j++)
60262b10
LO		 6167 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6168 "hns-ceq-%d", j - other_num - aeq_num);
33db6f94
YL		 6169
6170 for (j = 0; j < irq_num; j++) {
6171 if (j < other_num)
6172 ret = request_irq(hr_dev->irq[j],
6173 hns_roce_v2_msix_interrupt_abn,
6174 0, hr_dev->irq_names[j], hr_dev);
6175
6176 else if (j < (other_num + comp_num))
6177 ret = request_irq(eq_table->eq[j - other_num].irq,
6178 hns_roce_v2_msix_interrupt_eq,
6179 0, hr_dev->irq_names[j + aeq_num],
6180 &eq_table->eq[j - other_num]);
6181 else
6182 ret = request_irq(eq_table->eq[j - other_num].irq,
6183 hns_roce_v2_msix_interrupt_eq,
6184 0, hr_dev->irq_names[j - comp_num],
6185 &eq_table->eq[j - other_num]);
6186 if (ret) {
6187 dev_err(hr_dev->dev, "Request irq error!\n");
6188 goto err_request_failed;
6189 }
6190 }
6191
6192 return 0;
6193
6194err_request_failed:
6195 for (j -= 1; j >= 0; j--)
6196 if (j < other_num)
6197 free_irq(hr_dev->irq[j], hr_dev);
6198 else
6199 free_irq(eq_table->eq[j - other_num].irq,
6200 &eq_table->eq[j - other_num]);
6201
6202err_kzalloc_failed:
6203 for (i -= 1; i >= 0; i--)
6204 kfree(hr_dev->irq_names[i]);
6205
6206 return ret;
6207}
6208
6209static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
6210{
6211 int irq_num;
6212 int eq_num;
6213 int i;
6214
6215 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6216 irq_num = eq_num + hr_dev->caps.num_other_vectors;
6217
6218 for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
6219 free_irq(hr_dev->irq[i], hr_dev);
6220
6221 for (i = 0; i < eq_num; i++)
6222 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
6223
6224 for (i = 0; i < irq_num; i++)
6225 kfree(hr_dev->irq_names[i]);
6226}
6227
a5073d60
YL		 6228static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
6229{
6230 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6231 struct device *dev = hr_dev->dev;
6232 struct hns_roce_eq *eq;
6233 unsigned int eq_cmd;
6234 int irq_num;
6235 int eq_num;
6236 int other_num;
6237 int comp_num;
6238 int aeq_num;
33db6f94 6239 int i;
a5073d60
YL		 6240 int ret;
6241
6242 other_num = hr_dev->caps.num_other_vectors;
6243 comp_num = hr_dev->caps.num_comp_vectors;
6244 aeq_num = hr_dev->caps.num_aeq_vectors;
6245
6246 eq_num = comp_num + aeq_num;
6247 irq_num = eq_num + other_num;
6248
6249 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
6250 if (!eq_table->eq)
6251 return -ENOMEM;
6252
a5073d60 6253 /* create eq */
33db6f94
YL		 6254 for (i = 0; i < eq_num; i++) {
6255 eq = &eq_table->eq[i];
a5073d60 6256 eq->hr_dev = hr_dev;
33db6f94
YL		 6257 eq->eqn = i;
6258 if (i < comp_num) {
a5073d60
YL		 6259 /* CEQ */
6260 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
6261 eq->type_flag = HNS_ROCE_CEQ;
6262 eq->entries = hr_dev->caps.ceqe_depth;
6263 eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
33db6f94 6264 eq->irq = hr_dev->irq[i + other_num + aeq_num];
a5073d60
YL		 6265 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
6266 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
6267 } else {
6268 /* AEQ */
6269 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
6270 eq->type_flag = HNS_ROCE_AEQ;
6271 eq->entries = hr_dev->caps.aeqe_depth;
6272 eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
33db6f94 6273 eq->irq = hr_dev->irq[i - comp_num + other_num];
a5073d60
YL		 6274 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
6275 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
6276 }
6277
6278 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
6279 if (ret) {
6280 dev_err(dev, "eq create failed.\n");
6281 goto err_create_eq_fail;
6282 }
6283 }
6284
6285 /* enable irq */
6286 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
6287
33db6f94
YL		 6288 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num,
6289 aeq_num, other_num);
6290 if (ret) {
6291 dev_err(dev, "Request irq failed.\n");
6292 goto err_request_irq_fail;
a5073d60
YL		 6293 }
6294
0425e3e6
YL		 6295 hr_dev->irq_workq =
6296 create_singlethread_workqueue("hns_roce_irq_workqueue");
6297 if (!hr_dev->irq_workq) {
6298 dev_err(dev, "Create irq workqueue failed!\n");
f1a31542 6299 ret = -ENOMEM;
33db6f94 6300 goto err_create_wq_fail;
0425e3e6
YL		 6301 }
6302
a5073d60
YL		 6303 return 0;
6304
33db6f94
YL		 6305err_create_wq_fail:
6306 __hns_roce_free_irq(hr_dev);
6307
a5073d60 6308err_request_irq_fail:
33db6f94 6309 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
a5073d60
YL		 6310
6311err_create_eq_fail:
a5073d60 6312 for (i -= 1; i >= 0; i--)
33db6f94 6313 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
a5073d60
YL		 6314 kfree(eq_table->eq);
6315
6316 return ret;
6317}
6318
6319static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
6320{
6321 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
a5073d60
YL		 6322 int eq_num;
6323 int i;
6324
6325 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
a5073d60
YL		 6326
6327 /* Disable irq */
6328 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6329
33db6f94 6330 __hns_roce_free_irq(hr_dev);
a5073d60
YL		 6331
6332 for (i = 0; i < eq_num; i++) {
6333 hns_roce_v2_destroy_eqc(hr_dev, i);
6334
a5073d60
YL		 6335 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
6336 }
6337
a5073d60 6338 kfree(eq_table->eq);
0425e3e6
YL		 6339
6340 flush_workqueue(hr_dev->irq_workq);
6341 destroy_workqueue(hr_dev->irq_workq);
a5073d60
YL		 6342}
6343
c7bcb134
LO		 6344static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
6345 struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
6346 u32 cqn, void *mb_buf, u64 *mtts_wqe,
6347 u64 *mtts_idx, dma_addr_t dma_handle_wqe,
6348 dma_addr_t dma_handle_idx)
6349{
6350 struct hns_roce_srq_context *srq_context;
6351
6352 srq_context = mb_buf;
6353 memset(srq_context, 0, sizeof(*srq_context));
6354
6355 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
6356 SRQC_BYTE_4_SRQ_ST_S, 1);
6357
6358 roce_set_field(srq_context->byte_4_srqn_srqst,
6359 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
6360 SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
6361 (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
6362 hr_dev->caps.srqwqe_hop_num));
6363 roce_set_field(srq_context->byte_4_srqn_srqst,
6364 SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
d938d785 6365 ilog2(srq->wqe_cnt));
c7bcb134
LO		 6366
6367 roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
6368 SRQC_BYTE_4_SRQN_S, srq->srqn);
6369
6370 roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6371 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
6372
6373 roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
6374 SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
6375
6376 srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
6377
6378 roce_set_field(srq_context->byte_24_wqe_bt_ba,
6379 SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
6380 SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
bfe86035 6381 dma_handle_wqe >> 35);
c7bcb134
LO		 6382
6383 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
6384 SRQC_BYTE_28_PD_S, pdn);
6385 roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
6386 SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
6387 fls(srq->max_gs - 1));
6388
bfe86035 6389 srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
c7bcb134
LO		 6390 roce_set_field(srq_context->rsv_idx_bt_ba,
6391 SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
6392 SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
bfe86035 6393 dma_handle_idx >> 35);
c7bcb134 6394
c7bcb134 6395 srq_context->idx_cur_blk_addr =
bfe86035 6396 cpu_to_le32(mtts_idx[0] >> PAGE_ADDR_SHIFT);
c7bcb134
LO		 6397 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6398 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
6399 SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
bfe86035 6400 mtts_idx[0] >> (32 + PAGE_ADDR_SHIFT));
c7bcb134
LO		 6401 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6402 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
6403 SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
6404 hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
6405 hr_dev->caps.idx_hop_num);
6406
6407 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6408 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
6409 SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
5c7e76fb 6410 hr_dev->caps.idx_ba_pg_sz + PG_SHIFT_OFFSET);
c7bcb134
LO		 6411 roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
6412 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
6413 SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
5c7e76fb 6414 hr_dev->caps.idx_buf_pg_sz + PG_SHIFT_OFFSET);
c7bcb134 6415
c7bcb134 6416 srq_context->idx_nxt_blk_addr =
bfe86035 6417 cpu_to_le32(mtts_idx[1] >> PAGE_ADDR_SHIFT);
c7bcb134
LO		 6418 roce_set_field(srq_context->rsv_idxnxtblkaddr,
6419 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
6420 SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
bfe86035 6421 mtts_idx[1] >> (32 + PAGE_ADDR_SHIFT));
c7bcb134
LO		 6422 roce_set_field(srq_context->byte_56_xrc_cqn,
6423 SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
6424 cqn);
6425 roce_set_field(srq_context->byte_56_xrc_cqn,
6426 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
6427 SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
6428 hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
6429 roce_set_field(srq_context->byte_56_xrc_cqn,
6430 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
6431 SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
6432 hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
6433
6434 roce_set_bit(srq_context->db_record_addr_record_en,
6435 SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
6436}
6437
6438static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
6439 struct ib_srq_attr *srq_attr,
6440 enum ib_srq_attr_mask srq_attr_mask,
6441 struct ib_udata *udata)
6442{
6443 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6444 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6445 struct hns_roce_srq_context *srq_context;
6446 struct hns_roce_srq_context *srqc_mask;
6447 struct hns_roce_cmd_mailbox *mailbox;
6448 int ret;
6449
6450 if (srq_attr_mask & IB_SRQ_LIMIT) {
d938d785 6451 if (srq_attr->srq_limit >= srq->wqe_cnt)
c7bcb134
LO		 6452 return -EINVAL;
6453
6454 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6455 if (IS_ERR(mailbox))
6456 return PTR_ERR(mailbox);
6457
6458 srq_context = mailbox->buf;
6459 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
6460
6461 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
6462
6463 roce_set_field(srq_context->byte_8_limit_wl,
6464 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6465 SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
6466 roce_set_field(srqc_mask->byte_8_limit_wl,
6467 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6468 SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
6469
6470 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
6471 HNS_ROCE_CMD_MODIFY_SRQC,
6472 HNS_ROCE_CMD_TIMEOUT_MSECS);
6473 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6474 if (ret) {
6475 dev_err(hr_dev->dev,
6476 "MODIFY SRQ Failed to cmd mailbox.\n");
6477 return ret;
6478 }
6479 }
6480
6481 return 0;
6482}
6483
c3c668e7 6484static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
c7bcb134
LO		 6485{
6486 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6487 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6488 struct hns_roce_srq_context *srq_context;
6489 struct hns_roce_cmd_mailbox *mailbox;
6490 int limit_wl;
6491 int ret;
6492
6493 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6494 if (IS_ERR(mailbox))
6495 return PTR_ERR(mailbox);
6496
6497 srq_context = mailbox->buf;
6498 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
6499 HNS_ROCE_CMD_QUERY_SRQC,
6500 HNS_ROCE_CMD_TIMEOUT_MSECS);
6501 if (ret) {
6502 dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
6503 goto out;
6504 }
6505
6506 limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
6507 SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6508 SRQC_BYTE_8_SRQ_LIMIT_WL_S);
6509
6510 attr->srq_limit = limit_wl;
d938d785 6511 attr->max_wr = srq->wqe_cnt - 1;
c7bcb134
LO		 6512 attr->max_sge = srq->max_gs;
6513
6514 memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
6515
6516out:
6517 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6518 return ret;
6519}
6520
97545b10
LO		 6521static int find_empty_entry(struct hns_roce_idx_que *idx_que,
6522 unsigned long size)
c7bcb134 6523{
97545b10 6524 int wqe_idx;
c7bcb134 6525
97545b10
LO		 6526 if (unlikely(bitmap_full(idx_que->bitmap, size)))
6527 return -ENOSPC;
6528
6529 wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
6530
6531 bitmap_set(idx_que->bitmap, wqe_idx, 1);
c7bcb134 6532
97545b10 6533 return wqe_idx;
c7bcb134
LO		 6534}
6535
6536static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
6537 int cur_idx, int wqe_idx)
6538{
6539 unsigned int *addr;
6540
6541 addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
6542 cur_idx * idx_que->entry_sz);
6543 *addr = wqe_idx;
6544}
6545
6546static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
6547 const struct ib_recv_wr *wr,
6548 const struct ib_recv_wr **bad_wr)
6549{
d3743fa9 6550 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
c7bcb134
LO		 6551 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6552 struct hns_roce_v2_wqe_data_seg *dseg;
6553 struct hns_roce_v2_db srq_db;
6554 unsigned long flags;
6555 int ret = 0;
6556 int wqe_idx;
6557 void *wqe;
6558 int nreq;
6559 int ind;
6560 int i;
6561
6562 spin_lock_irqsave(&srq->lock, flags);
6563
d938d785 6564 ind = srq->head & (srq->wqe_cnt - 1);
c7bcb134
LO		 6565
6566 for (nreq = 0; wr; ++nreq, wr = wr->next) {
6567 if (unlikely(wr->num_sge > srq->max_gs)) {
6568 ret = -EINVAL;
6569 *bad_wr = wr;
6570 break;
6571 }
6572
6573 if (unlikely(srq->head == srq->tail)) {
6574 ret = -ENOMEM;
6575 *bad_wr = wr;
6576 break;
6577 }
6578
d938d785 6579 wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
97545b10
LO		 6580 if (wqe_idx < 0) {
6581 ret = -ENOMEM;
6582 *bad_wr = wr;
6583 break;
6584 }
6585
c7bcb134
LO		 6586 fill_idx_queue(&srq->idx_que, ind, wqe_idx);
6587 wqe = get_srq_wqe(srq, wqe_idx);
6588 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
6589
6590 for (i = 0; i < wr->num_sge; ++i) {
6591 dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
6592 dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
6593 dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
6594 }
6595
6596 if (i < srq->max_gs) {
4f18904c
LO		 6597 dseg[i].len = 0;
6598 dseg[i].lkey = cpu_to_le32(0x100);
6599 dseg[i].addr = 0;
c7bcb134
LO		 6600 }
6601
6602 srq->wrid[wqe_idx] = wr->wr_id;
d938d785 6603 ind = (ind + 1) & (srq->wqe_cnt - 1);
c7bcb134
LO		 6604 }
6605
6606 if (likely(nreq)) {
6607 srq->head += nreq;
6608
6609 /*
6610 * Make sure that descriptors are written before
6611 * doorbell record.
6612 */
6613 wmb();
6614
bfe86035
LC		 6615 srq_db.byte_4 =
6616 cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
6617 (srq->srqn & V2_DB_BYTE_4_TAG_M));
6618 srq_db.parameter = cpu_to_le32(srq->head);
c7bcb134 6619
d3743fa9 6620 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
c7bcb134
LO		 6621
6622 }
6623
6624 spin_unlock_irqrestore(&srq->lock, flags);
6625
6626 return ret;
6627}
6628
e1c9a0dc
LO		 6629static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6630 .query_cqc_info = hns_roce_v2_query_cqc_info,
6631};
6632
7f645a58
KH		 6633static const struct ib_device_ops hns_roce_v2_dev_ops = {
6634 .destroy_qp = hns_roce_v2_destroy_qp,
6635 .modify_cq = hns_roce_v2_modify_cq,
6636 .poll_cq = hns_roce_v2_poll_cq,
6637 .post_recv = hns_roce_v2_post_recv,
6638 .post_send = hns_roce_v2_post_send,
6639 .query_qp = hns_roce_v2_query_qp,
6640 .req_notify_cq = hns_roce_v2_req_notify_cq,
6641};
6642
6643static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6644 .modify_srq = hns_roce_v2_modify_srq,
6645 .post_srq_recv = hns_roce_v2_post_srq_recv,
6646 .query_srq = hns_roce_v2_query_srq,
6647};
6648
a04ff739
WHX		 6649static const struct hns_roce_hw hns_roce_hw_v2 = {
6650 .cmq_init = hns_roce_v2_cmq_init,
6651 .cmq_exit = hns_roce_v2_cmq_exit,
cfc85f3e 6652 .hw_profile = hns_roce_v2_profile,
6b63597d 6653 .hw_init = hns_roce_v2_init,
6654 .hw_exit = hns_roce_v2_exit,
a680f2f3
WHX		 6655 .post_mbox = hns_roce_v2_post_mbox,
6656 .chk_mbox = hns_roce_v2_chk_mbox,
6a04aed6 6657 .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
7afddafa
WHX		 6658 .set_gid = hns_roce_v2_set_gid,
6659 .set_mac = hns_roce_v2_set_mac,
3958cc56 6660 .write_mtpt = hns_roce_v2_write_mtpt,
a2c80b7b 6661 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
68a997c5 6662 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
c7c28191 6663 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
93aa2187 6664 .write_cqc = hns_roce_v2_write_cqc,
a81fba28
WHX		 6665 .set_hem = hns_roce_v2_set_hem,
6666 .clear_hem = hns_roce_v2_clear_hem,
926a01dc
WHX		 6667 .modify_qp = hns_roce_v2_modify_qp,
6668 .query_qp = hns_roce_v2_query_qp,
6669 .destroy_qp = hns_roce_v2_destroy_qp,
aa84fa18 6670 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
b156269d 6671 .modify_cq = hns_roce_v2_modify_cq,
2d407888
WHX		 6672 .post_send = hns_roce_v2_post_send,
6673 .post_recv = hns_roce_v2_post_recv,
93aa2187
WHX		 6674 .req_notify_cq = hns_roce_v2_req_notify_cq,
6675 .poll_cq = hns_roce_v2_poll_cq,
a5073d60
YL		 6676 .init_eq = hns_roce_v2_init_eq_table,
6677 .cleanup_eq = hns_roce_v2_cleanup_eq_table,
c7bcb134
LO		 6678 .write_srqc = hns_roce_v2_write_srqc,
6679 .modify_srq = hns_roce_v2_modify_srq,
6680 .query_srq = hns_roce_v2_query_srq,
6681 .post_srq_recv = hns_roce_v2_post_srq_recv,
7f645a58
KH		 6682 .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6683 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
a04ff739 6684};
dd74282d
WHX		 6685
6686static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6687 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6688 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
aaa31567
LO		 6689 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6690 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
dd74282d
WHX		 6691 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6692 /* required last entry */
6693 {0, }
6694};
6695
f97a62c3 6696MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6697
301cc7eb 6698static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
dd74282d
WHX		 6699 struct hnae3_handle *handle)
6700{
d061effc 6701 struct hns_roce_v2_priv *priv = hr_dev->priv;
a5073d60 6702 int i;
dd74282d 6703
301cc7eb
LC		 6704 hr_dev->pci_dev = handle->pdev;
6705 hr_dev->dev = &handle->pdev->dev;
dd74282d 6706 hr_dev->hw = &hns_roce_hw_v2;
e1c9a0dc 6707 hr_dev->dfx = &hns_roce_dfx_hw_v2;
2d407888
WHX		 6708 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6709 hr_dev->odb_offset = hr_dev->sdb_offset;
dd74282d
WHX		 6710
6711 /* Get info from NIC driver. */
6712 hr_dev->reg_base = handle->rinfo.roce_io_base;
6713 hr_dev->caps.num_ports = 1;
6714 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6715 hr_dev->iboe.phy_port[0] = 0;
6716
d4994d2f 6717 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6718 hr_dev->iboe.netdevs[0]->dev_addr);
6719
a5073d60
YL		 6720 for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
6721 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6722 i + handle->rinfo.base_vector);
6723
dd74282d 6724 /* cmd issue mode: 0 is poll, 1 is event */
a5073d60 6725 hr_dev->cmd_mod = 1;
dd74282d
WHX		 6726 hr_dev->loop_idc = 0;
6727
d061effc
WHX		 6728 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6729 priv->handle = handle;
dd74282d
WHX		 6730}
6731
d061effc 6732static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
dd74282d
WHX		 6733{
6734 struct hns_roce_dev *hr_dev;
6735 int ret;
6736
459cc69f 6737 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
dd74282d
WHX		 6738 if (!hr_dev)
6739 return -ENOMEM;
6740
a04ff739
WHX		 6741 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6742 if (!hr_dev->priv) {
6743 ret = -ENOMEM;
6744 goto error_failed_kzalloc;
6745 }
6746
301cc7eb 6747 hns_roce_hw_v2_get_cfg(hr_dev, handle);
dd74282d
WHX		 6748
6749 ret = hns_roce_init(hr_dev);
6750 if (ret) {
6751 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6752 goto error_failed_get_cfg;
6753 }
6754
d061effc
WHX		 6755 handle->priv = hr_dev;
6756
dd74282d
WHX		 6757 return 0;
6758
6759error_failed_get_cfg:
a04ff739
WHX		 6760 kfree(hr_dev->priv);
6761
6762error_failed_kzalloc:
dd74282d
WHX		 6763 ib_dealloc_device(&hr_dev->ib_dev);
6764
6765 return ret;
6766}
6767
d061effc 6768static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
dd74282d
WHX		 6769 bool reset)
6770{
6771 struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
6772
cb7a94c9
WHX		 6773 if (!hr_dev)
6774 return;
6775
d061effc 6776 handle->priv = NULL;
626903e9
XW		 6777
6778 hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
6779 hns_roce_handle_device_err(hr_dev);
6780
dd74282d 6781 hns_roce_exit(hr_dev);
a04ff739 6782 kfree(hr_dev->priv);
dd74282d
WHX		 6783 ib_dealloc_device(&hr_dev->ib_dev);
6784}
6785
d061effc
WHX		 6786static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6787{
6788 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
07c2339a 6789 const struct pci_device_id *id;
d061effc
WHX		 6790 struct device *dev = &handle->pdev->dev;
6791 int ret;
6792
6793 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6794
6795 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6796 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6797 goto reset_chk_err;
6798 }
6799
07c2339a
LO		 6800 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6801 if (!id)
6802 return 0;
6803
d061effc
WHX		 6804 ret = __hns_roce_hw_v2_init_instance(handle);
6805 if (ret) {
6806 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6807 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6808 if (ops->ae_dev_resetting(handle) ||
6809 ops->get_hw_reset_stat(handle))
6810 goto reset_chk_err;
6811 else
6812 return ret;
6813 }
6814
6815 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6816
6817
6818 return 0;
6819
6820reset_chk_err:
6821 dev_err(dev, "Device is busy in resetting state.\n"
6822 "please retry later.\n");
6823
6824 return -EBUSY;
6825}
6826
6827static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6828 bool reset)
6829{
6830 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6831 return;
6832
6833 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6834
6835 __hns_roce_hw_v2_uninit_instance(handle, reset);
6836
6837 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6838}
cb7a94c9
WHX		 6839static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6840{
d061effc 6841 struct hns_roce_dev *hr_dev;
cb7a94c9 6842
d061effc
WHX		 6843 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6844 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6845 return 0;
cb7a94c9
WHX		 6846 }
6847
d061effc
WHX		 6848 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6849 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6850
6851 hr_dev = (struct hns_roce_dev *)handle->priv;
6852 if (!hr_dev)
6853 return 0;
6854
726be12f 6855 hr_dev->is_reset = true;
cb7a94c9 6856 hr_dev->active = false;
d3743fa9 6857 hr_dev->dis_db = true;
cb7a94c9 6858
626903e9 6859 hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
cb7a94c9
WHX		 6860
6861 return 0;
6862}
6863
6864static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6865{
d061effc 6866 struct device *dev = &handle->pdev->dev;
cb7a94c9
WHX		 6867 int ret;
6868
d061effc
WHX		 6869 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6870 &handle->rinfo.state)) {
6871 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6872 return 0;
6873 }
6874
6875 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6876
6877 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6878 ret = __hns_roce_hw_v2_init_instance(handle);
cb7a94c9
WHX		 6879 if (ret) {
6880 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6881 * callback function, RoCE Engine reinitialize. If RoCE reinit
6882 * failed, we should inform NIC driver.
6883 */
6884 handle->priv = NULL;
d061effc
WHX		 6885 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6886 } else {
6887 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6888 dev_info(dev, "Reset done, RoCE client reinit finished.\n");
cb7a94c9
WHX		 6889 }
6890
6891 return ret;
6892}
6893
6894static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6895{
d061effc
WHX		 6896 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6897 return 0;
6898
6899 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6900 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
90c559b1 6901 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
d061effc
WHX		 6902 __hns_roce_hw_v2_uninit_instance(handle, false);
6903
cb7a94c9
WHX		 6904 return 0;
6905}
6906
6907static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6908 enum hnae3_reset_notify_type type)
6909{
6910 int ret = 0;
6911
6912 switch (type) {
6913 case HNAE3_DOWN_CLIENT:
6914 ret = hns_roce_hw_v2_reset_notify_down(handle);
6915 break;
6916 case HNAE3_INIT_CLIENT:
6917 ret = hns_roce_hw_v2_reset_notify_init(handle);
6918 break;
6919 case HNAE3_UNINIT_CLIENT:
6920 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6921 break;
6922 default:
6923 break;
6924 }
6925
6926 return ret;
6927}
6928
dd74282d
WHX		 6929static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6930 .init_instance = hns_roce_hw_v2_init_instance,
6931 .uninit_instance = hns_roce_hw_v2_uninit_instance,
cb7a94c9 6932 .reset_notify = hns_roce_hw_v2_reset_notify,
dd74282d
WHX		 6933};
6934
6935static struct hnae3_client hns_roce_hw_v2_client = {
6936 .name = "hns_roce_hw_v2",
6937 .type = HNAE3_CLIENT_ROCE,
6938 .ops = &hns_roce_hw_v2_ops,
6939};
6940
6941static int __init hns_roce_hw_v2_init(void)
6942{
6943 return hnae3_register_client(&hns_roce_hw_v2_client);
6944}
6945
6946static void __exit hns_roce_hw_v2_exit(void)
6947{
6948 hnae3_unregister_client(&hns_roce_hw_v2_client);
6949}
6950
6951module_init(hns_roce_hw_v2_init);
6952module_exit(hns_roce_hw_v2_exit);
6953
6954MODULE_LICENSE("Dual BSD/GPL");
6955MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
6956MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
6957MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
6958MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");
Linux 6.x block layer and io_uring tree(s)