2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/mlx5/driver.h>
34 #include "mlx5_core.h"
35 #include <linux/mlx5/transobj.h>
37 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
39 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
40 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
43 MLX5_SET(alloc_transport_domain_in, in, opcode,
44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
46 err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);
48 *tdn = MLX5_GET(alloc_transport_domain_out, out,
53 EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
55 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};
59 MLX5_SET(dealloc_transport_domain_in, in, opcode,
60 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
61 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
62 mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);
64 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
66 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
68 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {};
71 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
72 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
74 *rqn = MLX5_GET(create_rq_out, out, rqn);
78 EXPORT_SYMBOL(mlx5_core_create_rq);
80 int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in)
82 MLX5_SET(modify_rq_in, in, rqn, rqn);
83 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
85 return mlx5_cmd_exec_in(dev, modify_rq, in);
87 EXPORT_SYMBOL(mlx5_core_modify_rq);
89 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
91 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};
93 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
94 MLX5_SET(destroy_rq_in, in, rqn, rqn);
95 mlx5_cmd_exec_in(dev, destroy_rq, in);
97 EXPORT_SYMBOL(mlx5_core_destroy_rq);
99 int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
101 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {};
103 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
104 MLX5_SET(query_rq_in, in, rqn, rqn);
106 return mlx5_cmd_exec_inout(dev, query_rq, in, out);
108 EXPORT_SYMBOL(mlx5_core_query_rq);
110 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
112 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};
115 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
116 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
118 *sqn = MLX5_GET(create_sq_out, out, sqn);
123 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in)
125 MLX5_SET(modify_sq_in, in, sqn, sqn);
126 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
127 return mlx5_cmd_exec_in(dev, modify_sq, in);
129 EXPORT_SYMBOL(mlx5_core_modify_sq);
131 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
133 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};
135 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
136 MLX5_SET(destroy_sq_in, in, sqn, sqn);
137 mlx5_cmd_exec_in(dev, destroy_sq, in);
140 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
142 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {};
144 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
145 MLX5_SET(query_sq_in, in, sqn, sqn);
146 return mlx5_cmd_exec_inout(dev, query_sq, in, out);
148 EXPORT_SYMBOL(mlx5_core_query_sq);
150 int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state)
157 inlen = MLX5_ST_SZ_BYTES(query_sq_out);
158 out = kvzalloc(inlen, GFP_KERNEL);
162 err = mlx5_core_query_sq(dev, sqn, out);
166 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
167 *state = MLX5_GET(sqc, sqc, state);
173 EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);
175 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, u32 *tirn)
177 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
180 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
181 err = mlx5_cmd_exec_inout(dev, create_tir, in, out);
183 *tirn = MLX5_GET(create_tir_out, out, tirn);
187 EXPORT_SYMBOL(mlx5_core_create_tir);
189 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in)
191 MLX5_SET(modify_tir_in, in, tirn, tirn);
192 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
193 return mlx5_cmd_exec_in(dev, modify_tir, in);
196 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
198 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
200 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
201 MLX5_SET(destroy_tir_in, in, tirn, tirn);
202 mlx5_cmd_exec_in(dev, destroy_tir, in);
204 EXPORT_SYMBOL(mlx5_core_destroy_tir);
206 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, u32 *tisn)
208 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
211 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
212 err = mlx5_cmd_exec_inout(dev, create_tis, in, out);
214 *tisn = MLX5_GET(create_tis_out, out, tisn);
218 EXPORT_SYMBOL(mlx5_core_create_tis);
220 int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in)
222 MLX5_SET(modify_tis_in, in, tisn, tisn);
223 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
225 return mlx5_cmd_exec_in(dev, modify_tis, in);
227 EXPORT_SYMBOL(mlx5_core_modify_tis);
229 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
231 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};
233 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
234 MLX5_SET(destroy_tis_in, in, tisn, tisn);
235 mlx5_cmd_exec_in(dev, destroy_tis, in);
237 EXPORT_SYMBOL(mlx5_core_destroy_tis);
239 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
242 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
245 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
246 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
248 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
252 EXPORT_SYMBOL(mlx5_core_create_rqt);
254 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
257 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {};
259 MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
260 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
261 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
264 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
266 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};
268 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
269 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
270 mlx5_cmd_exec_in(dev, destroy_rqt, in);
272 EXPORT_SYMBOL(mlx5_core_destroy_rqt);
274 static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
275 struct mlx5_hairpin_params *params, u32 *rqn)
277 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
280 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
281 wq = MLX5_ADDR_OF(rqc, rqc, wq);
283 MLX5_SET(rqc, rqc, hairpin, 1);
284 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
285 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
287 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
288 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
290 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
293 static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
294 struct mlx5_hairpin_params *params, u32 *sqn)
296 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
299 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
300 wq = MLX5_ADDR_OF(sqc, sqc, wq);
302 MLX5_SET(sqc, sqc, hairpin, 1);
303 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
305 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
306 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
308 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
311 static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
312 struct mlx5_hairpin_params *params)
316 for (i = 0; i < hp->num_channels; i++) {
317 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
322 for (i = 0; i < hp->num_channels; i++) {
323 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
331 for (j = 0; j < i; j++)
332 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
333 i = hp->num_channels;
335 for (j = 0; j < i; j++)
336 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
340 static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
344 for (i = 0; i < hp->num_channels; i++) {
345 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
347 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
351 static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
352 int curr_state, int next_state,
353 u16 peer_vhca, u32 peer_sq)
355 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
358 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
360 if (next_state == MLX5_RQC_STATE_RDY) {
361 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
362 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
365 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
366 MLX5_SET(rqc, rqc, state, next_state);
368 return mlx5_core_modify_rq(func_mdev, rqn, in);
371 static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
372 int curr_state, int next_state,
373 u16 peer_vhca, u32 peer_rq)
375 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
378 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
380 if (next_state == MLX5_SQC_STATE_RDY) {
381 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
382 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
385 MLX5_SET(modify_sq_in, in, sq_state, curr_state);
386 MLX5_SET(sqc, sqc, state, next_state);
388 return mlx5_core_modify_sq(peer_mdev, sqn, in);
391 static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
396 for (i = 0; i < hp->num_channels; i++) {
397 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
398 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
399 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
405 for (i = 0; i < hp->num_channels; i++) {
406 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
407 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
408 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
416 for (j = 0; j < i; j++)
417 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
418 MLX5_RQC_STATE_RST, 0, 0);
419 i = hp->num_channels;
421 for (j = 0; j < i; j++)
422 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
423 MLX5_SQC_STATE_RST, 0, 0);
427 static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp)
431 for (i = 0; i < hp->num_channels; i++)
432 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
433 MLX5_SQC_STATE_RST, 0, 0);
436 static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
441 for (i = 0; i < hp->num_channels; i++)
442 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
443 MLX5_RQC_STATE_RST, 0, 0);
446 mlx5_hairpin_unpair_peer_sq(hp);
449 struct mlx5_hairpin *
450 mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
451 struct mlx5_core_dev *peer_mdev,
452 struct mlx5_hairpin_params *params)
454 struct mlx5_hairpin *hp;
457 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
458 hp = kzalloc(size, GFP_KERNEL);
460 return ERR_PTR(-ENOMEM);
462 hp->func_mdev = func_mdev;
463 hp->peer_mdev = peer_mdev;
464 hp->num_channels = params->num_channels;
466 hp->rqn = (void *)hp + sizeof(*hp);
467 hp->sqn = hp->rqn + params->num_channels;
469 /* alloc and pair func --> peer hairpin */
470 err = mlx5_hairpin_create_queues(hp, params);
472 goto err_create_queues;
474 err = mlx5_hairpin_pair_queues(hp);
476 goto err_pair_queues;
481 mlx5_hairpin_destroy_queues(hp);
487 void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
489 mlx5_hairpin_unpair_queues(hp);
490 mlx5_hairpin_destroy_queues(hp);
494 void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp)
498 mlx5_hairpin_unpair_peer_sq(hp);
500 /* destroy peer SQ */
501 for (i = 0; i < hp->num_channels; i++)
502 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
504 hp->peer_gone = true;