projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mlx5: Remove checksum on command interface commands
[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / cmd.c
CommitLineData
e126ba97
EC		 1/*
2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <asm-generic/kmap_types.h>
34#include <linux/module.h>
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/delay.h>
41#include <linux/random.h>
42#include <linux/io-mapping.h>
43#include <linux/mlx5/driver.h>
44#include <linux/debugfs.h>
45
46#include "mlx5_core.h"
47
48enum {
0a324f31 49 CMD_IF_REV = 5,
e126ba97
EC		 50};
51
52enum {
53 CMD_MODE_POLLING,
54 CMD_MODE_EVENTS
55};
56
57enum {
58 NUM_LONG_LISTS = 2,
59 NUM_MED_LISTS = 64,
60 LONG_LIST_SIZE = (2ULL * 1024 * 1024 * 1024 / PAGE_SIZE) * 8 + 16 +
61 MLX5_CMD_DATA_BLOCK_SIZE,
62 MED_LIST_SIZE = 16 + MLX5_CMD_DATA_BLOCK_SIZE,
63};
64
65enum {
66 MLX5_CMD_DELIVERY_STAT_OK = 0x0,
67 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,
68 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,
69 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,
70 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,
71 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,
72 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,
73 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,
74 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,
75 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,
76 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
77};
78
79enum {
80 MLX5_CMD_STAT_OK = 0x0,
81 MLX5_CMD_STAT_INT_ERR = 0x1,
82 MLX5_CMD_STAT_BAD_OP_ERR = 0x2,
83 MLX5_CMD_STAT_BAD_PARAM_ERR = 0x3,
84 MLX5_CMD_STAT_BAD_SYS_STATE_ERR = 0x4,
85 MLX5_CMD_STAT_BAD_RES_ERR = 0x5,
86 MLX5_CMD_STAT_RES_BUSY = 0x6,
87 MLX5_CMD_STAT_LIM_ERR = 0x8,
88 MLX5_CMD_STAT_BAD_RES_STATE_ERR = 0x9,
89 MLX5_CMD_STAT_IX_ERR = 0xa,
90 MLX5_CMD_STAT_NO_RES_ERR = 0xf,
91 MLX5_CMD_STAT_BAD_INP_LEN_ERR = 0x50,
92 MLX5_CMD_STAT_BAD_OUTP_LEN_ERR = 0x51,
93 MLX5_CMD_STAT_BAD_QP_STATE_ERR = 0x10,
94 MLX5_CMD_STAT_BAD_PKT_ERR = 0x30,
95 MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR = 0x40,
96};
97
98static struct mlx5_cmd_work_ent *alloc_cmd(struct mlx5_cmd *cmd,
99 struct mlx5_cmd_msg *in,
100 struct mlx5_cmd_msg *out,
101 mlx5_cmd_cbk_t cbk,
102 void *context, int page_queue)
103{
104 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
105 struct mlx5_cmd_work_ent *ent;
106
107 ent = kzalloc(sizeof(*ent), alloc_flags);
108 if (!ent)
109 return ERR_PTR(-ENOMEM);
110
111 ent->in = in;
112 ent->out = out;
113 ent->callback = cbk;
114 ent->context = context;
115 ent->cmd = cmd;
116 ent->page_queue = page_queue;
117
118 return ent;
119}
120
121static u8 alloc_token(struct mlx5_cmd *cmd)
122{
123 u8 token;
124
125 spin_lock(&cmd->token_lock);
126 token = cmd->token++ % 255 + 1;
127 spin_unlock(&cmd->token_lock);
128
129 return token;
130}
131
132static int alloc_ent(struct mlx5_cmd *cmd)
133{
134 unsigned long flags;
135 int ret;
136
137 spin_lock_irqsave(&cmd->alloc_lock, flags);
138 ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds);
139 if (ret < cmd->max_reg_cmds)
140 clear_bit(ret, &cmd->bitmask);
141 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
142
143 return ret < cmd->max_reg_cmds ? ret : -ENOMEM;
144}
145
146static void free_ent(struct mlx5_cmd *cmd, int idx)
147{
148 unsigned long flags;
149
150 spin_lock_irqsave(&cmd->alloc_lock, flags);
151 set_bit(idx, &cmd->bitmask);
152 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
153}
154
155static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
156{
157 return cmd->cmd_buf + (idx << cmd->log_stride);
158}
159
160static u8 xor8_buf(void *buf, int len)
161{
162 u8 *ptr = buf;
163 u8 sum = 0;
164 int i;
165
166 for (i = 0; i < len; i++)
167 sum ^= ptr[i];
168
169 return sum;
170}
171
172static int verify_block_sig(struct mlx5_cmd_prot_block *block)
173{
174 if (xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 1) != 0xff)
175 return -EINVAL;
176
177 if (xor8_buf(block, sizeof(*block)) != 0xff)
178 return -EINVAL;
179
180 return 0;
181}
182
c1868b82
EC		 183static void calc_block_sig(struct mlx5_cmd_prot_block *block, u8 token,
184 int csum)
e126ba97
EC		 185{
186 block->token = token;
c1868b82
EC		 187 if (csum) {
188 block->ctrl_sig = ~xor8_buf(block->rsvd0, sizeof(*block) -
189 sizeof(block->data) - 2);
190 block->sig = ~xor8_buf(block, sizeof(*block) - 1);
191 }
e126ba97
EC		 192}
193
c1868b82 194static void calc_chain_sig(struct mlx5_cmd_msg *msg, u8 token, int csum)
e126ba97
EC		 195{
196 struct mlx5_cmd_mailbox *next = msg->next;
197
198 while (next) {
c1868b82 199 calc_block_sig(next->buf, token, csum);
e126ba97
EC		 200 next = next->next;
201 }
202}
203
c1868b82 204static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
e126ba97
EC		 205{
206 ent->lay->sig = ~xor8_buf(ent->lay, sizeof(*ent->lay));
c1868b82
EC		 207 calc_chain_sig(ent->in, ent->token, csum);
208 calc_chain_sig(ent->out, ent->token, csum);
e126ba97
EC		 209}
210
211static void poll_timeout(struct mlx5_cmd_work_ent *ent)
212{
213 unsigned long poll_end = jiffies + msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC + 1000);
214 u8 own;
215
216 do {
217 own = ent->lay->status_own;
218 if (!(own & CMD_OWNER_HW)) {
219 ent->ret = 0;
220 return;
221 }
222 usleep_range(5000, 10000);
223 } while (time_before(jiffies, poll_end));
224
225 ent->ret = -ETIMEDOUT;
226}
227
228static void free_cmd(struct mlx5_cmd_work_ent *ent)
229{
230 kfree(ent);
231}
232
233
234static int verify_signature(struct mlx5_cmd_work_ent *ent)
235{
236 struct mlx5_cmd_mailbox *next = ent->out->next;
237 int err;
238 u8 sig;
239
240 sig = xor8_buf(ent->lay, sizeof(*ent->lay));
241 if (sig != 0xff)
242 return -EINVAL;
243
244 while (next) {
245 err = verify_block_sig(next->buf);
246 if (err)
247 return err;
248
249 next = next->next;
250 }
251
252 return 0;
253}
254
255static void dump_buf(void *buf, int size, int data_only, int offset)
256{
257 __be32 *p = buf;
258 int i;
259
260 for (i = 0; i < size; i += 16) {
261 pr_debug("%03x: %08x %08x %08x %08x\n", offset, be32_to_cpu(p[0]),
262 be32_to_cpu(p[1]), be32_to_cpu(p[2]),
263 be32_to_cpu(p[3]));
264 p += 4;
265 offset += 16;
266 }
267 if (!data_only)
268 pr_debug("\n");
269}
270
271const char *mlx5_command_str(int command)
272{
273 switch (command) {
274 case MLX5_CMD_OP_QUERY_HCA_CAP:
275 return "QUERY_HCA_CAP";
276
277 case MLX5_CMD_OP_SET_HCA_CAP:
278 return "SET_HCA_CAP";
279
280 case MLX5_CMD_OP_QUERY_ADAPTER:
281 return "QUERY_ADAPTER";
282
283 case MLX5_CMD_OP_INIT_HCA:
284 return "INIT_HCA";
285
286 case MLX5_CMD_OP_TEARDOWN_HCA:
287 return "TEARDOWN_HCA";
288
cd23b14b
EC		 289 case MLX5_CMD_OP_ENABLE_HCA:
290 return "MLX5_CMD_OP_ENABLE_HCA";
291
292 case MLX5_CMD_OP_DISABLE_HCA:
293 return "MLX5_CMD_OP_DISABLE_HCA";
294
e126ba97
EC		 295 case MLX5_CMD_OP_QUERY_PAGES:
296 return "QUERY_PAGES";
297
298 case MLX5_CMD_OP_MANAGE_PAGES:
299 return "MANAGE_PAGES";
300
301 case MLX5_CMD_OP_CREATE_MKEY:
302 return "CREATE_MKEY";
303
304 case MLX5_CMD_OP_QUERY_MKEY:
305 return "QUERY_MKEY";
306
307 case MLX5_CMD_OP_DESTROY_MKEY:
308 return "DESTROY_MKEY";
309
310 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
311 return "QUERY_SPECIAL_CONTEXTS";
312
313 case MLX5_CMD_OP_CREATE_EQ:
314 return "CREATE_EQ";
315
316 case MLX5_CMD_OP_DESTROY_EQ:
317 return "DESTROY_EQ";
318
319 case MLX5_CMD_OP_QUERY_EQ:
320 return "QUERY_EQ";
321
322 case MLX5_CMD_OP_CREATE_CQ:
323 return "CREATE_CQ";
324
325 case MLX5_CMD_OP_DESTROY_CQ:
326 return "DESTROY_CQ";
327
328 case MLX5_CMD_OP_QUERY_CQ:
329 return "QUERY_CQ";
330
331 case MLX5_CMD_OP_MODIFY_CQ:
332 return "MODIFY_CQ";
333
334 case MLX5_CMD_OP_CREATE_QP:
335 return "CREATE_QP";
336
337 case MLX5_CMD_OP_DESTROY_QP:
338 return "DESTROY_QP";
339
340 case MLX5_CMD_OP_RST2INIT_QP:
341 return "RST2INIT_QP";
342
343 case MLX5_CMD_OP_INIT2RTR_QP:
344 return "INIT2RTR_QP";
345
346 case MLX5_CMD_OP_RTR2RTS_QP:
347 return "RTR2RTS_QP";
348
349 case MLX5_CMD_OP_RTS2RTS_QP:
350 return "RTS2RTS_QP";
351
352 case MLX5_CMD_OP_SQERR2RTS_QP:
353 return "SQERR2RTS_QP";
354
355 case MLX5_CMD_OP_2ERR_QP:
356 return "2ERR_QP";
357
358 case MLX5_CMD_OP_RTS2SQD_QP:
359 return "RTS2SQD_QP";
360
361 case MLX5_CMD_OP_SQD2RTS_QP:
362 return "SQD2RTS_QP";
363
364 case MLX5_CMD_OP_2RST_QP:
365 return "2RST_QP";
366
367 case MLX5_CMD_OP_QUERY_QP:
368 return "QUERY_QP";
369
370 case MLX5_CMD_OP_CONF_SQP:
371 return "CONF_SQP";
372
373 case MLX5_CMD_OP_MAD_IFC:
374 return "MAD_IFC";
375
376 case MLX5_CMD_OP_INIT2INIT_QP:
377 return "INIT2INIT_QP";
378
379 case MLX5_CMD_OP_SUSPEND_QP:
380 return "SUSPEND_QP";
381
382 case MLX5_CMD_OP_UNSUSPEND_QP:
383 return "UNSUSPEND_QP";
384
385 case MLX5_CMD_OP_SQD2SQD_QP:
386 return "SQD2SQD_QP";
387
388 case MLX5_CMD_OP_ALLOC_QP_COUNTER_SET:
389 return "ALLOC_QP_COUNTER_SET";
390
391 case MLX5_CMD_OP_DEALLOC_QP_COUNTER_SET:
392 return "DEALLOC_QP_COUNTER_SET";
393
394 case MLX5_CMD_OP_QUERY_QP_COUNTER_SET:
395 return "QUERY_QP_COUNTER_SET";
396
397 case MLX5_CMD_OP_CREATE_PSV:
398 return "CREATE_PSV";
399
400 case MLX5_CMD_OP_DESTROY_PSV:
401 return "DESTROY_PSV";
402
403 case MLX5_CMD_OP_QUERY_PSV:
404 return "QUERY_PSV";
405
406 case MLX5_CMD_OP_QUERY_SIG_RULE_TABLE:
407 return "QUERY_SIG_RULE_TABLE";
408
409 case MLX5_CMD_OP_QUERY_BLOCK_SIZE_TABLE:
410 return "QUERY_BLOCK_SIZE_TABLE";
411
412 case MLX5_CMD_OP_CREATE_SRQ:
413 return "CREATE_SRQ";
414
415 case MLX5_CMD_OP_DESTROY_SRQ:
416 return "DESTROY_SRQ";
417
418 case MLX5_CMD_OP_QUERY_SRQ:
419 return "QUERY_SRQ";
420
421 case MLX5_CMD_OP_ARM_RQ:
422 return "ARM_RQ";
423
424 case MLX5_CMD_OP_RESIZE_SRQ:
425 return "RESIZE_SRQ";
426
427 case MLX5_CMD_OP_ALLOC_PD:
428 return "ALLOC_PD";
429
430 case MLX5_CMD_OP_DEALLOC_PD:
431 return "DEALLOC_PD";
432
433 case MLX5_CMD_OP_ALLOC_UAR:
434 return "ALLOC_UAR";
435
436 case MLX5_CMD_OP_DEALLOC_UAR:
437 return "DEALLOC_UAR";
438
439 case MLX5_CMD_OP_ATTACH_TO_MCG:
440 return "ATTACH_TO_MCG";
441
442 case MLX5_CMD_OP_DETACH_FROM_MCG:
443 return "DETACH_FROM_MCG";
444
445 case MLX5_CMD_OP_ALLOC_XRCD:
446 return "ALLOC_XRCD";
447
448 case MLX5_CMD_OP_DEALLOC_XRCD:
449 return "DEALLOC_XRCD";
450
451 case MLX5_CMD_OP_ACCESS_REG:
452 return "MLX5_CMD_OP_ACCESS_REG";
453
454 default: return "unknown command opcode";
455 }
456}
457
458static void dump_command(struct mlx5_core_dev *dev,
459 struct mlx5_cmd_work_ent *ent, int input)
460{
461 u16 op = be16_to_cpu(((struct mlx5_inbox_hdr *)(ent->lay->in))->opcode);
462 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
463 struct mlx5_cmd_mailbox *next = msg->next;
464 int data_only;
465 int offset = 0;
466 int dump_len;
467
468 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
469
470 if (data_only)
471 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
472 "dump command data %s(0x%x) %s\n",
473 mlx5_command_str(op), op,
474 input ? "INPUT" : "OUTPUT");
475 else
476 mlx5_core_dbg(dev, "dump command %s(0x%x) %s\n",
477 mlx5_command_str(op), op,
478 input ? "INPUT" : "OUTPUT");
479
480 if (data_only) {
481 if (input) {
482 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset);
483 offset += sizeof(ent->lay->in);
484 } else {
485 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset);
486 offset += sizeof(ent->lay->out);
487 }
488 } else {
489 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset);
490 offset += sizeof(*ent->lay);
491 }
492
493 while (next && offset < msg->len) {
494 if (data_only) {
495 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
496 dump_buf(next->buf, dump_len, 1, offset);
497 offset += MLX5_CMD_DATA_BLOCK_SIZE;
498 } else {
499 mlx5_core_dbg(dev, "command block:\n");
500 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset);
501 offset += sizeof(struct mlx5_cmd_prot_block);
502 }
503 next = next->next;
504 }
505
506 if (data_only)
507 pr_debug("\n");
508}
509
510static void cmd_work_handler(struct work_struct *work)
511{
512 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
513 struct mlx5_cmd *cmd = ent->cmd;
514 struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd);
515 struct mlx5_cmd_layout *lay;
516 struct semaphore *sem;
517
518 sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
519 down(sem);
520 if (!ent->page_queue) {
521 ent->idx = alloc_ent(cmd);
522 if (ent->idx < 0) {
523 mlx5_core_err(dev, "failed to allocate command entry\n");
524 up(sem);
525 return;
526 }
527 } else {
528 ent->idx = cmd->max_reg_cmds;
529 }
530
531 ent->token = alloc_token(cmd);
532 cmd->ent_arr[ent->idx] = ent;
533 lay = get_inst(cmd, ent->idx);
534 ent->lay = lay;
535 memset(lay, 0, sizeof(*lay));
536 memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
537 if (ent->in->next)
538 lay->in_ptr = cpu_to_be64(ent->in->next->dma);
539 lay->inlen = cpu_to_be32(ent->in->len);
540 if (ent->out->next)
541 lay->out_ptr = cpu_to_be64(ent->out->next->dma);
542 lay->outlen = cpu_to_be32(ent->out->len);
543 lay->type = MLX5_PCI_CMD_XPORT;
544 lay->token = ent->token;
545 lay->status_own = CMD_OWNER_HW;
c1868b82 546 set_signature(ent, !cmd->checksum_disabled);
e126ba97
EC		 547 dump_command(dev, ent, 1);
548 ktime_get_ts(&ent->ts1);
549
550 /* ring doorbell after the descriptor is valid */
551 wmb();
552 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
553 mlx5_core_dbg(dev, "write 0x%x to command doorbell\n", 1 << ent->idx);
554 mmiowb();
555 if (cmd->mode == CMD_MODE_POLLING) {
556 poll_timeout(ent);
557 /* make sure we read the descriptor after ownership is SW */
558 rmb();
559 mlx5_cmd_comp_handler(dev, 1UL << ent->idx);
560 }
561}
562
563static const char *deliv_status_to_str(u8 status)
564{
565 switch (status) {
566 case MLX5_CMD_DELIVERY_STAT_OK:
567 return "no errors";
568 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
569 return "signature error";
570 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
571 return "token error";
572 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
573 return "bad block number";
574 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
575 return "output pointer not aligned to block size";
576 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
577 return "input pointer not aligned to block size";
578 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
579 return "firmware internal error";
580 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
581 return "command input length error";
582 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
583 return "command ouput length error";
584 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
585 return "reserved fields not cleared";
586 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
587 return "bad command descriptor type";
588 default:
589 return "unknown status code";
590 }
591}
592
593static u16 msg_to_opcode(struct mlx5_cmd_msg *in)
594{
595 struct mlx5_inbox_hdr *hdr = (struct mlx5_inbox_hdr *)(in->first.data);
596
597 return be16_to_cpu(hdr->opcode);
598}
599
600static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
601{
602 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC);
603 struct mlx5_cmd *cmd = &dev->cmd;
604 int err;
605
606 if (cmd->mode == CMD_MODE_POLLING) {
607 wait_for_completion(&ent->done);
608 err = ent->ret;
609 } else {
610 if (!wait_for_completion_timeout(&ent->done, timeout))
611 err = -ETIMEDOUT;
612 else
613 err = 0;
614 }
615 if (err == -ETIMEDOUT) {
616 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
617 mlx5_command_str(msg_to_opcode(ent->in)),
618 msg_to_opcode(ent->in));
619 }
620 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n", err,
621 deliv_status_to_str(ent->status), ent->status);
622
623 return err;
624}
625
626/* Notes:
627 * 1. Callback functions may not sleep
628 * 2. page queue commands do not support asynchrous completion
629 */
630static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
631 struct mlx5_cmd_msg *out, mlx5_cmd_cbk_t callback,
632 void *context, int page_queue, u8 *status)
633{
634 struct mlx5_cmd *cmd = &dev->cmd;
635 struct mlx5_cmd_work_ent *ent;
636 ktime_t t1, t2, delta;
637 struct mlx5_cmd_stats *stats;
638 int err = 0;
639 s64 ds;
640 u16 op;
641
642 if (callback && page_queue)
643 return -EINVAL;
644
645 ent = alloc_cmd(cmd, in, out, callback, context, page_queue);
646 if (IS_ERR(ent))
647 return PTR_ERR(ent);
648
649 if (!callback)
650 init_completion(&ent->done);
651
652 INIT_WORK(&ent->work, cmd_work_handler);
653 if (page_queue) {
654 cmd_work_handler(&ent->work);
655 } else if (!queue_work(cmd->wq, &ent->work)) {
656 mlx5_core_warn(dev, "failed to queue work\n");
657 err = -ENOMEM;
658 goto out_free;
659 }
660
661 if (!callback) {
662 err = wait_func(dev, ent);
663 if (err == -ETIMEDOUT)
664 goto out;
665
666 t1 = timespec_to_ktime(ent->ts1);
667 t2 = timespec_to_ktime(ent->ts2);
668 delta = ktime_sub(t2, t1);
669 ds = ktime_to_ns(delta);
670 op = be16_to_cpu(((struct mlx5_inbox_hdr *)in->first.data)->opcode);
671 if (op < ARRAY_SIZE(cmd->stats)) {
672 stats = &cmd->stats[op];
673 spin_lock(&stats->lock);
674 stats->sum += ds;
675 ++stats->n;
676 spin_unlock(&stats->lock);
677 }
678 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
679 "fw exec time for %s is %lld nsec\n",
680 mlx5_command_str(op), ds);
681 *status = ent->status;
682 free_cmd(ent);
683 }
684
685 return err;
686
687out_free:
688 free_cmd(ent);
689out:
690 return err;
691}
692
693static ssize_t dbg_write(struct file *filp, const char __user *buf,
694 size_t count, loff_t *pos)
695{
696 struct mlx5_core_dev *dev = filp->private_data;
697 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
698 char lbuf[3];
699 int err;
700
701 if (!dbg->in_msg || !dbg->out_msg)
702 return -ENOMEM;
703
704 if (copy_from_user(lbuf, buf, sizeof(lbuf)))
5e631a03 705 return -EFAULT;
e126ba97
EC		 706
707 lbuf[sizeof(lbuf) - 1] = 0;
708
709 if (strcmp(lbuf, "go"))
710 return -EINVAL;
711
712 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);
713
714 return err ? err : count;
715}
716
717
718static const struct file_operations fops = {
719 .owner = THIS_MODULE,
720 .open = simple_open,
721 .write = dbg_write,
722};
723
724static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size)
725{
726 struct mlx5_cmd_prot_block *block;
727 struct mlx5_cmd_mailbox *next;
728 int copy;
729
730 if (!to || !from)
731 return -ENOMEM;
732
733 copy = min_t(int, size, sizeof(to->first.data));
734 memcpy(to->first.data, from, copy);
735 size -= copy;
736 from += copy;
737
738 next = to->next;
739 while (size) {
740 if (!next) {
741 /* this is a BUG */
742 return -ENOMEM;
743 }
744
745 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
746 block = next->buf;
747 memcpy(block->data, from, copy);
748 from += copy;
749 size -= copy;
750 next = next->next;
751 }
752
753 return 0;
754}
755
756static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
757{
758 struct mlx5_cmd_prot_block *block;
759 struct mlx5_cmd_mailbox *next;
760 int copy;
761
762 if (!to || !from)
763 return -ENOMEM;
764
765 copy = min_t(int, size, sizeof(from->first.data));
766 memcpy(to, from->first.data, copy);
767 size -= copy;
768 to += copy;
769
770 next = from->next;
771 while (size) {
772 if (!next) {
773 /* this is a BUG */
774 return -ENOMEM;
775 }
776
777 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
778 block = next->buf;
e126ba97
EC		 779
780 memcpy(to, block->data, copy);
781 to += copy;
782 size -= copy;
783 next = next->next;
784 }
785
786 return 0;
787}
788
789static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
790 gfp_t flags)
791{
792 struct mlx5_cmd_mailbox *mailbox;
793
794 mailbox = kmalloc(sizeof(*mailbox), flags);
795 if (!mailbox)
796 return ERR_PTR(-ENOMEM);
797
798 mailbox->buf = pci_pool_alloc(dev->cmd.pool, flags,
799 &mailbox->dma);
800 if (!mailbox->buf) {
801 mlx5_core_dbg(dev, "failed allocation\n");
802 kfree(mailbox);
803 return ERR_PTR(-ENOMEM);
804 }
805 memset(mailbox->buf, 0, sizeof(struct mlx5_cmd_prot_block));
806 mailbox->next = NULL;
807
808 return mailbox;
809}
810
811static void free_cmd_box(struct mlx5_core_dev *dev,
812 struct mlx5_cmd_mailbox *mailbox)
813{
814 pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
815 kfree(mailbox);
816}
817
818static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
819 gfp_t flags, int size)
820{
821 struct mlx5_cmd_mailbox *tmp, *head = NULL;
822 struct mlx5_cmd_prot_block *block;
823 struct mlx5_cmd_msg *msg;
824 int blen;
825 int err;
826 int n;
827 int i;
828
829 msg = kzalloc(sizeof(*msg), GFP_KERNEL);
830 if (!msg)
831 return ERR_PTR(-ENOMEM);
832
833 blen = size - min_t(int, sizeof(msg->first.data), size);
834 n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE;
835
836 for (i = 0; i < n; i++) {
837 tmp = alloc_cmd_box(dev, flags);
838 if (IS_ERR(tmp)) {
839 mlx5_core_warn(dev, "failed allocating block\n");
840 err = PTR_ERR(tmp);
841 goto err_alloc;
842 }
843
844 block = tmp->buf;
845 tmp->next = head;
846 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
847 block->block_num = cpu_to_be32(n - i - 1);
848 head = tmp;
849 }
850 msg->next = head;
851 msg->len = size;
852 return msg;
853
854err_alloc:
855 while (head) {
856 tmp = head->next;
857 free_cmd_box(dev, head);
858 head = tmp;
859 }
860 kfree(msg);
861
862 return ERR_PTR(err);
863}
864
865static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
866 struct mlx5_cmd_msg *msg)
867{
868 struct mlx5_cmd_mailbox *head = msg->next;
869 struct mlx5_cmd_mailbox *next;
870
871 while (head) {
872 next = head->next;
873 free_cmd_box(dev, head);
874 head = next;
875 }
876 kfree(msg);
877}
878
879static ssize_t data_write(struct file *filp, const char __user *buf,
880 size_t count, loff_t *pos)
881{
882 struct mlx5_core_dev *dev = filp->private_data;
883 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
884 void *ptr;
885 int err;
886
887 if (*pos != 0)
888 return -EINVAL;
889
890 kfree(dbg->in_msg);
891 dbg->in_msg = NULL;
892 dbg->inlen = 0;
893
894 ptr = kzalloc(count, GFP_KERNEL);
895 if (!ptr)
896 return -ENOMEM;
897
898 if (copy_from_user(ptr, buf, count)) {
5e631a03 899 err = -EFAULT;
e126ba97
EC		 900 goto out;
901 }
902 dbg->in_msg = ptr;
903 dbg->inlen = count;
904
905 *pos = count;
906
907 return count;
908
909out:
910 kfree(ptr);
911 return err;
912}
913
914static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
915 loff_t *pos)
916{
917 struct mlx5_core_dev *dev = filp->private_data;
918 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
919 int copy;
920
921 if (*pos)
922 return 0;
923
924 if (!dbg->out_msg)
925 return -ENOMEM;
926
927 copy = min_t(int, count, dbg->outlen);
928 if (copy_to_user(buf, dbg->out_msg, copy))
5e631a03 929 return -EFAULT;
e126ba97
EC		 930
931 *pos += copy;
932
933 return copy;
934}
935
936static const struct file_operations dfops = {
937 .owner = THIS_MODULE,
938 .open = simple_open,
939 .write = data_write,
940 .read = data_read,
941};
942
943static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
944 loff_t *pos)
945{
946 struct mlx5_core_dev *dev = filp->private_data;
947 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
948 char outlen[8];
949 int err;
950
951 if (*pos)
952 return 0;
953
954 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);
955 if (err < 0)
956 return err;
957
958 if (copy_to_user(buf, &outlen, err))
5e631a03 959 return -EFAULT;
e126ba97
EC		 960
961 *pos += err;
962
963 return err;
964}
965
966static ssize_t outlen_write(struct file *filp, const char __user *buf,
967 size_t count, loff_t *pos)
968{
969 struct mlx5_core_dev *dev = filp->private_data;
970 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
971 char outlen_str[8];
972 int outlen;
973 void *ptr;
974 int err;
975
976 if (*pos != 0 || count > 6)
977 return -EINVAL;
978
979 kfree(dbg->out_msg);
980 dbg->out_msg = NULL;
981 dbg->outlen = 0;
982
983 if (copy_from_user(outlen_str, buf, count))
5e631a03 984 return -EFAULT;
e126ba97
EC		 985
986 outlen_str[7] = 0;
987
988 err = sscanf(outlen_str, "%d", &outlen);
989 if (err < 0)
990 return err;
991
992 ptr = kzalloc(outlen, GFP_KERNEL);
993 if (!ptr)
994 return -ENOMEM;
995
996 dbg->out_msg = ptr;
997 dbg->outlen = outlen;
998
999 *pos = count;
1000
1001 return count;
1002}
1003
1004static const struct file_operations olfops = {
1005 .owner = THIS_MODULE,
1006 .open = simple_open,
1007 .write = outlen_write,
1008 .read = outlen_read,
1009};
1010
1011static void set_wqname(struct mlx5_core_dev *dev)
1012{
1013 struct mlx5_cmd *cmd = &dev->cmd;
1014
1015 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
1016 dev_name(&dev->pdev->dev));
1017}
1018
1019static void clean_debug_files(struct mlx5_core_dev *dev)
1020{
1021 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1022
1023 if (!mlx5_debugfs_root)
1024 return;
1025
1026 mlx5_cmdif_debugfs_cleanup(dev);
1027 debugfs_remove_recursive(dbg->dbg_root);
1028}
1029
1030static int create_debugfs_files(struct mlx5_core_dev *dev)
1031{
1032 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1033 int err = -ENOMEM;
1034
1035 if (!mlx5_debugfs_root)
1036 return 0;
1037
1038 dbg->dbg_root = debugfs_create_dir("cmd", dev->priv.dbg_root);
1039 if (!dbg->dbg_root)
1040 return err;
1041
1042 dbg->dbg_in = debugfs_create_file("in", 0400, dbg->dbg_root,
1043 dev, &dfops);
1044 if (!dbg->dbg_in)
1045 goto err_dbg;
1046
1047 dbg->dbg_out = debugfs_create_file("out", 0200, dbg->dbg_root,
1048 dev, &dfops);
1049 if (!dbg->dbg_out)
1050 goto err_dbg;
1051
1052 dbg->dbg_outlen = debugfs_create_file("out_len", 0600, dbg->dbg_root,
1053 dev, &olfops);
1054 if (!dbg->dbg_outlen)
1055 goto err_dbg;
1056
1057 dbg->dbg_status = debugfs_create_u8("status", 0600, dbg->dbg_root,
1058 &dbg->status);
1059 if (!dbg->dbg_status)
1060 goto err_dbg;
1061
1062 dbg->dbg_run = debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1063 if (!dbg->dbg_run)
1064 goto err_dbg;
1065
1066 mlx5_cmdif_debugfs_init(dev);
1067
1068 return 0;
1069
1070err_dbg:
1071 clean_debug_files(dev);
1072 return err;
1073}
1074
1075void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1076{
1077 struct mlx5_cmd *cmd = &dev->cmd;
1078 int i;
1079
1080 for (i = 0; i < cmd->max_reg_cmds; i++)
1081 down(&cmd->sem);
1082
1083 down(&cmd->pages_sem);
1084
1085 flush_workqueue(cmd->wq);
1086
1087 cmd->mode = CMD_MODE_EVENTS;
1088
1089 up(&cmd->pages_sem);
1090 for (i = 0; i < cmd->max_reg_cmds; i++)
1091 up(&cmd->sem);
1092}
1093
1094void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1095{
1096 struct mlx5_cmd *cmd = &dev->cmd;
1097 int i;
1098
1099 for (i = 0; i < cmd->max_reg_cmds; i++)
1100 down(&cmd->sem);
1101
1102 down(&cmd->pages_sem);
1103
1104 flush_workqueue(cmd->wq);
1105 cmd->mode = CMD_MODE_POLLING;
1106
1107 up(&cmd->pages_sem);
1108 for (i = 0; i < cmd->max_reg_cmds; i++)
1109 up(&cmd->sem);
1110}
1111
1112void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector)
1113{
1114 struct mlx5_cmd *cmd = &dev->cmd;
1115 struct mlx5_cmd_work_ent *ent;
1116 mlx5_cmd_cbk_t callback;
1117 void *context;
1118 int err;
1119 int i;
1120
1121 for (i = 0; i < (1 << cmd->log_sz); i++) {
1122 if (test_bit(i, &vector)) {
11940c87
DC		 1123 struct semaphore *sem;
1124
e126ba97 1125 ent = cmd->ent_arr[i];
11940c87
DC		 1126 if (ent->page_queue)
1127 sem = &cmd->pages_sem;
1128 else
1129 sem = &cmd->sem;
e126ba97
EC		 1130 ktime_get_ts(&ent->ts2);
1131 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1132 dump_command(dev, ent, 0);
1133 if (!ent->ret) {
1134 if (!cmd->checksum_disabled)
1135 ent->ret = verify_signature(ent);
1136 else
1137 ent->ret = 0;
1138 ent->status = ent->lay->status_own >> 1;
1139 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1140 ent->ret, deliv_status_to_str(ent->status), ent->status);
1141 }
1142 free_ent(cmd, ent->idx);
1143 if (ent->callback) {
1144 callback = ent->callback;
1145 context = ent->context;
1146 err = ent->ret;
1147 free_cmd(ent);
1148 callback(err, context);
1149 } else {
1150 complete(&ent->done);
1151 }
11940c87 1152 up(sem);
e126ba97
EC		 1153 }
1154 }
1155}
1156EXPORT_SYMBOL(mlx5_cmd_comp_handler);
1157
1158static int status_to_err(u8 status)
1159{
1160 return status ? -1 : 0; /* TBD more meaningful codes */
1161}
1162
1163static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size)
1164{
1165 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
1166 struct mlx5_cmd *cmd = &dev->cmd;
1167 struct cache_ent *ent = NULL;
1168
1169 if (in_size > MED_LIST_SIZE && in_size <= LONG_LIST_SIZE)
1170 ent = &cmd->cache.large;
1171 else if (in_size > 16 && in_size <= MED_LIST_SIZE)
1172 ent = &cmd->cache.med;
1173
1174 if (ent) {
1175 spin_lock(&ent->lock);
1176 if (!list_empty(&ent->head)) {
1177 msg = list_entry(ent->head.next, typeof(*msg), list);
1178 /* For cached lists, we must explicitly state what is
1179 * the real size
1180 */
1181 msg->len = in_size;
1182 list_del(&msg->list);
1183 }
1184 spin_unlock(&ent->lock);
1185 }
1186
1187 if (IS_ERR(msg))
1188 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, in_size);
1189
1190 return msg;
1191}
1192
1193static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1194{
1195 if (msg->cache) {
1196 spin_lock(&msg->cache->lock);
1197 list_add_tail(&msg->list, &msg->cache->head);
1198 spin_unlock(&msg->cache->lock);
1199 } else {
1200 mlx5_free_cmd_msg(dev, msg);
1201 }
1202}
1203
1204static int is_manage_pages(struct mlx5_inbox_hdr *in)
1205{
1206 return be16_to_cpu(in->opcode) == MLX5_CMD_OP_MANAGE_PAGES;
1207}
1208
1209int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1210 int out_size)
1211{
1212 struct mlx5_cmd_msg *inb;
1213 struct mlx5_cmd_msg *outb;
1214 int pages_queue;
1215 int err;
1216 u8 status = 0;
1217
1218 pages_queue = is_manage_pages(in);
1219
1220 inb = alloc_msg(dev, in_size);
1221 if (IS_ERR(inb)) {
1222 err = PTR_ERR(inb);
1223 return err;
1224 }
1225
1226 err = mlx5_copy_to_msg(inb, in, in_size);
1227 if (err) {
1228 mlx5_core_warn(dev, "err %d\n", err);
1229 goto out_in;
1230 }
1231
1232 outb = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, out_size);
1233 if (IS_ERR(outb)) {
1234 err = PTR_ERR(outb);
1235 goto out_in;
1236 }
1237
1238 err = mlx5_cmd_invoke(dev, inb, outb, NULL, NULL, pages_queue, &status);
1239 if (err)
1240 goto out_out;
1241
1242 mlx5_core_dbg(dev, "err %d, status %d\n", err, status);
1243 if (status) {
1244 err = status_to_err(status);
1245 goto out_out;
1246 }
1247
1248 err = mlx5_copy_from_msg(out, outb, out_size);
1249
1250out_out:
1251 mlx5_free_cmd_msg(dev, outb);
1252
1253out_in:
1254 free_msg(dev, inb);
1255 return err;
1256}
1257EXPORT_SYMBOL(mlx5_cmd_exec);
1258
1259static void destroy_msg_cache(struct mlx5_core_dev *dev)
1260{
1261 struct mlx5_cmd *cmd = &dev->cmd;
1262 struct mlx5_cmd_msg *msg;
1263 struct mlx5_cmd_msg *n;
1264
1265 list_for_each_entry_safe(msg, n, &cmd->cache.large.head, list) {
1266 list_del(&msg->list);
1267 mlx5_free_cmd_msg(dev, msg);
1268 }
1269
1270 list_for_each_entry_safe(msg, n, &cmd->cache.med.head, list) {
1271 list_del(&msg->list);
1272 mlx5_free_cmd_msg(dev, msg);
1273 }
1274}
1275
1276static int create_msg_cache(struct mlx5_core_dev *dev)
1277{
1278 struct mlx5_cmd *cmd = &dev->cmd;
1279 struct mlx5_cmd_msg *msg;
1280 int err;
1281 int i;
1282
1283 spin_lock_init(&cmd->cache.large.lock);
1284 INIT_LIST_HEAD(&cmd->cache.large.head);
1285 spin_lock_init(&cmd->cache.med.lock);
1286 INIT_LIST_HEAD(&cmd->cache.med.head);
1287
1288 for (i = 0; i < NUM_LONG_LISTS; i++) {
1289 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, LONG_LIST_SIZE);
1290 if (IS_ERR(msg)) {
1291 err = PTR_ERR(msg);
1292 goto ex_err;
1293 }
1294 msg->cache = &cmd->cache.large;
1295 list_add_tail(&msg->list, &cmd->cache.large.head);
1296 }
1297
1298 for (i = 0; i < NUM_MED_LISTS; i++) {
1299 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, MED_LIST_SIZE);
1300 if (IS_ERR(msg)) {
1301 err = PTR_ERR(msg);
1302 goto ex_err;
1303 }
1304 msg->cache = &cmd->cache.med;
1305 list_add_tail(&msg->list, &cmd->cache.med.head);
1306 }
1307
1308 return 0;
1309
1310ex_err:
1311 destroy_msg_cache(dev);
1312 return err;
1313}
1314
1315int mlx5_cmd_init(struct mlx5_core_dev *dev)
1316{
1317 int size = sizeof(struct mlx5_cmd_prot_block);
1318 int align = roundup_pow_of_two(size);
1319 struct mlx5_cmd *cmd = &dev->cmd;
1320 u32 cmd_h, cmd_l;
1321 u16 cmd_if_rev;
1322 int err;
1323 int i;
1324
1325 cmd_if_rev = cmdif_rev(dev);
1326 if (cmd_if_rev != CMD_IF_REV) {
1327 dev_err(&dev->pdev->dev,
1328 "Driver cmdif rev(%d) differs from firmware's(%d)\n",
1329 CMD_IF_REV, cmd_if_rev);
1330 return -EINVAL;
1331 }
1332
1333 cmd->pool = pci_pool_create("mlx5_cmd", dev->pdev, size, align, 0);
1334 if (!cmd->pool)
1335 return -ENOMEM;
1336
1337 cmd->cmd_buf = (void *)__get_free_pages(GFP_ATOMIC, 0);
1338 if (!cmd->cmd_buf) {
1339 err = -ENOMEM;
1340 goto err_free_pool;
1341 }
1342 cmd->dma = dma_map_single(&dev->pdev->dev, cmd->cmd_buf, PAGE_SIZE,
1343 DMA_BIDIRECTIONAL);
1344 if (dma_mapping_error(&dev->pdev->dev, cmd->dma)) {
1345 err = -ENOMEM;
1346 goto err_free;
1347 }
1348
1349 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
1350 cmd->log_sz = cmd_l >> 4 & 0xf;
1351 cmd->log_stride = cmd_l & 0xf;
1352 if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) {
1353 dev_err(&dev->pdev->dev, "firmware reports too many outstanding commands %d\n",
1354 1 << cmd->log_sz);
1355 err = -EINVAL;
1356 goto err_map;
1357 }
1358
1359 if (cmd->log_sz + cmd->log_stride > PAGE_SHIFT) {
1360 dev_err(&dev->pdev->dev, "command queue size overflow\n");
1361 err = -EINVAL;
1362 goto err_map;
1363 }
1364
c1868b82 1365 cmd->checksum_disabled = 1;
e126ba97
EC		 1366 cmd->max_reg_cmds = (1 << cmd->log_sz) - 1;
1367 cmd->bitmask = (1 << cmd->max_reg_cmds) - 1;
1368
1369 cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
1370 if (cmd->cmdif_rev > CMD_IF_REV) {
1371 dev_err(&dev->pdev->dev, "driver does not support command interface version. driver %d, firmware %d\n",
1372 CMD_IF_REV, cmd->cmdif_rev);
1373 err = -ENOTSUPP;
1374 goto err_map;
1375 }
1376
1377 spin_lock_init(&cmd->alloc_lock);
1378 spin_lock_init(&cmd->token_lock);
1379 for (i = 0; i < ARRAY_SIZE(cmd->stats); i++)
1380 spin_lock_init(&cmd->stats[i].lock);
1381
1382 sema_init(&cmd->sem, cmd->max_reg_cmds);
1383 sema_init(&cmd->pages_sem, 1);
1384
1385 cmd_h = (u32)((u64)(cmd->dma) >> 32);
1386 cmd_l = (u32)(cmd->dma);
1387 if (cmd_l & 0xfff) {
1388 dev_err(&dev->pdev->dev, "invalid command queue address\n");
1389 err = -ENOMEM;
1390 goto err_map;
1391 }
1392
1393 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
1394 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
1395
1396 /* Make sure firmware sees the complete address before we proceed */
1397 wmb();
1398
1399 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
1400
1401 cmd->mode = CMD_MODE_POLLING;
1402
1403 err = create_msg_cache(dev);
1404 if (err) {
1405 dev_err(&dev->pdev->dev, "failed to create command cache\n");
1406 goto err_map;
1407 }
1408
1409 set_wqname(dev);
1410 cmd->wq = create_singlethread_workqueue(cmd->wq_name);
1411 if (!cmd->wq) {
1412 dev_err(&dev->pdev->dev, "failed to create command workqueue\n");
1413 err = -ENOMEM;
1414 goto err_cache;
1415 }
1416
1417 err = create_debugfs_files(dev);
1418 if (err) {
1419 err = -ENOMEM;
1420 goto err_wq;
1421 }
1422
1423 return 0;
1424
1425err_wq:
1426 destroy_workqueue(cmd->wq);
1427
1428err_cache:
1429 destroy_msg_cache(dev);
1430
1431err_map:
1432 dma_unmap_single(&dev->pdev->dev, cmd->dma, PAGE_SIZE,
1433 DMA_BIDIRECTIONAL);
1434err_free:
1435 free_pages((unsigned long)cmd->cmd_buf, 0);
1436
1437err_free_pool:
1438 pci_pool_destroy(cmd->pool);
1439
1440 return err;
1441}
1442EXPORT_SYMBOL(mlx5_cmd_init);
1443
1444void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
1445{
1446 struct mlx5_cmd *cmd = &dev->cmd;
1447
1448 clean_debug_files(dev);
1449 destroy_workqueue(cmd->wq);
1450 destroy_msg_cache(dev);
1451 dma_unmap_single(&dev->pdev->dev, cmd->dma, PAGE_SIZE,
1452 DMA_BIDIRECTIONAL);
1453 free_pages((unsigned long)cmd->cmd_buf, 0);
1454 pci_pool_destroy(cmd->pool);
1455}
1456EXPORT_SYMBOL(mlx5_cmd_cleanup);
1457
1458static const char *cmd_status_str(u8 status)
1459{
1460 switch (status) {
1461 case MLX5_CMD_STAT_OK:
1462 return "OK";
1463 case MLX5_CMD_STAT_INT_ERR:
1464 return "internal error";
1465 case MLX5_CMD_STAT_BAD_OP_ERR:
1466 return "bad operation";
1467 case MLX5_CMD_STAT_BAD_PARAM_ERR:
1468 return "bad parameter";
1469 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
1470 return "bad system state";
1471 case MLX5_CMD_STAT_BAD_RES_ERR:
1472 return "bad resource";
1473 case MLX5_CMD_STAT_RES_BUSY:
1474 return "resource busy";
1475 case MLX5_CMD_STAT_LIM_ERR:
1476 return "limits exceeded";
1477 case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
1478 return "bad resource state";
1479 case MLX5_CMD_STAT_IX_ERR:
1480 return "bad index";
1481 case MLX5_CMD_STAT_NO_RES_ERR:
1482 return "no resources";
1483 case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
1484 return "bad input length";
1485 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
1486 return "bad output length";
1487 case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
1488 return "bad QP state";
1489 case MLX5_CMD_STAT_BAD_PKT_ERR:
1490 return "bad packet (discarded)";
1491 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
1492 return "bad size too many outstanding CQEs";
1493 default:
1494 return "unknown status";
1495 }
1496}
1497
1498int mlx5_cmd_status_to_err(struct mlx5_outbox_hdr *hdr)
1499{
1500 if (!hdr->status)
1501 return 0;
1502
1503 pr_warn("command failed, status %s(0x%x), syndrome 0x%x\n",
1504 cmd_status_str(hdr->status), hdr->status,
1505 be32_to_cpu(hdr->syndrome));
1506
1507 switch (hdr->status) {
1508 case MLX5_CMD_STAT_OK: return 0;
1509 case MLX5_CMD_STAT_INT_ERR: return -EIO;
1510 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;
1511 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;
1512 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;
1513 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;
1514 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;
1515 case MLX5_CMD_STAT_LIM_ERR: return -EINVAL;
1516 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;
1517 case MLX5_CMD_STAT_IX_ERR: return -EINVAL;
1518 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;
1519 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;
1520 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;
1521 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;
1522 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;
1523 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;
1524 default: return -EIO;
1525 }
1526}
Linux 6.x block layer and io_uring tree(s)