projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] be2iscsi: Fix port speed typo in driver.
[linux-2.6-block.git] / drivers / scsi / be2iscsi / be_main.c
CommitLineData
6733b39a 1/**
533c165f 2 * Copyright (C) 2005 - 2013 Emulex
6733b39a
JK		 3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
255fa9a3 10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
6733b39a
JK		 11 *
12 * Contact Information:
255fa9a3 13 * linux-drivers@emulex.com
6733b39a 14 *
255fa9a3
JK		 15 * Emulex
16 * 3333 Susan Street
17 * Costa Mesa, CA 92626
6733b39a 18 */
255fa9a3 19
6733b39a
JK		 20#include <linux/reboot.h>
21#include <linux/delay.h>
5a0e3ad6 22#include <linux/slab.h>
6733b39a
JK		 23#include <linux/interrupt.h>
24#include <linux/blkdev.h>
25#include <linux/pci.h>
26#include <linux/string.h>
27#include <linux/kernel.h>
28#include <linux/semaphore.h>
c7acc5b8 29#include <linux/iscsi_boot_sysfs.h>
acf3368f 30#include <linux/module.h>
ffce3e2e 31#include <linux/bsg-lib.h>
6733b39a
JK		 32
33#include <scsi/libiscsi.h>
ffce3e2e
JK		 34#include <scsi/scsi_bsg_iscsi.h>
35#include <scsi/scsi_netlink.h>
6733b39a
JK		 36#include <scsi/scsi_transport_iscsi.h>
37#include <scsi/scsi_transport.h>
38#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_device.h>
40#include <scsi/scsi_host.h>
41#include <scsi/scsi.h>
42#include "be_main.h"
43#include "be_iscsi.h"
44#include "be_mgmt.h"
0a513dd8 45#include "be_cmds.h"
6733b39a
JK		 46
47static unsigned int be_iopoll_budget = 10;
48static unsigned int be_max_phys_size = 64;
bfead3b2 49static unsigned int enable_msix = 1;
6733b39a
JK		 50
51MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
52MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
76d15dbd 53MODULE_VERSION(BUILD_STR);
2f635883 54MODULE_AUTHOR("Emulex Corporation");
6733b39a
JK		 55MODULE_LICENSE("GPL");
56module_param(be_iopoll_budget, int, 0);
57module_param(enable_msix, int, 0);
58module_param(be_max_phys_size, uint, S_IRUGO);
99bc5d55
JSJ		 59MODULE_PARM_DESC(be_max_phys_size,
60 "Maximum Size (In Kilobytes) of physically contiguous "
61 "memory that can be allocated. Range is 16 - 128");
62
63#define beiscsi_disp_param(_name)\
64ssize_t \
65beiscsi_##_name##_disp(struct device *dev,\
66 struct device_attribute *attrib, char *buf) \
67{ \
68 struct Scsi_Host *shost = class_to_shost(dev);\
69 struct beiscsi_hba *phba = iscsi_host_priv(shost); \
70 uint32_t param_val = 0; \
71 param_val = phba->attr_##_name;\
72 return snprintf(buf, PAGE_SIZE, "%d\n",\
73 phba->attr_##_name);\
74}
75
76#define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
77int \
78beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
79{\
80 if (val >= _minval && val <= _maxval) {\
81 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
82 "BA_%d : beiscsi_"#_name" updated "\
83 "from 0x%x ==> 0x%x\n",\
84 phba->attr_##_name, val); \
85 phba->attr_##_name = val;\
86 return 0;\
87 } \
88 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
89 "BA_%d beiscsi_"#_name" attribute "\
90 "cannot be updated to 0x%x, "\
91 "range allowed is ["#_minval" - "#_maxval"]\n", val);\
92 return -EINVAL;\
93}
94
95#define beiscsi_store_param(_name) \
96ssize_t \
97beiscsi_##_name##_store(struct device *dev,\
98 struct device_attribute *attr, const char *buf,\
99 size_t count) \
100{ \
101 struct Scsi_Host *shost = class_to_shost(dev);\
102 struct beiscsi_hba *phba = iscsi_host_priv(shost);\
103 uint32_t param_val = 0;\
104 if (!isdigit(buf[0]))\
105 return -EINVAL;\
106 if (sscanf(buf, "%i", &param_val) != 1)\
107 return -EINVAL;\
108 if (beiscsi_##_name##_change(phba, param_val) == 0) \
109 return strlen(buf);\
110 else \
111 return -EINVAL;\
112}
113
114#define beiscsi_init_param(_name, _minval, _maxval, _defval) \
115int \
116beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
117{ \
118 if (val >= _minval && val <= _maxval) {\
119 phba->attr_##_name = val;\
120 return 0;\
121 } \
122 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
123 "BA_%d beiscsi_"#_name" attribute " \
124 "cannot be updated to 0x%x, "\
125 "range allowed is ["#_minval" - "#_maxval"]\n", val);\
126 phba->attr_##_name = _defval;\
127 return -EINVAL;\
128}
129
130#define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
131static uint beiscsi_##_name = _defval;\
132module_param(beiscsi_##_name, uint, S_IRUGO);\
133MODULE_PARM_DESC(beiscsi_##_name, _descp);\
134beiscsi_disp_param(_name)\
135beiscsi_change_param(_name, _minval, _maxval, _defval)\
136beiscsi_store_param(_name)\
137beiscsi_init_param(_name, _minval, _maxval, _defval)\
138DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
139 beiscsi_##_name##_disp, beiscsi_##_name##_store)
140
141/*
142 * When new log level added update the
143 * the MAX allowed value for log_enable
144 */
145BEISCSI_RW_ATTR(log_enable, 0x00,
146 0xFF, 0x00, "Enable logging Bit Mask\n"
147 "\t\t\t\tInitialization Events : 0x01\n"
148 "\t\t\t\tMailbox Events : 0x02\n"
149 "\t\t\t\tMiscellaneous Events : 0x04\n"
150 "\t\t\t\tError Handling : 0x08\n"
151 "\t\t\t\tIO Path Events : 0x10\n"
afb96058
JK		 152 "\t\t\t\tConfiguration Path : 0x20\n"
153 "\t\t\t\tiSCSI Protocol : 0x40\n");
99bc5d55 154
5cac7596 155DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
26000db7 156DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
22661e25 157DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
d3fea9af 158DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
6103c1f7
JK		 159DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
160 beiscsi_active_session_disp, NULL);
161DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
162 beiscsi_free_session_disp, NULL);
99bc5d55
JSJ		 163struct device_attribute *beiscsi_attrs[] = {
164 &dev_attr_beiscsi_log_enable,
5cac7596 165 &dev_attr_beiscsi_drvr_ver,
26000db7 166 &dev_attr_beiscsi_adapter_family,
22661e25 167 &dev_attr_beiscsi_fw_ver,
6103c1f7
JK		 168 &dev_attr_beiscsi_active_session_count,
169 &dev_attr_beiscsi_free_session_count,
d3fea9af 170 &dev_attr_beiscsi_phys_port,
99bc5d55
JSJ		 171 NULL,
172};
6733b39a 173
6763daae
JSJ		 174static char const *cqe_desc[] = {
175 "RESERVED_DESC",
176 "SOL_CMD_COMPLETE",
177 "SOL_CMD_KILLED_DATA_DIGEST_ERR",
178 "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
179 "CXN_KILLED_BURST_LEN_MISMATCH",
180 "CXN_KILLED_AHS_RCVD",
181 "CXN_KILLED_HDR_DIGEST_ERR",
182 "CXN_KILLED_UNKNOWN_HDR",
183 "CXN_KILLED_STALE_ITT_TTT_RCVD",
184 "CXN_KILLED_INVALID_ITT_TTT_RCVD",
185 "CXN_KILLED_RST_RCVD",
186 "CXN_KILLED_TIMED_OUT",
187 "CXN_KILLED_RST_SENT",
188 "CXN_KILLED_FIN_RCVD",
189 "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
190 "CXN_KILLED_BAD_WRB_INDEX_ERROR",
191 "CXN_KILLED_OVER_RUN_RESIDUAL",
192 "CXN_KILLED_UNDER_RUN_RESIDUAL",
193 "CMD_KILLED_INVALID_STATSN_RCVD",
194 "CMD_KILLED_INVALID_R2T_RCVD",
195 "CMD_CXN_KILLED_LUN_INVALID",
196 "CMD_CXN_KILLED_ICD_INVALID",
197 "CMD_CXN_KILLED_ITT_INVALID",
198 "CMD_CXN_KILLED_SEQ_OUTOFORDER",
199 "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
200 "CXN_INVALIDATE_NOTIFY",
201 "CXN_INVALIDATE_INDEX_NOTIFY",
202 "CMD_INVALIDATED_NOTIFY",
203 "UNSOL_HDR_NOTIFY",
204 "UNSOL_DATA_NOTIFY",
205 "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
206 "DRIVERMSG_NOTIFY",
207 "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
208 "SOL_CMD_KILLED_DIF_ERR",
209 "CXN_KILLED_SYN_RCVD",
210 "CXN_KILLED_IMM_DATA_RCVD"
211};
212
6733b39a
JK		 213static int beiscsi_slave_configure(struct scsi_device *sdev)
214{
215 blk_queue_max_segment_size(sdev->request_queue, 65536);
216 return 0;
217}
218
4183122d
JK		 219static int beiscsi_eh_abort(struct scsi_cmnd *sc)
220{
221 struct iscsi_cls_session *cls_session;
222 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
223 struct beiscsi_io_task *aborted_io_task;
224 struct iscsi_conn *conn;
225 struct beiscsi_conn *beiscsi_conn;
226 struct beiscsi_hba *phba;
227 struct iscsi_session *session;
228 struct invalidate_command_table *inv_tbl;
3cbb7a74 229 struct be_dma_mem nonemb_cmd;
4183122d 230 unsigned int cid, tag, num_invalidate;
1957aa7f 231 int rc;
4183122d
JK		 232
233 cls_session = starget_to_session(scsi_target(sc->device));
234 session = cls_session->dd_data;
235
236 spin_lock_bh(&session->lock);
237 if (!aborted_task || !aborted_task->sc) {
238 /* we raced */
239 spin_unlock_bh(&session->lock);
240 return SUCCESS;
241 }
242
243 aborted_io_task = aborted_task->dd_data;
244 if (!aborted_io_task->scsi_cmnd) {
245 /* raced or invalid command */
246 spin_unlock_bh(&session->lock);
247 return SUCCESS;
248 }
249 spin_unlock_bh(&session->lock);
7626c06b
JK		 250 /* Invalidate WRB Posted for this Task */
251 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
252 aborted_io_task->pwrb_handle->pwrb,
253 1);
254
4183122d
JK		 255 conn = aborted_task->conn;
256 beiscsi_conn = conn->dd_data;
257 phba = beiscsi_conn->phba;
258
259 /* invalidate iocb */
260 cid = beiscsi_conn->beiscsi_conn_cid;
261 inv_tbl = phba->inv_tbl;
262 memset(inv_tbl, 0x0, sizeof(*inv_tbl));
263 inv_tbl->cid = cid;
264 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
265 num_invalidate = 1;
3cbb7a74
JK		 266 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
267 sizeof(struct invalidate_commands_params_in),
268 &nonemb_cmd.dma);
269 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 270 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
271 "BM_%d : Failed to allocate memory for"
272 "mgmt_invalidate_icds\n");
3cbb7a74
JK		 273 return FAILED;
274 }
275 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
276
277 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
278 cid, &nonemb_cmd);
4183122d 279 if (!tag) {
99bc5d55
JSJ		 280 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
281 "BM_%d : mgmt_invalidate_icds could not be"
282 "submitted\n");
3cbb7a74
JK		 283 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
284 nonemb_cmd.va, nonemb_cmd.dma);
285
4183122d 286 return FAILED;
4183122d 287 }
e175defe 288
1957aa7f
JK		 289 rc = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd);
290 if (rc != -EBUSY)
291 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
292 nonemb_cmd.va, nonemb_cmd.dma);
293
4183122d
JK		 294 return iscsi_eh_abort(sc);
295}
296
297static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
298{
299 struct iscsi_task *abrt_task;
300 struct beiscsi_io_task *abrt_io_task;
301 struct iscsi_conn *conn;
302 struct beiscsi_conn *beiscsi_conn;
303 struct beiscsi_hba *phba;
304 struct iscsi_session *session;
305 struct iscsi_cls_session *cls_session;
306 struct invalidate_command_table *inv_tbl;
3cbb7a74 307 struct be_dma_mem nonemb_cmd;
4183122d 308 unsigned int cid, tag, i, num_invalidate;
1957aa7f 309 int rc;
4183122d
JK		 310
311 /* invalidate iocbs */
312 cls_session = starget_to_session(scsi_target(sc->device));
313 session = cls_session->dd_data;
314 spin_lock_bh(&session->lock);
db7f7709
JK		 315 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
316 spin_unlock_bh(&session->lock);
317 return FAILED;
318 }
4183122d
JK		 319 conn = session->leadconn;
320 beiscsi_conn = conn->dd_data;
321 phba = beiscsi_conn->phba;
322 cid = beiscsi_conn->beiscsi_conn_cid;
323 inv_tbl = phba->inv_tbl;
324 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
325 num_invalidate = 0;
326 for (i = 0; i < conn->session->cmds_max; i++) {
327 abrt_task = conn->session->cmds[i];
328 abrt_io_task = abrt_task->dd_data;
329 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
330 continue;
331
126e964a 332 if (sc->device->lun != abrt_task->sc->device->lun)
4183122d
JK		 333 continue;
334
7626c06b
JK		 335 /* Invalidate WRB Posted for this Task */
336 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
337 abrt_io_task->pwrb_handle->pwrb,
338 1);
339
4183122d
JK		 340 inv_tbl->cid = cid;
341 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
342 num_invalidate++;
343 inv_tbl++;
344 }
345 spin_unlock_bh(&session->lock);
346 inv_tbl = phba->inv_tbl;
347
3cbb7a74
JK		 348 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
349 sizeof(struct invalidate_commands_params_in),
350 &nonemb_cmd.dma);
351 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 352 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
353 "BM_%d : Failed to allocate memory for"
354 "mgmt_invalidate_icds\n");
3cbb7a74
JK		 355 return FAILED;
356 }
357 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
358 memset(nonemb_cmd.va, 0, nonemb_cmd.size);
359 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
360 cid, &nonemb_cmd);
4183122d 361 if (!tag) {
99bc5d55
JSJ		 362 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
363 "BM_%d : mgmt_invalidate_icds could not be"
364 " submitted\n");
3cbb7a74
JK		 365 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
366 nonemb_cmd.va, nonemb_cmd.dma);
4183122d 367 return FAILED;
4183122d 368 }
e175defe 369
1957aa7f
JK		 370 rc = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd);
371 if (rc != -EBUSY)
372 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
373 nonemb_cmd.va, nonemb_cmd.dma);
4183122d 374 return iscsi_eh_device_reset(sc);
4183122d
JK		 375}
376
c7acc5b8
JK		 377static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
378{
379 struct beiscsi_hba *phba = data;
f457a46f
MC		 380 struct mgmt_session_info *boot_sess = &phba->boot_sess;
381 struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
c7acc5b8
JK		 382 char *str = buf;
383 int rc;
384
385 switch (type) {
386 case ISCSI_BOOT_TGT_NAME:
387 rc = sprintf(buf, "%.*s\n",
f457a46f
MC		 388 (int)strlen(boot_sess->target_name),
389 (char *)&boot_sess->target_name);
c7acc5b8
JK		 390 break;
391 case ISCSI_BOOT_TGT_IP_ADDR:
f457a46f 392 if (boot_conn->dest_ipaddr.ip_type == 0x1)
c7acc5b8 393 rc = sprintf(buf, "%pI4\n",
0e43895e 394 (char *)&boot_conn->dest_ipaddr.addr);
c7acc5b8
JK		 395 else
396 rc = sprintf(str, "%pI6\n",
0e43895e 397 (char *)&boot_conn->dest_ipaddr.addr);
c7acc5b8
JK		 398 break;
399 case ISCSI_BOOT_TGT_PORT:
f457a46f 400 rc = sprintf(str, "%d\n", boot_conn->dest_port);
c7acc5b8
JK		 401 break;
402
403 case ISCSI_BOOT_TGT_CHAP_NAME:
404 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 405 boot_conn->negotiated_login_options.auth_data.chap.
406 target_chap_name_length,
407 (char *)&boot_conn->negotiated_login_options.
408 auth_data.chap.target_chap_name);
c7acc5b8
JK		 409 break;
410 case ISCSI_BOOT_TGT_CHAP_SECRET:
411 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 412 boot_conn->negotiated_login_options.auth_data.chap.
413 target_secret_length,
414 (char *)&boot_conn->negotiated_login_options.
415 auth_data.chap.target_secret);
c7acc5b8
JK		 416 break;
417 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
418 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 419 boot_conn->negotiated_login_options.auth_data.chap.
420 intr_chap_name_length,
421 (char *)&boot_conn->negotiated_login_options.
422 auth_data.chap.intr_chap_name);
c7acc5b8
JK		 423 break;
424 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
f457a46f
MC		 425 rc = sprintf(str, "%.*s\n",
426 boot_conn->negotiated_login_options.auth_data.chap.
427 intr_secret_length,
428 (char *)&boot_conn->negotiated_login_options.
429 auth_data.chap.intr_secret);
c7acc5b8
JK		 430 break;
431 case ISCSI_BOOT_TGT_FLAGS:
f457a46f 432 rc = sprintf(str, "2\n");
c7acc5b8
JK		 433 break;
434 case ISCSI_BOOT_TGT_NIC_ASSOC:
f457a46f 435 rc = sprintf(str, "0\n");
c7acc5b8
JK		 436 break;
437 default:
438 rc = -ENOSYS;
439 break;
440 }
441 return rc;
442}
443
444static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
445{
446 struct beiscsi_hba *phba = data;
447 char *str = buf;
448 int rc;
449
450 switch (type) {
451 case ISCSI_BOOT_INI_INITIATOR_NAME:
452 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
453 break;
454 default:
455 rc = -ENOSYS;
456 break;
457 }
458 return rc;
459}
460
461static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
462{
463 struct beiscsi_hba *phba = data;
464 char *str = buf;
465 int rc;
466
467 switch (type) {
468 case ISCSI_BOOT_ETH_FLAGS:
f457a46f 469 rc = sprintf(str, "2\n");
c7acc5b8
JK		 470 break;
471 case ISCSI_BOOT_ETH_INDEX:
f457a46f 472 rc = sprintf(str, "0\n");
c7acc5b8
JK		 473 break;
474 case ISCSI_BOOT_ETH_MAC:
0e43895e
MC		 475 rc = beiscsi_get_macaddr(str, phba);
476 break;
c7acc5b8
JK		 477 default:
478 rc = -ENOSYS;
479 break;
480 }
481 return rc;
482}
483
484
587a1f16 485static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
c7acc5b8 486{
587a1f16 487 umode_t rc;
c7acc5b8
JK		 488
489 switch (type) {
490 case ISCSI_BOOT_TGT_NAME:
491 case ISCSI_BOOT_TGT_IP_ADDR:
492 case ISCSI_BOOT_TGT_PORT:
493 case ISCSI_BOOT_TGT_CHAP_NAME:
494 case ISCSI_BOOT_TGT_CHAP_SECRET:
495 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
496 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
497 case ISCSI_BOOT_TGT_NIC_ASSOC:
498 case ISCSI_BOOT_TGT_FLAGS:
499 rc = S_IRUGO;
500 break;
501 default:
502 rc = 0;
503 break;
504 }
505 return rc;
506}
507
587a1f16 508static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
c7acc5b8 509{
587a1f16 510 umode_t rc;
c7acc5b8
JK		 511
512 switch (type) {
513 case ISCSI_BOOT_INI_INITIATOR_NAME:
514 rc = S_IRUGO;
515 break;
516 default:
517 rc = 0;
518 break;
519 }
520 return rc;
521}
522
523
587a1f16 524static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
c7acc5b8 525{
587a1f16 526 umode_t rc;
c7acc5b8
JK		 527
528 switch (type) {
529 case ISCSI_BOOT_ETH_FLAGS:
530 case ISCSI_BOOT_ETH_MAC:
531 case ISCSI_BOOT_ETH_INDEX:
532 rc = S_IRUGO;
533 break;
534 default:
535 rc = 0;
536 break;
537 }
538 return rc;
539}
540
bfead3b2
JK		 541/*------------------- PCI Driver operations and data ----------------- */
542static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
543 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
f98c96b0 544 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
bfead3b2
JK		 545 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
546 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
547 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
139a1b1e 548 { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
bfead3b2
JK		 549 { 0 }
550};
551MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
552
99bc5d55 553
6733b39a
JK		 554static struct scsi_host_template beiscsi_sht = {
555 .module = THIS_MODULE,
2f635883 556 .name = "Emulex 10Gbe open-iscsi Initiator Driver",
6733b39a
JK		 557 .proc_name = DRV_NAME,
558 .queuecommand = iscsi_queuecommand,
6733b39a
JK		 559 .change_queue_depth = iscsi_change_queue_depth,
560 .slave_configure = beiscsi_slave_configure,
561 .target_alloc = iscsi_target_alloc,
4183122d
JK		 562 .eh_abort_handler = beiscsi_eh_abort,
563 .eh_device_reset_handler = beiscsi_eh_device_reset,
309ce156 564 .eh_target_reset_handler = iscsi_eh_session_reset,
99bc5d55 565 .shost_attrs = beiscsi_attrs,
6733b39a
JK		 566 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
567 .can_queue = BE2_IO_DEPTH,
568 .this_id = -1,
569 .max_sectors = BEISCSI_MAX_SECTORS,
570 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
571 .use_clustering = ENABLE_CLUSTERING,
ffce3e2e
JK		 572 .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
573
6733b39a 574};
6733b39a 575
bfead3b2 576static struct scsi_transport_template *beiscsi_scsi_transport;
6733b39a
JK		 577
578static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
579{
580 struct beiscsi_hba *phba;
581 struct Scsi_Host *shost;
582
583 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
584 if (!shost) {
99bc5d55
JSJ		 585 dev_err(&pcidev->dev,
586 "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
6733b39a
JK		 587 return NULL;
588 }
589 shost->dma_boundary = pcidev->dma_mask;
590 shost->max_id = BE2_MAX_SESSIONS;
591 shost->max_channel = 0;
592 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
593 shost->max_lun = BEISCSI_NUM_MAX_LUN;
594 shost->transportt = beiscsi_scsi_transport;
6733b39a
JK		 595 phba = iscsi_host_priv(shost);
596 memset(phba, 0, sizeof(*phba));
597 phba->shost = shost;
598 phba->pcidev = pci_dev_get(pcidev);
2807afb7 599 pci_set_drvdata(pcidev, phba);
0e43895e 600 phba->interface_handle = 0xFFFFFFFF;
6733b39a
JK		 601
602 if (iscsi_host_add(shost, &phba->pcidev->dev))
603 goto free_devices;
c7acc5b8 604
6733b39a
JK		 605 return phba;
606
607free_devices:
608 pci_dev_put(phba->pcidev);
609 iscsi_host_free(phba->shost);
610 return NULL;
611}
612
613static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
614{
615 if (phba->csr_va) {
616 iounmap(phba->csr_va);
617 phba->csr_va = NULL;
618 }
619 if (phba->db_va) {
620 iounmap(phba->db_va);
621 phba->db_va = NULL;
622 }
623 if (phba->pci_va) {
624 iounmap(phba->pci_va);
625 phba->pci_va = NULL;
626 }
627}
628
629static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
630 struct pci_dev *pcidev)
631{
632 u8 __iomem *addr;
f98c96b0 633 int pcicfg_reg;
6733b39a
JK		 634
635 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
636 pci_resource_len(pcidev, 2));
637 if (addr == NULL)
638 return -ENOMEM;
639 phba->ctrl.csr = addr;
640 phba->csr_va = addr;
641 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
642
643 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
644 if (addr == NULL)
645 goto pci_map_err;
646 phba->ctrl.db = addr;
647 phba->db_va = addr;
648 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
649
f98c96b0
JK		 650 if (phba->generation == BE_GEN2)
651 pcicfg_reg = 1;
652 else
653 pcicfg_reg = 0;
654
655 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
656 pci_resource_len(pcidev, pcicfg_reg));
657
6733b39a
JK		 658 if (addr == NULL)
659 goto pci_map_err;
660 phba->ctrl.pcicfg = addr;
661 phba->pci_va = addr;
f98c96b0 662 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
6733b39a
JK		 663 return 0;
664
665pci_map_err:
666 beiscsi_unmap_pci_function(phba);
667 return -ENOMEM;
668}
669
670static int beiscsi_enable_pci(struct pci_dev *pcidev)
671{
672 int ret;
673
674 ret = pci_enable_device(pcidev);
675 if (ret) {
99bc5d55
JSJ		 676 dev_err(&pcidev->dev,
677 "beiscsi_enable_pci - enable device failed\n");
6733b39a
JK		 678 return ret;
679 }
680
bfead3b2 681 pci_set_master(pcidev);
6733b39a
JK		 682 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
683 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
684 if (ret) {
685 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
686 pci_disable_device(pcidev);
687 return ret;
688 }
689 }
690 return 0;
691}
692
693static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
694{
695 struct be_ctrl_info *ctrl = &phba->ctrl;
696 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
697 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
698 int status = 0;
699
700 ctrl->pdev = pdev;
701 status = beiscsi_map_pci_bars(phba, pdev);
702 if (status)
703 return status;
6733b39a
JK		 704 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
705 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
706 mbox_mem_alloc->size,
707 &mbox_mem_alloc->dma);
708 if (!mbox_mem_alloc->va) {
709 beiscsi_unmap_pci_function(phba);
a49e06d5 710 return -ENOMEM;
6733b39a
JK		 711 }
712
713 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
714 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
715 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
716 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
717 spin_lock_init(&ctrl->mbox_lock);
bfead3b2
JK		 718 spin_lock_init(&phba->ctrl.mcc_lock);
719 spin_lock_init(&phba->ctrl.mcc_cq_lock);
720
6733b39a
JK		 721 return status;
722}
723
843ae752
JK		 724/**
725 * beiscsi_get_params()- Set the config paramters
726 * @phba: ptr device priv structure
727 **/
6733b39a
JK		 728static void beiscsi_get_params(struct beiscsi_hba *phba)
729{
843ae752
JK		 730 uint32_t total_cid_count = 0;
731 uint32_t total_icd_count = 0;
732 uint8_t ulp_num = 0;
733
734 total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
735 BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
736
cf987b79
JK		 737 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
738 uint32_t align_mask = 0;
739 uint32_t icd_post_per_page = 0;
740 uint32_t icd_count_unavailable = 0;
741 uint32_t icd_start = 0, icd_count = 0;
742 uint32_t icd_start_align = 0, icd_count_align = 0;
743
843ae752 744 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
cf987b79
JK		 745 icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
746 icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
747
748 /* Get ICD count that can be posted on each page */
749 icd_post_per_page = (PAGE_SIZE / (BE2_SGE *
750 sizeof(struct iscsi_sge)));
751 align_mask = (icd_post_per_page - 1);
752
753 /* Check if icd_start is aligned ICD per page posting */
754 if (icd_start % icd_post_per_page) {
755 icd_start_align = ((icd_start +
756 icd_post_per_page) &
757 ~(align_mask));
758 phba->fw_config.
759 iscsi_icd_start[ulp_num] =
760 icd_start_align;
761 }
762
763 icd_count_align = (icd_count & ~align_mask);
764
765 /* ICD discarded in the process of alignment */
766 if (icd_start_align)
767 icd_count_unavailable = ((icd_start_align -
768 icd_start) +
769 (icd_count -
770 icd_count_align));
771
772 /* Updated ICD count available */
773 phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count -
774 icd_count_unavailable);
775
776 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
777 "BM_%d : Aligned ICD values\n"
778 "\t ICD Start : %d\n"
779 "\t ICD Count : %d\n"
780 "\t ICD Discarded : %d\n",
781 phba->fw_config.
782 iscsi_icd_start[ulp_num],
783 phba->fw_config.
784 iscsi_icd_count[ulp_num],
785 icd_count_unavailable);
843ae752
JK		 786 break;
787 }
cf987b79 788 }
843ae752 789
cf987b79 790 total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
843ae752
JK		 791 phba->params.ios_per_ctrl = (total_icd_count -
792 (total_cid_count +
793 BE2_TMFS + BE2_NOPOUT_REQ));
794 phba->params.cxns_per_ctrl = total_cid_count;
795 phba->params.asyncpdus_per_ctrl = total_cid_count;
796 phba->params.icds_per_ctrl = total_icd_count;
6733b39a
JK		 797 phba->params.num_sge_per_io = BE2_SGE;
798 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
799 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
800 phba->params.eq_timer = 64;
843ae752
JK		 801 phba->params.num_eq_entries = 1024;
802 phba->params.num_cq_entries = 1024;
6733b39a
JK		 803 phba->params.wrbs_per_cxn = 256;
804}
805
806static void hwi_ring_eq_db(struct beiscsi_hba *phba,
807 unsigned int id, unsigned int clr_interrupt,
808 unsigned int num_processed,
809 unsigned char rearm, unsigned char event)
810{
811 u32 val = 0;
812 val |= id & DB_EQ_RING_ID_MASK;
813 if (rearm)
814 val |= 1 << DB_EQ_REARM_SHIFT;
815 if (clr_interrupt)
816 val |= 1 << DB_EQ_CLR_SHIFT;
817 if (event)
818 val |= 1 << DB_EQ_EVNT_SHIFT;
819 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
820 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
821}
822
bfead3b2
JK		 823/**
824 * be_isr_mcc - The isr routine of the driver.
825 * @irq: Not used
826 * @dev_id: Pointer to host adapter structure
827 */
828static irqreturn_t be_isr_mcc(int irq, void *dev_id)
829{
830 struct beiscsi_hba *phba;
831 struct be_eq_entry *eqe = NULL;
832 struct be_queue_info *eq;
833 struct be_queue_info *mcc;
834 unsigned int num_eq_processed;
835 struct be_eq_obj *pbe_eq;
836 unsigned long flags;
837
838 pbe_eq = dev_id;
839 eq = &pbe_eq->q;
840 phba = pbe_eq->phba;
841 mcc = &phba->ctrl.mcc_obj.cq;
842 eqe = queue_tail_node(eq);
bfead3b2
JK		 843
844 num_eq_processed = 0;
845
846 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
847 & EQE_VALID_MASK) {
848 if (((eqe->dw[offsetof(struct amap_eq_entry,
849 resource_id) / 32] &
850 EQE_RESID_MASK) >> 16) == mcc->id) {
851 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 852 pbe_eq->todo_mcc_cq = true;
bfead3b2
JK		 853 spin_unlock_irqrestore(&phba->isr_lock, flags);
854 }
855 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
856 queue_tail_inc(eq);
857 eqe = queue_tail_node(eq);
858 num_eq_processed++;
859 }
72fb46a9
JSJ		 860 if (pbe_eq->todo_mcc_cq)
861 queue_work(phba->wq, &pbe_eq->work_cqs);
bfead3b2
JK		 862 if (num_eq_processed)
863 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
864
865 return IRQ_HANDLED;
866}
867
868/**
869 * be_isr_msix - The isr routine of the driver.
870 * @irq: Not used
871 * @dev_id: Pointer to host adapter structure
872 */
873static irqreturn_t be_isr_msix(int irq, void *dev_id)
874{
875 struct beiscsi_hba *phba;
876 struct be_eq_entry *eqe = NULL;
877 struct be_queue_info *eq;
878 struct be_queue_info *cq;
879 unsigned int num_eq_processed;
880 struct be_eq_obj *pbe_eq;
881 unsigned long flags;
882
883 pbe_eq = dev_id;
884 eq = &pbe_eq->q;
885 cq = pbe_eq->cq;
886 eqe = queue_tail_node(eq);
bfead3b2
JK		 887
888 phba = pbe_eq->phba;
889 num_eq_processed = 0;
890 if (blk_iopoll_enabled) {
891 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
892 & EQE_VALID_MASK) {
893 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
894 blk_iopoll_sched(&pbe_eq->iopoll);
895
896 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
897 queue_tail_inc(eq);
898 eqe = queue_tail_node(eq);
899 num_eq_processed++;
900 }
bfead3b2
JK		 901 } else {
902 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
903 & EQE_VALID_MASK) {
904 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 905 pbe_eq->todo_cq = true;
bfead3b2
JK		 906 spin_unlock_irqrestore(&phba->isr_lock, flags);
907 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
908 queue_tail_inc(eq);
909 eqe = queue_tail_node(eq);
910 num_eq_processed++;
911 }
bfead3b2 912
72fb46a9
JSJ		 913 if (pbe_eq->todo_cq)
914 queue_work(phba->wq, &pbe_eq->work_cqs);
bfead3b2 915 }
72fb46a9
JSJ		 916
917 if (num_eq_processed)
918 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
919
920 return IRQ_HANDLED;
bfead3b2
JK		 921}
922
6733b39a
JK		 923/**
924 * be_isr - The isr routine of the driver.
925 * @irq: Not used
926 * @dev_id: Pointer to host adapter structure
927 */
928static irqreturn_t be_isr(int irq, void *dev_id)
929{
930 struct beiscsi_hba *phba;
931 struct hwi_controller *phwi_ctrlr;
932 struct hwi_context_memory *phwi_context;
933 struct be_eq_entry *eqe = NULL;
934 struct be_queue_info *eq;
935 struct be_queue_info *cq;
bfead3b2 936 struct be_queue_info *mcc;
6733b39a 937 unsigned long flags, index;
bfead3b2 938 unsigned int num_mcceq_processed, num_ioeq_processed;
6733b39a 939 struct be_ctrl_info *ctrl;
bfead3b2 940 struct be_eq_obj *pbe_eq;
6733b39a
JK		 941 int isr;
942
943 phba = dev_id;
6eab04a8 944 ctrl = &phba->ctrl;
bfead3b2
JK		 945 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
946 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
947 if (!isr)
948 return IRQ_NONE;
6733b39a
JK		 949
950 phwi_ctrlr = phba->phwi_ctrlr;
951 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 952 pbe_eq = &phwi_context->be_eq[0];
953
954 eq = &phwi_context->be_eq[0].q;
955 mcc = &phba->ctrl.mcc_obj.cq;
6733b39a
JK		 956 index = 0;
957 eqe = queue_tail_node(eq);
6733b39a 958
bfead3b2
JK		 959 num_ioeq_processed = 0;
960 num_mcceq_processed = 0;
6733b39a
JK		 961 if (blk_iopoll_enabled) {
962 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
963 & EQE_VALID_MASK) {
bfead3b2
JK		 964 if (((eqe->dw[offsetof(struct amap_eq_entry,
965 resource_id) / 32] &
966 EQE_RESID_MASK) >> 16) == mcc->id) {
967 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 968 pbe_eq->todo_mcc_cq = true;
bfead3b2
JK		 969 spin_unlock_irqrestore(&phba->isr_lock, flags);
970 num_mcceq_processed++;
971 } else {
972 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
973 blk_iopoll_sched(&pbe_eq->iopoll);
974 num_ioeq_processed++;
975 }
6733b39a
JK		 976 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
977 queue_tail_inc(eq);
978 eqe = queue_tail_node(eq);
6733b39a 979 }
bfead3b2 980 if (num_ioeq_processed || num_mcceq_processed) {
72fb46a9
JSJ		 981 if (pbe_eq->todo_mcc_cq)
982 queue_work(phba->wq, &pbe_eq->work_cqs);
bfead3b2 983
756d29c8 984 if ((num_mcceq_processed) && (!num_ioeq_processed))
bfead3b2
JK		 985 hwi_ring_eq_db(phba, eq->id, 0,
986 (num_ioeq_processed +
987 num_mcceq_processed) , 1, 1);
988 else
989 hwi_ring_eq_db(phba, eq->id, 0,
990 (num_ioeq_processed +
991 num_mcceq_processed), 0, 1);
992
6733b39a
JK		 993 return IRQ_HANDLED;
994 } else
995 return IRQ_NONE;
996 } else {
bfead3b2 997 cq = &phwi_context->be_cq[0];
6733b39a
JK		 998 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
999 & EQE_VALID_MASK) {
1000
1001 if (((eqe->dw[offsetof(struct amap_eq_entry,
1002 resource_id) / 32] &
1003 EQE_RESID_MASK) >> 16) != cq->id) {
1004 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 1005 pbe_eq->todo_mcc_cq = true;
6733b39a
JK		 1006 spin_unlock_irqrestore(&phba->isr_lock, flags);
1007 } else {
1008 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 1009 pbe_eq->todo_cq = true;
6733b39a
JK		 1010 spin_unlock_irqrestore(&phba->isr_lock, flags);
1011 }
1012 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
1013 queue_tail_inc(eq);
1014 eqe = queue_tail_node(eq);
bfead3b2 1015 num_ioeq_processed++;
6733b39a 1016 }
72fb46a9
JSJ		 1017 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq)
1018 queue_work(phba->wq, &pbe_eq->work_cqs);
6733b39a 1019
bfead3b2
JK		 1020 if (num_ioeq_processed) {
1021 hwi_ring_eq_db(phba, eq->id, 0,
1022 num_ioeq_processed, 1, 1);
6733b39a
JK		 1023 return IRQ_HANDLED;
1024 } else
1025 return IRQ_NONE;
1026 }
1027}
1028
1029static int beiscsi_init_irqs(struct beiscsi_hba *phba)
1030{
1031 struct pci_dev *pcidev = phba->pcidev;
bfead3b2
JK		 1032 struct hwi_controller *phwi_ctrlr;
1033 struct hwi_context_memory *phwi_context;
4f5af07e 1034 int ret, msix_vec, i, j;
6733b39a 1035
bfead3b2
JK		 1036 phwi_ctrlr = phba->phwi_ctrlr;
1037 phwi_context = phwi_ctrlr->phwi_ctxt;
1038
1039 if (phba->msix_enabled) {
1040 for (i = 0; i < phba->num_cpus; i++) {
8fcfb210
JK		 1041 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
1042 GFP_KERNEL);
1043 if (!phba->msi_name[i]) {
1044 ret = -ENOMEM;
1045 goto free_msix_irqs;
1046 }
1047
1048 sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
1049 phba->shost->host_no, i);
bfead3b2 1050 msix_vec = phba->msix_entries[i].vector;
8fcfb210
JK		 1051 ret = request_irq(msix_vec, be_isr_msix, 0,
1052 phba->msi_name[i],
bfead3b2 1053 &phwi_context->be_eq[i]);
4f5af07e 1054 if (ret) {
99bc5d55
JSJ		 1055 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1056 "BM_%d : beiscsi_init_irqs-Failed to"
1057 "register msix for i = %d\n",
1058 i);
8fcfb210 1059 kfree(phba->msi_name[i]);
4f5af07e
JK		 1060 goto free_msix_irqs;
1061 }
bfead3b2 1062 }
8fcfb210
JK		 1063 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
1064 if (!phba->msi_name[i]) {
1065 ret = -ENOMEM;
1066 goto free_msix_irqs;
1067 }
1068 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
1069 phba->shost->host_no);
bfead3b2 1070 msix_vec = phba->msix_entries[i].vector;
8fcfb210 1071 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
bfead3b2 1072 &phwi_context->be_eq[i]);
4f5af07e 1073 if (ret) {
99bc5d55
JSJ		 1074 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
1075 "BM_%d : beiscsi_init_irqs-"
1076 "Failed to register beiscsi_msix_mcc\n");
8fcfb210 1077 kfree(phba->msi_name[i]);
4f5af07e
JK		 1078 goto free_msix_irqs;
1079 }
1080
bfead3b2
JK		 1081 } else {
1082 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1083 "beiscsi", phba);
1084 if (ret) {
99bc5d55
JSJ		 1085 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1086 "BM_%d : beiscsi_init_irqs-"
1087 "Failed to register irq\\n");
bfead3b2
JK		 1088 return ret;
1089 }
6733b39a
JK		 1090 }
1091 return 0;
4f5af07e 1092free_msix_irqs:
8fcfb210
JK		 1093 for (j = i - 1; j >= 0; j--) {
1094 kfree(phba->msi_name[j]);
1095 msix_vec = phba->msix_entries[j].vector;
4f5af07e 1096 free_irq(msix_vec, &phwi_context->be_eq[j]);
8fcfb210 1097 }
4f5af07e 1098 return ret;
6733b39a
JK		 1099}
1100
1101static void hwi_ring_cq_db(struct beiscsi_hba *phba,
1102 unsigned int id, unsigned int num_processed,
1103 unsigned char rearm, unsigned char event)
1104{
1105 u32 val = 0;
1106 val |= id & DB_CQ_RING_ID_MASK;
1107 if (rearm)
1108 val |= 1 << DB_CQ_REARM_SHIFT;
1109 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1110 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1111}
1112
6733b39a
JK		 1113static unsigned int
1114beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1115 struct beiscsi_hba *phba,
6733b39a
JK		 1116 struct pdu_base *ppdu,
1117 unsigned long pdu_len,
1118 void *pbuffer, unsigned long buf_len)
1119{
1120 struct iscsi_conn *conn = beiscsi_conn->conn;
1121 struct iscsi_session *session = conn->session;
bfead3b2
JK		 1122 struct iscsi_task *task;
1123 struct beiscsi_io_task *io_task;
1124 struct iscsi_hdr *login_hdr;
6733b39a
JK		 1125
1126 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1127 PDUBASE_OPCODE_MASK) {
1128 case ISCSI_OP_NOOP_IN:
1129 pbuffer = NULL;
1130 buf_len = 0;
1131 break;
1132 case ISCSI_OP_ASYNC_EVENT:
1133 break;
1134 case ISCSI_OP_REJECT:
1135 WARN_ON(!pbuffer);
1136 WARN_ON(!(buf_len == 48));
99bc5d55
JSJ		 1137 beiscsi_log(phba, KERN_ERR,
1138 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1139 "BM_%d : In ISCSI_OP_REJECT\n");
6733b39a
JK		 1140 break;
1141 case ISCSI_OP_LOGIN_RSP:
7bd6e25c 1142 case ISCSI_OP_TEXT_RSP:
bfead3b2
JK		 1143 task = conn->login_task;
1144 io_task = task->dd_data;
1145 login_hdr = (struct iscsi_hdr *)ppdu;
1146 login_hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1147 break;
1148 default:
99bc5d55
JSJ		 1149 beiscsi_log(phba, KERN_WARNING,
1150 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1151 "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1152 (ppdu->
6733b39a 1153 dw[offsetof(struct amap_pdu_base, opcode) / 32]
99bc5d55 1154 & PDUBASE_OPCODE_MASK));
6733b39a
JK		 1155 return 1;
1156 }
1157
1158 spin_lock_bh(&session->lock);
1159 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1160 spin_unlock_bh(&session->lock);
1161 return 0;
1162}
1163
1164static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1165{
1166 struct sgl_handle *psgl_handle;
1167
1168 if (phba->io_sgl_hndl_avbl) {
99bc5d55
JSJ		 1169 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1170 "BM_%d : In alloc_io_sgl_handle,"
1171 " io_sgl_alloc_index=%d\n",
1172 phba->io_sgl_alloc_index);
1173
6733b39a
JK		 1174 psgl_handle = phba->io_sgl_hndl_base[phba->
1175 io_sgl_alloc_index];
1176 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1177 phba->io_sgl_hndl_avbl--;
bfead3b2
JK		 1178 if (phba->io_sgl_alloc_index == (phba->params.
1179 ios_per_ctrl - 1))
6733b39a
JK		 1180 phba->io_sgl_alloc_index = 0;
1181 else
1182 phba->io_sgl_alloc_index++;
1183 } else
1184 psgl_handle = NULL;
1185 return psgl_handle;
1186}
1187
1188static void
1189free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1190{
99bc5d55
JSJ		 1191 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1192 "BM_%d : In free_,io_sgl_free_index=%d\n",
1193 phba->io_sgl_free_index);
1194
6733b39a
JK		 1195 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1196 /*
1197 * this can happen if clean_task is called on a task that
1198 * failed in xmit_task or alloc_pdu.
1199 */
99bc5d55
JSJ		 1200 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1201 "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1202 "value there=%p\n", phba->io_sgl_free_index,
1203 phba->io_sgl_hndl_base
1204 [phba->io_sgl_free_index]);
6733b39a
JK		 1205 return;
1206 }
1207 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1208 phba->io_sgl_hndl_avbl++;
1209 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1210 phba->io_sgl_free_index = 0;
1211 else
1212 phba->io_sgl_free_index++;
1213}
1214
1215/**
1216 * alloc_wrb_handle - To allocate a wrb handle
1217 * @phba: The hba pointer
1218 * @cid: The cid to use for allocation
6733b39a
JK		 1219 *
1220 * This happens under session_lock until submission to chip
1221 */
d5431488 1222struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
6733b39a
JK		 1223{
1224 struct hwi_wrb_context *pwrb_context;
1225 struct hwi_controller *phwi_ctrlr;
d5431488 1226 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
a7909b39 1227 uint16_t cri_index = BE_GET_CRI_FROM_CID(cid);
6733b39a
JK		 1228
1229 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39 1230 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
d5431488 1231 if (pwrb_context->wrb_handles_available >= 2) {
bfead3b2
JK		 1232 pwrb_handle = pwrb_context->pwrb_handle_base[
1233 pwrb_context->alloc_index];
1234 pwrb_context->wrb_handles_available--;
bfead3b2
JK		 1235 if (pwrb_context->alloc_index ==
1236 (phba->params.wrbs_per_cxn - 1))
1237 pwrb_context->alloc_index = 0;
1238 else
1239 pwrb_context->alloc_index++;
d5431488
JK		 1240 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1241 pwrb_context->alloc_index];
1242 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
bfead3b2
JK		 1243 } else
1244 pwrb_handle = NULL;
6733b39a
JK		 1245 return pwrb_handle;
1246}
1247
1248/**
1249 * free_wrb_handle - To free the wrb handle back to pool
1250 * @phba: The hba pointer
1251 * @pwrb_context: The context to free from
1252 * @pwrb_handle: The wrb_handle to free
1253 *
1254 * This happens under session_lock until submission to chip
1255 */
1256static void
1257free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1258 struct wrb_handle *pwrb_handle)
1259{
32951dd8 1260 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
bfead3b2
JK		 1261 pwrb_context->wrb_handles_available++;
1262 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1263 pwrb_context->free_index = 0;
1264 else
1265 pwrb_context->free_index++;
1266
99bc5d55
JSJ		 1267 beiscsi_log(phba, KERN_INFO,
1268 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1269 "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1270 "wrb_handles_available=%d\n",
1271 pwrb_handle, pwrb_context->free_index,
1272 pwrb_context->wrb_handles_available);
6733b39a
JK		 1273}
1274
1275static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1276{
1277 struct sgl_handle *psgl_handle;
1278
1279 if (phba->eh_sgl_hndl_avbl) {
1280 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1281 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
99bc5d55
JSJ		 1282 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1283 "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1284 phba->eh_sgl_alloc_index,
1285 phba->eh_sgl_alloc_index);
1286
6733b39a
JK		 1287 phba->eh_sgl_hndl_avbl--;
1288 if (phba->eh_sgl_alloc_index ==
1289 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1290 1))
1291 phba->eh_sgl_alloc_index = 0;
1292 else
1293 phba->eh_sgl_alloc_index++;
1294 } else
1295 psgl_handle = NULL;
1296 return psgl_handle;
1297}
1298
1299void
1300free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1301{
1302
99bc5d55
JSJ		 1303 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1304 "BM_%d : In free_mgmt_sgl_handle,"
1305 "eh_sgl_free_index=%d\n",
1306 phba->eh_sgl_free_index);
1307
6733b39a
JK		 1308 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1309 /*
1310 * this can happen if clean_task is called on a task that
1311 * failed in xmit_task or alloc_pdu.
1312 */
99bc5d55
JSJ		 1313 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1314 "BM_%d : Double Free in eh SGL ,"
1315 "eh_sgl_free_index=%d\n",
1316 phba->eh_sgl_free_index);
6733b39a
JK		 1317 return;
1318 }
1319 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1320 phba->eh_sgl_hndl_avbl++;
1321 if (phba->eh_sgl_free_index ==
1322 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1323 phba->eh_sgl_free_index = 0;
1324 else
1325 phba->eh_sgl_free_index++;
1326}
1327
1328static void
1329be_complete_io(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1330 struct iscsi_task *task,
1331 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1332{
1333 struct beiscsi_io_task *io_task = task->dd_data;
1334 struct be_status_bhs *sts_bhs =
1335 (struct be_status_bhs *)io_task->cmd_bhs;
1336 struct iscsi_conn *conn = beiscsi_conn->conn;
6733b39a
JK		 1337 unsigned char *sense;
1338 u32 resid = 0, exp_cmdsn, max_cmdsn;
1339 u8 rsp, status, flags;
1340
73133261
JSJ		 1341 exp_cmdsn = csol_cqe->exp_cmdsn;
1342 max_cmdsn = (csol_cqe->exp_cmdsn +
1343 csol_cqe->cmd_wnd - 1);
1344 rsp = csol_cqe->i_resp;
1345 status = csol_cqe->i_sts;
1346 flags = csol_cqe->i_flags;
1347 resid = csol_cqe->res_cnt;
1348
bd535451
JK		 1349 if (!task->sc) {
1350 if (io_task->scsi_cmnd)
1351 scsi_dma_unmap(io_task->scsi_cmnd);
6733b39a 1352
bd535451
JK		 1353 return;
1354 }
6733b39a
JK		 1355 task->sc->result = (DID_OK << 16) | status;
1356 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1357 task->sc->result = DID_ERROR << 16;
1358 goto unmap;
1359 }
1360
1361 /* bidi not initially supported */
1362 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
6733b39a
JK		 1363 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1364 task->sc->result = DID_ERROR << 16;
1365
1366 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1367 scsi_set_resid(task->sc, resid);
1368 if (!status && (scsi_bufflen(task->sc) - resid <
1369 task->sc->underflow))
1370 task->sc->result = DID_ERROR << 16;
1371 }
1372 }
1373
1374 if (status == SAM_STAT_CHECK_CONDITION) {
4053a4be 1375 u16 sense_len;
bfead3b2 1376 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
4053a4be 1377
6733b39a 1378 sense = sts_bhs->sense_info + sizeof(unsigned short);
4053a4be 1379 sense_len = be16_to_cpu(*slen);
6733b39a
JK		 1380 memcpy(task->sc->sense_buffer, sense,
1381 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1382 }
756d29c8 1383
73133261
JSJ		 1384 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1385 conn->rxdata_octets += resid;
6733b39a
JK		 1386unmap:
1387 scsi_dma_unmap(io_task->scsi_cmnd);
1388 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1389}
1390
1391static void
1392be_complete_logout(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1393 struct iscsi_task *task,
1394 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1395{
1396 struct iscsi_logout_rsp *hdr;
bfead3b2 1397 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1398 struct iscsi_conn *conn = beiscsi_conn->conn;
1399
1400 hdr = (struct iscsi_logout_rsp *)task->hdr;
7bd6e25c 1401 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
6733b39a
JK		 1402 hdr->t2wait = 5;
1403 hdr->t2retain = 0;
73133261
JSJ		 1404 hdr->flags = csol_cqe->i_flags;
1405 hdr->response = csol_cqe->i_resp;
702dc5e8
JK		 1406 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1407 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1408 csol_cqe->cmd_wnd - 1);
73133261 1409
7bd6e25c
JK		 1410 hdr->dlength[0] = 0;
1411 hdr->dlength[1] = 0;
1412 hdr->dlength[2] = 0;
6733b39a 1413 hdr->hlength = 0;
bfead3b2 1414 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1415 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1416}
1417
1418static void
1419be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1420 struct iscsi_task *task,
1421 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1422{
1423 struct iscsi_tm_rsp *hdr;
1424 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1425 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1426
1427 hdr = (struct iscsi_tm_rsp *)task->hdr;
7bd6e25c 1428 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
73133261
JSJ		 1429 hdr->flags = csol_cqe->i_flags;
1430 hdr->response = csol_cqe->i_resp;
702dc5e8
JK		 1431 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1432 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1433 csol_cqe->cmd_wnd - 1);
73133261 1434
bfead3b2 1435 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1436 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1437}
1438
1439static void
1440hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1441 struct beiscsi_hba *phba, struct sol_cqe *psol)
1442{
1443 struct hwi_wrb_context *pwrb_context;
bfead3b2 1444 struct wrb_handle *pwrb_handle = NULL;
6733b39a 1445 struct hwi_controller *phwi_ctrlr;
bfead3b2
JK		 1446 struct iscsi_task *task;
1447 struct beiscsi_io_task *io_task;
a7909b39 1448 uint16_t wrb_index, cid, cri_index;
6733b39a
JK		 1449
1450 phwi_ctrlr = phba->phwi_ctrlr;
2c9dfd36
JK		 1451 if (is_chip_be2_be3r(phba)) {
1452 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
73133261 1453 wrb_idx, psol);
2c9dfd36 1454 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
73133261
JSJ		 1455 cid, psol);
1456 } else {
2c9dfd36 1457 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
73133261 1458 wrb_idx, psol);
2c9dfd36 1459 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
73133261
JSJ		 1460 cid, psol);
1461 }
1462
a7909b39
JK		 1463 cri_index = BE_GET_CRI_FROM_CID(cid);
1464 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
73133261 1465 pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
32951dd8 1466 task = pwrb_handle->pio_handle;
35e66019 1467
bfead3b2 1468 io_task = task->dd_data;
4a4a11b9
JK		 1469 memset(io_task->pwrb_handle->pwrb, 0, sizeof(struct iscsi_wrb));
1470 iscsi_put_task(task);
6733b39a
JK		 1471}
1472
1473static void
1474be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1475 struct iscsi_task *task,
1476 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1477{
1478 struct iscsi_nopin *hdr;
1479 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1480 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1481
1482 hdr = (struct iscsi_nopin *)task->hdr;
73133261
JSJ		 1483 hdr->flags = csol_cqe->i_flags;
1484 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
702dc5e8
JK		 1485 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1486 csol_cqe->cmd_wnd - 1);
73133261 1487
6733b39a 1488 hdr->opcode = ISCSI_OP_NOOP_IN;
bfead3b2 1489 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1490 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1491}
1492
73133261
JSJ		 1493static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1494 struct sol_cqe *psol,
1495 struct common_sol_cqe *csol_cqe)
1496{
2c9dfd36
JK		 1497 if (is_chip_be2_be3r(phba)) {
1498 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1499 i_exp_cmd_sn, psol);
1500 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1501 i_res_cnt, psol);
1502 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1503 i_cmd_wnd, psol);
1504 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1505 wrb_index, psol);
1506 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1507 cid, psol);
1508 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1509 hw_sts, psol);
1510 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1511 i_resp, psol);
1512 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1513 i_sts, psol);
1514 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1515 i_flags, psol);
1516 } else {
73133261
JSJ		 1517 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1518 i_exp_cmd_sn, psol);
1519 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1520 i_res_cnt, psol);
1521 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1522 wrb_index, psol);
1523 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1524 cid, psol);
1525 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1526 hw_sts, psol);
702dc5e8 1527 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2,
73133261
JSJ		 1528 i_cmd_wnd, psol);
1529 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1530 cmd_cmpl, psol))
1531 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1532 i_sts, psol);
1533 else
1534 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1535 i_sts, psol);
1536 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1537 u, psol))
1538 csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1539
1540 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1541 o, psol))
1542 csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
73133261
JSJ		 1543 }
1544}
1545
1546
6733b39a
JK		 1547static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1548 struct beiscsi_hba *phba, struct sol_cqe *psol)
1549{
1550 struct hwi_wrb_context *pwrb_context;
1551 struct wrb_handle *pwrb_handle;
1552 struct iscsi_wrb *pwrb = NULL;
1553 struct hwi_controller *phwi_ctrlr;
1554 struct iscsi_task *task;
bfead3b2 1555 unsigned int type;
6733b39a
JK		 1556 struct iscsi_conn *conn = beiscsi_conn->conn;
1557 struct iscsi_session *session = conn->session;
73133261 1558 struct common_sol_cqe csol_cqe = {0};
a7909b39 1559 uint16_t cri_index = 0;
6733b39a
JK		 1560
1561 phwi_ctrlr = phba->phwi_ctrlr;
73133261
JSJ		 1562
1563 /* Copy the elements to a common structure */
1564 adapter_get_sol_cqe(phba, psol, &csol_cqe);
1565
a7909b39
JK		 1566 cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid);
1567 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
73133261
JSJ		 1568
1569 pwrb_handle = pwrb_context->pwrb_handle_basestd[
1570 csol_cqe.wrb_index];
1571
32951dd8
JK		 1572 task = pwrb_handle->pio_handle;
1573 pwrb = pwrb_handle->pwrb;
73133261 1574 type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
32951dd8 1575
bfead3b2
JK		 1576 spin_lock_bh(&session->lock);
1577 switch (type) {
6733b39a
JK		 1578 case HWH_TYPE_IO:
1579 case HWH_TYPE_IO_RD:
1580 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
dafab8e0 1581 ISCSI_OP_NOOP_OUT)
73133261 1582 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
dafab8e0 1583 else
73133261 1584 be_complete_io(beiscsi_conn, task, &csol_cqe);
6733b39a
JK		 1585 break;
1586
1587 case HWH_TYPE_LOGOUT:
dafab8e0 1588 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
73133261 1589 be_complete_logout(beiscsi_conn, task, &csol_cqe);
dafab8e0 1590 else
73133261 1591 be_complete_tmf(beiscsi_conn, task, &csol_cqe);
6733b39a
JK		 1592 break;
1593
1594 case HWH_TYPE_LOGIN:
99bc5d55
JSJ		 1595 beiscsi_log(phba, KERN_ERR,
1596 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1597 "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1598 " hwi_complete_cmd- Solicited path\n");
6733b39a
JK		 1599 break;
1600
6733b39a 1601 case HWH_TYPE_NOP:
73133261 1602 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
6733b39a
JK		 1603 break;
1604
1605 default:
99bc5d55
JSJ		 1606 beiscsi_log(phba, KERN_WARNING,
1607 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1608 "BM_%d : In hwi_complete_cmd, unknown type = %d"
1609 "wrb_index 0x%x CID 0x%x\n", type,
73133261
JSJ		 1610 csol_cqe.wrb_index,
1611 csol_cqe.cid);
6733b39a
JK		 1612 break;
1613 }
35e66019 1614
6733b39a
JK		 1615 spin_unlock_bh(&session->lock);
1616}
1617
1618static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1619 *pasync_ctx, unsigned int is_header,
1620 unsigned int host_write_ptr)
1621{
1622 if (is_header)
1623 return &pasync_ctx->async_entry[host_write_ptr].
1624 header_busy_list;
1625 else
1626 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1627}
1628
1629static struct async_pdu_handle *
1630hwi_get_async_handle(struct beiscsi_hba *phba,
1631 struct beiscsi_conn *beiscsi_conn,
1632 struct hwi_async_pdu_context *pasync_ctx,
1633 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1634{
1635 struct be_bus_address phys_addr;
1636 struct list_head *pbusy_list;
1637 struct async_pdu_handle *pasync_handle = NULL;
6733b39a 1638 unsigned char is_header = 0;
73133261
JSJ		 1639 unsigned int index, dpl;
1640
2c9dfd36
JK		 1641 if (is_chip_be2_be3r(phba)) {
1642 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
73133261 1643 dpl, pdpdu_cqe);
2c9dfd36 1644 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
73133261
JSJ		 1645 index, pdpdu_cqe);
1646 } else {
2c9dfd36 1647 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
73133261 1648 dpl, pdpdu_cqe);
2c9dfd36 1649 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
73133261
JSJ		 1650 index, pdpdu_cqe);
1651 }
6733b39a
JK		 1652
1653 phys_addr.u.a32.address_lo =
73133261
JSJ		 1654 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1655 db_addr_lo) / 32] - dpl);
6733b39a 1656 phys_addr.u.a32.address_hi =
73133261
JSJ		 1657 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1658 db_addr_hi) / 32];
6733b39a
JK		 1659
1660 phys_addr.u.a64.address =
1661 *((unsigned long long *)(&phys_addr.u.a64.address));
1662
1663 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1664 & PDUCQE_CODE_MASK) {
1665 case UNSOL_HDR_NOTIFY:
1666 is_header = 1;
1667
73133261
JSJ		 1668 pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1669 is_header, index);
6733b39a
JK		 1670 break;
1671 case UNSOL_DATA_NOTIFY:
73133261
JSJ		 1672 pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1673 is_header, index);
6733b39a
JK		 1674 break;
1675 default:
1676 pbusy_list = NULL;
99bc5d55
JSJ		 1677 beiscsi_log(phba, KERN_WARNING,
1678 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1679 "BM_%d : Unexpected code=%d\n",
1680 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1681 code) / 32] & PDUCQE_CODE_MASK);
6733b39a
JK		 1682 return NULL;
1683 }
1684
6733b39a
JK		 1685 WARN_ON(list_empty(pbusy_list));
1686 list_for_each_entry(pasync_handle, pbusy_list, link) {
dc63aac6 1687 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
6733b39a
JK		 1688 break;
1689 }
1690
1691 WARN_ON(!pasync_handle);
1692
8a86e833
JK		 1693 pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID(
1694 beiscsi_conn->beiscsi_conn_cid);
6733b39a 1695 pasync_handle->is_header = is_header;
73133261
JSJ		 1696 pasync_handle->buffer_len = dpl;
1697 *pcq_index = index;
6733b39a 1698
6733b39a
JK		 1699 return pasync_handle;
1700}
1701
1702static unsigned int
99bc5d55
JSJ		 1703hwi_update_async_writables(struct beiscsi_hba *phba,
1704 struct hwi_async_pdu_context *pasync_ctx,
1705 unsigned int is_header, unsigned int cq_index)
6733b39a
JK		 1706{
1707 struct list_head *pbusy_list;
1708 struct async_pdu_handle *pasync_handle;
1709 unsigned int num_entries, writables = 0;
1710 unsigned int *pep_read_ptr, *pwritables;
1711
dc63aac6 1712 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1713 if (is_header) {
1714 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1715 pwritables = &pasync_ctx->async_header.writables;
6733b39a
JK		 1716 } else {
1717 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1718 pwritables = &pasync_ctx->async_data.writables;
6733b39a
JK		 1719 }
1720
1721 while ((*pep_read_ptr) != cq_index) {
1722 (*pep_read_ptr)++;
1723 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1724
1725 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1726 *pep_read_ptr);
1727 if (writables == 0)
1728 WARN_ON(list_empty(pbusy_list));
1729
1730 if (!list_empty(pbusy_list)) {
1731 pasync_handle = list_entry(pbusy_list->next,
1732 struct async_pdu_handle,
1733 link);
1734 WARN_ON(!pasync_handle);
1735 pasync_handle->consumed = 1;
1736 }
1737
1738 writables++;
1739 }
1740
1741 if (!writables) {
99bc5d55
JSJ		 1742 beiscsi_log(phba, KERN_ERR,
1743 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1744 "BM_%d : Duplicate notification received - index 0x%x!!\n",
1745 cq_index);
6733b39a
JK		 1746 WARN_ON(1);
1747 }
1748
1749 *pwritables = *pwritables + writables;
1750 return 0;
1751}
1752
9728d8d0 1753static void hwi_free_async_msg(struct beiscsi_hba *phba,
8a86e833
JK		 1754 struct hwi_async_pdu_context *pasync_ctx,
1755 unsigned int cri)
6733b39a 1756{
6733b39a
JK		 1757 struct async_pdu_handle *pasync_handle, *tmp_handle;
1758 struct list_head *plist;
6733b39a 1759
6733b39a 1760 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
6733b39a
JK		 1761 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1762 list_del(&pasync_handle->link);
1763
9728d8d0 1764 if (pasync_handle->is_header) {
6733b39a
JK		 1765 list_add_tail(&pasync_handle->link,
1766 &pasync_ctx->async_header.free_list);
1767 pasync_ctx->async_header.free_entries++;
6733b39a
JK		 1768 } else {
1769 list_add_tail(&pasync_handle->link,
1770 &pasync_ctx->async_data.free_list);
1771 pasync_ctx->async_data.free_entries++;
6733b39a
JK		 1772 }
1773 }
1774
1775 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1776 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1777 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
6733b39a
JK		 1778}
1779
1780static struct phys_addr *
1781hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1782 unsigned int is_header, unsigned int host_write_ptr)
1783{
1784 struct phys_addr *pasync_sge = NULL;
1785
1786 if (is_header)
1787 pasync_sge = pasync_ctx->async_header.ring_base;
1788 else
1789 pasync_sge = pasync_ctx->async_data.ring_base;
1790
1791 return pasync_sge + host_write_ptr;
1792}
1793
1794static void hwi_post_async_buffers(struct beiscsi_hba *phba,
8a86e833 1795 unsigned int is_header, uint8_t ulp_num)
6733b39a
JK		 1796{
1797 struct hwi_controller *phwi_ctrlr;
1798 struct hwi_async_pdu_context *pasync_ctx;
1799 struct async_pdu_handle *pasync_handle;
1800 struct list_head *pfree_link, *pbusy_list;
1801 struct phys_addr *pasync_sge;
1802 unsigned int ring_id, num_entries;
8a86e833 1803 unsigned int host_write_num, doorbell_offset;
6733b39a
JK		 1804 unsigned int writables;
1805 unsigned int i = 0;
1806 u32 doorbell = 0;
1807
1808 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833 1809 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
dc63aac6 1810 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1811
1812 if (is_header) {
6733b39a
JK		 1813 writables = min(pasync_ctx->async_header.writables,
1814 pasync_ctx->async_header.free_entries);
1815 pfree_link = pasync_ctx->async_header.free_list.next;
1816 host_write_num = pasync_ctx->async_header.host_write_ptr;
8a86e833
JK		 1817 ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;
1818 doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].
1819 doorbell_offset;
6733b39a 1820 } else {
6733b39a
JK		 1821 writables = min(pasync_ctx->async_data.writables,
1822 pasync_ctx->async_data.free_entries);
1823 pfree_link = pasync_ctx->async_data.free_list.next;
1824 host_write_num = pasync_ctx->async_data.host_write_ptr;
8a86e833
JK		 1825 ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;
1826 doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].
1827 doorbell_offset;
6733b39a
JK		 1828 }
1829
1830 writables = (writables / 8) * 8;
1831 if (writables) {
1832 for (i = 0; i < writables; i++) {
1833 pbusy_list =
1834 hwi_get_async_busy_list(pasync_ctx, is_header,
1835 host_write_num);
1836 pasync_handle =
1837 list_entry(pfree_link, struct async_pdu_handle,
1838 link);
1839 WARN_ON(!pasync_handle);
1840 pasync_handle->consumed = 0;
1841
1842 pfree_link = pfree_link->next;
1843
1844 pasync_sge = hwi_get_ring_address(pasync_ctx,
1845 is_header, host_write_num);
1846
1847 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1848 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1849
1850 list_move(&pasync_handle->link, pbusy_list);
1851
1852 host_write_num++;
1853 host_write_num = host_write_num % num_entries;
1854 }
1855
1856 if (is_header) {
1857 pasync_ctx->async_header.host_write_ptr =
1858 host_write_num;
1859 pasync_ctx->async_header.free_entries -= writables;
1860 pasync_ctx->async_header.writables -= writables;
1861 pasync_ctx->async_header.busy_entries += writables;
1862 } else {
1863 pasync_ctx->async_data.host_write_ptr = host_write_num;
1864 pasync_ctx->async_data.free_entries -= writables;
1865 pasync_ctx->async_data.writables -= writables;
1866 pasync_ctx->async_data.busy_entries += writables;
1867 }
1868
1869 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1870 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1871 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1872 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1873 << DB_DEF_PDU_CQPROC_SHIFT;
1874
8a86e833 1875 iowrite32(doorbell, phba->db_va + doorbell_offset);
6733b39a
JK		 1876 }
1877}
1878
1879static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1880 struct beiscsi_conn *beiscsi_conn,
1881 struct i_t_dpdu_cqe *pdpdu_cqe)
1882{
1883 struct hwi_controller *phwi_ctrlr;
1884 struct hwi_async_pdu_context *pasync_ctx;
1885 struct async_pdu_handle *pasync_handle = NULL;
1886 unsigned int cq_index = -1;
8a86e833
JK		 1887 uint16_t cri_index = BE_GET_CRI_FROM_CID(
1888 beiscsi_conn->beiscsi_conn_cid);
6733b39a
JK		 1889
1890 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833
JK		 1891 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
1892 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1893 cri_index));
6733b39a
JK		 1894
1895 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1896 pdpdu_cqe, &cq_index);
1897 BUG_ON(pasync_handle->is_header != 0);
1898 if (pasync_handle->consumed == 0)
99bc5d55
JSJ		 1899 hwi_update_async_writables(phba, pasync_ctx,
1900 pasync_handle->is_header, cq_index);
6733b39a 1901
8a86e833
JK		 1902 hwi_free_async_msg(phba, pasync_ctx, pasync_handle->cri);
1903 hwi_post_async_buffers(phba, pasync_handle->is_header,
1904 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1905 cri_index));
6733b39a
JK		 1906}
1907
1908static unsigned int
1909hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1910 struct beiscsi_hba *phba,
1911 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1912{
1913 struct list_head *plist;
1914 struct async_pdu_handle *pasync_handle;
1915 void *phdr = NULL;
1916 unsigned int hdr_len = 0, buf_len = 0;
1917 unsigned int status, index = 0, offset = 0;
1918 void *pfirst_buffer = NULL;
1919 unsigned int num_buf = 0;
1920
1921 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1922
1923 list_for_each_entry(pasync_handle, plist, link) {
1924 if (index == 0) {
1925 phdr = pasync_handle->pbuffer;
1926 hdr_len = pasync_handle->buffer_len;
1927 } else {
1928 buf_len = pasync_handle->buffer_len;
1929 if (!num_buf) {
1930 pfirst_buffer = pasync_handle->pbuffer;
1931 num_buf++;
1932 }
1933 memcpy(pfirst_buffer + offset,
1934 pasync_handle->pbuffer, buf_len);
f2ba02b8 1935 offset += buf_len;
6733b39a
JK		 1936 }
1937 index++;
1938 }
1939
1940 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
7da50879 1941 phdr, hdr_len, pfirst_buffer,
f2ba02b8 1942 offset);
6733b39a 1943
8a86e833 1944 hwi_free_async_msg(phba, pasync_ctx, cri);
6733b39a
JK		 1945 return 0;
1946}
1947
1948static unsigned int
1949hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1950 struct beiscsi_hba *phba,
1951 struct async_pdu_handle *pasync_handle)
1952{
1953 struct hwi_async_pdu_context *pasync_ctx;
1954 struct hwi_controller *phwi_ctrlr;
1955 unsigned int bytes_needed = 0, status = 0;
1956 unsigned short cri = pasync_handle->cri;
1957 struct pdu_base *ppdu;
1958
1959 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833
JK		 1960 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
1961 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1962 BE_GET_CRI_FROM_CID(beiscsi_conn->
1963 beiscsi_conn_cid)));
6733b39a
JK		 1964
1965 list_del(&pasync_handle->link);
1966 if (pasync_handle->is_header) {
1967 pasync_ctx->async_header.busy_entries--;
1968 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
8a86e833 1969 hwi_free_async_msg(phba, pasync_ctx, cri);
6733b39a
JK		 1970 BUG();
1971 }
1972
1973 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1974 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1975 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1976 (unsigned short)pasync_handle->buffer_len;
1977 list_add_tail(&pasync_handle->link,
1978 &pasync_ctx->async_entry[cri].wait_queue.list);
1979
1980 ppdu = pasync_handle->pbuffer;
1981 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1982 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1983 0xFFFF0000) | ((be16_to_cpu((ppdu->
1984 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1985 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1986
1987 if (status == 0) {
1988 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1989 bytes_needed;
1990
1991 if (bytes_needed == 0)
1992 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1993 pasync_ctx, cri);
1994 }
1995 } else {
1996 pasync_ctx->async_data.busy_entries--;
1997 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1998 list_add_tail(&pasync_handle->link,
1999 &pasync_ctx->async_entry[cri].wait_queue.
2000 list);
2001 pasync_ctx->async_entry[cri].wait_queue.
2002 bytes_received +=
2003 (unsigned short)pasync_handle->buffer_len;
2004
2005 if (pasync_ctx->async_entry[cri].wait_queue.
2006 bytes_received >=
2007 pasync_ctx->async_entry[cri].wait_queue.
2008 bytes_needed)
2009 status = hwi_fwd_async_msg(beiscsi_conn, phba,
2010 pasync_ctx, cri);
2011 }
2012 }
2013 return status;
2014}
2015
2016static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
2017 struct beiscsi_hba *phba,
2018 struct i_t_dpdu_cqe *pdpdu_cqe)
2019{
2020 struct hwi_controller *phwi_ctrlr;
2021 struct hwi_async_pdu_context *pasync_ctx;
2022 struct async_pdu_handle *pasync_handle = NULL;
2023 unsigned int cq_index = -1;
8a86e833
JK		 2024 uint16_t cri_index = BE_GET_CRI_FROM_CID(
2025 beiscsi_conn->beiscsi_conn_cid);
6733b39a
JK		 2026
2027 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833
JK		 2028 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
2029 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
2030 cri_index));
2031
6733b39a
JK		 2032 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
2033 pdpdu_cqe, &cq_index);
2034
2035 if (pasync_handle->consumed == 0)
99bc5d55
JSJ		 2036 hwi_update_async_writables(phba, pasync_ctx,
2037 pasync_handle->is_header, cq_index);
2038
6733b39a 2039 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
8a86e833
JK		 2040 hwi_post_async_buffers(phba, pasync_handle->is_header,
2041 BEISCSI_GET_ULP_FROM_CRI(
2042 phwi_ctrlr, cri_index));
6733b39a
JK		 2043}
2044
756d29c8
JK		 2045static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
2046{
2047 struct be_queue_info *mcc_cq;
2048 struct be_mcc_compl *mcc_compl;
2049 unsigned int num_processed = 0;
2050
2051 mcc_cq = &phba->ctrl.mcc_obj.cq;
2052 mcc_compl = queue_tail_node(mcc_cq);
2053 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
2054 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
2055
2056 if (num_processed >= 32) {
2057 hwi_ring_cq_db(phba, mcc_cq->id,
2058 num_processed, 0, 0);
2059 num_processed = 0;
2060 }
2061 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
2062 /* Interpret flags as an async trailer */
2063 if (is_link_state_evt(mcc_compl->flags))
2064 /* Interpret compl as a async link evt */
2065 beiscsi_async_link_state_process(phba,
2066 (struct be_async_event_link_state *) mcc_compl);
2067 else
99bc5d55
JSJ		 2068 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
2069 "BM_%d : Unsupported Async Event, flags"
2070 " = 0x%08x\n",
2071 mcc_compl->flags);
756d29c8
JK		 2072 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
2073 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
2074 atomic_dec(&phba->ctrl.mcc_obj.q.used);
2075 }
2076
2077 mcc_compl->flags = 0;
2078 queue_tail_inc(mcc_cq);
2079 mcc_compl = queue_tail_node(mcc_cq);
2080 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
2081 num_processed++;
2082 }
2083
2084 if (num_processed > 0)
2085 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
2086
2087}
bfead3b2 2088
6763daae
JSJ		 2089/**
2090 * beiscsi_process_cq()- Process the Completion Queue
2091 * @pbe_eq: Event Q on which the Completion has come
2092 *
2093 * return
2094 * Number of Completion Entries processed.
2095 **/
bfead3b2 2096static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
6733b39a 2097{
6733b39a
JK		 2098 struct be_queue_info *cq;
2099 struct sol_cqe *sol;
2100 struct dmsg_cqe *dmsg;
2101 unsigned int num_processed = 0;
2102 unsigned int tot_nump = 0;
0a513dd8 2103 unsigned short code = 0, cid = 0;
a7909b39 2104 uint16_t cri_index = 0;
6733b39a 2105 struct beiscsi_conn *beiscsi_conn;
c2462288
JK		 2106 struct beiscsi_endpoint *beiscsi_ep;
2107 struct iscsi_endpoint *ep;
bfead3b2 2108 struct beiscsi_hba *phba;
6733b39a 2109
bfead3b2 2110 cq = pbe_eq->cq;
6733b39a 2111 sol = queue_tail_node(cq);
bfead3b2 2112 phba = pbe_eq->phba;
6733b39a
JK		 2113
2114 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
2115 CQE_VALID_MASK) {
2116 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
2117
73133261
JSJ		 2118 code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
2119 32] & CQE_CODE_MASK);
2120
2121 /* Get the CID */
2c9dfd36
JK		 2122 if (is_chip_be2_be3r(phba)) {
2123 cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
2124 } else {
73133261
JSJ		 2125 if ((code == DRIVERMSG_NOTIFY) ||
2126 (code == UNSOL_HDR_NOTIFY) ||
2127 (code == UNSOL_DATA_NOTIFY))
2128 cid = AMAP_GET_BITS(
2129 struct amap_i_t_dpdu_cqe_v2,
2130 cid, sol);
2131 else
2132 cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
2133 cid, sol);
2c9dfd36 2134 }
32951dd8 2135
a7909b39
JK		 2136 cri_index = BE_GET_CRI_FROM_CID(cid);
2137 ep = phba->ep_array[cri_index];
c2462288
JK		 2138 beiscsi_ep = ep->dd_data;
2139 beiscsi_conn = beiscsi_ep->conn;
756d29c8 2140
6733b39a 2141 if (num_processed >= 32) {
bfead3b2 2142 hwi_ring_cq_db(phba, cq->id,
6733b39a
JK		 2143 num_processed, 0, 0);
2144 tot_nump += num_processed;
2145 num_processed = 0;
2146 }
2147
0a513dd8 2148 switch (code) {
6733b39a
JK		 2149 case SOL_CMD_COMPLETE:
2150 hwi_complete_cmd(beiscsi_conn, phba, sol);
2151 break;
2152 case DRIVERMSG_NOTIFY:
99bc5d55
JSJ		 2153 beiscsi_log(phba, KERN_INFO,
2154 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2155 "BM_%d : Received %s[%d] on CID : %d\n",
2156 cqe_desc[code], code, cid);
99bc5d55 2157
6733b39a
JK		 2158 dmsg = (struct dmsg_cqe *)sol;
2159 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2160 break;
2161 case UNSOL_HDR_NOTIFY:
99bc5d55
JSJ		 2162 beiscsi_log(phba, KERN_INFO,
2163 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2164 "BM_%d : Received %s[%d] on CID : %d\n",
2165 cqe_desc[code], code, cid);
99bc5d55 2166
8f09a3b9 2167 spin_lock_bh(&phba->async_pdu_lock);
bfead3b2
JK		 2168 hwi_process_default_pdu_ring(beiscsi_conn, phba,
2169 (struct i_t_dpdu_cqe *)sol);
8f09a3b9 2170 spin_unlock_bh(&phba->async_pdu_lock);
bfead3b2 2171 break;
6733b39a 2172 case UNSOL_DATA_NOTIFY:
99bc5d55
JSJ		 2173 beiscsi_log(phba, KERN_INFO,
2174 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
6763daae
JSJ		 2175 "BM_%d : Received %s[%d] on CID : %d\n",
2176 cqe_desc[code], code, cid);
99bc5d55 2177
8f09a3b9 2178 spin_lock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2179 hwi_process_default_pdu_ring(beiscsi_conn, phba,
2180 (struct i_t_dpdu_cqe *)sol);
8f09a3b9 2181 spin_unlock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2182 break;
2183 case CXN_INVALIDATE_INDEX_NOTIFY:
2184 case CMD_INVALIDATED_NOTIFY:
2185 case CXN_INVALIDATE_NOTIFY:
99bc5d55
JSJ		 2186 beiscsi_log(phba, KERN_ERR,
2187 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2188 "BM_%d : Ignoring %s[%d] on CID : %d\n",
2189 cqe_desc[code], code, cid);
6733b39a
JK		 2190 break;
2191 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2192 case CMD_KILLED_INVALID_STATSN_RCVD:
2193 case CMD_KILLED_INVALID_R2T_RCVD:
2194 case CMD_CXN_KILLED_LUN_INVALID:
2195 case CMD_CXN_KILLED_ICD_INVALID:
2196 case CMD_CXN_KILLED_ITT_INVALID:
2197 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2198 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
99bc5d55
JSJ		 2199 beiscsi_log(phba, KERN_ERR,
2200 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
6763daae
JSJ		 2201 "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2202 cqe_desc[code], code, cid);
6733b39a
JK		 2203 break;
2204 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
99bc5d55
JSJ		 2205 beiscsi_log(phba, KERN_ERR,
2206 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2207 "BM_%d : Dropping %s[%d] on DPDU ring on CID : %d\n",
2208 cqe_desc[code], code, cid);
8f09a3b9 2209 spin_lock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2210 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2211 (struct i_t_dpdu_cqe *) sol);
8f09a3b9 2212 spin_unlock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2213 break;
2214 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2215 case CXN_KILLED_BURST_LEN_MISMATCH:
2216 case CXN_KILLED_AHS_RCVD:
2217 case CXN_KILLED_HDR_DIGEST_ERR:
2218 case CXN_KILLED_UNKNOWN_HDR:
2219 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2220 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2221 case CXN_KILLED_TIMED_OUT:
2222 case CXN_KILLED_FIN_RCVD:
6763daae
JSJ		 2223 case CXN_KILLED_RST_SENT:
2224 case CXN_KILLED_RST_RCVD:
6733b39a
JK		 2225 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2226 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2227 case CXN_KILLED_OVER_RUN_RESIDUAL:
2228 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2229 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
99bc5d55
JSJ		 2230 beiscsi_log(phba, KERN_ERR,
2231 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2232 "BM_%d : Event %s[%d] received on CID : %d\n",
2233 cqe_desc[code], code, cid);
0a513dd8
JSJ		 2234 if (beiscsi_conn)
2235 iscsi_conn_failure(beiscsi_conn->conn,
2236 ISCSI_ERR_CONN_FAILED);
6733b39a
JK		 2237 break;
2238 default:
99bc5d55
JSJ		 2239 beiscsi_log(phba, KERN_ERR,
2240 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2241 "BM_%d : Invalid CQE Event Received Code : %d"
2242 "CID 0x%x...\n",
0a513dd8 2243 code, cid);
6733b39a
JK		 2244 break;
2245 }
2246
2247 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2248 queue_tail_inc(cq);
2249 sol = queue_tail_node(cq);
2250 num_processed++;
2251 }
2252
2253 if (num_processed > 0) {
2254 tot_nump += num_processed;
bfead3b2 2255 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
6733b39a
JK		 2256 }
2257 return tot_nump;
2258}
2259
756d29c8 2260void beiscsi_process_all_cqs(struct work_struct *work)
6733b39a
JK		 2261{
2262 unsigned long flags;
bfead3b2
JK		 2263 struct hwi_controller *phwi_ctrlr;
2264 struct hwi_context_memory *phwi_context;
72fb46a9
JSJ		 2265 struct beiscsi_hba *phba;
2266 struct be_eq_obj *pbe_eq =
2267 container_of(work, struct be_eq_obj, work_cqs);
6733b39a 2268
72fb46a9 2269 phba = pbe_eq->phba;
bfead3b2
JK		 2270 phwi_ctrlr = phba->phwi_ctrlr;
2271 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2 2272
72fb46a9 2273 if (pbe_eq->todo_mcc_cq) {
6733b39a 2274 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 2275 pbe_eq->todo_mcc_cq = false;
6733b39a 2276 spin_unlock_irqrestore(&phba->isr_lock, flags);
756d29c8 2277 beiscsi_process_mcc_isr(phba);
6733b39a
JK		 2278 }
2279
72fb46a9 2280 if (pbe_eq->todo_cq) {
6733b39a 2281 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 2282 pbe_eq->todo_cq = false;
6733b39a 2283 spin_unlock_irqrestore(&phba->isr_lock, flags);
bfead3b2 2284 beiscsi_process_cq(pbe_eq);
6733b39a 2285 }
72fb46a9
JSJ		 2286
2287 /* rearm EQ for further interrupts */
2288 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 2289}
2290
2291static int be_iopoll(struct blk_iopoll *iop, int budget)
2292{
ad3f428e 2293 unsigned int ret;
6733b39a 2294 struct beiscsi_hba *phba;
bfead3b2 2295 struct be_eq_obj *pbe_eq;
6733b39a 2296
bfead3b2
JK		 2297 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2298 ret = beiscsi_process_cq(pbe_eq);
6733b39a 2299 if (ret < budget) {
bfead3b2 2300 phba = pbe_eq->phba;
6733b39a 2301 blk_iopoll_complete(iop);
99bc5d55
JSJ		 2302 beiscsi_log(phba, KERN_INFO,
2303 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2304 "BM_%d : rearm pbe_eq->q.id =%d\n",
2305 pbe_eq->q.id);
bfead3b2 2306 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 2307 }
2308 return ret;
2309}
2310
09a1093a
JSJ		 2311static void
2312hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2313 unsigned int num_sg, struct beiscsi_io_task *io_task)
2314{
2315 struct iscsi_sge *psgl;
2316 unsigned int sg_len, index;
2317 unsigned int sge_len = 0;
2318 unsigned long long addr;
2319 struct scatterlist *l_sg;
2320 unsigned int offset;
2321
2322 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2323 io_task->bhs_pa.u.a32.address_lo);
2324 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2325 io_task->bhs_pa.u.a32.address_hi);
2326
2327 l_sg = sg;
2328 for (index = 0; (index < num_sg) && (index < 2); index++,
2329 sg = sg_next(sg)) {
2330 if (index == 0) {
2331 sg_len = sg_dma_len(sg);
2332 addr = (u64) sg_dma_address(sg);
2333 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2334 sge0_addr_lo, pwrb,
2335 lower_32_bits(addr));
2336 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2337 sge0_addr_hi, pwrb,
2338 upper_32_bits(addr));
2339 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2340 sge0_len, pwrb,
2341 sg_len);
2342 sge_len = sg_len;
2343 } else {
2344 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2345 pwrb, sge_len);
2346 sg_len = sg_dma_len(sg);
2347 addr = (u64) sg_dma_address(sg);
2348 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2349 sge1_addr_lo, pwrb,
2350 lower_32_bits(addr));
2351 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2352 sge1_addr_hi, pwrb,
2353 upper_32_bits(addr));
2354 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2355 sge1_len, pwrb,
2356 sg_len);
2357 }
2358 }
2359 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2360 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2361
2362 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2363
2364 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2365 io_task->bhs_pa.u.a32.address_hi);
2366 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2367 io_task->bhs_pa.u.a32.address_lo);
2368
2369 if (num_sg == 1) {
2370 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2371 1);
2372 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2373 0);
2374 } else if (num_sg == 2) {
2375 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2376 0);
2377 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2378 1);
2379 } else {
2380 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2381 0);
2382 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2383 0);
2384 }
2385
2386 sg = l_sg;
2387 psgl++;
2388 psgl++;
2389 offset = 0;
2390 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2391 sg_len = sg_dma_len(sg);
2392 addr = (u64) sg_dma_address(sg);
2393 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2394 lower_32_bits(addr));
2395 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2396 upper_32_bits(addr));
2397 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2398 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2399 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2400 offset += sg_len;
2401 }
2402 psgl--;
2403 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2404}
2405
6733b39a
JK		 2406static void
2407hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2408 unsigned int num_sg, struct beiscsi_io_task *io_task)
2409{
2410 struct iscsi_sge *psgl;
58ff4bd0 2411 unsigned int sg_len, index;
6733b39a
JK		 2412 unsigned int sge_len = 0;
2413 unsigned long long addr;
2414 struct scatterlist *l_sg;
2415 unsigned int offset;
2416
2417 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2418 io_task->bhs_pa.u.a32.address_lo);
2419 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2420 io_task->bhs_pa.u.a32.address_hi);
2421
2422 l_sg = sg;
48bd86cf
JK		 2423 for (index = 0; (index < num_sg) && (index < 2); index++,
2424 sg = sg_next(sg)) {
6733b39a
JK		 2425 if (index == 0) {
2426 sg_len = sg_dma_len(sg);
2427 addr = (u64) sg_dma_address(sg);
2428 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
457ff3b7 2429 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2430 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
457ff3b7 2431 ((u32)(addr >> 32)));
6733b39a
JK		 2432 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2433 sg_len);
2434 sge_len = sg_len;
6733b39a 2435 } else {
6733b39a
JK		 2436 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2437 pwrb, sge_len);
2438 sg_len = sg_dma_len(sg);
2439 addr = (u64) sg_dma_address(sg);
2440 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
457ff3b7 2441 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2442 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
457ff3b7 2443 ((u32)(addr >> 32)));
6733b39a
JK		 2444 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2445 sg_len);
2446 }
2447 }
2448 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2449 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2450
2451 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2452
2453 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2454 io_task->bhs_pa.u.a32.address_hi);
2455 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2456 io_task->bhs_pa.u.a32.address_lo);
2457
caf818f1
JK		 2458 if (num_sg == 1) {
2459 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2460 1);
2461 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2462 0);
2463 } else if (num_sg == 2) {
2464 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2465 0);
2466 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2467 1);
2468 } else {
2469 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2470 0);
2471 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2472 0);
2473 }
6733b39a
JK		 2474 sg = l_sg;
2475 psgl++;
2476 psgl++;
2477 offset = 0;
48bd86cf 2478 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
6733b39a
JK		 2479 sg_len = sg_dma_len(sg);
2480 addr = (u64) sg_dma_address(sg);
2481 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2482 (addr & 0xFFFFFFFF));
2483 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2484 (addr >> 32));
2485 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2486 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2487 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2488 offset += sg_len;
2489 }
2490 psgl--;
2491 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2492}
2493
d629c471
JSJ		 2494/**
2495 * hwi_write_buffer()- Populate the WRB with task info
2496 * @pwrb: ptr to the WRB entry
2497 * @task: iscsi task which is to be executed
2498 **/
6733b39a
JK		 2499static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2500{
2501 struct iscsi_sge *psgl;
6733b39a
JK		 2502 struct beiscsi_io_task *io_task = task->dd_data;
2503 struct beiscsi_conn *beiscsi_conn = io_task->conn;
2504 struct beiscsi_hba *phba = beiscsi_conn->phba;
09a1093a 2505 uint8_t dsp_value = 0;
6733b39a
JK		 2506
2507 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2508 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2509 io_task->bhs_pa.u.a32.address_lo);
2510 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2511 io_task->bhs_pa.u.a32.address_hi);
2512
2513 if (task->data) {
09a1093a
JSJ		 2514
2515 /* Check for the data_count */
2516 dsp_value = (task->data_count) ? 1 : 0;
2517
2c9dfd36
JK		 2518 if (is_chip_be2_be3r(phba))
2519 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
09a1093a
JSJ		 2520 pwrb, dsp_value);
2521 else
2c9dfd36 2522 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
09a1093a
JSJ		 2523 pwrb, dsp_value);
2524
2525 /* Map addr only if there is data_count */
2526 if (dsp_value) {
d629c471
JSJ		 2527 io_task->mtask_addr = pci_map_single(phba->pcidev,
2528 task->data,
2529 task->data_count,
2530 PCI_DMA_TODEVICE);
d629c471 2531 io_task->mtask_data_count = task->data_count;
09a1093a 2532 } else
d629c471 2533 io_task->mtask_addr = 0;
09a1093a 2534
6733b39a 2535 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
d629c471 2536 lower_32_bits(io_task->mtask_addr));
6733b39a 2537 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
d629c471 2538 upper_32_bits(io_task->mtask_addr));
6733b39a
JK		 2539 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2540 task->data_count);
2541
2542 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2543 } else {
2544 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
d629c471 2545 io_task->mtask_addr = 0;
6733b39a
JK		 2546 }
2547
2548 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2549
2550 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2551
2552 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2553 io_task->bhs_pa.u.a32.address_hi);
2554 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2555 io_task->bhs_pa.u.a32.address_lo);
2556 if (task->data) {
2557 psgl++;
2558 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2559 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2560 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2561 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2562 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2563 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2564
2565 psgl++;
2566 if (task->data) {
2567 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
d629c471 2568 lower_32_bits(io_task->mtask_addr));
6733b39a 2569 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
d629c471 2570 upper_32_bits(io_task->mtask_addr));
6733b39a
JK		 2571 }
2572 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2573 }
2574 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2575}
2576
843ae752
JK		 2577/**
2578 * beiscsi_find_mem_req()- Find mem needed
2579 * @phba: ptr to HBA struct
2580 **/
6733b39a
JK		 2581static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2582{
8a86e833 2583 uint8_t mem_descr_index, ulp_num;
bfead3b2 2584 unsigned int num_cq_pages, num_async_pdu_buf_pages;
6733b39a
JK		 2585 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2586 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2587
2588 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2589 sizeof(struct sol_cqe));
6733b39a
JK		 2590
2591 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2592
2593 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2594 BE_ISCSI_PDU_HEADER_SIZE;
2595 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2596 sizeof(struct hwi_context_memory);
2597
6733b39a
JK		 2598
2599 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2600 * (phba->params.wrbs_per_cxn)
2601 * phba->params.cxns_per_ctrl;
2602 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
2603 (phba->params.wrbs_per_cxn);
2604 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2605 phba->params.cxns_per_ctrl);
2606
2607 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2608 phba->params.icds_per_ctrl;
2609 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2610 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
8a86e833
JK		 2611 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2612 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 2613
8a86e833
JK		 2614 num_async_pdu_buf_sgl_pages =
2615 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2616 phba, ulp_num) *
2617 sizeof(struct phys_addr));
2618
2619 num_async_pdu_buf_pages =
2620 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2621 phba, ulp_num) *
2622 phba->params.defpdu_hdr_sz);
2623
2624 num_async_pdu_data_pages =
2625 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2626 phba, ulp_num) *
2627 phba->params.defpdu_data_sz);
2628
2629 num_async_pdu_data_sgl_pages =
2630 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2631 phba, ulp_num) *
2632 sizeof(struct phys_addr));
2633
a129d92f
JK		 2634 mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 +
2635 (ulp_num * MEM_DESCR_OFFSET));
2636 phba->mem_req[mem_descr_index] =
2637 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2638 BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE;
2639
8a86e833
JK		 2640 mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2641 (ulp_num * MEM_DESCR_OFFSET));
2642 phba->mem_req[mem_descr_index] =
2643 num_async_pdu_buf_pages *
2644 PAGE_SIZE;
2645
2646 mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2647 (ulp_num * MEM_DESCR_OFFSET));
2648 phba->mem_req[mem_descr_index] =
2649 num_async_pdu_data_pages *
2650 PAGE_SIZE;
2651
2652 mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2653 (ulp_num * MEM_DESCR_OFFSET));
2654 phba->mem_req[mem_descr_index] =
2655 num_async_pdu_buf_sgl_pages *
2656 PAGE_SIZE;
2657
2658 mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 +
2659 (ulp_num * MEM_DESCR_OFFSET));
2660 phba->mem_req[mem_descr_index] =
2661 num_async_pdu_data_sgl_pages *
2662 PAGE_SIZE;
2663
2664 mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2665 (ulp_num * MEM_DESCR_OFFSET));
2666 phba->mem_req[mem_descr_index] =
2667 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2668 sizeof(struct async_pdu_handle);
2669
2670 mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2671 (ulp_num * MEM_DESCR_OFFSET));
2672 phba->mem_req[mem_descr_index] =
2673 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2674 sizeof(struct async_pdu_handle);
2675
2676 mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2677 (ulp_num * MEM_DESCR_OFFSET));
2678 phba->mem_req[mem_descr_index] =
2679 sizeof(struct hwi_async_pdu_context) +
2680 (BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2681 sizeof(struct hwi_async_entry));
2682 }
2683 }
6733b39a
JK		 2684}
2685
2686static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2687{
6733b39a 2688 dma_addr_t bus_add;
a7909b39
JK		 2689 struct hwi_controller *phwi_ctrlr;
2690 struct be_mem_descriptor *mem_descr;
6733b39a
JK		 2691 struct mem_array *mem_arr, *mem_arr_orig;
2692 unsigned int i, j, alloc_size, curr_alloc_size;
2693
3ec78271 2694 phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
6733b39a
JK		 2695 if (!phba->phwi_ctrlr)
2696 return -ENOMEM;
2697
a7909b39
JK		 2698 /* Allocate memory for wrb_context */
2699 phwi_ctrlr = phba->phwi_ctrlr;
2700 phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) *
2701 phba->params.cxns_per_ctrl,
2702 GFP_KERNEL);
2703 if (!phwi_ctrlr->wrb_context)
2704 return -ENOMEM;
2705
6733b39a
JK		 2706 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2707 GFP_KERNEL);
2708 if (!phba->init_mem) {
a7909b39 2709 kfree(phwi_ctrlr->wrb_context);
6733b39a
JK		 2710 kfree(phba->phwi_ctrlr);
2711 return -ENOMEM;
2712 }
2713
2714 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2715 GFP_KERNEL);
2716 if (!mem_arr_orig) {
2717 kfree(phba->init_mem);
a7909b39 2718 kfree(phwi_ctrlr->wrb_context);
6733b39a
JK		 2719 kfree(phba->phwi_ctrlr);
2720 return -ENOMEM;
2721 }
2722
2723 mem_descr = phba->init_mem;
2724 for (i = 0; i < SE_MEM_MAX; i++) {
8a86e833
JK		 2725 if (!phba->mem_req[i]) {
2726 mem_descr->mem_array = NULL;
2727 mem_descr++;
2728 continue;
2729 }
2730
6733b39a
JK		 2731 j = 0;
2732 mem_arr = mem_arr_orig;
2733 alloc_size = phba->mem_req[i];
2734 memset(mem_arr, 0, sizeof(struct mem_array) *
2735 BEISCSI_MAX_FRAGS_INIT);
2736 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2737 do {
2738 mem_arr->virtual_address = pci_alloc_consistent(
2739 phba->pcidev,
2740 curr_alloc_size,
2741 &bus_add);
2742 if (!mem_arr->virtual_address) {
2743 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2744 goto free_mem;
2745 if (curr_alloc_size -
2746 rounddown_pow_of_two(curr_alloc_size))
2747 curr_alloc_size = rounddown_pow_of_two
2748 (curr_alloc_size);
2749 else
2750 curr_alloc_size = curr_alloc_size / 2;
2751 } else {
2752 mem_arr->bus_address.u.
2753 a64.address = (__u64) bus_add;
2754 mem_arr->size = curr_alloc_size;
2755 alloc_size -= curr_alloc_size;
2756 curr_alloc_size = min(be_max_phys_size *
2757 1024, alloc_size);
2758 j++;
2759 mem_arr++;
2760 }
2761 } while (alloc_size);
2762 mem_descr->num_elements = j;
2763 mem_descr->size_in_bytes = phba->mem_req[i];
2764 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2765 GFP_KERNEL);
2766 if (!mem_descr->mem_array)
2767 goto free_mem;
2768
2769 memcpy(mem_descr->mem_array, mem_arr_orig,
2770 sizeof(struct mem_array) * j);
2771 mem_descr++;
2772 }
2773 kfree(mem_arr_orig);
2774 return 0;
2775free_mem:
2776 mem_descr->num_elements = j;
2777 while ((i) || (j)) {
2778 for (j = mem_descr->num_elements; j > 0; j--) {
2779 pci_free_consistent(phba->pcidev,
2780 mem_descr->mem_array[j - 1].size,
2781 mem_descr->mem_array[j - 1].
2782 virtual_address,
457ff3b7
JK		 2783 (unsigned long)mem_descr->
2784 mem_array[j - 1].
6733b39a
JK		 2785 bus_address.u.a64.address);
2786 }
2787 if (i) {
2788 i--;
2789 kfree(mem_descr->mem_array);
2790 mem_descr--;
2791 }
2792 }
2793 kfree(mem_arr_orig);
2794 kfree(phba->init_mem);
a7909b39 2795 kfree(phba->phwi_ctrlr->wrb_context);
6733b39a
JK		 2796 kfree(phba->phwi_ctrlr);
2797 return -ENOMEM;
2798}
2799
2800static int beiscsi_get_memory(struct beiscsi_hba *phba)
2801{
2802 beiscsi_find_mem_req(phba);
2803 return beiscsi_alloc_mem(phba);
2804}
2805
2806static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2807{
2808 struct pdu_data_out *pdata_out;
2809 struct pdu_nop_out *pnop_out;
2810 struct be_mem_descriptor *mem_descr;
2811
2812 mem_descr = phba->init_mem;
2813 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2814 pdata_out =
2815 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2816 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2817
2818 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2819 IIOC_SCSI_DATA);
2820
2821 pnop_out =
2822 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2823 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2824
2825 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2826 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2827 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2828 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2829}
2830
3ec78271 2831static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
6733b39a
JK		 2832{
2833 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
a7909b39 2834 struct hwi_context_memory *phwi_ctxt;
3ec78271 2835 struct wrb_handle *pwrb_handle = NULL;
6733b39a
JK		 2836 struct hwi_controller *phwi_ctrlr;
2837 struct hwi_wrb_context *pwrb_context;
3ec78271
JK		 2838 struct iscsi_wrb *pwrb = NULL;
2839 unsigned int num_cxn_wrbh = 0;
2840 unsigned int num_cxn_wrb = 0, j, idx = 0, index;
6733b39a
JK		 2841
2842 mem_descr_wrbh = phba->init_mem;
2843 mem_descr_wrbh += HWI_MEM_WRBH;
2844
2845 mem_descr_wrb = phba->init_mem;
2846 mem_descr_wrb += HWI_MEM_WRB;
6733b39a
JK		 2847 phwi_ctrlr = phba->phwi_ctrlr;
2848
a7909b39
JK		 2849 /* Allocate memory for WRBQ */
2850 phwi_ctxt = phwi_ctrlr->phwi_ctxt;
2851 phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) *
843ae752 2852 phba->params.cxns_per_ctrl,
a7909b39
JK		 2853 GFP_KERNEL);
2854 if (!phwi_ctxt->be_wrbq) {
2855 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2856 "BM_%d : WRBQ Mem Alloc Failed\n");
2857 return -ENOMEM;
2858 }
2859
2860 for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
6733b39a 2861 pwrb_context = &phwi_ctrlr->wrb_context[index];
6733b39a
JK		 2862 pwrb_context->pwrb_handle_base =
2863 kzalloc(sizeof(struct wrb_handle *) *
2864 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271 2865 if (!pwrb_context->pwrb_handle_base) {
99bc5d55
JSJ		 2866 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2867 "BM_%d : Mem Alloc Failed. Failing to load\n");
3ec78271
JK		 2868 goto init_wrb_hndl_failed;
2869 }
6733b39a
JK		 2870 pwrb_context->pwrb_handle_basestd =
2871 kzalloc(sizeof(struct wrb_handle *) *
2872 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271 2873 if (!pwrb_context->pwrb_handle_basestd) {
99bc5d55
JSJ		 2874 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2875 "BM_%d : Mem Alloc Failed. Failing to load\n");
3ec78271
JK		 2876 goto init_wrb_hndl_failed;
2877 }
2878 if (!num_cxn_wrbh) {
2879 pwrb_handle =
2880 mem_descr_wrbh->mem_array[idx].virtual_address;
2881 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2882 ((sizeof(struct wrb_handle)) *
2883 phba->params.wrbs_per_cxn));
2884 idx++;
2885 }
2886 pwrb_context->alloc_index = 0;
2887 pwrb_context->wrb_handles_available = 0;
2888 pwrb_context->free_index = 0;
2889
6733b39a 2890 if (num_cxn_wrbh) {
6733b39a
JK		 2891 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2892 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2893 pwrb_context->pwrb_handle_basestd[j] =
2894 pwrb_handle;
2895 pwrb_context->wrb_handles_available++;
bfead3b2 2896 pwrb_handle->wrb_index = j;
6733b39a
JK		 2897 pwrb_handle++;
2898 }
6733b39a
JK		 2899 num_cxn_wrbh--;
2900 }
2901 }
2902 idx = 0;
a7909b39 2903 for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
6733b39a 2904 pwrb_context = &phwi_ctrlr->wrb_context[index];
3ec78271 2905 if (!num_cxn_wrb) {
6733b39a 2906 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
7c56533c 2907 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
3ec78271
JK		 2908 ((sizeof(struct iscsi_wrb) *
2909 phba->params.wrbs_per_cxn));
2910 idx++;
2911 }
2912
2913 if (num_cxn_wrb) {
6733b39a
JK		 2914 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2915 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2916 pwrb_handle->pwrb = pwrb;
2917 pwrb++;
2918 }
2919 num_cxn_wrb--;
2920 }
2921 }
3ec78271
JK		 2922 return 0;
2923init_wrb_hndl_failed:
2924 for (j = index; j > 0; j--) {
2925 pwrb_context = &phwi_ctrlr->wrb_context[j];
2926 kfree(pwrb_context->pwrb_handle_base);
2927 kfree(pwrb_context->pwrb_handle_basestd);
2928 }
2929 return -ENOMEM;
6733b39a
JK		 2930}
2931
a7909b39 2932static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
6733b39a 2933{
8a86e833 2934 uint8_t ulp_num;
6733b39a
JK		 2935 struct hwi_controller *phwi_ctrlr;
2936 struct hba_parameters *p = &phba->params;
2937 struct hwi_async_pdu_context *pasync_ctx;
2938 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
dc63aac6 2939 unsigned int index, idx, num_per_mem, num_async_data;
6733b39a
JK		 2940 struct be_mem_descriptor *mem_descr;
2941
8a86e833
JK		 2942 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2943 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 2944
8a86e833
JK		 2945 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2946 mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2947 (ulp_num * MEM_DESCR_OFFSET));
2948
2949 phwi_ctrlr = phba->phwi_ctrlr;
2950 phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =
2951 (struct hwi_async_pdu_context *)
2952 mem_descr->mem_array[0].virtual_address;
2953
2954 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
2955 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2956
2957 pasync_ctx->async_entry =
2958 (struct hwi_async_entry *)
2959 ((long unsigned int)pasync_ctx +
2960 sizeof(struct hwi_async_pdu_context));
2961
2962 pasync_ctx->num_entries = BEISCSI_GET_CID_COUNT(phba,
2963 ulp_num);
2964 pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2965
2966 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2967 mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2968 (ulp_num * MEM_DESCR_OFFSET);
2969 if (mem_descr->mem_array[0].virtual_address) {
2970 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2971 "BM_%d : hwi_init_async_pdu_ctx"
2972 " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n",
2973 ulp_num,
2974 mem_descr->mem_array[0].
2975 virtual_address);
2976 } else
2977 beiscsi_log(phba, KERN_WARNING,
2978 BEISCSI_LOG_INIT,
2979 "BM_%d : No Virtual address for ULP : %d\n",
2980 ulp_num);
2981
2982 pasync_ctx->async_header.va_base =
6733b39a 2983 mem_descr->mem_array[0].virtual_address;
6733b39a 2984
8a86e833
JK		 2985 pasync_ctx->async_header.pa_base.u.a64.address =
2986 mem_descr->mem_array[0].
2987 bus_address.u.a64.address;
6733b39a 2988
8a86e833
JK		 2989 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2990 mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2991 (ulp_num * MEM_DESCR_OFFSET);
2992 if (mem_descr->mem_array[0].virtual_address) {
2993 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2994 "BM_%d : hwi_init_async_pdu_ctx"
2995 " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n",
2996 ulp_num,
2997 mem_descr->mem_array[0].
2998 virtual_address);
2999 } else
3000 beiscsi_log(phba, KERN_WARNING,
3001 BEISCSI_LOG_INIT,
3002 "BM_%d : No Virtual address for ULP : %d\n",
3003 ulp_num);
3004
3005 pasync_ctx->async_header.ring_base =
3006 mem_descr->mem_array[0].virtual_address;
6733b39a 3007
8a86e833
JK		 3008 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3009 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
3010 (ulp_num * MEM_DESCR_OFFSET);
3011 if (mem_descr->mem_array[0].virtual_address) {
3012 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3013 "BM_%d : hwi_init_async_pdu_ctx"
3014 " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n",
3015 ulp_num,
3016 mem_descr->mem_array[0].
3017 virtual_address);
3018 } else
3019 beiscsi_log(phba, KERN_WARNING,
3020 BEISCSI_LOG_INIT,
3021 "BM_%d : No Virtual address for ULP : %d\n",
3022 ulp_num);
3023
3024 pasync_ctx->async_header.handle_base =
3025 mem_descr->mem_array[0].virtual_address;
3026 pasync_ctx->async_header.writables = 0;
3027 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
3028
3029 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3030 mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3031 (ulp_num * MEM_DESCR_OFFSET);
3032 if (mem_descr->mem_array[0].virtual_address) {
3033 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3034 "BM_%d : hwi_init_async_pdu_ctx"
3035 " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n",
3036 ulp_num,
3037 mem_descr->mem_array[0].
3038 virtual_address);
3039 } else
3040 beiscsi_log(phba, KERN_WARNING,
3041 BEISCSI_LOG_INIT,
3042 "BM_%d : No Virtual address for ULP : %d\n",
3043 ulp_num);
3044
3045 pasync_ctx->async_data.ring_base =
3046 mem_descr->mem_array[0].virtual_address;
6733b39a 3047
8a86e833
JK		 3048 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3049 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
3050 (ulp_num * MEM_DESCR_OFFSET);
3051 if (!mem_descr->mem_array[0].virtual_address)
3052 beiscsi_log(phba, KERN_WARNING,
3053 BEISCSI_LOG_INIT,
3054 "BM_%d : No Virtual address for ULP : %d\n",
3055 ulp_num);
99bc5d55 3056
8a86e833
JK		 3057 pasync_ctx->async_data.handle_base =
3058 mem_descr->mem_array[0].virtual_address;
3059 pasync_ctx->async_data.writables = 0;
3060 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
3061
3062 pasync_header_h =
3063 (struct async_pdu_handle *)
3064 pasync_ctx->async_header.handle_base;
3065 pasync_data_h =
3066 (struct async_pdu_handle *)
3067 pasync_ctx->async_data.handle_base;
3068
3069 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3070 mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 +
3071 (ulp_num * MEM_DESCR_OFFSET);
3072 if (mem_descr->mem_array[0].virtual_address) {
3073 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3074 "BM_%d : hwi_init_async_pdu_ctx"
3075 " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n",
3076 ulp_num,
3077 mem_descr->mem_array[0].
3078 virtual_address);
3079 } else
3080 beiscsi_log(phba, KERN_WARNING,
3081 BEISCSI_LOG_INIT,
3082 "BM_%d : No Virtual address for ULP : %d\n",
3083 ulp_num);
3084
3085 idx = 0;
dc63aac6
JK		 3086 pasync_ctx->async_data.va_base =
3087 mem_descr->mem_array[idx].virtual_address;
3088 pasync_ctx->async_data.pa_base.u.a64.address =
3089 mem_descr->mem_array[idx].
3090 bus_address.u.a64.address;
3091
3092 num_async_data = ((mem_descr->mem_array[idx].size) /
3093 phba->params.defpdu_data_sz);
8a86e833 3094 num_per_mem = 0;
6733b39a 3095
8a86e833
JK		 3096 for (index = 0; index < BEISCSI_GET_CID_COUNT
3097 (phba, ulp_num); index++) {
3098 pasync_header_h->cri = -1;
3099 pasync_header_h->index = (char)index;
3100 INIT_LIST_HEAD(&pasync_header_h->link);
3101 pasync_header_h->pbuffer =
3102 (void *)((unsigned long)
3103 (pasync_ctx->
3104 async_header.va_base) +
3105 (p->defpdu_hdr_sz * index));
3106
3107 pasync_header_h->pa.u.a64.address =
3108 pasync_ctx->async_header.pa_base.u.a64.
3109 address + (p->defpdu_hdr_sz * index);
3110
3111 list_add_tail(&pasync_header_h->link,
3112 &pasync_ctx->async_header.
3113 free_list);
3114 pasync_header_h++;
3115 pasync_ctx->async_header.free_entries++;
3116 pasync_ctx->async_header.writables++;
3117
3118 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3119 wait_queue.list);
3120 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3121 header_busy_list);
3122 pasync_data_h->cri = -1;
3123 pasync_data_h->index = (char)index;
3124 INIT_LIST_HEAD(&pasync_data_h->link);
3125
3126 if (!num_async_data) {
3127 num_per_mem = 0;
3128 idx++;
3129 pasync_ctx->async_data.va_base =
3130 mem_descr->mem_array[idx].
3131 virtual_address;
3132 pasync_ctx->async_data.pa_base.u.
3133 a64.address =
3134 mem_descr->mem_array[idx].
3135 bus_address.u.a64.address;
3136 num_async_data =
3137 ((mem_descr->mem_array[idx].
3138 size) /
3139 phba->params.defpdu_data_sz);
3140 }
3141 pasync_data_h->pbuffer =
3142 (void *)((unsigned long)
3143 (pasync_ctx->async_data.va_base) +
3144 (p->defpdu_data_sz * num_per_mem));
3145
3146 pasync_data_h->pa.u.a64.address =
3147 pasync_ctx->async_data.pa_base.u.a64.
3148 address + (p->defpdu_data_sz *
3149 num_per_mem);
3150 num_per_mem++;
3151 num_async_data--;
3152
3153 list_add_tail(&pasync_data_h->link,
3154 &pasync_ctx->async_data.
3155 free_list);
3156 pasync_data_h++;
3157 pasync_ctx->async_data.free_entries++;
3158 pasync_ctx->async_data.writables++;
3159
3160 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3161 data_busy_list);
3162 }
6733b39a 3163
8a86e833
JK		 3164 pasync_ctx->async_header.host_write_ptr = 0;
3165 pasync_ctx->async_header.ep_read_ptr = -1;
3166 pasync_ctx->async_data.host_write_ptr = 0;
3167 pasync_ctx->async_data.ep_read_ptr = -1;
3168 }
6733b39a
JK		 3169 }
3170
a7909b39 3171 return 0;
6733b39a
JK		 3172}
3173
3174static int
3175be_sgl_create_contiguous(void *virtual_address,
3176 u64 physical_address, u32 length,
3177 struct be_dma_mem *sgl)
3178{
3179 WARN_ON(!virtual_address);
3180 WARN_ON(!physical_address);
3181 WARN_ON(!length > 0);
3182 WARN_ON(!sgl);
3183
3184 sgl->va = virtual_address;
457ff3b7 3185 sgl->dma = (unsigned long)physical_address;
6733b39a
JK		 3186 sgl->size = length;
3187
3188 return 0;
3189}
3190
3191static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
3192{
3193 memset(sgl, 0, sizeof(*sgl));
3194}
3195
3196static void
3197hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
3198 struct mem_array *pmem, struct be_dma_mem *sgl)
3199{
3200 if (sgl->va)
3201 be_sgl_destroy_contiguous(sgl);
3202
3203 be_sgl_create_contiguous(pmem->virtual_address,
3204 pmem->bus_address.u.a64.address,
3205 pmem->size, sgl);
3206}
3207
3208static void
3209hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
3210 struct mem_array *pmem, struct be_dma_mem *sgl)
3211{
3212 if (sgl->va)
3213 be_sgl_destroy_contiguous(sgl);
3214
3215 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
3216 pmem->bus_address.u.a64.address,
3217 pmem->size, sgl);
3218}
3219
3220static int be_fill_queue(struct be_queue_info *q,
3221 u16 len, u16 entry_size, void *vaddress)
3222{
3223 struct be_dma_mem *mem = &q->dma_mem;
3224
3225 memset(q, 0, sizeof(*q));
3226 q->len = len;
3227 q->entry_size = entry_size;
3228 mem->size = len * entry_size;
3229 mem->va = vaddress;
3230 if (!mem->va)
3231 return -ENOMEM;
3232 memset(mem->va, 0, mem->size);
3233 return 0;
3234}
3235
bfead3b2 3236static int beiscsi_create_eqs(struct beiscsi_hba *phba,
6733b39a
JK		 3237 struct hwi_context_memory *phwi_context)
3238{
bfead3b2 3239 unsigned int i, num_eq_pages;
99bc5d55 3240 int ret = 0, eq_for_mcc;
6733b39a
JK		 3241 struct be_queue_info *eq;
3242 struct be_dma_mem *mem;
6733b39a 3243 void *eq_vaddress;
bfead3b2 3244 dma_addr_t paddr;
6733b39a 3245
bfead3b2
JK		 3246 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3247 sizeof(struct be_eq_entry));
6733b39a 3248
bfead3b2
JK		 3249 if (phba->msix_enabled)
3250 eq_for_mcc = 1;
3251 else
3252 eq_for_mcc = 0;
3253 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3254 eq = &phwi_context->be_eq[i].q;
3255 mem = &eq->dma_mem;
3256 phwi_context->be_eq[i].phba = phba;
3257 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3258 num_eq_pages * PAGE_SIZE,
3259 &paddr);
3260 if (!eq_vaddress)
3261 goto create_eq_error;
3262
3263 mem->va = eq_vaddress;
3264 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3265 sizeof(struct be_eq_entry), eq_vaddress);
3266 if (ret) {
99bc5d55
JSJ		 3267 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3268 "BM_%d : be_fill_queue Failed for EQ\n");
bfead3b2
JK		 3269 goto create_eq_error;
3270 }
6733b39a 3271
bfead3b2
JK		 3272 mem->dma = paddr;
3273 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3274 phwi_context->cur_eqd);
3275 if (ret) {
99bc5d55
JSJ		 3276 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3277 "BM_%d : beiscsi_cmd_eq_create"
3278 "Failed for EQ\n");
bfead3b2
JK		 3279 goto create_eq_error;
3280 }
99bc5d55
JSJ		 3281
3282 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3283 "BM_%d : eqid = %d\n",
3284 phwi_context->be_eq[i].q.id);
6733b39a 3285 }
6733b39a 3286 return 0;
bfead3b2 3287create_eq_error:
107dfcba 3288 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
bfead3b2
JK		 3289 eq = &phwi_context->be_eq[i].q;
3290 mem = &eq->dma_mem;
3291 if (mem->va)
3292 pci_free_consistent(phba->pcidev, num_eq_pages
3293 * PAGE_SIZE,
3294 mem->va, mem->dma);
3295 }
3296 return ret;
6733b39a
JK		 3297}
3298
bfead3b2 3299static int beiscsi_create_cqs(struct beiscsi_hba *phba,
6733b39a
JK		 3300 struct hwi_context_memory *phwi_context)
3301{
bfead3b2 3302 unsigned int i, num_cq_pages;
99bc5d55 3303 int ret = 0;
6733b39a
JK		 3304 struct be_queue_info *cq, *eq;
3305 struct be_dma_mem *mem;
bfead3b2 3306 struct be_eq_obj *pbe_eq;
6733b39a 3307 void *cq_vaddress;
bfead3b2 3308 dma_addr_t paddr;
6733b39a 3309
bfead3b2
JK		 3310 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3311 sizeof(struct sol_cqe));
6733b39a 3312
bfead3b2
JK		 3313 for (i = 0; i < phba->num_cpus; i++) {
3314 cq = &phwi_context->be_cq[i];
3315 eq = &phwi_context->be_eq[i].q;
3316 pbe_eq = &phwi_context->be_eq[i];
3317 pbe_eq->cq = cq;
3318 pbe_eq->phba = phba;
3319 mem = &cq->dma_mem;
3320 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3321 num_cq_pages * PAGE_SIZE,
3322 &paddr);
3323 if (!cq_vaddress)
3324 goto create_cq_error;
7da50879 3325 ret = be_fill_queue(cq, phba->params.num_cq_entries,
bfead3b2
JK		 3326 sizeof(struct sol_cqe), cq_vaddress);
3327 if (ret) {
99bc5d55
JSJ		 3328 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3329 "BM_%d : be_fill_queue Failed "
3330 "for ISCSI CQ\n");
bfead3b2
JK		 3331 goto create_cq_error;
3332 }
3333
3334 mem->dma = paddr;
3335 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3336 false, 0);
3337 if (ret) {
99bc5d55
JSJ		 3338 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3339 "BM_%d : beiscsi_cmd_eq_create"
3340 "Failed for ISCSI CQ\n");
bfead3b2
JK		 3341 goto create_cq_error;
3342 }
99bc5d55
JSJ		 3343 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3344 "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3345 "iSCSI CQ CREATED\n", cq->id, eq->id);
6733b39a 3346 }
6733b39a 3347 return 0;
bfead3b2
JK		 3348
3349create_cq_error:
3350 for (i = 0; i < phba->num_cpus; i++) {
3351 cq = &phwi_context->be_cq[i];
3352 mem = &cq->dma_mem;
3353 if (mem->va)
3354 pci_free_consistent(phba->pcidev, num_cq_pages
3355 * PAGE_SIZE,
3356 mem->va, mem->dma);
3357 }
3358 return ret;
3359
6733b39a
JK		 3360}
3361
3362static int
3363beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3364 struct hwi_context_memory *phwi_context,
3365 struct hwi_controller *phwi_ctrlr,
8a86e833 3366 unsigned int def_pdu_ring_sz, uint8_t ulp_num)
6733b39a
JK		 3367{
3368 unsigned int idx;
3369 int ret;
3370 struct be_queue_info *dq, *cq;
3371 struct be_dma_mem *mem;
3372 struct be_mem_descriptor *mem_descr;
3373 void *dq_vaddress;
3374
3375 idx = 0;
8a86e833 3376 dq = &phwi_context->be_def_hdrq[ulp_num];
bfead3b2 3377 cq = &phwi_context->be_cq[0];
6733b39a
JK		 3378 mem = &dq->dma_mem;
3379 mem_descr = phba->init_mem;
8a86e833
JK		 3380 mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
3381 (ulp_num * MEM_DESCR_OFFSET);
6733b39a
JK		 3382 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3383 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3384 sizeof(struct phys_addr),
3385 sizeof(struct phys_addr), dq_vaddress);
3386 if (ret) {
99bc5d55 3387 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
8a86e833
JK		 3388 "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n",
3389 ulp_num);
3390
6733b39a
JK		 3391 return ret;
3392 }
457ff3b7
JK		 3393 mem->dma = (unsigned long)mem_descr->mem_array[idx].
3394 bus_address.u.a64.address;
6733b39a
JK		 3395 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3396 def_pdu_ring_sz,
8a86e833
JK		 3397 phba->params.defpdu_hdr_sz,
3398 BEISCSI_DEFQ_HDR, ulp_num);
6733b39a 3399 if (ret) {
99bc5d55 3400 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
8a86e833
JK		 3401 "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n",
3402 ulp_num);
3403
6733b39a
JK		 3404 return ret;
3405 }
99bc5d55 3406
8a86e833
JK		 3407 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3408 "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
3409 ulp_num,
3410 phwi_context->be_def_hdrq[ulp_num].id);
3411 hwi_post_async_buffers(phba, BEISCSI_DEFQ_HDR, ulp_num);
6733b39a
JK		 3412 return 0;
3413}
3414
3415static int
3416beiscsi_create_def_data(struct beiscsi_hba *phba,
3417 struct hwi_context_memory *phwi_context,
3418 struct hwi_controller *phwi_ctrlr,
8a86e833 3419 unsigned int def_pdu_ring_sz, uint8_t ulp_num)
6733b39a
JK		 3420{
3421 unsigned int idx;
3422 int ret;
3423 struct be_queue_info *dataq, *cq;
3424 struct be_dma_mem *mem;
3425 struct be_mem_descriptor *mem_descr;
3426 void *dq_vaddress;
3427
3428 idx = 0;
8a86e833 3429 dataq = &phwi_context->be_def_dataq[ulp_num];
bfead3b2 3430 cq = &phwi_context->be_cq[0];
6733b39a
JK		 3431 mem = &dataq->dma_mem;
3432 mem_descr = phba->init_mem;
8a86e833
JK		 3433 mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3434 (ulp_num * MEM_DESCR_OFFSET);
6733b39a
JK		 3435 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3436 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3437 sizeof(struct phys_addr),
3438 sizeof(struct phys_addr), dq_vaddress);
3439 if (ret) {
99bc5d55 3440 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
8a86e833
JK		 3441 "BM_%d : be_fill_queue Failed for DEF PDU "
3442 "DATA on ULP : %d\n",
3443 ulp_num);
3444
6733b39a
JK		 3445 return ret;
3446 }
457ff3b7
JK		 3447 mem->dma = (unsigned long)mem_descr->mem_array[idx].
3448 bus_address.u.a64.address;
6733b39a
JK		 3449 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3450 def_pdu_ring_sz,
8a86e833
JK		 3451 phba->params.defpdu_data_sz,
3452 BEISCSI_DEFQ_DATA, ulp_num);
6733b39a 3453 if (ret) {
99bc5d55
JSJ		 3454 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3455 "BM_%d be_cmd_create_default_pdu_queue"
8a86e833
JK		 3456 " Failed for DEF PDU DATA on ULP : %d\n",
3457 ulp_num);
6733b39a
JK		 3458 return ret;
3459 }
8a86e833 3460
99bc5d55 3461 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
8a86e833
JK		 3462 "BM_%d : iscsi def data id on ULP : %d is %d\n",
3463 ulp_num,
3464 phwi_context->be_def_dataq[ulp_num].id);
99bc5d55 3465
8a86e833 3466 hwi_post_async_buffers(phba, BEISCSI_DEFQ_DATA, ulp_num);
99bc5d55 3467 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
8a86e833
JK		 3468 "BM_%d : DEFAULT PDU DATA RING CREATED"
3469 "on ULP : %d\n", ulp_num);
99bc5d55 3470
6733b39a
JK		 3471 return 0;
3472}
3473
15a90fe0
JK		 3474
3475static int
3476beiscsi_post_template_hdr(struct beiscsi_hba *phba)
3477{
3478 struct be_mem_descriptor *mem_descr;
3479 struct mem_array *pm_arr;
3480 struct be_dma_mem sgl;
a129d92f 3481 int status, ulp_num;
15a90fe0 3482
a129d92f
JK		 3483 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3484 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3485 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3486 mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 +
3487 (ulp_num * MEM_DESCR_OFFSET);
3488 pm_arr = mem_descr->mem_array;
15a90fe0 3489
a129d92f
JK		 3490 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3491 status = be_cmd_iscsi_post_template_hdr(
3492 &phba->ctrl, &sgl);
15a90fe0 3493
a129d92f
JK		 3494 if (status != 0) {
3495 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3496 "BM_%d : Post Template HDR Failed for"
3497 "ULP_%d\n", ulp_num);
3498 return status;
3499 }
3500
3501 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3502 "BM_%d : Template HDR Pages Posted for"
3503 "ULP_%d\n", ulp_num);
15a90fe0
JK		 3504 }
3505 }
15a90fe0
JK		 3506 return 0;
3507}
3508
6733b39a
JK		 3509static int
3510beiscsi_post_pages(struct beiscsi_hba *phba)
3511{
3512 struct be_mem_descriptor *mem_descr;
3513 struct mem_array *pm_arr;
3514 unsigned int page_offset, i;
3515 struct be_dma_mem sgl;
843ae752 3516 int status, ulp_num = 0;
6733b39a
JK		 3517
3518 mem_descr = phba->init_mem;
3519 mem_descr += HWI_MEM_SGE;
3520 pm_arr = mem_descr->mem_array;
3521
90622db3
JK		 3522 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3523 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3524 break;
3525
6733b39a 3526 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
843ae752 3527 phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE;
6733b39a
JK		 3528 for (i = 0; i < mem_descr->num_elements; i++) {
3529 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3530 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3531 page_offset,
3532 (pm_arr->size / PAGE_SIZE));
3533 page_offset += pm_arr->size / PAGE_SIZE;
3534 if (status != 0) {
99bc5d55
JSJ		 3535 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3536 "BM_%d : post sgl failed.\n");
6733b39a
JK		 3537 return status;
3538 }
3539 pm_arr++;
3540 }
99bc5d55
JSJ		 3541 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3542 "BM_%d : POSTED PAGES\n");
6733b39a
JK		 3543 return 0;
3544}
3545
bfead3b2
JK		 3546static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3547{
3548 struct be_dma_mem *mem = &q->dma_mem;
c8b25598 3549 if (mem->va) {
bfead3b2
JK		 3550 pci_free_consistent(phba->pcidev, mem->size,
3551 mem->va, mem->dma);
c8b25598
JK		 3552 mem->va = NULL;
3553 }
bfead3b2
JK		 3554}
3555
3556static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3557 u16 len, u16 entry_size)
3558{
3559 struct be_dma_mem *mem = &q->dma_mem;
3560
3561 memset(q, 0, sizeof(*q));
3562 q->len = len;
3563 q->entry_size = entry_size;
3564 mem->size = len * entry_size;
3565 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
3566 if (!mem->va)
d3ad2bb3 3567 return -ENOMEM;
bfead3b2
JK		 3568 memset(mem->va, 0, mem->size);
3569 return 0;
3570}
3571
6733b39a
JK		 3572static int
3573beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3574 struct hwi_context_memory *phwi_context,
3575 struct hwi_controller *phwi_ctrlr)
3576{
3577 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3578 u64 pa_addr_lo;
4eea99d5 3579 unsigned int idx, num, i, ulp_num;
6733b39a
JK		 3580 struct mem_array *pwrb_arr;
3581 void *wrb_vaddr;
3582 struct be_dma_mem sgl;
3583 struct be_mem_descriptor *mem_descr;
a7909b39 3584 struct hwi_wrb_context *pwrb_context;
6733b39a 3585 int status;
4eea99d5
JK		 3586 uint8_t ulp_count = 0, ulp_base_num = 0;
3587 uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 };
6733b39a
JK		 3588
3589 idx = 0;
3590 mem_descr = phba->init_mem;
3591 mem_descr += HWI_MEM_WRB;
3592 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3593 GFP_KERNEL);
3594 if (!pwrb_arr) {
99bc5d55
JSJ		 3595 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3596 "BM_%d : Memory alloc failed in create wrb ring.\n");
6733b39a
JK		 3597 return -ENOMEM;
3598 }
3599 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3600 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3601 num_wrb_rings = mem_descr->mem_array[idx].size /
3602 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3603
3604 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3605 if (num_wrb_rings) {
3606 pwrb_arr[num].virtual_address = wrb_vaddr;
3607 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3608 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3609 sizeof(struct iscsi_wrb);
3610 wrb_vaddr += pwrb_arr[num].size;
3611 pa_addr_lo += pwrb_arr[num].size;
3612 num_wrb_rings--;
3613 } else {
3614 idx++;
3615 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3616 pa_addr_lo = mem_descr->mem_array[idx].\
3617 bus_address.u.a64.address;
3618 num_wrb_rings = mem_descr->mem_array[idx].size /
3619 (phba->params.wrbs_per_cxn *
3620 sizeof(struct iscsi_wrb));
3621 pwrb_arr[num].virtual_address = wrb_vaddr;
3622 pwrb_arr[num].bus_address.u.a64.address\
3623 = pa_addr_lo;
3624 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3625 sizeof(struct iscsi_wrb);
3626 wrb_vaddr += pwrb_arr[num].size;
3627 pa_addr_lo += pwrb_arr[num].size;
3628 num_wrb_rings--;
3629 }
3630 }
4eea99d5
JK		 3631
3632 /* Get the ULP Count */
3633 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3634 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3635 ulp_count++;
3636 ulp_base_num = ulp_num;
3637 cid_count_ulp[ulp_num] =
3638 BEISCSI_GET_CID_COUNT(phba, ulp_num);
3639 }
3640
6733b39a
JK		 3641 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3642 wrb_mem_index = 0;
3643 offset = 0;
3644 size = 0;
3645
4eea99d5
JK		 3646 if (ulp_count > 1) {
3647 ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT;
3648
3649 if (!cid_count_ulp[ulp_base_num])
3650 ulp_base_num = (ulp_base_num + 1) %
3651 BEISCSI_ULP_COUNT;
3652
3653 cid_count_ulp[ulp_base_num]--;
3654 }
3655
3656
6733b39a
JK		 3657 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3658 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
4eea99d5
JK		 3659 &phwi_context->be_wrbq[i],
3660 &phwi_ctrlr->wrb_context[i],
3661 ulp_base_num);
6733b39a 3662 if (status != 0) {
99bc5d55
JSJ		 3663 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3664 "BM_%d : wrbq create failed.");
1462b8ff 3665 kfree(pwrb_arr);
6733b39a
JK		 3666 return status;
3667 }
a7909b39 3668 pwrb_context = &phwi_ctrlr->wrb_context[i];
a7909b39 3669 BE_SET_CID_TO_CRI(i, pwrb_context->cid);
6733b39a
JK		 3670 }
3671 kfree(pwrb_arr);
3672 return 0;
3673}
3674
3675static void free_wrb_handles(struct beiscsi_hba *phba)
3676{
3677 unsigned int index;
3678 struct hwi_controller *phwi_ctrlr;
3679 struct hwi_wrb_context *pwrb_context;
3680
3681 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39 3682 for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
6733b39a
JK		 3683 pwrb_context = &phwi_ctrlr->wrb_context[index];
3684 kfree(pwrb_context->pwrb_handle_base);
3685 kfree(pwrb_context->pwrb_handle_basestd);
3686 }
3687}
3688
bfead3b2
JK		 3689static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3690{
3691 struct be_queue_info *q;
3692 struct be_ctrl_info *ctrl = &phba->ctrl;
3693
3694 q = &phba->ctrl.mcc_obj.q;
3695 if (q->created)
3696 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3697 be_queue_free(phba, q);
3698
3699 q = &phba->ctrl.mcc_obj.cq;
3700 if (q->created)
3701 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3702 be_queue_free(phba, q);
3703}
3704
6733b39a
JK		 3705static void hwi_cleanup(struct beiscsi_hba *phba)
3706{
3707 struct be_queue_info *q;
3708 struct be_ctrl_info *ctrl = &phba->ctrl;
3709 struct hwi_controller *phwi_ctrlr;
3710 struct hwi_context_memory *phwi_context;
a7909b39 3711 struct hwi_async_pdu_context *pasync_ctx;
8a86e833 3712 int i, eq_num, ulp_num;
6733b39a
JK		 3713
3714 phwi_ctrlr = phba->phwi_ctrlr;
3715 phwi_context = phwi_ctrlr->phwi_ctxt;
15a90fe0
JK		 3716
3717 be_cmd_iscsi_remove_template_hdr(ctrl);
3718
6733b39a
JK		 3719 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3720 q = &phwi_context->be_wrbq[i];
3721 if (q->created)
3722 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3723 }
a7909b39 3724 kfree(phwi_context->be_wrbq);
6733b39a
JK		 3725 free_wrb_handles(phba);
3726
8a86e833
JK		 3727 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3728 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 3729
8a86e833
JK		 3730 q = &phwi_context->be_def_hdrq[ulp_num];
3731 if (q->created)
3732 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3733
3734 q = &phwi_context->be_def_dataq[ulp_num];
3735 if (q->created)
3736 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3737
3738 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
3739 }
3740 }
6733b39a
JK		 3741
3742 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3743
bfead3b2
JK		 3744 for (i = 0; i < (phba->num_cpus); i++) {
3745 q = &phwi_context->be_cq[i];
3746 if (q->created)
3747 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3748 }
3749 if (phba->msix_enabled)
3750 eq_num = 1;
3751 else
3752 eq_num = 0;
3753 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3754 q = &phwi_context->be_eq[i].q;
3755 if (q->created)
3756 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3757 }
3758 be_mcc_queues_destroy(phba);
0283fbb1 3759 be_cmd_fw_uninit(ctrl);
bfead3b2 3760}
6733b39a 3761
bfead3b2
JK		 3762static int be_mcc_queues_create(struct beiscsi_hba *phba,
3763 struct hwi_context_memory *phwi_context)
3764{
3765 struct be_queue_info *q, *cq;
3766 struct be_ctrl_info *ctrl = &phba->ctrl;
3767
3768 /* Alloc MCC compl queue */
3769 cq = &phba->ctrl.mcc_obj.cq;
3770 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3771 sizeof(struct be_mcc_compl)))
3772 goto err;
3773 /* Ask BE to create MCC compl queue; */
3774 if (phba->msix_enabled) {
3775 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3776 [phba->num_cpus].q, false, true, 0))
3777 goto mcc_cq_free;
3778 } else {
3779 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3780 false, true, 0))
3781 goto mcc_cq_free;
3782 }
3783
3784 /* Alloc MCC queue */
3785 q = &phba->ctrl.mcc_obj.q;
3786 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3787 goto mcc_cq_destroy;
3788
3789 /* Ask BE to create MCC queue */
35e66019 3790 if (beiscsi_cmd_mccq_create(phba, q, cq))
bfead3b2
JK		 3791 goto mcc_q_free;
3792
3793 return 0;
3794
3795mcc_q_free:
3796 be_queue_free(phba, q);
3797mcc_cq_destroy:
3798 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3799mcc_cq_free:
3800 be_queue_free(phba, cq);
3801err:
d3ad2bb3 3802 return -ENOMEM;
bfead3b2
JK		 3803}
3804
107dfcba
JSJ		 3805/**
3806 * find_num_cpus()- Get the CPU online count
3807 * @phba: ptr to priv structure
3808 *
3809 * CPU count is used for creating EQ.
3810 **/
3811static void find_num_cpus(struct beiscsi_hba *phba)
bfead3b2
JK		 3812{
3813 int num_cpus = 0;
3814
3815 num_cpus = num_online_cpus();
bfead3b2 3816
22abeef0
JSJ		 3817 switch (phba->generation) {
3818 case BE_GEN2:
3819 case BE_GEN3:
3820 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3821 BEISCSI_MAX_NUM_CPUS : num_cpus;
3822 break;
3823 case BE_GEN4:
68c26a3a
JK		 3824 /*
3825 * If eqid_count == 1 fall back to
3826 * INTX mechanism
3827 **/
3828 if (phba->fw_config.eqid_count == 1) {
3829 enable_msix = 0;
3830 phba->num_cpus = 1;
3831 return;
3832 }
3833
3834 phba->num_cpus =
3835 (num_cpus > (phba->fw_config.eqid_count - 1)) ?
3836 (phba->fw_config.eqid_count - 1) : num_cpus;
22abeef0
JSJ		 3837 break;
3838 default:
3839 phba->num_cpus = 1;
3840 }
6733b39a
JK		 3841}
3842
3843static int hwi_init_port(struct beiscsi_hba *phba)
3844{
3845 struct hwi_controller *phwi_ctrlr;
3846 struct hwi_context_memory *phwi_context;
3847 unsigned int def_pdu_ring_sz;
3848 struct be_ctrl_info *ctrl = &phba->ctrl;
8a86e833 3849 int status, ulp_num;
6733b39a 3850
6733b39a 3851 phwi_ctrlr = phba->phwi_ctrlr;
6733b39a 3852 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3853 phwi_context->max_eqd = 0;
3854 phwi_context->min_eqd = 0;
3855 phwi_context->cur_eqd = 64;
6733b39a 3856 be_cmd_fw_initialize(&phba->ctrl);
bfead3b2
JK		 3857
3858 status = beiscsi_create_eqs(phba, phwi_context);
6733b39a 3859 if (status != 0) {
99bc5d55
JSJ		 3860 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3861 "BM_%d : EQ not created\n");
6733b39a
JK		 3862 goto error;
3863 }
3864
bfead3b2
JK		 3865 status = be_mcc_queues_create(phba, phwi_context);
3866 if (status != 0)
3867 goto error;
3868
3869 status = mgmt_check_supported_fw(ctrl, phba);
6733b39a 3870 if (status != 0) {
99bc5d55
JSJ		 3871 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3872 "BM_%d : Unsupported fw version\n");
6733b39a
JK		 3873 goto error;
3874 }
3875
bfead3b2 3876 status = beiscsi_create_cqs(phba, phwi_context);
6733b39a 3877 if (status != 0) {
99bc5d55
JSJ		 3878 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3879 "BM_%d : CQ not created\n");
6733b39a
JK		 3880 goto error;
3881 }
3882
8a86e833
JK		 3883 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3884 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 3885
8a86e833
JK		 3886 def_pdu_ring_sz =
3887 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
3888 sizeof(struct phys_addr);
3889
3890 status = beiscsi_create_def_hdr(phba, phwi_context,
3891 phwi_ctrlr,
3892 def_pdu_ring_sz,
3893 ulp_num);
3894 if (status != 0) {
3895 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3896 "BM_%d : Default Header not created for ULP : %d\n",
3897 ulp_num);
3898 goto error;
3899 }
3900
3901 status = beiscsi_create_def_data(phba, phwi_context,
3902 phwi_ctrlr,
3903 def_pdu_ring_sz,
3904 ulp_num);
3905 if (status != 0) {
3906 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3907 "BM_%d : Default Data not created for ULP : %d\n",
3908 ulp_num);
3909 goto error;
3910 }
3911 }
6733b39a
JK		 3912 }
3913
3914 status = beiscsi_post_pages(phba);
3915 if (status != 0) {
99bc5d55
JSJ		 3916 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3917 "BM_%d : Post SGL Pages Failed\n");
6733b39a
JK		 3918 goto error;
3919 }
3920
15a90fe0
JK		 3921 status = beiscsi_post_template_hdr(phba);
3922 if (status != 0) {
3923 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3924 "BM_%d : Template HDR Posting for CXN Failed\n");
3925 }
3926
6733b39a
JK		 3927 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3928 if (status != 0) {
99bc5d55
JSJ		 3929 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3930 "BM_%d : WRB Rings not created\n");
6733b39a
JK		 3931 goto error;
3932 }
3933
8a86e833
JK		 3934 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3935 uint16_t async_arr_idx = 0;
3936
3937 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3938 uint16_t cri = 0;
3939 struct hwi_async_pdu_context *pasync_ctx;
3940
3941 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(
3942 phwi_ctrlr, ulp_num);
3943 for (cri = 0; cri <
3944 phba->params.cxns_per_ctrl; cri++) {
3945 if (ulp_num == BEISCSI_GET_ULP_FROM_CRI
3946 (phwi_ctrlr, cri))
3947 pasync_ctx->cid_to_async_cri_map[
3948 phwi_ctrlr->wrb_context[cri].cid] =
3949 async_arr_idx++;
3950 }
3951 }
3952 }
3953
99bc5d55
JSJ		 3954 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3955 "BM_%d : hwi_init_port success\n");
6733b39a
JK		 3956 return 0;
3957
3958error:
99bc5d55
JSJ		 3959 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3960 "BM_%d : hwi_init_port failed");
6733b39a 3961 hwi_cleanup(phba);
a49e06d5 3962 return status;
6733b39a
JK		 3963}
3964
6733b39a
JK		 3965static int hwi_init_controller(struct beiscsi_hba *phba)
3966{
3967 struct hwi_controller *phwi_ctrlr;
3968
3969 phwi_ctrlr = phba->phwi_ctrlr;
3970 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3971 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3972 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
99bc5d55
JSJ		 3973 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3974 "BM_%d : phwi_ctrlr->phwi_ctxt=%p\n",
3975 phwi_ctrlr->phwi_ctxt);
6733b39a 3976 } else {
99bc5d55
JSJ		 3977 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3978 "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3979 "than one element.Failing to load\n");
6733b39a
JK		 3980 return -ENOMEM;
3981 }
3982
3983 iscsi_init_global_templates(phba);
3ec78271
JK		 3984 if (beiscsi_init_wrb_handle(phba))
3985 return -ENOMEM;
3986
a7909b39
JK		 3987 if (hwi_init_async_pdu_ctx(phba)) {
3988 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3989 "BM_%d : hwi_init_async_pdu_ctx failed\n");
3990 return -ENOMEM;
3991 }
3992
6733b39a 3993 if (hwi_init_port(phba) != 0) {
99bc5d55
JSJ		 3994 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3995 "BM_%d : hwi_init_controller failed\n");
3996
6733b39a
JK		 3997 return -ENOMEM;
3998 }
3999 return 0;
4000}
4001
4002static void beiscsi_free_mem(struct beiscsi_hba *phba)
4003{
4004 struct be_mem_descriptor *mem_descr;
4005 int i, j;
4006
4007 mem_descr = phba->init_mem;
4008 i = 0;
4009 j = 0;
4010 for (i = 0; i < SE_MEM_MAX; i++) {
4011 for (j = mem_descr->num_elements; j > 0; j--) {
4012 pci_free_consistent(phba->pcidev,
4013 mem_descr->mem_array[j - 1].size,
4014 mem_descr->mem_array[j - 1].virtual_address,
457ff3b7
JK		 4015 (unsigned long)mem_descr->mem_array[j - 1].
4016 bus_address.u.a64.address);
6733b39a 4017 }
8a86e833 4018
6733b39a
JK		 4019 kfree(mem_descr->mem_array);
4020 mem_descr++;
4021 }
4022 kfree(phba->init_mem);
a7909b39 4023 kfree(phba->phwi_ctrlr->wrb_context);
6733b39a
JK		 4024 kfree(phba->phwi_ctrlr);
4025}
4026
4027static int beiscsi_init_controller(struct beiscsi_hba *phba)
4028{
4029 int ret = -ENOMEM;
4030
4031 ret = beiscsi_get_memory(phba);
4032 if (ret < 0) {
99bc5d55
JSJ		 4033 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4034 "BM_%d : beiscsi_dev_probe -"
4035 "Failed in beiscsi_alloc_memory\n");
6733b39a
JK		 4036 return ret;
4037 }
4038
4039 ret = hwi_init_controller(phba);
4040 if (ret)
4041 goto free_init;
99bc5d55
JSJ		 4042 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4043 "BM_%d : Return success from beiscsi_init_controller");
4044
6733b39a
JK		 4045 return 0;
4046
4047free_init:
4048 beiscsi_free_mem(phba);
a49e06d5 4049 return ret;
6733b39a
JK		 4050}
4051
4052static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
4053{
4054 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
4055 struct sgl_handle *psgl_handle;
4056 struct iscsi_sge *pfrag;
90622db3
JK		 4057 unsigned int arr_index, i, idx;
4058 unsigned int ulp_icd_start, ulp_num = 0;
6733b39a
JK		 4059
4060 phba->io_sgl_hndl_avbl = 0;
4061 phba->eh_sgl_hndl_avbl = 0;
bfead3b2 4062
6733b39a
JK		 4063 mem_descr_sglh = phba->init_mem;
4064 mem_descr_sglh += HWI_MEM_SGLH;
4065 if (1 == mem_descr_sglh->num_elements) {
4066 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
4067 phba->params.ios_per_ctrl,
4068 GFP_KERNEL);
4069 if (!phba->io_sgl_hndl_base) {
99bc5d55
JSJ		 4070 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4071 "BM_%d : Mem Alloc Failed. Failing to load\n");
6733b39a
JK		 4072 return -ENOMEM;
4073 }
4074 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
4075 (phba->params.icds_per_ctrl -
4076 phba->params.ios_per_ctrl),
4077 GFP_KERNEL);
4078 if (!phba->eh_sgl_hndl_base) {
4079 kfree(phba->io_sgl_hndl_base);
99bc5d55
JSJ		 4080 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4081 "BM_%d : Mem Alloc Failed. Failing to load\n");
6733b39a
JK		 4082 return -ENOMEM;
4083 }
4084 } else {
99bc5d55
JSJ		 4085 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4086 "BM_%d : HWI_MEM_SGLH is more than one element."
4087 "Failing to load\n");
6733b39a
JK		 4088 return -ENOMEM;
4089 }
4090
4091 arr_index = 0;
4092 idx = 0;
4093 while (idx < mem_descr_sglh->num_elements) {
4094 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
4095
4096 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
4097 sizeof(struct sgl_handle)); i++) {
4098 if (arr_index < phba->params.ios_per_ctrl) {
4099 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
4100 phba->io_sgl_hndl_avbl++;
4101 arr_index++;
4102 } else {
4103 phba->eh_sgl_hndl_base[arr_index -
4104 phba->params.ios_per_ctrl] =
4105 psgl_handle;
4106 arr_index++;
4107 phba->eh_sgl_hndl_avbl++;
4108 }
4109 psgl_handle++;
4110 }
4111 idx++;
4112 }
99bc5d55
JSJ		 4113 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4114 "BM_%d : phba->io_sgl_hndl_avbl=%d"
4115 "phba->eh_sgl_hndl_avbl=%d\n",
4116 phba->io_sgl_hndl_avbl,
4117 phba->eh_sgl_hndl_avbl);
4118
6733b39a
JK		 4119 mem_descr_sg = phba->init_mem;
4120 mem_descr_sg += HWI_MEM_SGE;
99bc5d55
JSJ		 4121 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4122 "\n BM_%d : mem_descr_sg->num_elements=%d\n",
4123 mem_descr_sg->num_elements);
4124
90622db3
JK		 4125 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
4126 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
4127 break;
4128
4129 ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
4130
6733b39a
JK		 4131 arr_index = 0;
4132 idx = 0;
4133 while (idx < mem_descr_sg->num_elements) {
4134 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
4135
4136 for (i = 0;
4137 i < (mem_descr_sg->mem_array[idx].size) /
4138 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
4139 i++) {
4140 if (arr_index < phba->params.ios_per_ctrl)
4141 psgl_handle = phba->io_sgl_hndl_base[arr_index];
4142 else
4143 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
4144 phba->params.ios_per_ctrl];
4145 psgl_handle->pfrag = pfrag;
4146 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
4147 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
4148 pfrag += phba->params.num_sge_per_io;
90622db3 4149 psgl_handle->sgl_index = ulp_icd_start + arr_index++;
6733b39a
JK		 4150 }
4151 idx++;
4152 }
4153 phba->io_sgl_free_index = 0;
4154 phba->io_sgl_alloc_index = 0;
4155 phba->eh_sgl_free_index = 0;
4156 phba->eh_sgl_alloc_index = 0;
4157 return 0;
4158}
4159
4160static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
4161{
0a3db7c0
JK		 4162 int ret;
4163 uint16_t i, ulp_num;
4164 struct ulp_cid_info *ptr_cid_info = NULL;
6733b39a 4165
0a3db7c0
JK		 4166 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4167 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4168 ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info),
4169 GFP_KERNEL);
4170
4171 if (!ptr_cid_info) {
4172 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4173 "BM_%d : Failed to allocate memory"
4174 "for ULP_CID_INFO for ULP : %d\n",
4175 ulp_num);
4176 ret = -ENOMEM;
4177 goto free_memory;
4178
4179 }
4180
4181 /* Allocate memory for CID array */
4182 ptr_cid_info->cid_array = kzalloc(sizeof(void *) *
4183 BEISCSI_GET_CID_COUNT(phba,
4184 ulp_num), GFP_KERNEL);
4185 if (!ptr_cid_info->cid_array) {
4186 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4187 "BM_%d : Failed to allocate memory"
4188 "for CID_ARRAY for ULP : %d\n",
4189 ulp_num);
4190 kfree(ptr_cid_info);
4191 ptr_cid_info = NULL;
4192 ret = -ENOMEM;
4193
4194 goto free_memory;
4195 }
4196 ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT(
4197 phba, ulp_num);
4198
4199 /* Save the cid_info_array ptr */
4200 phba->cid_array_info[ulp_num] = ptr_cid_info;
4201 }
6733b39a 4202 }
c2462288 4203 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
a7909b39 4204 phba->params.cxns_per_ctrl, GFP_KERNEL);
6733b39a 4205 if (!phba->ep_array) {
99bc5d55
JSJ		 4206 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4207 "BM_%d : Failed to allocate memory in "
4208 "hba_setup_cid_tbls\n");
0a3db7c0
JK		 4209 ret = -ENOMEM;
4210
4211 goto free_memory;
6733b39a 4212 }
a7909b39
JK		 4213
4214 phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) *
4215 phba->params.cxns_per_ctrl, GFP_KERNEL);
4216 if (!phba->conn_table) {
4217 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4218 "BM_%d : Failed to allocate memory in"
4219 "hba_setup_cid_tbls\n");
4220
a7909b39 4221 kfree(phba->ep_array);
a7909b39 4222 phba->ep_array = NULL;
0a3db7c0 4223 ret = -ENOMEM;
6733b39a 4224 }
a7909b39 4225
0a3db7c0
JK		 4226 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
4227 ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num;
4228
4229 ptr_cid_info = phba->cid_array_info[ulp_num];
4230 ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] =
4231 phba->phwi_ctrlr->wrb_context[i].cid;
4232
4233 }
4234
4235 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4236 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4237 ptr_cid_info = phba->cid_array_info[ulp_num];
a7909b39 4238
0a3db7c0
JK		 4239 ptr_cid_info->cid_alloc = 0;
4240 ptr_cid_info->cid_free = 0;
4241 }
4242 }
6733b39a 4243 return 0;
0a3db7c0
JK		 4244
4245free_memory:
4246 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4247 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4248 ptr_cid_info = phba->cid_array_info[ulp_num];
4249
4250 if (ptr_cid_info) {
4251 kfree(ptr_cid_info->cid_array);
4252 kfree(ptr_cid_info);
4253 phba->cid_array_info[ulp_num] = NULL;
4254 }
4255 }
4256 }
4257
4258 return ret;
6733b39a
JK		 4259}
4260
238f6b72 4261static void hwi_enable_intr(struct beiscsi_hba *phba)
6733b39a
JK		 4262{
4263 struct be_ctrl_info *ctrl = &phba->ctrl;
4264 struct hwi_controller *phwi_ctrlr;
4265 struct hwi_context_memory *phwi_context;
4266 struct be_queue_info *eq;
4267 u8 __iomem *addr;
bfead3b2 4268 u32 reg, i;
6733b39a
JK		 4269 u32 enabled;
4270
4271 phwi_ctrlr = phba->phwi_ctrlr;
4272 phwi_context = phwi_ctrlr->phwi_ctxt;
4273
6733b39a
JK		 4274 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
4275 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
4276 reg = ioread32(addr);
6733b39a
JK		 4277
4278 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4279 if (!enabled) {
4280 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
99bc5d55
JSJ		 4281 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4282 "BM_%d : reg =x%08x addr=%p\n", reg, addr);
6733b39a 4283 iowrite32(reg, addr);
665d6d94
JK		 4284 }
4285
4286 if (!phba->msix_enabled) {
4287 eq = &phwi_context->be_eq[0].q;
99bc5d55
JSJ		 4288 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4289 "BM_%d : eq->id=%d\n", eq->id);
4290
665d6d94
JK		 4291 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4292 } else {
4293 for (i = 0; i <= phba->num_cpus; i++) {
4294 eq = &phwi_context->be_eq[i].q;
99bc5d55
JSJ		 4295 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4296 "BM_%d : eq->id=%d\n", eq->id);
bfead3b2
JK		 4297 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4298 }
c03af1ae 4299 }
6733b39a
JK		 4300}
4301
4302static void hwi_disable_intr(struct beiscsi_hba *phba)
4303{
4304 struct be_ctrl_info *ctrl = &phba->ctrl;
4305
4306 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
4307 u32 reg = ioread32(addr);
4308
4309 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4310 if (enabled) {
4311 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4312 iowrite32(reg, addr);
4313 } else
99bc5d55
JSJ		 4314 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4315 "BM_%d : In hwi_disable_intr, Already Disabled\n");
6733b39a
JK		 4316}
4317
9aef4200
JSJ		 4318/**
4319 * beiscsi_get_boot_info()- Get the boot session info
4320 * @phba: The device priv structure instance
4321 *
4322 * Get the boot target info and store in driver priv structure
4323 *
4324 * return values
4325 * Success: 0
4326 * Failure: Non-Zero Value
4327 **/
c7acc5b8
JK		 4328static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
4329{
0e43895e 4330 struct be_cmd_get_session_resp *session_resp;
c7acc5b8 4331 struct be_dma_mem nonemb_cmd;
e175defe 4332 unsigned int tag;
9aef4200 4333 unsigned int s_handle;
f457a46f 4334 int ret = -ENOMEM;
c7acc5b8 4335
9aef4200
JSJ		 4336 /* Get the session handle of the boot target */
4337 ret = be_mgmt_get_boot_shandle(phba, &s_handle);
4338 if (ret) {
99bc5d55
JSJ		 4339 beiscsi_log(phba, KERN_ERR,
4340 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4341 "BM_%d : No boot session\n");
9aef4200 4342 return ret;
c7acc5b8 4343 }
c7acc5b8
JK		 4344 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4345 sizeof(*session_resp),
4346 &nonemb_cmd.dma);
4347 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 4348 beiscsi_log(phba, KERN_ERR,
4349 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4350 "BM_%d : Failed to allocate memory for"
4351 "beiscsi_get_session_info\n");
4352
c7acc5b8
JK		 4353 return -ENOMEM;
4354 }
4355
4356 memset(nonemb_cmd.va, 0, sizeof(*session_resp));
9aef4200 4357 tag = mgmt_get_session_info(phba, s_handle,
0e43895e 4358 &nonemb_cmd);
c7acc5b8 4359 if (!tag) {
99bc5d55
JSJ		 4360 beiscsi_log(phba, KERN_ERR,
4361 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4362 "BM_%d : beiscsi_get_session_info"
4363 " Failed\n");
4364
c7acc5b8 4365 goto boot_freemem;
e175defe 4366 }
c7acc5b8 4367
1957aa7f 4368 ret = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd);
e175defe 4369 if (ret) {
99bc5d55
JSJ		 4370 beiscsi_log(phba, KERN_ERR,
4371 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
e175defe 4372 "BM_%d : beiscsi_get_session_info Failed");
1957aa7f
JK		 4373
4374 if (ret != -EBUSY)
4375 goto boot_freemem;
4376 else
4377 return ret;
c7acc5b8 4378 }
e175defe 4379
c7acc5b8 4380 session_resp = nonemb_cmd.va ;
f457a46f 4381
c7acc5b8
JK		 4382 memcpy(&phba->boot_sess, &session_resp->session_info,
4383 sizeof(struct mgmt_session_info));
f457a46f
MC		 4384 ret = 0;
4385
c7acc5b8
JK		 4386boot_freemem:
4387 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4388 nonemb_cmd.va, nonemb_cmd.dma);
f457a46f
MC		 4389 return ret;
4390}
4391
4392static void beiscsi_boot_release(void *data)
4393{
4394 struct beiscsi_hba *phba = data;
4395
4396 scsi_host_put(phba->shost);
4397}
4398
4399static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
4400{
4401 struct iscsi_boot_kobj *boot_kobj;
4402
4403 /* get boot info using mgmt cmd */
4404 if (beiscsi_get_boot_info(phba))
4405 /* Try to see if we can carry on without this */
4406 return 0;
4407
4408 phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
4409 if (!phba->boot_kset)
4410 return -ENOMEM;
4411
4412 /* get a ref because the show function will ref the phba */
4413 if (!scsi_host_get(phba->shost))
4414 goto free_kset;
4415 boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
4416 beiscsi_show_boot_tgt_info,
4417 beiscsi_tgt_get_attr_visibility,
4418 beiscsi_boot_release);
4419 if (!boot_kobj)
4420 goto put_shost;
4421
4422 if (!scsi_host_get(phba->shost))
4423 goto free_kset;
4424 boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
4425 beiscsi_show_boot_ini_info,
4426 beiscsi_ini_get_attr_visibility,
4427 beiscsi_boot_release);
4428 if (!boot_kobj)
4429 goto put_shost;
4430
4431 if (!scsi_host_get(phba->shost))
4432 goto free_kset;
4433 boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
4434 beiscsi_show_boot_eth_info,
4435 beiscsi_eth_get_attr_visibility,
4436 beiscsi_boot_release);
4437 if (!boot_kobj)
4438 goto put_shost;
4439 return 0;
4440
4441put_shost:
4442 scsi_host_put(phba->shost);
4443free_kset:
4444 iscsi_boot_destroy_kset(phba->boot_kset);
c7acc5b8
JK		 4445 return -ENOMEM;
4446}
4447
6733b39a
JK		 4448static int beiscsi_init_port(struct beiscsi_hba *phba)
4449{
4450 int ret;
4451
4452 ret = beiscsi_init_controller(phba);
4453 if (ret < 0) {
99bc5d55
JSJ		 4454 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4455 "BM_%d : beiscsi_dev_probe - Failed in"
4456 "beiscsi_init_controller\n");
6733b39a
JK		 4457 return ret;
4458 }
4459 ret = beiscsi_init_sgl_handle(phba);
4460 if (ret < 0) {
99bc5d55
JSJ		 4461 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4462 "BM_%d : beiscsi_dev_probe - Failed in"
4463 "beiscsi_init_sgl_handle\n");
6733b39a
JK		 4464 goto do_cleanup_ctrlr;
4465 }
4466
4467 if (hba_setup_cid_tbls(phba)) {
99bc5d55
JSJ		 4468 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4469 "BM_%d : Failed in hba_setup_cid_tbls\n");
6733b39a
JK		 4470 kfree(phba->io_sgl_hndl_base);
4471 kfree(phba->eh_sgl_hndl_base);
4472 goto do_cleanup_ctrlr;
4473 }
4474
4475 return ret;
4476
4477do_cleanup_ctrlr:
4478 hwi_cleanup(phba);
4479 return ret;
4480}
4481
4482static void hwi_purge_eq(struct beiscsi_hba *phba)
4483{
4484 struct hwi_controller *phwi_ctrlr;
4485 struct hwi_context_memory *phwi_context;
4486 struct be_queue_info *eq;
4487 struct be_eq_entry *eqe = NULL;
bfead3b2 4488 int i, eq_msix;
756d29c8 4489 unsigned int num_processed;
6733b39a
JK		 4490
4491 phwi_ctrlr = phba->phwi_ctrlr;
4492 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 4493 if (phba->msix_enabled)
4494 eq_msix = 1;
4495 else
4496 eq_msix = 0;
6733b39a 4497
bfead3b2
JK		 4498 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
4499 eq = &phwi_context->be_eq[i].q;
6733b39a 4500 eqe = queue_tail_node(eq);
756d29c8 4501 num_processed = 0;
bfead3b2
JK		 4502 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
4503 & EQE_VALID_MASK) {
4504 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
4505 queue_tail_inc(eq);
4506 eqe = queue_tail_node(eq);
756d29c8 4507 num_processed++;
bfead3b2 4508 }
756d29c8
JK		 4509
4510 if (num_processed)
4511 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
6733b39a
JK		 4512 }
4513}
4514
4515static void beiscsi_clean_port(struct beiscsi_hba *phba)
4516{
0a3db7c0
JK		 4517 int mgmt_status, ulp_num;
4518 struct ulp_cid_info *ptr_cid_info = NULL;
6733b39a 4519
bd41c2bd
JK		 4520 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4521 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4522 mgmt_status = mgmt_epfw_cleanup(phba, ulp_num);
4523 if (mgmt_status)
4524 beiscsi_log(phba, KERN_WARNING,
4525 BEISCSI_LOG_INIT,
4526 "BM_%d : mgmt_epfw_cleanup FAILED"
4527 " for ULP_%d\n", ulp_num);
4528 }
4529 }
756d29c8 4530
6733b39a 4531 hwi_purge_eq(phba);
756d29c8 4532 hwi_cleanup(phba);
6733b39a
JK		 4533 kfree(phba->io_sgl_hndl_base);
4534 kfree(phba->eh_sgl_hndl_base);
6733b39a 4535 kfree(phba->ep_array);
a7909b39 4536 kfree(phba->conn_table);
0a3db7c0
JK		 4537
4538 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4539 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4540 ptr_cid_info = phba->cid_array_info[ulp_num];
4541
4542 if (ptr_cid_info) {
4543 kfree(ptr_cid_info->cid_array);
4544 kfree(ptr_cid_info);
4545 phba->cid_array_info[ulp_num] = NULL;
4546 }
4547 }
4548 }
4549
6733b39a
JK		 4550}
4551
43f388b0
JK		 4552/**
4553 * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4554 * @beiscsi_conn: ptr to the conn to be cleaned up
4a4a11b9 4555 * @task: ptr to iscsi_task resource to be freed.
43f388b0
JK		 4556 *
4557 * Free driver mgmt resources binded to CXN.
4558 **/
4559void
4a4a11b9
JK		 4560beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
4561 struct iscsi_task *task)
43f388b0
JK		 4562{
4563 struct beiscsi_io_task *io_task;
4564 struct beiscsi_hba *phba = beiscsi_conn->phba;
4565 struct hwi_wrb_context *pwrb_context;
4566 struct hwi_controller *phwi_ctrlr;
a7909b39
JK		 4567 uint16_t cri_index = BE_GET_CRI_FROM_CID(
4568 beiscsi_conn->beiscsi_conn_cid);
43f388b0
JK		 4569
4570 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39
JK		 4571 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4572
4a4a11b9 4573 io_task = task->dd_data;
43f388b0
JK		 4574
4575 if (io_task->pwrb_handle) {
4576 memset(io_task->pwrb_handle->pwrb, 0,
4577 sizeof(struct iscsi_wrb));
4578 free_wrb_handle(phba, pwrb_context,
4579 io_task->pwrb_handle);
4580 io_task->pwrb_handle = NULL;
4581 }
4582
4583 if (io_task->psgl_handle) {
4584 spin_lock_bh(&phba->mgmt_sgl_lock);
4585 free_mgmt_sgl_handle(phba,
4586 io_task->psgl_handle);
43f388b0 4587 io_task->psgl_handle = NULL;
4a4a11b9 4588 spin_unlock_bh(&phba->mgmt_sgl_lock);
43f388b0
JK		 4589 }
4590
4591 if (io_task->mtask_addr)
4592 pci_unmap_single(phba->pcidev,
4593 io_task->mtask_addr,
4594 io_task->mtask_data_count,
4595 PCI_DMA_TODEVICE);
4596}
4597
d629c471
JSJ		 4598/**
4599 * beiscsi_cleanup_task()- Free driver resources of the task
4600 * @task: ptr to the iscsi task
4601 *
4602 **/
1282ab76
MC		 4603static void beiscsi_cleanup_task(struct iscsi_task *task)
4604{
4605 struct beiscsi_io_task *io_task = task->dd_data;
4606 struct iscsi_conn *conn = task->conn;
4607 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4608 struct beiscsi_hba *phba = beiscsi_conn->phba;
4609 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4610 struct hwi_wrb_context *pwrb_context;
4611 struct hwi_controller *phwi_ctrlr;
a7909b39
JK		 4612 uint16_t cri_index = BE_GET_CRI_FROM_CID(
4613 beiscsi_conn->beiscsi_conn_cid);
1282ab76
MC		 4614
4615 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39 4616 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1282ab76
MC		 4617
4618 if (io_task->cmd_bhs) {
4619 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4620 io_task->bhs_pa.u.a64.address);
4621 io_task->cmd_bhs = NULL;
4622 }
4623
4624 if (task->sc) {
4625 if (io_task->pwrb_handle) {
4626 free_wrb_handle(phba, pwrb_context,
4627 io_task->pwrb_handle);
4628 io_task->pwrb_handle = NULL;
4629 }
4630
4631 if (io_task->psgl_handle) {
4632 spin_lock(&phba->io_sgl_lock);
4633 free_io_sgl_handle(phba, io_task->psgl_handle);
4634 spin_unlock(&phba->io_sgl_lock);
4635 io_task->psgl_handle = NULL;
4636 }
4637 } else {
43f388b0 4638 if (!beiscsi_conn->login_in_progress)
4a4a11b9 4639 beiscsi_free_mgmt_task_handles(beiscsi_conn, task);
1282ab76
MC		 4640 }
4641}
4642
6733b39a
JK		 4643void
4644beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4645 struct beiscsi_offload_params *params)
4646{
4647 struct wrb_handle *pwrb_handle;
6733b39a 4648 struct beiscsi_hba *phba = beiscsi_conn->phba;
1282ab76
MC		 4649 struct iscsi_task *task = beiscsi_conn->task;
4650 struct iscsi_session *session = task->conn->session;
6733b39a
JK		 4651 u32 doorbell = 0;
4652
4653 /*
4654 * We can always use 0 here because it is reserved by libiscsi for
4655 * login/startup related tasks.
4656 */
1282ab76
MC		 4657 beiscsi_conn->login_in_progress = 0;
4658 spin_lock_bh(&session->lock);
4659 beiscsi_cleanup_task(task);
4660 spin_unlock_bh(&session->lock);
4661
a7909b39 4662 pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid);
6733b39a 4663
acb9693c 4664 /* Check for the adapter family */
2c9dfd36 4665 if (is_chip_be2_be3r(phba))
acb9693c
JSJ		 4666 beiscsi_offload_cxn_v0(params, pwrb_handle,
4667 phba->init_mem);
2c9dfd36
JK		 4668 else
4669 beiscsi_offload_cxn_v2(params, pwrb_handle);
6733b39a 4670
acb9693c
JSJ		 4671 be_dws_le_to_cpu(pwrb_handle->pwrb,
4672 sizeof(struct iscsi_target_context_update_wrb));
6733b39a
JK		 4673
4674 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 4675 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
bfead3b2 4676 << DB_DEF_PDU_WRB_INDEX_SHIFT;
6733b39a 4677 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
1e4be6ff
JK		 4678 iowrite32(doorbell, phba->db_va +
4679 beiscsi_conn->doorbell_offset);
6733b39a
JK		 4680}
4681
4682static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4683 int *index, int *age)
4684{
bfead3b2 4685 *index = (int)itt;
6733b39a
JK		 4686 if (age)
4687 *age = conn->session->age;
4688}
4689
4690/**
4691 * beiscsi_alloc_pdu - allocates pdu and related resources
4692 * @task: libiscsi task
4693 * @opcode: opcode of pdu for task
4694 *
4695 * This is called with the session lock held. It will allocate
4696 * the wrb and sgl if needed for the command. And it will prep
4697 * the pdu's itt. beiscsi_parse_pdu will later translate
4698 * the pdu itt to the libiscsi task itt.
4699 */
4700static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4701{
4702 struct beiscsi_io_task *io_task = task->dd_data;
4703 struct iscsi_conn *conn = task->conn;
4704 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4705 struct beiscsi_hba *phba = beiscsi_conn->phba;
4706 struct hwi_wrb_context *pwrb_context;
4707 struct hwi_controller *phwi_ctrlr;
4708 itt_t itt;
a7909b39 4709 uint16_t cri_index = 0;
2afc95bf
JK		 4710 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4711 dma_addr_t paddr;
6733b39a 4712
2afc95bf 4713 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
bc7accec 4714 GFP_ATOMIC, &paddr);
2afc95bf
JK		 4715 if (!io_task->cmd_bhs)
4716 return -ENOMEM;
2afc95bf 4717 io_task->bhs_pa.u.a64.address = paddr;
bfead3b2 4718 io_task->libiscsi_itt = (itt_t)task->itt;
6733b39a
JK		 4719 io_task->conn = beiscsi_conn;
4720
4721 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4722 task->hdr_max = sizeof(struct be_cmd_bhs);
d2cecf0d 4723 io_task->psgl_handle = NULL;
3ec78271 4724 io_task->pwrb_handle = NULL;
6733b39a
JK		 4725
4726 if (task->sc) {
4727 spin_lock(&phba->io_sgl_lock);
4728 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4729 spin_unlock(&phba->io_sgl_lock);
8359c79b
JSJ		 4730 if (!io_task->psgl_handle) {
4731 beiscsi_log(phba, KERN_ERR,
4732 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4733 "BM_%d : Alloc of IO_SGL_ICD Failed"
4734 "for the CID : %d\n",
4735 beiscsi_conn->beiscsi_conn_cid);
2afc95bf 4736 goto free_hndls;
8359c79b 4737 }
d2cecf0d 4738 io_task->pwrb_handle = alloc_wrb_handle(phba,
a7909b39 4739 beiscsi_conn->beiscsi_conn_cid);
8359c79b
JSJ		 4740 if (!io_task->pwrb_handle) {
4741 beiscsi_log(phba, KERN_ERR,
4742 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4743 "BM_%d : Alloc of WRB_HANDLE Failed"
4744 "for the CID : %d\n",
4745 beiscsi_conn->beiscsi_conn_cid);
d2cecf0d 4746 goto free_io_hndls;
8359c79b 4747 }
6733b39a
JK		 4748 } else {
4749 io_task->scsi_cmnd = NULL;
d7aea67b 4750 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
43f388b0 4751 beiscsi_conn->task = task;
6733b39a
JK		 4752 if (!beiscsi_conn->login_in_progress) {
4753 spin_lock(&phba->mgmt_sgl_lock);
4754 io_task->psgl_handle = (struct sgl_handle *)
4755 alloc_mgmt_sgl_handle(phba);
4756 spin_unlock(&phba->mgmt_sgl_lock);
8359c79b
JSJ		 4757 if (!io_task->psgl_handle) {
4758 beiscsi_log(phba, KERN_ERR,
4759 BEISCSI_LOG_IO |
4760 BEISCSI_LOG_CONFIG,
4761 "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4762 "for the CID : %d\n",
4763 beiscsi_conn->
4764 beiscsi_conn_cid);
2afc95bf 4765 goto free_hndls;
8359c79b 4766 }
2afc95bf 4767
6733b39a
JK		 4768 beiscsi_conn->login_in_progress = 1;
4769 beiscsi_conn->plogin_sgl_handle =
4770 io_task->psgl_handle;
d2cecf0d
JK		 4771 io_task->pwrb_handle =
4772 alloc_wrb_handle(phba,
a7909b39 4773 beiscsi_conn->beiscsi_conn_cid);
8359c79b
JSJ		 4774 if (!io_task->pwrb_handle) {
4775 beiscsi_log(phba, KERN_ERR,
4776 BEISCSI_LOG_IO |
4777 BEISCSI_LOG_CONFIG,
4778 "BM_%d : Alloc of WRB_HANDLE Failed"
4779 "for the CID : %d\n",
4780 beiscsi_conn->
4781 beiscsi_conn_cid);
4782 goto free_mgmt_hndls;
4783 }
d2cecf0d
JK		 4784 beiscsi_conn->plogin_wrb_handle =
4785 io_task->pwrb_handle;
4786
6733b39a
JK		 4787 } else {
4788 io_task->psgl_handle =
4789 beiscsi_conn->plogin_sgl_handle;
d2cecf0d
JK		 4790 io_task->pwrb_handle =
4791 beiscsi_conn->plogin_wrb_handle;
6733b39a
JK		 4792 }
4793 } else {
4794 spin_lock(&phba->mgmt_sgl_lock);
4795 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4796 spin_unlock(&phba->mgmt_sgl_lock);
8359c79b
JSJ		 4797 if (!io_task->psgl_handle) {
4798 beiscsi_log(phba, KERN_ERR,
4799 BEISCSI_LOG_IO |
4800 BEISCSI_LOG_CONFIG,
4801 "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4802 "for the CID : %d\n",
4803 beiscsi_conn->
4804 beiscsi_conn_cid);
2afc95bf 4805 goto free_hndls;
8359c79b 4806 }
d2cecf0d
JK		 4807 io_task->pwrb_handle =
4808 alloc_wrb_handle(phba,
a7909b39 4809 beiscsi_conn->beiscsi_conn_cid);
8359c79b
JSJ		 4810 if (!io_task->pwrb_handle) {
4811 beiscsi_log(phba, KERN_ERR,
4812 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4813 "BM_%d : Alloc of WRB_HANDLE Failed"
4814 "for the CID : %d\n",
4815 beiscsi_conn->beiscsi_conn_cid);
d2cecf0d 4816 goto free_mgmt_hndls;
8359c79b 4817 }
d2cecf0d 4818
6733b39a
JK		 4819 }
4820 }
bfead3b2
JK		 4821 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4822 wrb_index << 16) | (unsigned int)
4823 (io_task->psgl_handle->sgl_index));
32951dd8 4824 io_task->pwrb_handle->pio_handle = task;
bfead3b2 4825
6733b39a
JK		 4826 io_task->cmd_bhs->iscsi_hdr.itt = itt;
4827 return 0;
2afc95bf 4828
d2cecf0d
JK		 4829free_io_hndls:
4830 spin_lock(&phba->io_sgl_lock);
4831 free_io_sgl_handle(phba, io_task->psgl_handle);
4832 spin_unlock(&phba->io_sgl_lock);
4833 goto free_hndls;
4834free_mgmt_hndls:
4835 spin_lock(&phba->mgmt_sgl_lock);
4836 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
a7909b39 4837 io_task->psgl_handle = NULL;
d2cecf0d 4838 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 4839free_hndls:
4840 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39
JK		 4841 cri_index = BE_GET_CRI_FROM_CID(
4842 beiscsi_conn->beiscsi_conn_cid);
4843 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
d2cecf0d
JK		 4844 if (io_task->pwrb_handle)
4845 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2afc95bf
JK		 4846 io_task->pwrb_handle = NULL;
4847 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4848 io_task->bhs_pa.u.a64.address);
1282ab76 4849 io_task->cmd_bhs = NULL;
2afc95bf 4850 return -ENOMEM;
6733b39a 4851}
09a1093a
JSJ		 4852int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4853 unsigned int num_sg, unsigned int xferlen,
4854 unsigned int writedir)
4855{
4856
4857 struct beiscsi_io_task *io_task = task->dd_data;
4858 struct iscsi_conn *conn = task->conn;
4859 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4860 struct beiscsi_hba *phba = beiscsi_conn->phba;
4861 struct iscsi_wrb *pwrb = NULL;
4862 unsigned int doorbell = 0;
4863
4864 pwrb = io_task->pwrb_handle->pwrb;
09a1093a
JSJ		 4865
4866 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4867 io_task->bhs_len = sizeof(struct be_cmd_bhs);
4868
4869 if (writedir) {
4870 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4871 INI_WR_CMD);
4872 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4873 } else {
4874 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4875 INI_RD_CMD);
4876 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4877 }
4878
4879 io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4880 type, pwrb);
4881
4882 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4883 cpu_to_be16(*(unsigned short *)
4884 &io_task->cmd_bhs->iscsi_hdr.lun));
4885 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4886 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4887 io_task->pwrb_handle->wrb_index);
4888 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4889 be32_to_cpu(task->cmdsn));
4890 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4891 io_task->psgl_handle->sgl_index);
4892
4893 hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4894 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4895 io_task->pwrb_handle->nxt_wrb_index);
4896
4897 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4898
4899 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4900 doorbell |= (io_task->pwrb_handle->wrb_index &
4901 DB_DEF_PDU_WRB_INDEX_MASK) <<
4902 DB_DEF_PDU_WRB_INDEX_SHIFT;
4903 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
1e4be6ff
JK		 4904 iowrite32(doorbell, phba->db_va +
4905 beiscsi_conn->doorbell_offset);
09a1093a
JSJ		 4906 return 0;
4907}
6733b39a 4908
6733b39a
JK		 4909static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4910 unsigned int num_sg, unsigned int xferlen,
4911 unsigned int writedir)
4912{
4913
4914 struct beiscsi_io_task *io_task = task->dd_data;
4915 struct iscsi_conn *conn = task->conn;
4916 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4917 struct beiscsi_hba *phba = beiscsi_conn->phba;
4918 struct iscsi_wrb *pwrb = NULL;
4919 unsigned int doorbell = 0;
4920
4921 pwrb = io_task->pwrb_handle->pwrb;
6733b39a
JK		 4922 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4923 io_task->bhs_len = sizeof(struct be_cmd_bhs);
4924
4925 if (writedir) {
32951dd8
JK		 4926 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4927 INI_WR_CMD);
6733b39a 4928 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
6733b39a 4929 } else {
32951dd8
JK		 4930 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4931 INI_RD_CMD);
6733b39a
JK		 4932 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4933 }
6733b39a 4934
09a1093a
JSJ		 4935 io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4936 type, pwrb);
4937
6733b39a 4938 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
dc63aac6
JK		 4939 cpu_to_be16(*(unsigned short *)
4940 &io_task->cmd_bhs->iscsi_hdr.lun));
6733b39a
JK		 4941 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4942 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4943 io_task->pwrb_handle->wrb_index);
4944 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4945 be32_to_cpu(task->cmdsn));
4946 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4947 io_task->psgl_handle->sgl_index);
4948
4949 hwi_write_sgl(pwrb, sg, num_sg, io_task);
4950
4951 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4952 io_task->pwrb_handle->nxt_wrb_index);
4953 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4954
4955 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 4956 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 4957 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4958 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4959
1e4be6ff
JK		 4960 iowrite32(doorbell, phba->db_va +
4961 beiscsi_conn->doorbell_offset);
6733b39a
JK		 4962 return 0;
4963}
4964
4965static int beiscsi_mtask(struct iscsi_task *task)
4966{
dafab8e0 4967 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 4968 struct iscsi_conn *conn = task->conn;
4969 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4970 struct beiscsi_hba *phba = beiscsi_conn->phba;
4971 struct iscsi_wrb *pwrb = NULL;
4972 unsigned int doorbell = 0;
dafab8e0 4973 unsigned int cid;
09a1093a 4974 unsigned int pwrb_typeoffset = 0;
6733b39a 4975
bfead3b2 4976 cid = beiscsi_conn->beiscsi_conn_cid;
6733b39a 4977 pwrb = io_task->pwrb_handle->pwrb;
caf818f1 4978 memset(pwrb, 0, sizeof(*pwrb));
09a1093a 4979
2c9dfd36 4980 if (is_chip_be2_be3r(phba)) {
09a1093a
JSJ		 4981 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4982 be32_to_cpu(task->cmdsn));
4983 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4984 io_task->pwrb_handle->wrb_index);
4985 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4986 io_task->psgl_handle->sgl_index);
4987 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4988 task->data_count);
4989 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4990 io_task->pwrb_handle->nxt_wrb_index);
4991 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
2c9dfd36
JK		 4992 } else {
4993 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4994 be32_to_cpu(task->cmdsn));
4995 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4996 io_task->pwrb_handle->wrb_index);
4997 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4998 io_task->psgl_handle->sgl_index);
4999 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
5000 task->data_count);
5001 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
5002 io_task->pwrb_handle->nxt_wrb_index);
5003 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
09a1093a
JSJ		 5004 }
5005
dafab8e0 5006
6733b39a
JK		 5007 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
5008 case ISCSI_OP_LOGIN:
6733b39a 5009 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
09a1093a 5010 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
6733b39a
JK		 5011 hwi_write_buffer(pwrb, task);
5012 break;
5013 case ISCSI_OP_NOOP_OUT:
1390b01b 5014 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
09a1093a 5015 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
2c9dfd36
JK		 5016 if (is_chip_be2_be3r(phba))
5017 AMAP_SET_BITS(struct amap_iscsi_wrb,
09a1093a
JSJ		 5018 dmsg, pwrb, 1);
5019 else
2c9dfd36 5020 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
09a1093a 5021 dmsg, pwrb, 1);
1390b01b 5022 } else {
09a1093a 5023 ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
2c9dfd36
JK		 5024 if (is_chip_be2_be3r(phba))
5025 AMAP_SET_BITS(struct amap_iscsi_wrb,
09a1093a
JSJ		 5026 dmsg, pwrb, 0);
5027 else
2c9dfd36 5028 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
09a1093a 5029 dmsg, pwrb, 0);
1390b01b 5030 }
6733b39a
JK		 5031 hwi_write_buffer(pwrb, task);
5032 break;
5033 case ISCSI_OP_TEXT:
09a1093a 5034 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
6733b39a
JK		 5035 hwi_write_buffer(pwrb, task);
5036 break;
5037 case ISCSI_OP_SCSI_TMFUNC:
09a1093a 5038 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
6733b39a
JK		 5039 hwi_write_buffer(pwrb, task);
5040 break;
5041 case ISCSI_OP_LOGOUT:
09a1093a 5042 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
6733b39a
JK		 5043 hwi_write_buffer(pwrb, task);
5044 break;
5045
5046 default:
99bc5d55
JSJ		 5047 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5048 "BM_%d : opcode =%d Not supported\n",
5049 task->hdr->opcode & ISCSI_OPCODE_MASK);
5050
6733b39a
JK		 5051 return -EINVAL;
5052 }
5053
09a1093a 5054 /* Set the task type */
2c9dfd36
JK		 5055 io_task->wrb_type = (is_chip_be2_be3r(phba)) ?
5056 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) :
5057 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb);
6733b39a 5058
bfead3b2 5059 doorbell |= cid & DB_WRB_POST_CID_MASK;
32951dd8 5060 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 5061 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
5062 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
1e4be6ff
JK		 5063 iowrite32(doorbell, phba->db_va +
5064 beiscsi_conn->doorbell_offset);
6733b39a
JK		 5065 return 0;
5066}
5067
5068static int beiscsi_task_xmit(struct iscsi_task *task)
5069{
6733b39a
JK		 5070 struct beiscsi_io_task *io_task = task->dd_data;
5071 struct scsi_cmnd *sc = task->sc;
09a1093a 5072 struct beiscsi_hba *phba = NULL;
6733b39a
JK		 5073 struct scatterlist *sg;
5074 int num_sg;
5075 unsigned int writedir = 0, xferlen = 0;
5076
09a1093a
JSJ		 5077 phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba;
5078
6733b39a
JK		 5079 if (!sc)
5080 return beiscsi_mtask(task);
5081
5082 io_task->scsi_cmnd = sc;
5083 num_sg = scsi_dma_map(sc);
5084 if (num_sg < 0) {
99bc5d55
JSJ		 5085 struct iscsi_conn *conn = task->conn;
5086 struct beiscsi_hba *phba = NULL;
5087
5088 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
afb96058
JK		 5089 beiscsi_log(phba, KERN_ERR,
5090 BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI,
5091 "BM_%d : scsi_dma_map Failed "
5092 "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n",
5093 be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt),
5094 io_task->libiscsi_itt, scsi_bufflen(sc));
99bc5d55 5095
6733b39a
JK		 5096 return num_sg;
5097 }
6733b39a
JK		 5098 xferlen = scsi_bufflen(sc);
5099 sg = scsi_sglist(sc);
99bc5d55 5100 if (sc->sc_data_direction == DMA_TO_DEVICE)
6733b39a 5101 writedir = 1;
99bc5d55 5102 else
6733b39a 5103 writedir = 0;
99bc5d55 5104
09a1093a 5105 return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
6733b39a
JK		 5106}
5107
ffce3e2e
JK		 5108/**
5109 * beiscsi_bsg_request - handle bsg request from ISCSI transport
5110 * @job: job to handle
5111 */
5112static int beiscsi_bsg_request(struct bsg_job *job)
5113{
5114 struct Scsi_Host *shost;
5115 struct beiscsi_hba *phba;
5116 struct iscsi_bsg_request *bsg_req = job->request;
5117 int rc = -EINVAL;
5118 unsigned int tag;
5119 struct be_dma_mem nonemb_cmd;
5120 struct be_cmd_resp_hdr *resp;
5121 struct iscsi_bsg_reply *bsg_reply = job->reply;
5122 unsigned short status, extd_status;
5123
5124 shost = iscsi_job_to_shost(job);
5125 phba = iscsi_host_priv(shost);
5126
5127 switch (bsg_req->msgcode) {
5128 case ISCSI_BSG_HST_VENDOR:
5129 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
5130 job->request_payload.payload_len,
5131 &nonemb_cmd.dma);
5132 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 5133 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5134 "BM_%d : Failed to allocate memory for "
5135 "beiscsi_bsg_request\n");
8359c79b 5136 return -ENOMEM;
ffce3e2e
JK		 5137 }
5138 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
5139 &nonemb_cmd);
5140 if (!tag) {
99bc5d55 5141 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
8359c79b 5142 "BM_%d : MBX Tag Allocation Failed\n");
99bc5d55 5143
ffce3e2e
JK		 5144 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
5145 nonemb_cmd.va, nonemb_cmd.dma);
5146 return -EAGAIN;
e175defe
JSJ		 5147 }
5148
5149 rc = wait_event_interruptible_timeout(
5150 phba->ctrl.mcc_wait[tag],
5151 phba->ctrl.mcc_numtag[tag],
5152 msecs_to_jiffies(
5153 BEISCSI_HOST_MBX_TIMEOUT));
ffce3e2e
JK		 5154 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
5155 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
5156 free_mcc_tag(&phba->ctrl, tag);
5157 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
5158 sg_copy_from_buffer(job->reply_payload.sg_list,
5159 job->reply_payload.sg_cnt,
5160 nonemb_cmd.va, (resp->response_length
5161 + sizeof(*resp)));
5162 bsg_reply->reply_payload_rcv_len = resp->response_length;
5163 bsg_reply->result = status;
5164 bsg_job_done(job, bsg_reply->result,
5165 bsg_reply->reply_payload_rcv_len);
5166 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
5167 nonemb_cmd.va, nonemb_cmd.dma);
5168 if (status || extd_status) {
99bc5d55 5169 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
8359c79b 5170 "BM_%d : MBX Cmd Failed"
99bc5d55
JSJ		 5171 " status = %d extd_status = %d\n",
5172 status, extd_status);
5173
ffce3e2e 5174 return -EIO;
8359c79b
JSJ		 5175 } else {
5176 rc = 0;
ffce3e2e
JK		 5177 }
5178 break;
5179
5180 default:
99bc5d55
JSJ		 5181 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5182 "BM_%d : Unsupported bsg command: 0x%x\n",
5183 bsg_req->msgcode);
ffce3e2e
JK		 5184 break;
5185 }
5186
5187 return rc;
5188}
5189
99bc5d55
JSJ		 5190void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
5191{
5192 /* Set the logging parameter */
5193 beiscsi_log_enable_init(phba, beiscsi_log_enable);
5194}
5195
4d4d1ef8
JSJ		 5196/*
5197 * beiscsi_quiesce()- Cleanup Driver resources
5198 * @phba: Instance Priv structure
3567f36a 5199 * @unload_state:i Clean or EEH unload state
4d4d1ef8
JSJ		 5200 *
5201 * Free the OS and HW resources held by the driver
5202 **/
3567f36a
JK		 5203static void beiscsi_quiesce(struct beiscsi_hba *phba,
5204 uint32_t unload_state)
6733b39a 5205{
bfead3b2
JK		 5206 struct hwi_controller *phwi_ctrlr;
5207 struct hwi_context_memory *phwi_context;
5208 struct be_eq_obj *pbe_eq;
5209 unsigned int i, msix_vec;
6733b39a 5210
bfead3b2
JK		 5211 phwi_ctrlr = phba->phwi_ctrlr;
5212 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 5213 hwi_disable_intr(phba);
bfead3b2
JK		 5214 if (phba->msix_enabled) {
5215 for (i = 0; i <= phba->num_cpus; i++) {
5216 msix_vec = phba->msix_entries[i].vector;
3567f36a 5217 synchronize_irq(msix_vec);
bfead3b2 5218 free_irq(msix_vec, &phwi_context->be_eq[i]);
8fcfb210 5219 kfree(phba->msi_name[i]);
bfead3b2
JK		 5220 }
5221 } else
3567f36a
JK		 5222 if (phba->pcidev->irq) {
5223 synchronize_irq(phba->pcidev->irq);
bfead3b2 5224 free_irq(phba->pcidev->irq, phba);
3567f36a 5225 }
bfead3b2 5226 pci_disable_msix(phba->pcidev);
3567f36a 5227
6733b39a 5228 if (blk_iopoll_enabled)
bfead3b2
JK		 5229 for (i = 0; i < phba->num_cpus; i++) {
5230 pbe_eq = &phwi_context->be_eq[i];
5231 blk_iopoll_disable(&pbe_eq->iopoll);
5232 }
6733b39a 5233
3567f36a
JK		 5234 if (unload_state == BEISCSI_CLEAN_UNLOAD) {
5235 destroy_workqueue(phba->wq);
5236 beiscsi_clean_port(phba);
5237 beiscsi_free_mem(phba);
e9b91193 5238
3567f36a
JK		 5239 beiscsi_unmap_pci_function(phba);
5240 pci_free_consistent(phba->pcidev,
5241 phba->ctrl.mbox_mem_alloced.size,
5242 phba->ctrl.mbox_mem_alloced.va,
5243 phba->ctrl.mbox_mem_alloced.dma);
5244 } else {
5245 hwi_purge_eq(phba);
5246 hwi_cleanup(phba);
5247 }
7a158003
JSJ		 5248
5249 cancel_delayed_work_sync(&phba->beiscsi_hw_check_task);
25602c97
JK		 5250}
5251
5252static void beiscsi_remove(struct pci_dev *pcidev)
5253{
5254
5255 struct beiscsi_hba *phba = NULL;
5256
5257 phba = pci_get_drvdata(pcidev);
5258 if (!phba) {
5259 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
5260 return;
5261 }
5262
0e43895e 5263 beiscsi_destroy_def_ifaces(phba);
3567f36a 5264 beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
9d045163 5265 iscsi_boot_destroy_kset(phba->boot_kset);
6733b39a
JK		 5266 iscsi_host_remove(phba->shost);
5267 pci_dev_put(phba->pcidev);
5268 iscsi_host_free(phba->shost);
3567f36a
JK		 5269 pci_disable_pcie_error_reporting(pcidev);
5270 pci_set_drvdata(pcidev, NULL);
8dce69ff 5271 pci_disable_device(pcidev);
6733b39a
JK		 5272}
5273
25602c97
JK		 5274static void beiscsi_shutdown(struct pci_dev *pcidev)
5275{
5276
5277 struct beiscsi_hba *phba = NULL;
5278
5279 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
5280 if (!phba) {
5281 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
5282 return;
5283 }
5284
3567f36a 5285 beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
8dce69ff 5286 pci_disable_device(pcidev);
25602c97
JK		 5287}
5288
bfead3b2
JK		 5289static void beiscsi_msix_enable(struct beiscsi_hba *phba)
5290{
5291 int i, status;
5292
5293 for (i = 0; i <= phba->num_cpus; i++)
5294 phba->msix_entries[i].entry = i;
5295
5296 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
5297 (phba->num_cpus + 1));
5298 if (!status)
5299 phba->msix_enabled = true;
5300
5301 return;
5302}
5303
7a158003
JSJ		 5304/*
5305 * beiscsi_hw_health_check()- Check adapter health
5306 * @work: work item to check HW health
5307 *
5308 * Check if adapter in an unrecoverable state or not.
5309 **/
5310static void
5311beiscsi_hw_health_check(struct work_struct *work)
5312{
5313 struct beiscsi_hba *phba =
5314 container_of(work, struct beiscsi_hba,
5315 beiscsi_hw_check_task.work);
5316
5317 beiscsi_ue_detect(phba);
5318
5319 schedule_delayed_work(&phba->beiscsi_hw_check_task,
5320 msecs_to_jiffies(1000));
5321}
5322
3567f36a
JK		 5323
5324static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
5325 pci_channel_state_t state)
5326{
5327 struct beiscsi_hba *phba = NULL;
5328
5329 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5330 phba->state |= BE_ADAPTER_PCI_ERR;
5331
5332 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5333 "BM_%d : EEH error detected\n");
5334
5335 beiscsi_quiesce(phba, BEISCSI_EEH_UNLOAD);
5336
5337 if (state == pci_channel_io_perm_failure) {
5338 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5339 "BM_%d : EEH : State PERM Failure");
5340 return PCI_ERS_RESULT_DISCONNECT;
5341 }
5342
5343 pci_disable_device(pdev);
5344
5345 /* The error could cause the FW to trigger a flash debug dump.
5346 * Resetting the card while flash dump is in progress
5347 * can cause it not to recover; wait for it to finish.
5348 * Wait only for first function as it is needed only once per
5349 * adapter.
5350 **/
5351 if (pdev->devfn == 0)
5352 ssleep(30);
5353
5354 return PCI_ERS_RESULT_NEED_RESET;
5355}
5356
5357static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
5358{
5359 struct beiscsi_hba *phba = NULL;
5360 int status = 0;
5361
5362 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5363
5364 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5365 "BM_%d : EEH Reset\n");
5366
5367 status = pci_enable_device(pdev);
5368 if (status)
5369 return PCI_ERS_RESULT_DISCONNECT;
5370
5371 pci_set_master(pdev);
5372 pci_set_power_state(pdev, PCI_D0);
5373 pci_restore_state(pdev);
5374
5375 /* Wait for the CHIP Reset to complete */
5376 status = be_chk_reset_complete(phba);
5377 if (!status) {
5378 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5379 "BM_%d : EEH Reset Completed\n");
5380 } else {
5381 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5382 "BM_%d : EEH Reset Completion Failure\n");
5383 return PCI_ERS_RESULT_DISCONNECT;
5384 }
5385
5386 pci_cleanup_aer_uncorrect_error_status(pdev);
5387 return PCI_ERS_RESULT_RECOVERED;
5388}
5389
5390static void beiscsi_eeh_resume(struct pci_dev *pdev)
5391{
5392 int ret = 0, i;
5393 struct be_eq_obj *pbe_eq;
5394 struct beiscsi_hba *phba = NULL;
5395 struct hwi_controller *phwi_ctrlr;
5396 struct hwi_context_memory *phwi_context;
5397
5398 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5399 pci_save_state(pdev);
5400
5401 if (enable_msix)
5402 find_num_cpus(phba);
5403 else
5404 phba->num_cpus = 1;
5405
5406 if (enable_msix) {
5407 beiscsi_msix_enable(phba);
5408 if (!phba->msix_enabled)
5409 phba->num_cpus = 1;
5410 }
5411
5412 ret = beiscsi_cmd_reset_function(phba);
5413 if (ret) {
5414 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5415 "BM_%d : Reset Failed\n");
5416 goto ret_err;
5417 }
5418
5419 ret = be_chk_reset_complete(phba);
5420 if (ret) {
5421 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5422 "BM_%d : Failed to get out of reset.\n");
5423 goto ret_err;
5424 }
5425
5426 beiscsi_get_params(phba);
5427 phba->shost->max_id = phba->params.cxns_per_ctrl;
5428 phba->shost->can_queue = phba->params.ios_per_ctrl;
5429 ret = hwi_init_controller(phba);
5430
5431 for (i = 0; i < MAX_MCC_CMD; i++) {
5432 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5433 phba->ctrl.mcc_tag[i] = i + 1;
5434 phba->ctrl.mcc_numtag[i + 1] = 0;
5435 phba->ctrl.mcc_tag_available++;
5436 }
5437
5438 phwi_ctrlr = phba->phwi_ctrlr;
5439 phwi_context = phwi_ctrlr->phwi_ctxt;
5440
5441 if (blk_iopoll_enabled) {
5442 for (i = 0; i < phba->num_cpus; i++) {
5443 pbe_eq = &phwi_context->be_eq[i];
5444 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5445 be_iopoll);
5446 blk_iopoll_enable(&pbe_eq->iopoll);
5447 }
5448
5449 i = (phba->msix_enabled) ? i : 0;
5450 /* Work item for MCC handling */
5451 pbe_eq = &phwi_context->be_eq[i];
5452 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5453 } else {
5454 if (phba->msix_enabled) {
5455 for (i = 0; i <= phba->num_cpus; i++) {
5456 pbe_eq = &phwi_context->be_eq[i];
5457 INIT_WORK(&pbe_eq->work_cqs,
5458 beiscsi_process_all_cqs);
5459 }
5460 } else {
5461 pbe_eq = &phwi_context->be_eq[0];
5462 INIT_WORK(&pbe_eq->work_cqs,
5463 beiscsi_process_all_cqs);
5464 }
5465 }
5466
5467 ret = beiscsi_init_irqs(phba);
5468 if (ret < 0) {
5469 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5470 "BM_%d : beiscsi_eeh_resume - "
5471 "Failed to beiscsi_init_irqs\n");
5472 goto ret_err;
5473 }
5474
5475 hwi_enable_intr(phba);
5476 phba->state &= ~BE_ADAPTER_PCI_ERR;
5477
5478 return;
5479ret_err:
5480 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5481 "BM_%d : AER EEH Resume Failed\n");
5482}
5483
6f039790
GKH		 5484static int beiscsi_dev_probe(struct pci_dev *pcidev,
5485 const struct pci_device_id *id)
6733b39a
JK		 5486{
5487 struct beiscsi_hba *phba = NULL;
bfead3b2
JK		 5488 struct hwi_controller *phwi_ctrlr;
5489 struct hwi_context_memory *phwi_context;
5490 struct be_eq_obj *pbe_eq;
3567f36a 5491 int ret = 0, i;
6733b39a
JK		 5492
5493 ret = beiscsi_enable_pci(pcidev);
5494 if (ret < 0) {
99bc5d55
JSJ		 5495 dev_err(&pcidev->dev,
5496 "beiscsi_dev_probe - Failed to enable pci device\n");
6733b39a
JK		 5497 return ret;
5498 }
5499
5500 phba = beiscsi_hba_alloc(pcidev);
5501 if (!phba) {
99bc5d55
JSJ		 5502 dev_err(&pcidev->dev,
5503 "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
6733b39a
JK		 5504 goto disable_pci;
5505 }
5506
3567f36a
JK		 5507 /* Enable EEH reporting */
5508 ret = pci_enable_pcie_error_reporting(pcidev);
5509 if (ret)
5510 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5511 "BM_%d : PCIe Error Reporting "
5512 "Enabling Failed\n");
5513
5514 pci_save_state(pcidev);
5515
99bc5d55
JSJ		 5516 /* Initialize Driver configuration Paramters */
5517 beiscsi_hba_attrs_init(phba);
5518
e175defe 5519 phba->fw_timeout = false;
6c83185a 5520 phba->mac_addr_set = false;
e175defe
JSJ		 5521
5522
f98c96b0
JK		 5523 switch (pcidev->device) {
5524 case BE_DEVICE_ID1:
5525 case OC_DEVICE_ID1:
5526 case OC_DEVICE_ID2:
5527 phba->generation = BE_GEN2;
09a1093a 5528 phba->iotask_fn = beiscsi_iotask;
f98c96b0
JK		 5529 break;
5530 case BE_DEVICE_ID2:
5531 case OC_DEVICE_ID3:
5532 phba->generation = BE_GEN3;
09a1093a 5533 phba->iotask_fn = beiscsi_iotask;
f98c96b0 5534 break;
139a1b1e
JSJ		 5535 case OC_SKH_ID1:
5536 phba->generation = BE_GEN4;
09a1093a 5537 phba->iotask_fn = beiscsi_iotask_v2;
bf9131cb 5538 break;
f98c96b0
JK		 5539 default:
5540 phba->generation = 0;
5541 }
5542
6733b39a
JK		 5543 ret = be_ctrl_init(phba, pcidev);
5544 if (ret) {
99bc5d55
JSJ		 5545 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5546 "BM_%d : beiscsi_dev_probe-"
5547 "Failed in be_ctrl_init\n");
6733b39a
JK		 5548 goto hba_free;
5549 }
5550
4d4d1ef8
JSJ		 5551 ret = beiscsi_cmd_reset_function(phba);
5552 if (ret) {
5553 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
92665a66 5554 "BM_%d : Reset Failed\n");
4d4d1ef8
JSJ		 5555 goto hba_free;
5556 }
5557 ret = be_chk_reset_complete(phba);
5558 if (ret) {
5559 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
92665a66 5560 "BM_%d : Failed to get out of reset.\n");
4d4d1ef8 5561 goto hba_free;
e9b91193
JK		 5562 }
5563
6733b39a
JK		 5564 spin_lock_init(&phba->io_sgl_lock);
5565 spin_lock_init(&phba->mgmt_sgl_lock);
5566 spin_lock_init(&phba->isr_lock);
8f09a3b9 5567 spin_lock_init(&phba->async_pdu_lock);
7da50879
JK		 5568 ret = mgmt_get_fw_config(&phba->ctrl, phba);
5569 if (ret != 0) {
99bc5d55
JSJ		 5570 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5571 "BM_%d : Error getting fw config\n");
7da50879
JK		 5572 goto free_port;
5573 }
68c26a3a
JK		 5574
5575 if (enable_msix)
5576 find_num_cpus(phba);
5577 else
5578 phba->num_cpus = 1;
5579
5580 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5581 "BM_%d : num_cpus = %d\n",
5582 phba->num_cpus);
5583
5584 if (enable_msix) {
5585 beiscsi_msix_enable(phba);
5586 if (!phba->msix_enabled)
5587 phba->num_cpus = 1;
5588 }
5589
843ae752 5590 phba->shost->max_id = phba->params.cxns_per_ctrl;
6733b39a 5591 beiscsi_get_params(phba);
aa874f07 5592 phba->shost->can_queue = phba->params.ios_per_ctrl;
6733b39a
JK		 5593 ret = beiscsi_init_port(phba);
5594 if (ret < 0) {
99bc5d55
JSJ		 5595 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5596 "BM_%d : beiscsi_dev_probe-"
5597 "Failed in beiscsi_init_port\n");
6733b39a
JK		 5598 goto free_port;
5599 }
5600
3567f36a 5601 for (i = 0; i < MAX_MCC_CMD; i++) {
756d29c8
JK		 5602 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5603 phba->ctrl.mcc_tag[i] = i + 1;
5604 phba->ctrl.mcc_numtag[i + 1] = 0;
5605 phba->ctrl.mcc_tag_available++;
1957aa7f
JK		 5606 memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0,
5607 sizeof(struct beiscsi_mcc_tag_state));
756d29c8
JK		 5608 }
5609
5610 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
5611
72fb46a9 5612 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
6733b39a 5613 phba->shost->host_no);
d8537548 5614 phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, phba->wq_name);
6733b39a 5615 if (!phba->wq) {
99bc5d55
JSJ		 5616 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5617 "BM_%d : beiscsi_dev_probe-"
5618 "Failed to allocate work queue\n");
6733b39a
JK		 5619 goto free_twq;
5620 }
5621
7a158003
JSJ		 5622 INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task,
5623 beiscsi_hw_health_check);
6733b39a 5624
bfead3b2
JK		 5625 phwi_ctrlr = phba->phwi_ctrlr;
5626 phwi_context = phwi_ctrlr->phwi_ctxt;
72fb46a9 5627
6733b39a 5628 if (blk_iopoll_enabled) {
bfead3b2
JK		 5629 for (i = 0; i < phba->num_cpus; i++) {
5630 pbe_eq = &phwi_context->be_eq[i];
5631 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5632 be_iopoll);
5633 blk_iopoll_enable(&pbe_eq->iopoll);
5634 }
72fb46a9
JSJ		 5635
5636 i = (phba->msix_enabled) ? i : 0;
5637 /* Work item for MCC handling */
5638 pbe_eq = &phwi_context->be_eq[i];
5639 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5640 } else {
5641 if (phba->msix_enabled) {
5642 for (i = 0; i <= phba->num_cpus; i++) {
5643 pbe_eq = &phwi_context->be_eq[i];
5644 INIT_WORK(&pbe_eq->work_cqs,
5645 beiscsi_process_all_cqs);
5646 }
5647 } else {
5648 pbe_eq = &phwi_context->be_eq[0];
5649 INIT_WORK(&pbe_eq->work_cqs,
5650 beiscsi_process_all_cqs);
5651 }
6733b39a 5652 }
72fb46a9 5653
6733b39a
JK		 5654 ret = beiscsi_init_irqs(phba);
5655 if (ret < 0) {
99bc5d55
JSJ		 5656 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5657 "BM_%d : beiscsi_dev_probe-"
5658 "Failed to beiscsi_init_irqs\n");
6733b39a
JK		 5659 goto free_blkenbld;
5660 }
238f6b72 5661 hwi_enable_intr(phba);
f457a46f
MC		 5662
5663 if (beiscsi_setup_boot_info(phba))
5664 /*
5665 * log error but continue, because we may not be using
5666 * iscsi boot.
5667 */
99bc5d55
JSJ		 5668 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5669 "BM_%d : Could not set up "
5670 "iSCSI boot info.\n");
f457a46f 5671
0e43895e 5672 beiscsi_create_def_ifaces(phba);
7a158003
JSJ		 5673 schedule_delayed_work(&phba->beiscsi_hw_check_task,
5674 msecs_to_jiffies(1000));
5675
99bc5d55
JSJ		 5676 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5677 "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
6733b39a
JK		 5678 return 0;
5679
6733b39a
JK		 5680free_blkenbld:
5681 destroy_workqueue(phba->wq);
5682 if (blk_iopoll_enabled)
bfead3b2
JK		 5683 for (i = 0; i < phba->num_cpus; i++) {
5684 pbe_eq = &phwi_context->be_eq[i];
5685 blk_iopoll_disable(&pbe_eq->iopoll);
5686 }
6733b39a
JK		 5687free_twq:
5688 beiscsi_clean_port(phba);
5689 beiscsi_free_mem(phba);
5690free_port:
5691 pci_free_consistent(phba->pcidev,
5692 phba->ctrl.mbox_mem_alloced.size,
5693 phba->ctrl.mbox_mem_alloced.va,
5694 phba->ctrl.mbox_mem_alloced.dma);
5695 beiscsi_unmap_pci_function(phba);
5696hba_free:
238f6b72
JK		 5697 if (phba->msix_enabled)
5698 pci_disable_msix(phba->pcidev);
6733b39a
JK		 5699 iscsi_host_remove(phba->shost);
5700 pci_dev_put(phba->pcidev);
5701 iscsi_host_free(phba->shost);
5702disable_pci:
5703 pci_disable_device(pcidev);
5704 return ret;
5705}
5706
3567f36a
JK		 5707static struct pci_error_handlers beiscsi_eeh_handlers = {
5708 .error_detected = beiscsi_eeh_err_detected,
5709 .slot_reset = beiscsi_eeh_reset,
5710 .resume = beiscsi_eeh_resume,
5711};
5712
6733b39a
JK		 5713struct iscsi_transport beiscsi_iscsi_transport = {
5714 .owner = THIS_MODULE,
5715 .name = DRV_NAME,
9db0fb3a 5716 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
6733b39a 5717 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
6733b39a
JK		 5718 .create_session = beiscsi_session_create,
5719 .destroy_session = beiscsi_session_destroy,
5720 .create_conn = beiscsi_conn_create,
5721 .bind_conn = beiscsi_conn_bind,
5722 .destroy_conn = iscsi_conn_teardown,
3128c6c7 5723 .attr_is_visible = be2iscsi_attr_is_visible,
0e43895e
MC		 5724 .set_iface_param = be2iscsi_iface_set_param,
5725 .get_iface_param = be2iscsi_iface_get_param,
6733b39a 5726 .set_param = beiscsi_set_param,
c7f7fd5b 5727 .get_conn_param = iscsi_conn_get_param,
6733b39a
JK		 5728 .get_session_param = iscsi_session_get_param,
5729 .get_host_param = beiscsi_get_host_param,
5730 .start_conn = beiscsi_conn_start,
fa95d206 5731 .stop_conn = iscsi_conn_stop,
6733b39a
JK		 5732 .send_pdu = iscsi_conn_send_pdu,
5733 .xmit_task = beiscsi_task_xmit,
5734 .cleanup_task = beiscsi_cleanup_task,
5735 .alloc_pdu = beiscsi_alloc_pdu,
5736 .parse_pdu_itt = beiscsi_parse_pdu,
5737 .get_stats = beiscsi_conn_get_stats,
c7f7fd5b 5738 .get_ep_param = beiscsi_ep_get_param,
6733b39a
JK		 5739 .ep_connect = beiscsi_ep_connect,
5740 .ep_poll = beiscsi_ep_poll,
5741 .ep_disconnect = beiscsi_ep_disconnect,
5742 .session_recovery_timedout = iscsi_session_recovery_timedout,
ffce3e2e 5743 .bsg_request = beiscsi_bsg_request,
6733b39a
JK		 5744};
5745
5746static struct pci_driver beiscsi_pci_driver = {
5747 .name = DRV_NAME,
5748 .probe = beiscsi_dev_probe,
5749 .remove = beiscsi_remove,
25602c97 5750 .shutdown = beiscsi_shutdown,
3567f36a
JK		 5751 .id_table = beiscsi_pci_id_table,
5752 .err_handler = &beiscsi_eeh_handlers
6733b39a
JK		 5753};
5754
bfead3b2 5755
6733b39a
JK		 5756static int __init beiscsi_module_init(void)
5757{
5758 int ret;
5759
5760 beiscsi_scsi_transport =
5761 iscsi_register_transport(&beiscsi_iscsi_transport);
5762 if (!beiscsi_scsi_transport) {
99bc5d55
JSJ		 5763 printk(KERN_ERR
5764 "beiscsi_module_init - Unable to register beiscsi transport.\n");
f55a24f2 5765 return -ENOMEM;
6733b39a 5766 }
99bc5d55
JSJ		 5767 printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5768 &beiscsi_iscsi_transport);
6733b39a
JK		 5769
5770 ret = pci_register_driver(&beiscsi_pci_driver);
5771 if (ret) {
99bc5d55
JSJ		 5772 printk(KERN_ERR
5773 "beiscsi_module_init - Unable to register beiscsi pci driver.\n");
6733b39a
JK		 5774 goto unregister_iscsi_transport;
5775 }
5776 return 0;
5777
5778unregister_iscsi_transport:
5779 iscsi_unregister_transport(&beiscsi_iscsi_transport);
5780 return ret;
5781}
5782
5783static void __exit beiscsi_module_exit(void)
5784{
5785 pci_unregister_driver(&beiscsi_pci_driver);
5786 iscsi_unregister_transport(&beiscsi_iscsi_transport);
5787}
5788
5789module_init(beiscsi_module_init);
5790module_exit(beiscsi_module_exit);
Linux 6.x block layer and io_uring tree(s)