projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] gdth: Convert to use regular kernel types.
[linux-2.6-block.git] / drivers / scsi / be2iscsi / be_main.c
CommitLineData
6733b39a
JK		 1/**
2 * Copyright (C) 2005 - 2009 ServerEngines
3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
10 * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
11 *
12 * Contact Information:
13 * linux-drivers@serverengines.com
14 *
15 * ServerEngines
16 * 209 N. Fair Oaks Ave
17 * Sunnyvale, CA 94085
18 *
19 */
20#include <linux/reboot.h>
21#include <linux/delay.h>
22#include <linux/interrupt.h>
23#include <linux/blkdev.h>
24#include <linux/pci.h>
25#include <linux/string.h>
26#include <linux/kernel.h>
27#include <linux/semaphore.h>
28
29#include <scsi/libiscsi.h>
30#include <scsi/scsi_transport_iscsi.h>
31#include <scsi/scsi_transport.h>
32#include <scsi/scsi_cmnd.h>
33#include <scsi/scsi_device.h>
34#include <scsi/scsi_host.h>
35#include <scsi/scsi.h>
36#include "be_main.h"
37#include "be_iscsi.h"
38#include "be_mgmt.h"
39
40static unsigned int be_iopoll_budget = 10;
41static unsigned int be_max_phys_size = 64;
bfead3b2 42static unsigned int enable_msix = 1;
35e66019 43static unsigned int ring_mode;
6733b39a
JK		 44
45MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
46MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
47MODULE_AUTHOR("ServerEngines Corporation");
48MODULE_LICENSE("GPL");
49module_param(be_iopoll_budget, int, 0);
50module_param(enable_msix, int, 0);
51module_param(be_max_phys_size, uint, S_IRUGO);
52MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
53 "contiguous memory that can be allocated."
54 "Range is 16 - 128");
55
56static int beiscsi_slave_configure(struct scsi_device *sdev)
57{
58 blk_queue_max_segment_size(sdev->request_queue, 65536);
59 return 0;
60}
61
bfead3b2
JK		 62/*------------------- PCI Driver operations and data ----------------- */
63static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
64 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
65 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
66 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
67 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
68 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID4) },
69 { 0 }
70};
71MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
72
6733b39a
JK		 73static struct scsi_host_template beiscsi_sht = {
74 .module = THIS_MODULE,
75 .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
76 .proc_name = DRV_NAME,
77 .queuecommand = iscsi_queuecommand,
78 .eh_abort_handler = iscsi_eh_abort,
79 .change_queue_depth = iscsi_change_queue_depth,
80 .slave_configure = beiscsi_slave_configure,
81 .target_alloc = iscsi_target_alloc,
82 .eh_device_reset_handler = iscsi_eh_device_reset,
83 .eh_target_reset_handler = iscsi_eh_target_reset,
84 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
85 .can_queue = BE2_IO_DEPTH,
86 .this_id = -1,
87 .max_sectors = BEISCSI_MAX_SECTORS,
88 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
89 .use_clustering = ENABLE_CLUSTERING,
90};
6733b39a 91
bfead3b2 92static struct scsi_transport_template *beiscsi_scsi_transport;
6733b39a
JK		 93
94static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
95{
96 struct beiscsi_hba *phba;
97 struct Scsi_Host *shost;
98
99 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
100 if (!shost) {
101 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
102 "iscsi_host_alloc failed \n");
103 return NULL;
104 }
105 shost->dma_boundary = pcidev->dma_mask;
106 shost->max_id = BE2_MAX_SESSIONS;
107 shost->max_channel = 0;
108 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
109 shost->max_lun = BEISCSI_NUM_MAX_LUN;
110 shost->transportt = beiscsi_scsi_transport;
6733b39a
JK		 111 phba = iscsi_host_priv(shost);
112 memset(phba, 0, sizeof(*phba));
113 phba->shost = shost;
114 phba->pcidev = pci_dev_get(pcidev);
115
116 if (iscsi_host_add(shost, &phba->pcidev->dev))
117 goto free_devices;
118 return phba;
119
120free_devices:
121 pci_dev_put(phba->pcidev);
122 iscsi_host_free(phba->shost);
123 return NULL;
124}
125
126static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
127{
128 if (phba->csr_va) {
129 iounmap(phba->csr_va);
130 phba->csr_va = NULL;
131 }
132 if (phba->db_va) {
133 iounmap(phba->db_va);
134 phba->db_va = NULL;
135 }
136 if (phba->pci_va) {
137 iounmap(phba->pci_va);
138 phba->pci_va = NULL;
139 }
140}
141
142static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
143 struct pci_dev *pcidev)
144{
145 u8 __iomem *addr;
146
147 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
148 pci_resource_len(pcidev, 2));
149 if (addr == NULL)
150 return -ENOMEM;
151 phba->ctrl.csr = addr;
152 phba->csr_va = addr;
153 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
154
155 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
156 if (addr == NULL)
157 goto pci_map_err;
158 phba->ctrl.db = addr;
159 phba->db_va = addr;
160 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
161
162 addr = ioremap_nocache(pci_resource_start(pcidev, 1),
163 pci_resource_len(pcidev, 1));
164 if (addr == NULL)
165 goto pci_map_err;
166 phba->ctrl.pcicfg = addr;
167 phba->pci_va = addr;
168 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, 1);
169 return 0;
170
171pci_map_err:
172 beiscsi_unmap_pci_function(phba);
173 return -ENOMEM;
174}
175
176static int beiscsi_enable_pci(struct pci_dev *pcidev)
177{
178 int ret;
179
180 ret = pci_enable_device(pcidev);
181 if (ret) {
182 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
183 "failed. Returning -ENODEV\n");
184 return ret;
185 }
186
bfead3b2 187 pci_set_master(pcidev);
6733b39a
JK		 188 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
189 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
190 if (ret) {
191 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
192 pci_disable_device(pcidev);
193 return ret;
194 }
195 }
196 return 0;
197}
198
199static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
200{
201 struct be_ctrl_info *ctrl = &phba->ctrl;
202 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
203 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
204 int status = 0;
205
206 ctrl->pdev = pdev;
207 status = beiscsi_map_pci_bars(phba, pdev);
208 if (status)
209 return status;
6733b39a
JK		 210 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
211 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
212 mbox_mem_alloc->size,
213 &mbox_mem_alloc->dma);
214 if (!mbox_mem_alloc->va) {
215 beiscsi_unmap_pci_function(phba);
216 status = -ENOMEM;
217 return status;
218 }
219
220 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
221 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
222 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
223 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
224 spin_lock_init(&ctrl->mbox_lock);
bfead3b2
JK		 225 spin_lock_init(&phba->ctrl.mcc_lock);
226 spin_lock_init(&phba->ctrl.mcc_cq_lock);
227
6733b39a
JK		 228 return status;
229}
230
231static void beiscsi_get_params(struct beiscsi_hba *phba)
232{
233 phba->params.ios_per_ctrl = BE2_IO_DEPTH;
234 phba->params.cxns_per_ctrl = BE2_MAX_SESSIONS;
235 phba->params.asyncpdus_per_ctrl = BE2_ASYNCPDUS;
236 phba->params.icds_per_ctrl = BE2_MAX_ICDS / 2;
237 phba->params.num_sge_per_io = BE2_SGE;
238 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
239 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
240 phba->params.eq_timer = 64;
241 phba->params.num_eq_entries =
242 (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
243 512) + 1) * 512;
244 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
245 ? 1024 : phba->params.num_eq_entries;
246 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d \n",
247 phba->params.num_eq_entries);
248 phba->params.num_cq_entries =
249 (((BE2_CMDS_PER_CXN * 2 + BE2_LOGOUTS + BE2_TMFS + BE2_ASYNCPDUS) /
250 512) + 1) * 512;
251 SE_DEBUG(DBG_LVL_8,
252 "phba->params.num_cq_entries=%d BE2_CMDS_PER_CXN=%d"
253 "BE2_LOGOUTS=%d BE2_TMFS=%d BE2_ASYNCPDUS=%d \n",
254 phba->params.num_cq_entries, BE2_CMDS_PER_CXN,
255 BE2_LOGOUTS, BE2_TMFS, BE2_ASYNCPDUS);
256 phba->params.wrbs_per_cxn = 256;
257}
258
259static void hwi_ring_eq_db(struct beiscsi_hba *phba,
260 unsigned int id, unsigned int clr_interrupt,
261 unsigned int num_processed,
262 unsigned char rearm, unsigned char event)
263{
264 u32 val = 0;
265 val |= id & DB_EQ_RING_ID_MASK;
266 if (rearm)
267 val |= 1 << DB_EQ_REARM_SHIFT;
268 if (clr_interrupt)
269 val |= 1 << DB_EQ_CLR_SHIFT;
270 if (event)
271 val |= 1 << DB_EQ_EVNT_SHIFT;
272 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
273 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
274}
275
bfead3b2
JK		 276/**
277 * be_isr_mcc - The isr routine of the driver.
278 * @irq: Not used
279 * @dev_id: Pointer to host adapter structure
280 */
281static irqreturn_t be_isr_mcc(int irq, void *dev_id)
282{
283 struct beiscsi_hba *phba;
284 struct be_eq_entry *eqe = NULL;
285 struct be_queue_info *eq;
286 struct be_queue_info *mcc;
287 unsigned int num_eq_processed;
288 struct be_eq_obj *pbe_eq;
289 unsigned long flags;
290
291 pbe_eq = dev_id;
292 eq = &pbe_eq->q;
293 phba = pbe_eq->phba;
294 mcc = &phba->ctrl.mcc_obj.cq;
295 eqe = queue_tail_node(eq);
296 if (!eqe)
297 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
298
299 num_eq_processed = 0;
300
301 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
302 & EQE_VALID_MASK) {
303 if (((eqe->dw[offsetof(struct amap_eq_entry,
304 resource_id) / 32] &
305 EQE_RESID_MASK) >> 16) == mcc->id) {
306 spin_lock_irqsave(&phba->isr_lock, flags);
307 phba->todo_mcc_cq = 1;
308 spin_unlock_irqrestore(&phba->isr_lock, flags);
309 }
310 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
311 queue_tail_inc(eq);
312 eqe = queue_tail_node(eq);
313 num_eq_processed++;
314 }
315 if (phba->todo_mcc_cq)
316 queue_work(phba->wq, &phba->work_cqs);
317 if (num_eq_processed)
318 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
319
320 return IRQ_HANDLED;
321}
322
323/**
324 * be_isr_msix - The isr routine of the driver.
325 * @irq: Not used
326 * @dev_id: Pointer to host adapter structure
327 */
328static irqreturn_t be_isr_msix(int irq, void *dev_id)
329{
330 struct beiscsi_hba *phba;
331 struct be_eq_entry *eqe = NULL;
332 struct be_queue_info *eq;
333 struct be_queue_info *cq;
334 unsigned int num_eq_processed;
335 struct be_eq_obj *pbe_eq;
336 unsigned long flags;
337
338 pbe_eq = dev_id;
339 eq = &pbe_eq->q;
340 cq = pbe_eq->cq;
341 eqe = queue_tail_node(eq);
342 if (!eqe)
343 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
344
345 phba = pbe_eq->phba;
346 num_eq_processed = 0;
347 if (blk_iopoll_enabled) {
348 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
349 & EQE_VALID_MASK) {
350 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
351 blk_iopoll_sched(&pbe_eq->iopoll);
352
353 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
354 queue_tail_inc(eq);
355 eqe = queue_tail_node(eq);
356 num_eq_processed++;
357 }
358 if (num_eq_processed)
359 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
360
361 return IRQ_HANDLED;
362 } else {
363 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
364 & EQE_VALID_MASK) {
365 spin_lock_irqsave(&phba->isr_lock, flags);
366 phba->todo_cq = 1;
367 spin_unlock_irqrestore(&phba->isr_lock, flags);
368 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
369 queue_tail_inc(eq);
370 eqe = queue_tail_node(eq);
371 num_eq_processed++;
372 }
373 if (phba->todo_cq)
374 queue_work(phba->wq, &phba->work_cqs);
375
376 if (num_eq_processed)
377 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
378
379 return IRQ_HANDLED;
380 }
381}
382
6733b39a
JK		 383/**
384 * be_isr - The isr routine of the driver.
385 * @irq: Not used
386 * @dev_id: Pointer to host adapter structure
387 */
388static irqreturn_t be_isr(int irq, void *dev_id)
389{
390 struct beiscsi_hba *phba;
391 struct hwi_controller *phwi_ctrlr;
392 struct hwi_context_memory *phwi_context;
393 struct be_eq_entry *eqe = NULL;
394 struct be_queue_info *eq;
395 struct be_queue_info *cq;
bfead3b2 396 struct be_queue_info *mcc;
6733b39a 397 unsigned long flags, index;
bfead3b2 398 unsigned int num_mcceq_processed, num_ioeq_processed;
6733b39a 399 struct be_ctrl_info *ctrl;
bfead3b2 400 struct be_eq_obj *pbe_eq;
6733b39a
JK		 401 int isr;
402
403 phba = dev_id;
bfead3b2
JK		 404 ctrl = &phba->ctrl;;
405 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
406 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
407 if (!isr)
408 return IRQ_NONE;
6733b39a
JK		 409
410 phwi_ctrlr = phba->phwi_ctrlr;
411 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 412 pbe_eq = &phwi_context->be_eq[0];
413
414 eq = &phwi_context->be_eq[0].q;
415 mcc = &phba->ctrl.mcc_obj.cq;
6733b39a
JK		 416 index = 0;
417 eqe = queue_tail_node(eq);
418 if (!eqe)
419 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
420
bfead3b2
JK		 421 num_ioeq_processed = 0;
422 num_mcceq_processed = 0;
6733b39a
JK		 423 if (blk_iopoll_enabled) {
424 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
425 & EQE_VALID_MASK) {
bfead3b2
JK		 426 if (((eqe->dw[offsetof(struct amap_eq_entry,
427 resource_id) / 32] &
428 EQE_RESID_MASK) >> 16) == mcc->id) {
429 spin_lock_irqsave(&phba->isr_lock, flags);
430 phba->todo_mcc_cq = 1;
431 spin_unlock_irqrestore(&phba->isr_lock, flags);
432 num_mcceq_processed++;
433 } else {
434 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
435 blk_iopoll_sched(&pbe_eq->iopoll);
436 num_ioeq_processed++;
437 }
6733b39a
JK		 438 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
439 queue_tail_inc(eq);
440 eqe = queue_tail_node(eq);
6733b39a 441 }
bfead3b2
JK		 442 if (num_ioeq_processed || num_mcceq_processed) {
443 if (phba->todo_mcc_cq)
444 queue_work(phba->wq, &phba->work_cqs);
445
446 if ((num_mcceq_processed) && (!num_ioeq_processed))
447 hwi_ring_eq_db(phba, eq->id, 0,
448 (num_ioeq_processed +
449 num_mcceq_processed) , 1, 1);
450 else
451 hwi_ring_eq_db(phba, eq->id, 0,
452 (num_ioeq_processed +
453 num_mcceq_processed), 0, 1);
454
6733b39a
JK		 455 return IRQ_HANDLED;
456 } else
457 return IRQ_NONE;
458 } else {
bfead3b2 459 cq = &phwi_context->be_cq[0];
6733b39a
JK		 460 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
461 & EQE_VALID_MASK) {
462
463 if (((eqe->dw[offsetof(struct amap_eq_entry,
464 resource_id) / 32] &
465 EQE_RESID_MASK) >> 16) != cq->id) {
466 spin_lock_irqsave(&phba->isr_lock, flags);
467 phba->todo_mcc_cq = 1;
468 spin_unlock_irqrestore(&phba->isr_lock, flags);
469 } else {
470 spin_lock_irqsave(&phba->isr_lock, flags);
471 phba->todo_cq = 1;
472 spin_unlock_irqrestore(&phba->isr_lock, flags);
473 }
474 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
475 queue_tail_inc(eq);
476 eqe = queue_tail_node(eq);
bfead3b2 477 num_ioeq_processed++;
6733b39a
JK		 478 }
479 if (phba->todo_cq || phba->todo_mcc_cq)
480 queue_work(phba->wq, &phba->work_cqs);
481
bfead3b2
JK		 482 if (num_ioeq_processed) {
483 hwi_ring_eq_db(phba, eq->id, 0,
484 num_ioeq_processed, 1, 1);
6733b39a
JK		 485 return IRQ_HANDLED;
486 } else
487 return IRQ_NONE;
488 }
489}
490
491static int beiscsi_init_irqs(struct beiscsi_hba *phba)
492{
493 struct pci_dev *pcidev = phba->pcidev;
bfead3b2
JK		 494 struct hwi_controller *phwi_ctrlr;
495 struct hwi_context_memory *phwi_context;
496 int ret, msix_vec, i = 0;
497 char desc[32];
6733b39a 498
bfead3b2
JK		 499 phwi_ctrlr = phba->phwi_ctrlr;
500 phwi_context = phwi_ctrlr->phwi_ctxt;
501
502 if (phba->msix_enabled) {
503 for (i = 0; i < phba->num_cpus; i++) {
504 sprintf(desc, "beiscsi_msix_%04x", i);
505 msix_vec = phba->msix_entries[i].vector;
506 ret = request_irq(msix_vec, be_isr_msix, 0, desc,
507 &phwi_context->be_eq[i]);
508 }
509 msix_vec = phba->msix_entries[i].vector;
510 ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc",
511 &phwi_context->be_eq[i]);
512 } else {
513 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
514 "beiscsi", phba);
515 if (ret) {
516 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
517 "Failed to register irq\\n");
518 return ret;
519 }
6733b39a
JK		 520 }
521 return 0;
522}
523
524static void hwi_ring_cq_db(struct beiscsi_hba *phba,
525 unsigned int id, unsigned int num_processed,
526 unsigned char rearm, unsigned char event)
527{
528 u32 val = 0;
529 val |= id & DB_CQ_RING_ID_MASK;
530 if (rearm)
531 val |= 1 << DB_CQ_REARM_SHIFT;
532 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
533 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
534}
535
6733b39a
JK		 536static unsigned int
537beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
538 struct beiscsi_hba *phba,
539 unsigned short cid,
540 struct pdu_base *ppdu,
541 unsigned long pdu_len,
542 void *pbuffer, unsigned long buf_len)
543{
544 struct iscsi_conn *conn = beiscsi_conn->conn;
545 struct iscsi_session *session = conn->session;
bfead3b2
JK		 546 struct iscsi_task *task;
547 struct beiscsi_io_task *io_task;
548 struct iscsi_hdr *login_hdr;
6733b39a
JK		 549
550 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
551 PDUBASE_OPCODE_MASK) {
552 case ISCSI_OP_NOOP_IN:
553 pbuffer = NULL;
554 buf_len = 0;
555 break;
556 case ISCSI_OP_ASYNC_EVENT:
557 break;
558 case ISCSI_OP_REJECT:
559 WARN_ON(!pbuffer);
560 WARN_ON(!(buf_len == 48));
561 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
562 break;
563 case ISCSI_OP_LOGIN_RSP:
bfead3b2
JK		 564 task = conn->login_task;
565 io_task = task->dd_data;
566 login_hdr = (struct iscsi_hdr *)ppdu;
567 login_hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 568 break;
569 default:
570 shost_printk(KERN_WARNING, phba->shost,
571 "Unrecognized opcode 0x%x in async msg \n",
572 (ppdu->
573 dw[offsetof(struct amap_pdu_base, opcode) / 32]
574 & PDUBASE_OPCODE_MASK));
575 return 1;
576 }
577
578 spin_lock_bh(&session->lock);
579 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
580 spin_unlock_bh(&session->lock);
581 return 0;
582}
583
584static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
585{
586 struct sgl_handle *psgl_handle;
587
588 if (phba->io_sgl_hndl_avbl) {
589 SE_DEBUG(DBG_LVL_8,
590 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d \n",
591 phba->io_sgl_alloc_index);
592 psgl_handle = phba->io_sgl_hndl_base[phba->
593 io_sgl_alloc_index];
594 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
595 phba->io_sgl_hndl_avbl--;
bfead3b2
JK		 596 if (phba->io_sgl_alloc_index == (phba->params.
597 ios_per_ctrl - 1))
6733b39a
JK		 598 phba->io_sgl_alloc_index = 0;
599 else
600 phba->io_sgl_alloc_index++;
601 } else
602 psgl_handle = NULL;
603 return psgl_handle;
604}
605
606static void
607free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
608{
609 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d \n",
610 phba->io_sgl_free_index);
611 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
612 /*
613 * this can happen if clean_task is called on a task that
614 * failed in xmit_task or alloc_pdu.
615 */
616 SE_DEBUG(DBG_LVL_8,
617 "Double Free in IO SGL io_sgl_free_index=%d,"
618 "value there=%p \n", phba->io_sgl_free_index,
619 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
620 return;
621 }
622 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
623 phba->io_sgl_hndl_avbl++;
624 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
625 phba->io_sgl_free_index = 0;
626 else
627 phba->io_sgl_free_index++;
628}
629
630/**
631 * alloc_wrb_handle - To allocate a wrb handle
632 * @phba: The hba pointer
633 * @cid: The cid to use for allocation
634 * @index: index allocation and wrb index
635 *
636 * This happens under session_lock until submission to chip
637 */
638struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
639 int index)
640{
641 struct hwi_wrb_context *pwrb_context;
642 struct hwi_controller *phwi_ctrlr;
643 struct wrb_handle *pwrb_handle;
644
645 phwi_ctrlr = phba->phwi_ctrlr;
646 pwrb_context = &phwi_ctrlr->wrb_context[cid];
bfead3b2
JK		 647 if (pwrb_context->wrb_handles_available) {
648 pwrb_handle = pwrb_context->pwrb_handle_base[
649 pwrb_context->alloc_index];
650 pwrb_context->wrb_handles_available--;
651 pwrb_handle->nxt_wrb_index = pwrb_handle->wrb_index;
652 if (pwrb_context->alloc_index ==
653 (phba->params.wrbs_per_cxn - 1))
654 pwrb_context->alloc_index = 0;
655 else
656 pwrb_context->alloc_index++;
657 } else
658 pwrb_handle = NULL;
6733b39a
JK		 659 return pwrb_handle;
660}
661
662/**
663 * free_wrb_handle - To free the wrb handle back to pool
664 * @phba: The hba pointer
665 * @pwrb_context: The context to free from
666 * @pwrb_handle: The wrb_handle to free
667 *
668 * This happens under session_lock until submission to chip
669 */
670static void
671free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
672 struct wrb_handle *pwrb_handle)
673{
35e66019
JK		 674 if (!ring_mode)
675 pwrb_context->pwrb_handle_base[pwrb_context->free_index] =
676 pwrb_handle;
bfead3b2
JK		 677 pwrb_context->wrb_handles_available++;
678 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
679 pwrb_context->free_index = 0;
680 else
681 pwrb_context->free_index++;
682
6733b39a 683 SE_DEBUG(DBG_LVL_8,
bfead3b2 684 "FREE WRB: pwrb_handle=%p free_index=0x%x"
6733b39a
JK		 685 "wrb_handles_available=%d \n",
686 pwrb_handle, pwrb_context->free_index,
bfead3b2 687 pwrb_context->wrb_handles_available);
6733b39a
JK		 688}
689
690static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
691{
692 struct sgl_handle *psgl_handle;
693
694 if (phba->eh_sgl_hndl_avbl) {
695 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
696 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
697 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x \n",
698 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
699 phba->eh_sgl_hndl_avbl--;
700 if (phba->eh_sgl_alloc_index ==
701 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
702 1))
703 phba->eh_sgl_alloc_index = 0;
704 else
705 phba->eh_sgl_alloc_index++;
706 } else
707 psgl_handle = NULL;
708 return psgl_handle;
709}
710
711void
712free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
713{
714
bfead3b2
JK		 715 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d \n",
716 phba->eh_sgl_free_index);
6733b39a
JK		 717 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
718 /*
719 * this can happen if clean_task is called on a task that
720 * failed in xmit_task or alloc_pdu.
721 */
722 SE_DEBUG(DBG_LVL_8,
723 "Double Free in eh SGL ,eh_sgl_free_index=%d \n",
724 phba->eh_sgl_free_index);
725 return;
726 }
727 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
728 phba->eh_sgl_hndl_avbl++;
729 if (phba->eh_sgl_free_index ==
730 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
731 phba->eh_sgl_free_index = 0;
732 else
733 phba->eh_sgl_free_index++;
734}
735
736static void
737be_complete_io(struct beiscsi_conn *beiscsi_conn,
738 struct iscsi_task *task, struct sol_cqe *psol)
739{
740 struct beiscsi_io_task *io_task = task->dd_data;
741 struct be_status_bhs *sts_bhs =
742 (struct be_status_bhs *)io_task->cmd_bhs;
743 struct iscsi_conn *conn = beiscsi_conn->conn;
744 unsigned int sense_len;
745 unsigned char *sense;
746 u32 resid = 0, exp_cmdsn, max_cmdsn;
747 u8 rsp, status, flags;
748
bfead3b2 749 exp_cmdsn = (psol->
6733b39a
JK		 750 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
751 & SOL_EXP_CMD_SN_MASK);
bfead3b2 752 max_cmdsn = ((psol->
6733b39a
JK		 753 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
754 & SOL_EXP_CMD_SN_MASK) +
755 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
756 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
757 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
758 & SOL_RESP_MASK) >> 16);
759 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
760 & SOL_STS_MASK) >> 8);
761 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
762 & SOL_FLAGS_MASK) >> 24) | 0x80;
763
764 task->sc->result = (DID_OK << 16) | status;
765 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
766 task->sc->result = DID_ERROR << 16;
767 goto unmap;
768 }
769
770 /* bidi not initially supported */
771 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
772 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
773 32] & SOL_RES_CNT_MASK);
774
775 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
776 task->sc->result = DID_ERROR << 16;
777
778 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
779 scsi_set_resid(task->sc, resid);
780 if (!status && (scsi_bufflen(task->sc) - resid <
781 task->sc->underflow))
782 task->sc->result = DID_ERROR << 16;
783 }
784 }
785
786 if (status == SAM_STAT_CHECK_CONDITION) {
bfead3b2 787 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
6733b39a 788 sense = sts_bhs->sense_info + sizeof(unsigned short);
bfead3b2 789 sense_len = cpu_to_be16(*slen);
6733b39a
JK		 790 memcpy(task->sc->sense_buffer, sense,
791 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
792 }
793 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
794 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
795 & SOL_RES_CNT_MASK)
796 conn->rxdata_octets += (psol->
bfead3b2
JK		 797 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
798 & SOL_RES_CNT_MASK);
6733b39a
JK		 799 }
800unmap:
801 scsi_dma_unmap(io_task->scsi_cmnd);
802 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
803}
804
805static void
806be_complete_logout(struct beiscsi_conn *beiscsi_conn,
807 struct iscsi_task *task, struct sol_cqe *psol)
808{
809 struct iscsi_logout_rsp *hdr;
bfead3b2 810 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 811 struct iscsi_conn *conn = beiscsi_conn->conn;
812
813 hdr = (struct iscsi_logout_rsp *)task->hdr;
814 hdr->t2wait = 5;
815 hdr->t2retain = 0;
816 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
817 & SOL_FLAGS_MASK) >> 24) | 0x80;
818 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
819 32] & SOL_RESP_MASK);
820 hdr->exp_cmdsn = cpu_to_be32(psol->
821 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
822 & SOL_EXP_CMD_SN_MASK);
823 hdr->max_cmdsn = be32_to_cpu((psol->
824 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
825 & SOL_EXP_CMD_SN_MASK) +
826 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
827 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
828 hdr->hlength = 0;
bfead3b2 829 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 830 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
831}
832
833static void
834be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
835 struct iscsi_task *task, struct sol_cqe *psol)
836{
837 struct iscsi_tm_rsp *hdr;
838 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 839 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 840
841 hdr = (struct iscsi_tm_rsp *)task->hdr;
842 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
843 & SOL_FLAGS_MASK) >> 24) | 0x80;
844 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
845 32] & SOL_RESP_MASK);
846 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
bfead3b2 847 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
6733b39a
JK		 848 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
849 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
850 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
851 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
bfead3b2 852 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 853 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
854}
855
856static void
857hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
858 struct beiscsi_hba *phba, struct sol_cqe *psol)
859{
860 struct hwi_wrb_context *pwrb_context;
bfead3b2 861 struct wrb_handle *pwrb_handle = NULL;
35e66019 862 struct sgl_handle *psgl_handle = NULL;
6733b39a 863 struct hwi_controller *phwi_ctrlr;
bfead3b2
JK		 864 struct iscsi_task *task;
865 struct beiscsi_io_task *io_task;
6733b39a
JK		 866 struct iscsi_conn *conn = beiscsi_conn->conn;
867 struct iscsi_session *session = conn->session;
868
869 phwi_ctrlr = phba->phwi_ctrlr;
35e66019
JK		 870 if (ring_mode) {
871 psgl_handle = phba->sgl_hndl_array[((psol->
872 dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
873 32] & SOL_ICD_INDEX_MASK) >> 6)];
874 pwrb_context = &phwi_ctrlr->wrb_context[psgl_handle->cid];
875 task = psgl_handle->task;
876 pwrb_handle = NULL;
877 } else {
878 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
879 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
880 SOL_CID_MASK) >> 6)];
881 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
882 dw[offsetof(struct amap_sol_cqe, wrb_index) /
883 32] & SOL_WRB_INDEX_MASK) >> 16)];
884 task = pwrb_handle->pio_handle;
885 }
886
bfead3b2
JK		 887 io_task = task->dd_data;
888 spin_lock(&phba->mgmt_sgl_lock);
889 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
890 spin_unlock(&phba->mgmt_sgl_lock);
6733b39a
JK		 891 spin_lock_bh(&session->lock);
892 free_wrb_handle(phba, pwrb_context, pwrb_handle);
893 spin_unlock_bh(&session->lock);
894}
895
896static void
897be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
898 struct iscsi_task *task, struct sol_cqe *psol)
899{
900 struct iscsi_nopin *hdr;
901 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 902 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 903
904 hdr = (struct iscsi_nopin *)task->hdr;
905 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
906 & SOL_FLAGS_MASK) >> 24) | 0x80;
907 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
908 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
909 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
910 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
911 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
912 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
913 hdr->opcode = ISCSI_OP_NOOP_IN;
bfead3b2 914 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 915 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
916}
917
918static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
919 struct beiscsi_hba *phba, struct sol_cqe *psol)
920{
921 struct hwi_wrb_context *pwrb_context;
922 struct wrb_handle *pwrb_handle;
923 struct iscsi_wrb *pwrb = NULL;
924 struct hwi_controller *phwi_ctrlr;
925 struct iscsi_task *task;
35e66019 926 struct sgl_handle *psgl_handle = NULL;
bfead3b2 927 unsigned int type;
6733b39a
JK		 928 struct iscsi_conn *conn = beiscsi_conn->conn;
929 struct iscsi_session *session = conn->session;
930
931 phwi_ctrlr = phba->phwi_ctrlr;
35e66019
JK		 932 if (ring_mode) {
933 psgl_handle = phba->sgl_hndl_array[((psol->
934 dw[offsetof(struct amap_sol_cqe_ring, icd_index) /
935 32] & SOL_ICD_INDEX_MASK) >> 6)];
936 task = psgl_handle->task;
937 type = psgl_handle->type;
938 } else {
939 pwrb_context = &phwi_ctrlr->
940 wrb_context[((psol->dw[offsetof
941 (struct amap_sol_cqe, cid) / 32]
942 & SOL_CID_MASK) >> 6)];
943 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
944 dw[offsetof(struct amap_sol_cqe, wrb_index) /
945 32] & SOL_WRB_INDEX_MASK) >> 16)];
946 task = pwrb_handle->pio_handle;
947 pwrb = pwrb_handle->pwrb;
948 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
bfead3b2 949 WRB_TYPE_MASK) >> 28;
35e66019 950 }
bfead3b2
JK		 951 spin_lock_bh(&session->lock);
952 switch (type) {
6733b39a
JK		 953 case HWH_TYPE_IO:
954 case HWH_TYPE_IO_RD:
955 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
956 ISCSI_OP_NOOP_OUT) {
957 be_complete_nopin_resp(beiscsi_conn, task, psol);
958 } else
959 be_complete_io(beiscsi_conn, task, psol);
960 break;
961
962 case HWH_TYPE_LOGOUT:
963 be_complete_logout(beiscsi_conn, task, psol);
964 break;
965
966 case HWH_TYPE_LOGIN:
967 SE_DEBUG(DBG_LVL_1,
968 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
969 "- Solicited path \n");
970 break;
971
972 case HWH_TYPE_TMF:
973 be_complete_tmf(beiscsi_conn, task, psol);
974 break;
975
976 case HWH_TYPE_NOP:
977 be_complete_nopin_resp(beiscsi_conn, task, psol);
978 break;
979
980 default:
35e66019
JK		 981 if (ring_mode)
982 shost_printk(KERN_WARNING, phba->shost,
983 "In hwi_complete_cmd, unknown type = %d"
984 "icd_index 0x%x CID 0x%x\n", type,
985 ((psol->dw[offsetof(struct amap_sol_cqe_ring,
986 icd_index) / 32] & SOL_ICD_INDEX_MASK) >> 6),
987 psgl_handle->cid);
988 else
989 shost_printk(KERN_WARNING, phba->shost,
990 "In hwi_complete_cmd, unknown type = %d"
991 "wrb_index 0x%x CID 0x%x\n", type,
992 ((psol->dw[offsetof(struct amap_iscsi_wrb,
993 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
994 ((psol->dw[offsetof(struct amap_sol_cqe,
995 cid) / 32] & SOL_CID_MASK) >> 6));
6733b39a
JK		 996 break;
997 }
35e66019 998
6733b39a
JK		 999 spin_unlock_bh(&session->lock);
1000}
1001
1002static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1003 *pasync_ctx, unsigned int is_header,
1004 unsigned int host_write_ptr)
1005{
1006 if (is_header)
1007 return &pasync_ctx->async_entry[host_write_ptr].
1008 header_busy_list;
1009 else
1010 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1011}
1012
1013static struct async_pdu_handle *
1014hwi_get_async_handle(struct beiscsi_hba *phba,
1015 struct beiscsi_conn *beiscsi_conn,
1016 struct hwi_async_pdu_context *pasync_ctx,
1017 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1018{
1019 struct be_bus_address phys_addr;
1020 struct list_head *pbusy_list;
1021 struct async_pdu_handle *pasync_handle = NULL;
1022 int buffer_len = 0;
1023 unsigned char buffer_index = -1;
1024 unsigned char is_header = 0;
1025
1026 phys_addr.u.a32.address_lo =
1027 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1028 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1029 & PDUCQE_DPL_MASK) >> 16);
1030 phys_addr.u.a32.address_hi =
1031 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1032
1033 phys_addr.u.a64.address =
1034 *((unsigned long long *)(&phys_addr.u.a64.address));
1035
1036 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1037 & PDUCQE_CODE_MASK) {
1038 case UNSOL_HDR_NOTIFY:
1039 is_header = 1;
1040
1041 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1042 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1043 index) / 32] & PDUCQE_INDEX_MASK));
1044
1045 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1046 pasync_ctx->async_header.pa_base.u.a64.address);
1047
1048 buffer_index = buffer_len /
1049 pasync_ctx->async_header.buffer_size;
1050
1051 break;
1052 case UNSOL_DATA_NOTIFY:
1053 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1054 dw[offsetof(struct amap_i_t_dpdu_cqe,
1055 index) / 32] & PDUCQE_INDEX_MASK));
1056 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1057 pasync_ctx->async_data.pa_base.u.
1058 a64.address);
1059 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1060 break;
1061 default:
1062 pbusy_list = NULL;
1063 shost_printk(KERN_WARNING, phba->shost,
1064 "Unexpected code=%d \n",
1065 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1066 code) / 32] & PDUCQE_CODE_MASK);
1067 return NULL;
1068 }
1069
1070 WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1071 WARN_ON(list_empty(pbusy_list));
1072 list_for_each_entry(pasync_handle, pbusy_list, link) {
1073 WARN_ON(pasync_handle->consumed);
1074 if (pasync_handle->index == buffer_index)
1075 break;
1076 }
1077
1078 WARN_ON(!pasync_handle);
1079
1080 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid;
1081 pasync_handle->is_header = is_header;
1082 pasync_handle->buffer_len = ((pdpdu_cqe->
1083 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1084 & PDUCQE_DPL_MASK) >> 16);
1085
1086 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1087 index) / 32] & PDUCQE_INDEX_MASK);
1088 return pasync_handle;
1089}
1090
1091static unsigned int
1092hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1093 unsigned int is_header, unsigned int cq_index)
1094{
1095 struct list_head *pbusy_list;
1096 struct async_pdu_handle *pasync_handle;
1097 unsigned int num_entries, writables = 0;
1098 unsigned int *pep_read_ptr, *pwritables;
1099
1100
1101 if (is_header) {
1102 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1103 pwritables = &pasync_ctx->async_header.writables;
1104 num_entries = pasync_ctx->async_header.num_entries;
1105 } else {
1106 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1107 pwritables = &pasync_ctx->async_data.writables;
1108 num_entries = pasync_ctx->async_data.num_entries;
1109 }
1110
1111 while ((*pep_read_ptr) != cq_index) {
1112 (*pep_read_ptr)++;
1113 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1114
1115 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1116 *pep_read_ptr);
1117 if (writables == 0)
1118 WARN_ON(list_empty(pbusy_list));
1119
1120 if (!list_empty(pbusy_list)) {
1121 pasync_handle = list_entry(pbusy_list->next,
1122 struct async_pdu_handle,
1123 link);
1124 WARN_ON(!pasync_handle);
1125 pasync_handle->consumed = 1;
1126 }
1127
1128 writables++;
1129 }
1130
1131 if (!writables) {
1132 SE_DEBUG(DBG_LVL_1,
1133 "Duplicate notification received - index 0x%x!!\n",
1134 cq_index);
1135 WARN_ON(1);
1136 }
1137
1138 *pwritables = *pwritables + writables;
1139 return 0;
1140}
1141
1142static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1143 unsigned int cri)
1144{
1145 struct hwi_controller *phwi_ctrlr;
1146 struct hwi_async_pdu_context *pasync_ctx;
1147 struct async_pdu_handle *pasync_handle, *tmp_handle;
1148 struct list_head *plist;
1149 unsigned int i = 0;
1150
1151 phwi_ctrlr = phba->phwi_ctrlr;
1152 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1153
1154 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1155
1156 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1157 list_del(&pasync_handle->link);
1158
1159 if (i == 0) {
1160 list_add_tail(&pasync_handle->link,
1161 &pasync_ctx->async_header.free_list);
1162 pasync_ctx->async_header.free_entries++;
1163 i++;
1164 } else {
1165 list_add_tail(&pasync_handle->link,
1166 &pasync_ctx->async_data.free_list);
1167 pasync_ctx->async_data.free_entries++;
1168 i++;
1169 }
1170 }
1171
1172 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1173 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1174 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1175 return 0;
1176}
1177
1178static struct phys_addr *
1179hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1180 unsigned int is_header, unsigned int host_write_ptr)
1181{
1182 struct phys_addr *pasync_sge = NULL;
1183
1184 if (is_header)
1185 pasync_sge = pasync_ctx->async_header.ring_base;
1186 else
1187 pasync_sge = pasync_ctx->async_data.ring_base;
1188
1189 return pasync_sge + host_write_ptr;
1190}
1191
1192static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1193 unsigned int is_header)
1194{
1195 struct hwi_controller *phwi_ctrlr;
1196 struct hwi_async_pdu_context *pasync_ctx;
1197 struct async_pdu_handle *pasync_handle;
1198 struct list_head *pfree_link, *pbusy_list;
1199 struct phys_addr *pasync_sge;
1200 unsigned int ring_id, num_entries;
1201 unsigned int host_write_num;
1202 unsigned int writables;
1203 unsigned int i = 0;
1204 u32 doorbell = 0;
1205
1206 phwi_ctrlr = phba->phwi_ctrlr;
1207 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1208
1209 if (is_header) {
1210 num_entries = pasync_ctx->async_header.num_entries;
1211 writables = min(pasync_ctx->async_header.writables,
1212 pasync_ctx->async_header.free_entries);
1213 pfree_link = pasync_ctx->async_header.free_list.next;
1214 host_write_num = pasync_ctx->async_header.host_write_ptr;
1215 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1216 } else {
1217 num_entries = pasync_ctx->async_data.num_entries;
1218 writables = min(pasync_ctx->async_data.writables,
1219 pasync_ctx->async_data.free_entries);
1220 pfree_link = pasync_ctx->async_data.free_list.next;
1221 host_write_num = pasync_ctx->async_data.host_write_ptr;
1222 ring_id = phwi_ctrlr->default_pdu_data.id;
1223 }
1224
1225 writables = (writables / 8) * 8;
1226 if (writables) {
1227 for (i = 0; i < writables; i++) {
1228 pbusy_list =
1229 hwi_get_async_busy_list(pasync_ctx, is_header,
1230 host_write_num);
1231 pasync_handle =
1232 list_entry(pfree_link, struct async_pdu_handle,
1233 link);
1234 WARN_ON(!pasync_handle);
1235 pasync_handle->consumed = 0;
1236
1237 pfree_link = pfree_link->next;
1238
1239 pasync_sge = hwi_get_ring_address(pasync_ctx,
1240 is_header, host_write_num);
1241
1242 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1243 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1244
1245 list_move(&pasync_handle->link, pbusy_list);
1246
1247 host_write_num++;
1248 host_write_num = host_write_num % num_entries;
1249 }
1250
1251 if (is_header) {
1252 pasync_ctx->async_header.host_write_ptr =
1253 host_write_num;
1254 pasync_ctx->async_header.free_entries -= writables;
1255 pasync_ctx->async_header.writables -= writables;
1256 pasync_ctx->async_header.busy_entries += writables;
1257 } else {
1258 pasync_ctx->async_data.host_write_ptr = host_write_num;
1259 pasync_ctx->async_data.free_entries -= writables;
1260 pasync_ctx->async_data.writables -= writables;
1261 pasync_ctx->async_data.busy_entries += writables;
1262 }
1263
1264 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1265 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1266 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1267 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1268 << DB_DEF_PDU_CQPROC_SHIFT;
1269
1270 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1271 }
1272}
1273
1274static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1275 struct beiscsi_conn *beiscsi_conn,
1276 struct i_t_dpdu_cqe *pdpdu_cqe)
1277{
1278 struct hwi_controller *phwi_ctrlr;
1279 struct hwi_async_pdu_context *pasync_ctx;
1280 struct async_pdu_handle *pasync_handle = NULL;
1281 unsigned int cq_index = -1;
1282
1283 phwi_ctrlr = phba->phwi_ctrlr;
1284 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1285
1286 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1287 pdpdu_cqe, &cq_index);
1288 BUG_ON(pasync_handle->is_header != 0);
1289 if (pasync_handle->consumed == 0)
1290 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1291 cq_index);
1292
1293 hwi_free_async_msg(phba, pasync_handle->cri);
1294 hwi_post_async_buffers(phba, pasync_handle->is_header);
1295}
1296
1297static unsigned int
1298hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1299 struct beiscsi_hba *phba,
1300 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1301{
1302 struct list_head *plist;
1303 struct async_pdu_handle *pasync_handle;
1304 void *phdr = NULL;
1305 unsigned int hdr_len = 0, buf_len = 0;
1306 unsigned int status, index = 0, offset = 0;
1307 void *pfirst_buffer = NULL;
1308 unsigned int num_buf = 0;
1309
1310 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1311
1312 list_for_each_entry(pasync_handle, plist, link) {
1313 if (index == 0) {
1314 phdr = pasync_handle->pbuffer;
1315 hdr_len = pasync_handle->buffer_len;
1316 } else {
1317 buf_len = pasync_handle->buffer_len;
1318 if (!num_buf) {
1319 pfirst_buffer = pasync_handle->pbuffer;
1320 num_buf++;
1321 }
1322 memcpy(pfirst_buffer + offset,
1323 pasync_handle->pbuffer, buf_len);
1324 offset = buf_len;
1325 }
1326 index++;
1327 }
1328
1329 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1330 beiscsi_conn->beiscsi_conn_cid,
1331 phdr, hdr_len, pfirst_buffer,
1332 buf_len);
1333
1334 if (status == 0)
1335 hwi_free_async_msg(phba, cri);
1336 return 0;
1337}
1338
1339static unsigned int
1340hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1341 struct beiscsi_hba *phba,
1342 struct async_pdu_handle *pasync_handle)
1343{
1344 struct hwi_async_pdu_context *pasync_ctx;
1345 struct hwi_controller *phwi_ctrlr;
1346 unsigned int bytes_needed = 0, status = 0;
1347 unsigned short cri = pasync_handle->cri;
1348 struct pdu_base *ppdu;
1349
1350 phwi_ctrlr = phba->phwi_ctrlr;
1351 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1352
1353 list_del(&pasync_handle->link);
1354 if (pasync_handle->is_header) {
1355 pasync_ctx->async_header.busy_entries--;
1356 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1357 hwi_free_async_msg(phba, cri);
1358 BUG();
1359 }
1360
1361 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1362 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1363 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1364 (unsigned short)pasync_handle->buffer_len;
1365 list_add_tail(&pasync_handle->link,
1366 &pasync_ctx->async_entry[cri].wait_queue.list);
1367
1368 ppdu = pasync_handle->pbuffer;
1369 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1370 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1371 0xFFFF0000) | ((be16_to_cpu((ppdu->
1372 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1373 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1374
1375 if (status == 0) {
1376 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1377 bytes_needed;
1378
1379 if (bytes_needed == 0)
1380 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1381 pasync_ctx, cri);
1382 }
1383 } else {
1384 pasync_ctx->async_data.busy_entries--;
1385 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1386 list_add_tail(&pasync_handle->link,
1387 &pasync_ctx->async_entry[cri].wait_queue.
1388 list);
1389 pasync_ctx->async_entry[cri].wait_queue.
1390 bytes_received +=
1391 (unsigned short)pasync_handle->buffer_len;
1392
1393 if (pasync_ctx->async_entry[cri].wait_queue.
1394 bytes_received >=
1395 pasync_ctx->async_entry[cri].wait_queue.
1396 bytes_needed)
1397 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1398 pasync_ctx, cri);
1399 }
1400 }
1401 return status;
1402}
1403
1404static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1405 struct beiscsi_hba *phba,
1406 struct i_t_dpdu_cqe *pdpdu_cqe)
1407{
1408 struct hwi_controller *phwi_ctrlr;
1409 struct hwi_async_pdu_context *pasync_ctx;
1410 struct async_pdu_handle *pasync_handle = NULL;
1411 unsigned int cq_index = -1;
1412
1413 phwi_ctrlr = phba->phwi_ctrlr;
1414 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1415 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1416 pdpdu_cqe, &cq_index);
1417
1418 if (pasync_handle->consumed == 0)
1419 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1420 cq_index);
1421 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1422 hwi_post_async_buffers(phba, pasync_handle->is_header);
1423}
1424
bfead3b2
JK		 1425
1426static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
6733b39a 1427{
6733b39a
JK		 1428 struct be_queue_info *cq;
1429 struct sol_cqe *sol;
1430 struct dmsg_cqe *dmsg;
1431 unsigned int num_processed = 0;
1432 unsigned int tot_nump = 0;
1433 struct beiscsi_conn *beiscsi_conn;
35e66019 1434 struct sgl_handle *psgl_handle = NULL;
bfead3b2 1435 struct beiscsi_hba *phba;
6733b39a 1436
bfead3b2 1437 cq = pbe_eq->cq;
6733b39a 1438 sol = queue_tail_node(cq);
bfead3b2 1439 phba = pbe_eq->phba;
6733b39a
JK		 1440
1441 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1442 CQE_VALID_MASK) {
1443 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1444
35e66019
JK		 1445 if (ring_mode) {
1446 psgl_handle = phba->sgl_hndl_array[((sol->
1447 dw[offsetof(struct amap_sol_cqe_ring,
1448 icd_index) / 32] & SOL_ICD_INDEX_MASK)
1449 >> 6)];
1450 beiscsi_conn = phba->conn_table[psgl_handle->cid];
1451 if (!beiscsi_conn || !beiscsi_conn->ep) {
1452 shost_printk(KERN_WARNING, phba->shost,
1453 "Connection table empty for cid = %d\n",
1454 psgl_handle->cid);
1455 return 0;
1456 }
1457
1458 } else {
1459 beiscsi_conn = phba->conn_table[(u32) (sol->
6733b39a
JK		 1460 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1461 SOL_CID_MASK) >> 6];
1462
35e66019
JK		 1463 if (!beiscsi_conn || !beiscsi_conn->ep) {
1464 shost_printk(KERN_WARNING, phba->shost,
6733b39a
JK		 1465 "Connection table empty for cid = %d\n",
1466 (u32)(sol->dw[offsetof(struct amap_sol_cqe,
1467 cid) / 32] & SOL_CID_MASK) >> 6);
bfead3b2
JK		 1468 return 0;
1469 }
35e66019 1470 }
6733b39a
JK		 1471
1472 if (num_processed >= 32) {
bfead3b2 1473 hwi_ring_cq_db(phba, cq->id,
6733b39a
JK		 1474 num_processed, 0, 0);
1475 tot_nump += num_processed;
1476 num_processed = 0;
1477 }
1478
1479 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1480 32] & CQE_CODE_MASK) {
1481 case SOL_CMD_COMPLETE:
1482 hwi_complete_cmd(beiscsi_conn, phba, sol);
1483 break;
1484 case DRIVERMSG_NOTIFY:
1485 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY \n");
1486 dmsg = (struct dmsg_cqe *)sol;
1487 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1488 break;
1489 case UNSOL_HDR_NOTIFY:
bfead3b2
JK		 1490 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1491 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1492 (struct i_t_dpdu_cqe *)sol);
1493 break;
6733b39a 1494 case UNSOL_DATA_NOTIFY:
bfead3b2 1495 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
6733b39a
JK		 1496 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1497 (struct i_t_dpdu_cqe *)sol);
1498 break;
1499 case CXN_INVALIDATE_INDEX_NOTIFY:
1500 case CMD_INVALIDATED_NOTIFY:
1501 case CXN_INVALIDATE_NOTIFY:
1502 SE_DEBUG(DBG_LVL_1,
1503 "Ignoring CQ Error notification for cmd/cxn"
1504 "invalidate\n");
1505 break;
1506 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1507 case CMD_KILLED_INVALID_STATSN_RCVD:
1508 case CMD_KILLED_INVALID_R2T_RCVD:
1509 case CMD_CXN_KILLED_LUN_INVALID:
1510 case CMD_CXN_KILLED_ICD_INVALID:
1511 case CMD_CXN_KILLED_ITT_INVALID:
1512 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1513 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
35e66019
JK		 1514 if (ring_mode) {
1515 SE_DEBUG(DBG_LVL_1,
1516 "CQ Error notification for cmd.. "
1517 "code %d cid 0x%x\n",
1518 sol->dw[offsetof(struct amap_sol_cqe, code) /
1519 32] & CQE_CODE_MASK, psgl_handle->cid);
1520 } else {
1521 SE_DEBUG(DBG_LVL_1,
6733b39a
JK		 1522 "CQ Error notification for cmd.. "
1523 "code %d cid 0x%x\n",
1524 sol->dw[offsetof(struct amap_sol_cqe, code) /
1525 32] & CQE_CODE_MASK,
1526 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1527 32] & SOL_CID_MASK));
35e66019 1528 }
6733b39a
JK		 1529 break;
1530 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1531 SE_DEBUG(DBG_LVL_1,
1532 "Digest error on def pdu ring, dropping..\n");
1533 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1534 (struct i_t_dpdu_cqe *) sol);
1535 break;
1536 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1537 case CXN_KILLED_BURST_LEN_MISMATCH:
1538 case CXN_KILLED_AHS_RCVD:
1539 case CXN_KILLED_HDR_DIGEST_ERR:
1540 case CXN_KILLED_UNKNOWN_HDR:
1541 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1542 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1543 case CXN_KILLED_TIMED_OUT:
1544 case CXN_KILLED_FIN_RCVD:
1545 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1546 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1547 case CXN_KILLED_OVER_RUN_RESIDUAL:
1548 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1549 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
35e66019
JK		 1550 if (ring_mode) {
1551 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1552 "0x%x...\n",
1553 sol->dw[offsetof(struct amap_sol_cqe, code) /
1554 32] & CQE_CODE_MASK, psgl_handle->cid);
1555 } else {
1556 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
6733b39a
JK		 1557 "0x%x...\n",
1558 sol->dw[offsetof(struct amap_sol_cqe, code) /
1559 32] & CQE_CODE_MASK,
1560 sol->dw[offsetof(struct amap_sol_cqe, cid) /
1561 32] & CQE_CID_MASK);
35e66019 1562 }
6733b39a
JK		 1563 iscsi_conn_failure(beiscsi_conn->conn,
1564 ISCSI_ERR_CONN_FAILED);
1565 break;
1566 case CXN_KILLED_RST_SENT:
1567 case CXN_KILLED_RST_RCVD:
35e66019
JK		 1568 if (ring_mode) {
1569 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1570 "received/sent on CID 0x%x...\n",
1571 sol->dw[offsetof(struct amap_sol_cqe, code) /
1572 32] & CQE_CODE_MASK, psgl_handle->cid);
1573 } else {
1574 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
bfead3b2 1575 "received/sent on CID 0x%x...\n",
6733b39a
JK		 1576 sol->dw[offsetof(struct amap_sol_cqe, code) /
1577 32] & CQE_CODE_MASK,
1578 sol->dw[offsetof(struct amap_sol_cqe, cid) /
1579 32] & CQE_CID_MASK);
35e66019 1580 }
6733b39a
JK		 1581 iscsi_conn_failure(beiscsi_conn->conn,
1582 ISCSI_ERR_CONN_FAILED);
1583 break;
1584 default:
1585 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1586 "received on CID 0x%x...\n",
1587 sol->dw[offsetof(struct amap_sol_cqe, code) /
1588 32] & CQE_CODE_MASK,
1589 sol->dw[offsetof(struct amap_sol_cqe, cid) /
1590 32] & CQE_CID_MASK);
1591 break;
1592 }
1593
1594 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1595 queue_tail_inc(cq);
1596 sol = queue_tail_node(cq);
1597 num_processed++;
1598 }
1599
1600 if (num_processed > 0) {
1601 tot_nump += num_processed;
bfead3b2 1602 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
6733b39a
JK		 1603 }
1604 return tot_nump;
1605}
1606
1607static void beiscsi_process_all_cqs(struct work_struct *work)
1608{
1609 unsigned long flags;
bfead3b2
JK		 1610 struct hwi_controller *phwi_ctrlr;
1611 struct hwi_context_memory *phwi_context;
1612 struct be_eq_obj *pbe_eq;
6733b39a
JK		 1613 struct beiscsi_hba *phba =
1614 container_of(work, struct beiscsi_hba, work_cqs);
1615
bfead3b2
JK		 1616 phwi_ctrlr = phba->phwi_ctrlr;
1617 phwi_context = phwi_ctrlr->phwi_ctxt;
1618 if (phba->msix_enabled)
1619 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1620 else
1621 pbe_eq = &phwi_context->be_eq[0];
1622
6733b39a
JK		 1623 if (phba->todo_mcc_cq) {
1624 spin_lock_irqsave(&phba->isr_lock, flags);
1625 phba->todo_mcc_cq = 0;
1626 spin_unlock_irqrestore(&phba->isr_lock, flags);
6733b39a
JK		 1627 }
1628
1629 if (phba->todo_cq) {
1630 spin_lock_irqsave(&phba->isr_lock, flags);
1631 phba->todo_cq = 0;
1632 spin_unlock_irqrestore(&phba->isr_lock, flags);
bfead3b2 1633 beiscsi_process_cq(pbe_eq);
6733b39a
JK		 1634 }
1635}
1636
1637static int be_iopoll(struct blk_iopoll *iop, int budget)
1638{
1639 static unsigned int ret;
1640 struct beiscsi_hba *phba;
bfead3b2 1641 struct be_eq_obj *pbe_eq;
6733b39a 1642
bfead3b2
JK		 1643 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1644 ret = beiscsi_process_cq(pbe_eq);
6733b39a 1645 if (ret < budget) {
bfead3b2 1646 phba = pbe_eq->phba;
6733b39a 1647 blk_iopoll_complete(iop);
bfead3b2
JK		 1648 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1649 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 1650 }
1651 return ret;
1652}
1653
1654static void
1655hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1656 unsigned int num_sg, struct beiscsi_io_task *io_task)
1657{
1658 struct iscsi_sge *psgl;
1659 unsigned short sg_len, index;
1660 unsigned int sge_len = 0;
1661 unsigned long long addr;
1662 struct scatterlist *l_sg;
1663 unsigned int offset;
1664
1665 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1666 io_task->bhs_pa.u.a32.address_lo);
1667 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1668 io_task->bhs_pa.u.a32.address_hi);
1669
1670 l_sg = sg;
1671 for (index = 0; (index < num_sg) && (index < 2); index++, sg_next(sg)) {
1672 if (index == 0) {
1673 sg_len = sg_dma_len(sg);
1674 addr = (u64) sg_dma_address(sg);
1675 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1676 (addr & 0xFFFFFFFF));
1677 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1678 (addr >> 32));
1679 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1680 sg_len);
1681 sge_len = sg_len;
1682 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1683 1);
1684 } else {
1685 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
1686 0);
1687 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
1688 pwrb, sge_len);
1689 sg_len = sg_dma_len(sg);
1690 addr = (u64) sg_dma_address(sg);
1691 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
1692 (addr & 0xFFFFFFFF));
1693 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
1694 (addr >> 32));
1695 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
1696 sg_len);
1697 }
1698 }
1699 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1700 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
1701
1702 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
1703
1704 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1705 io_task->bhs_pa.u.a32.address_hi);
1706 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1707 io_task->bhs_pa.u.a32.address_lo);
1708
1709 if (num_sg == 2)
1710 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, 1);
1711 sg = l_sg;
1712 psgl++;
1713 psgl++;
1714 offset = 0;
1715 for (index = 0; index < num_sg; index++, sg_next(sg), psgl++) {
1716 sg_len = sg_dma_len(sg);
1717 addr = (u64) sg_dma_address(sg);
1718 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1719 (addr & 0xFFFFFFFF));
1720 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1721 (addr >> 32));
1722 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
1723 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
1724 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1725 offset += sg_len;
1726 }
1727 psgl--;
1728 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1729}
1730
1731static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
1732{
1733 struct iscsi_sge *psgl;
1734 unsigned long long addr;
1735 struct beiscsi_io_task *io_task = task->dd_data;
1736 struct beiscsi_conn *beiscsi_conn = io_task->conn;
1737 struct beiscsi_hba *phba = beiscsi_conn->phba;
1738
1739 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
1740 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1741 io_task->bhs_pa.u.a32.address_lo);
1742 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
1743 io_task->bhs_pa.u.a32.address_hi);
1744
1745 if (task->data) {
1746 if (task->data_count) {
1747 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
1748 addr = (u64) pci_map_single(phba->pcidev,
1749 task->data,
1750 task->data_count, 1);
1751 } else {
1752 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1753 addr = 0;
1754 }
1755 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
1756 (addr & 0xFFFFFFFF));
1757 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
1758 (addr >> 32));
1759 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
1760 task->data_count);
1761
1762 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
1763 } else {
1764 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
1765 addr = 0;
1766 }
1767
1768 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
1769
1770 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
1771
1772 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1773 io_task->bhs_pa.u.a32.address_hi);
1774 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1775 io_task->bhs_pa.u.a32.address_lo);
1776 if (task->data) {
1777 psgl++;
1778 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
1779 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
1780 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
1781 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
1782 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
1783 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
1784
1785 psgl++;
1786 if (task->data) {
1787 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
1788 (addr & 0xFFFFFFFF));
1789 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
1790 (addr >> 32));
1791 }
1792 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
1793 }
1794 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
1795}
1796
1797static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
1798{
bfead3b2 1799 unsigned int num_cq_pages, num_async_pdu_buf_pages;
6733b39a
JK		 1800 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
1801 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
1802
1803 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
1804 sizeof(struct sol_cqe));
6733b39a
JK		 1805 num_async_pdu_buf_pages =
1806 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1807 phba->params.defpdu_hdr_sz);
1808 num_async_pdu_buf_sgl_pages =
1809 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1810 sizeof(struct phys_addr));
1811 num_async_pdu_data_pages =
1812 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1813 phba->params.defpdu_data_sz);
1814 num_async_pdu_data_sgl_pages =
1815 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
1816 sizeof(struct phys_addr));
1817
1818 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
1819
1820 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
1821 BE_ISCSI_PDU_HEADER_SIZE;
1822 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
1823 sizeof(struct hwi_context_memory);
1824
6733b39a
JK		 1825
1826 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
1827 * (phba->params.wrbs_per_cxn)
1828 * phba->params.cxns_per_ctrl;
1829 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
1830 (phba->params.wrbs_per_cxn);
1831 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
1832 phba->params.cxns_per_ctrl);
1833
1834 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
1835 phba->params.icds_per_ctrl;
1836 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
1837 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
1838
1839 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
1840 num_async_pdu_buf_pages * PAGE_SIZE;
1841 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
1842 num_async_pdu_data_pages * PAGE_SIZE;
1843 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
1844 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
1845 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
1846 num_async_pdu_data_sgl_pages * PAGE_SIZE;
1847 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
1848 phba->params.asyncpdus_per_ctrl *
1849 sizeof(struct async_pdu_handle);
1850 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
1851 phba->params.asyncpdus_per_ctrl *
1852 sizeof(struct async_pdu_handle);
1853 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
1854 sizeof(struct hwi_async_pdu_context) +
1855 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
1856}
1857
1858static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
1859{
1860 struct be_mem_descriptor *mem_descr;
1861 dma_addr_t bus_add;
1862 struct mem_array *mem_arr, *mem_arr_orig;
1863 unsigned int i, j, alloc_size, curr_alloc_size;
1864
1865 phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
1866 if (!phba->phwi_ctrlr)
1867 return -ENOMEM;
1868
1869 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
1870 GFP_KERNEL);
1871 if (!phba->init_mem) {
1872 kfree(phba->phwi_ctrlr);
1873 return -ENOMEM;
1874 }
1875
1876 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
1877 GFP_KERNEL);
1878 if (!mem_arr_orig) {
1879 kfree(phba->init_mem);
1880 kfree(phba->phwi_ctrlr);
1881 return -ENOMEM;
1882 }
1883
1884 mem_descr = phba->init_mem;
1885 for (i = 0; i < SE_MEM_MAX; i++) {
1886 j = 0;
1887 mem_arr = mem_arr_orig;
1888 alloc_size = phba->mem_req[i];
1889 memset(mem_arr, 0, sizeof(struct mem_array) *
1890 BEISCSI_MAX_FRAGS_INIT);
1891 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
1892 do {
1893 mem_arr->virtual_address = pci_alloc_consistent(
1894 phba->pcidev,
1895 curr_alloc_size,
1896 &bus_add);
1897 if (!mem_arr->virtual_address) {
1898 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
1899 goto free_mem;
1900 if (curr_alloc_size -
1901 rounddown_pow_of_two(curr_alloc_size))
1902 curr_alloc_size = rounddown_pow_of_two
1903 (curr_alloc_size);
1904 else
1905 curr_alloc_size = curr_alloc_size / 2;
1906 } else {
1907 mem_arr->bus_address.u.
1908 a64.address = (__u64) bus_add;
1909 mem_arr->size = curr_alloc_size;
1910 alloc_size -= curr_alloc_size;
1911 curr_alloc_size = min(be_max_phys_size *
1912 1024, alloc_size);
1913 j++;
1914 mem_arr++;
1915 }
1916 } while (alloc_size);
1917 mem_descr->num_elements = j;
1918 mem_descr->size_in_bytes = phba->mem_req[i];
1919 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
1920 GFP_KERNEL);
1921 if (!mem_descr->mem_array)
1922 goto free_mem;
1923
1924 memcpy(mem_descr->mem_array, mem_arr_orig,
1925 sizeof(struct mem_array) * j);
1926 mem_descr++;
1927 }
1928 kfree(mem_arr_orig);
1929 return 0;
1930free_mem:
1931 mem_descr->num_elements = j;
1932 while ((i) || (j)) {
1933 for (j = mem_descr->num_elements; j > 0; j--) {
1934 pci_free_consistent(phba->pcidev,
1935 mem_descr->mem_array[j - 1].size,
1936 mem_descr->mem_array[j - 1].
1937 virtual_address,
1938 mem_descr->mem_array[j - 1].
1939 bus_address.u.a64.address);
1940 }
1941 if (i) {
1942 i--;
1943 kfree(mem_descr->mem_array);
1944 mem_descr--;
1945 }
1946 }
1947 kfree(mem_arr_orig);
1948 kfree(phba->init_mem);
1949 kfree(phba->phwi_ctrlr);
1950 return -ENOMEM;
1951}
1952
1953static int beiscsi_get_memory(struct beiscsi_hba *phba)
1954{
1955 beiscsi_find_mem_req(phba);
1956 return beiscsi_alloc_mem(phba);
1957}
1958
1959static void iscsi_init_global_templates(struct beiscsi_hba *phba)
1960{
1961 struct pdu_data_out *pdata_out;
1962 struct pdu_nop_out *pnop_out;
1963 struct be_mem_descriptor *mem_descr;
1964
1965 mem_descr = phba->init_mem;
1966 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
1967 pdata_out =
1968 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
1969 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1970
1971 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
1972 IIOC_SCSI_DATA);
1973
1974 pnop_out =
1975 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
1976 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
1977
1978 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
1979 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
1980 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
1981 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
1982}
1983
1984static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
1985{
1986 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
1987 struct wrb_handle *pwrb_handle;
1988 struct hwi_controller *phwi_ctrlr;
1989 struct hwi_wrb_context *pwrb_context;
1990 struct iscsi_wrb *pwrb;
1991 unsigned int num_cxn_wrbh;
1992 unsigned int num_cxn_wrb, j, idx, index;
1993
1994 mem_descr_wrbh = phba->init_mem;
1995 mem_descr_wrbh += HWI_MEM_WRBH;
1996
1997 mem_descr_wrb = phba->init_mem;
1998 mem_descr_wrb += HWI_MEM_WRB;
1999
2000 idx = 0;
2001 pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2002 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2003 ((sizeof(struct wrb_handle)) *
2004 phba->params.wrbs_per_cxn));
2005 phwi_ctrlr = phba->phwi_ctrlr;
2006
2007 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2008 pwrb_context = &phwi_ctrlr->wrb_context[index];
6733b39a
JK		 2009 pwrb_context->pwrb_handle_base =
2010 kzalloc(sizeof(struct wrb_handle *) *
2011 phba->params.wrbs_per_cxn, GFP_KERNEL);
2012 pwrb_context->pwrb_handle_basestd =
2013 kzalloc(sizeof(struct wrb_handle *) *
2014 phba->params.wrbs_per_cxn, GFP_KERNEL);
2015 if (num_cxn_wrbh) {
2016 pwrb_context->alloc_index = 0;
2017 pwrb_context->wrb_handles_available = 0;
2018 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2019 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2020 pwrb_context->pwrb_handle_basestd[j] =
2021 pwrb_handle;
2022 pwrb_context->wrb_handles_available++;
bfead3b2 2023 pwrb_handle->wrb_index = j;
6733b39a
JK		 2024 pwrb_handle++;
2025 }
2026 pwrb_context->free_index = 0;
2027 num_cxn_wrbh--;
2028 } else {
2029 idx++;
2030 pwrb_handle =
2031 mem_descr_wrbh->mem_array[idx].virtual_address;
2032 num_cxn_wrbh =
2033 ((mem_descr_wrbh->mem_array[idx].size) /
2034 ((sizeof(struct wrb_handle)) *
2035 phba->params.wrbs_per_cxn));
2036 pwrb_context->alloc_index = 0;
2037 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2038 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2039 pwrb_context->pwrb_handle_basestd[j] =
2040 pwrb_handle;
2041 pwrb_context->wrb_handles_available++;
bfead3b2 2042 pwrb_handle->wrb_index = j;
6733b39a
JK		 2043 pwrb_handle++;
2044 }
2045 pwrb_context->free_index = 0;
2046 num_cxn_wrbh--;
2047 }
2048 }
2049 idx = 0;
2050 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2051 num_cxn_wrb =
2052 ((mem_descr_wrb->mem_array[idx].size) / (sizeof(struct iscsi_wrb)) *
2053 phba->params.wrbs_per_cxn);
2054
2055 for (index = 0; index < phba->params.cxns_per_ctrl; index += 2) {
2056 pwrb_context = &phwi_ctrlr->wrb_context[index];
2057 if (num_cxn_wrb) {
2058 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2059 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2060 pwrb_handle->pwrb = pwrb;
2061 pwrb++;
2062 }
2063 num_cxn_wrb--;
2064 } else {
2065 idx++;
2066 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2067 num_cxn_wrb = ((mem_descr_wrb->mem_array[idx].size) /
2068 (sizeof(struct iscsi_wrb)) *
2069 phba->params.wrbs_per_cxn);
2070 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2071 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2072 pwrb_handle->pwrb = pwrb;
2073 pwrb++;
2074 }
2075 num_cxn_wrb--;
2076 }
2077 }
2078}
2079
2080static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2081{
2082 struct hwi_controller *phwi_ctrlr;
2083 struct hba_parameters *p = &phba->params;
2084 struct hwi_async_pdu_context *pasync_ctx;
2085 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2086 unsigned int index;
2087 struct be_mem_descriptor *mem_descr;
2088
2089 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2090 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2091
2092 phwi_ctrlr = phba->phwi_ctrlr;
2093 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2094 mem_descr->mem_array[0].virtual_address;
2095 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2096 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2097
2098 pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2099 pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2100 pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2101 pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2102
2103 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2104 mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2105 if (mem_descr->mem_array[0].virtual_address) {
2106 SE_DEBUG(DBG_LVL_8,
2107 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2108 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2109 } else
2110 shost_printk(KERN_WARNING, phba->shost,
2111 "No Virtual address \n");
2112
2113 pasync_ctx->async_header.va_base =
2114 mem_descr->mem_array[0].virtual_address;
2115
2116 pasync_ctx->async_header.pa_base.u.a64.address =
2117 mem_descr->mem_array[0].bus_address.u.a64.address;
2118
2119 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2120 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2121 if (mem_descr->mem_array[0].virtual_address) {
2122 SE_DEBUG(DBG_LVL_8,
2123 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2124 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2125 } else
2126 shost_printk(KERN_WARNING, phba->shost,
2127 "No Virtual address \n");
2128 pasync_ctx->async_header.ring_base =
2129 mem_descr->mem_array[0].virtual_address;
2130
2131 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2132 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2133 if (mem_descr->mem_array[0].virtual_address) {
2134 SE_DEBUG(DBG_LVL_8,
2135 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2136 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2137 } else
2138 shost_printk(KERN_WARNING, phba->shost,
2139 "No Virtual address \n");
2140
2141 pasync_ctx->async_header.handle_base =
2142 mem_descr->mem_array[0].virtual_address;
2143 pasync_ctx->async_header.writables = 0;
2144 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2145
2146 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2147 mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2148 if (mem_descr->mem_array[0].virtual_address) {
2149 SE_DEBUG(DBG_LVL_8,
2150 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2151 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2152 } else
2153 shost_printk(KERN_WARNING, phba->shost,
2154 "No Virtual address \n");
2155 pasync_ctx->async_data.va_base =
2156 mem_descr->mem_array[0].virtual_address;
2157 pasync_ctx->async_data.pa_base.u.a64.address =
2158 mem_descr->mem_array[0].bus_address.u.a64.address;
2159
2160 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2161 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2162 if (mem_descr->mem_array[0].virtual_address) {
2163 SE_DEBUG(DBG_LVL_8,
2164 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2165 "va=%p \n", mem_descr->mem_array[0].virtual_address);
2166 } else
2167 shost_printk(KERN_WARNING, phba->shost,
2168 "No Virtual address \n");
2169
2170 pasync_ctx->async_data.ring_base =
2171 mem_descr->mem_array[0].virtual_address;
2172
2173 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2174 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2175 if (!mem_descr->mem_array[0].virtual_address)
2176 shost_printk(KERN_WARNING, phba->shost,
2177 "No Virtual address \n");
2178
2179 pasync_ctx->async_data.handle_base =
2180 mem_descr->mem_array[0].virtual_address;
2181 pasync_ctx->async_data.writables = 0;
2182 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2183
2184 pasync_header_h =
2185 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2186 pasync_data_h =
2187 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2188
2189 for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2190 pasync_header_h->cri = -1;
2191 pasync_header_h->index = (char)index;
2192 INIT_LIST_HEAD(&pasync_header_h->link);
2193 pasync_header_h->pbuffer =
2194 (void *)((unsigned long)
2195 (pasync_ctx->async_header.va_base) +
2196 (p->defpdu_hdr_sz * index));
2197
2198 pasync_header_h->pa.u.a64.address =
2199 pasync_ctx->async_header.pa_base.u.a64.address +
2200 (p->defpdu_hdr_sz * index);
2201
2202 list_add_tail(&pasync_header_h->link,
2203 &pasync_ctx->async_header.free_list);
2204 pasync_header_h++;
2205 pasync_ctx->async_header.free_entries++;
2206 pasync_ctx->async_header.writables++;
2207
2208 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2209 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2210 header_busy_list);
2211 pasync_data_h->cri = -1;
2212 pasync_data_h->index = (char)index;
2213 INIT_LIST_HEAD(&pasync_data_h->link);
2214 pasync_data_h->pbuffer =
2215 (void *)((unsigned long)
2216 (pasync_ctx->async_data.va_base) +
2217 (p->defpdu_data_sz * index));
2218
2219 pasync_data_h->pa.u.a64.address =
2220 pasync_ctx->async_data.pa_base.u.a64.address +
2221 (p->defpdu_data_sz * index);
2222
2223 list_add_tail(&pasync_data_h->link,
2224 &pasync_ctx->async_data.free_list);
2225 pasync_data_h++;
2226 pasync_ctx->async_data.free_entries++;
2227 pasync_ctx->async_data.writables++;
2228
2229 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2230 }
2231
2232 pasync_ctx->async_header.host_write_ptr = 0;
2233 pasync_ctx->async_header.ep_read_ptr = -1;
2234 pasync_ctx->async_data.host_write_ptr = 0;
2235 pasync_ctx->async_data.ep_read_ptr = -1;
2236}
2237
2238static int
2239be_sgl_create_contiguous(void *virtual_address,
2240 u64 physical_address, u32 length,
2241 struct be_dma_mem *sgl)
2242{
2243 WARN_ON(!virtual_address);
2244 WARN_ON(!physical_address);
2245 WARN_ON(!length > 0);
2246 WARN_ON(!sgl);
2247
2248 sgl->va = virtual_address;
2249 sgl->dma = physical_address;
2250 sgl->size = length;
2251
2252 return 0;
2253}
2254
2255static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2256{
2257 memset(sgl, 0, sizeof(*sgl));
2258}
2259
2260static void
2261hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2262 struct mem_array *pmem, struct be_dma_mem *sgl)
2263{
2264 if (sgl->va)
2265 be_sgl_destroy_contiguous(sgl);
2266
2267 be_sgl_create_contiguous(pmem->virtual_address,
2268 pmem->bus_address.u.a64.address,
2269 pmem->size, sgl);
2270}
2271
2272static void
2273hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2274 struct mem_array *pmem, struct be_dma_mem *sgl)
2275{
2276 if (sgl->va)
2277 be_sgl_destroy_contiguous(sgl);
2278
2279 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2280 pmem->bus_address.u.a64.address,
2281 pmem->size, sgl);
2282}
2283
2284static int be_fill_queue(struct be_queue_info *q,
2285 u16 len, u16 entry_size, void *vaddress)
2286{
2287 struct be_dma_mem *mem = &q->dma_mem;
2288
2289 memset(q, 0, sizeof(*q));
2290 q->len = len;
2291 q->entry_size = entry_size;
2292 mem->size = len * entry_size;
2293 mem->va = vaddress;
2294 if (!mem->va)
2295 return -ENOMEM;
2296 memset(mem->va, 0, mem->size);
2297 return 0;
2298}
2299
bfead3b2 2300static int beiscsi_create_eqs(struct beiscsi_hba *phba,
6733b39a
JK		 2301 struct hwi_context_memory *phwi_context)
2302{
bfead3b2
JK		 2303 unsigned int i, num_eq_pages;
2304 int ret, eq_for_mcc;
6733b39a
JK		 2305 struct be_queue_info *eq;
2306 struct be_dma_mem *mem;
6733b39a 2307 void *eq_vaddress;
bfead3b2 2308 dma_addr_t paddr;
6733b39a 2309
bfead3b2
JK		 2310 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2311 sizeof(struct be_eq_entry));
6733b39a 2312
bfead3b2
JK		 2313 if (phba->msix_enabled)
2314 eq_for_mcc = 1;
2315 else
2316 eq_for_mcc = 0;
2317 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2318 eq = &phwi_context->be_eq[i].q;
2319 mem = &eq->dma_mem;
2320 phwi_context->be_eq[i].phba = phba;
2321 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2322 num_eq_pages * PAGE_SIZE,
2323 &paddr);
2324 if (!eq_vaddress)
2325 goto create_eq_error;
2326
2327 mem->va = eq_vaddress;
2328 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2329 sizeof(struct be_eq_entry), eq_vaddress);
2330 if (ret) {
2331 shost_printk(KERN_ERR, phba->shost,
2332 "be_fill_queue Failed for EQ \n");
2333 goto create_eq_error;
2334 }
6733b39a 2335
bfead3b2
JK		 2336 mem->dma = paddr;
2337 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2338 phwi_context->cur_eqd);
2339 if (ret) {
2340 shost_printk(KERN_ERR, phba->shost,
2341 "beiscsi_cmd_eq_create"
2342 "Failedfor EQ \n");
2343 goto create_eq_error;
2344 }
2345 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
6733b39a 2346 }
6733b39a 2347 return 0;
bfead3b2
JK		 2348create_eq_error:
2349 for (i = 0; i < (phba->num_cpus + 1); i++) {
2350 eq = &phwi_context->be_eq[i].q;
2351 mem = &eq->dma_mem;
2352 if (mem->va)
2353 pci_free_consistent(phba->pcidev, num_eq_pages
2354 * PAGE_SIZE,
2355 mem->va, mem->dma);
2356 }
2357 return ret;
6733b39a
JK		 2358}
2359
bfead3b2 2360static int beiscsi_create_cqs(struct beiscsi_hba *phba,
6733b39a
JK		 2361 struct hwi_context_memory *phwi_context)
2362{
bfead3b2 2363 unsigned int i, num_cq_pages;
6733b39a
JK		 2364 int ret;
2365 struct be_queue_info *cq, *eq;
2366 struct be_dma_mem *mem;
bfead3b2 2367 struct be_eq_obj *pbe_eq;
6733b39a 2368 void *cq_vaddress;
bfead3b2 2369 dma_addr_t paddr;
6733b39a 2370
bfead3b2
JK		 2371 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2372 sizeof(struct sol_cqe));
6733b39a 2373
bfead3b2
JK		 2374 for (i = 0; i < phba->num_cpus; i++) {
2375 cq = &phwi_context->be_cq[i];
2376 eq = &phwi_context->be_eq[i].q;
2377 pbe_eq = &phwi_context->be_eq[i];
2378 pbe_eq->cq = cq;
2379 pbe_eq->phba = phba;
2380 mem = &cq->dma_mem;
2381 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2382 num_cq_pages * PAGE_SIZE,
2383 &paddr);
2384 if (!cq_vaddress)
2385 goto create_cq_error;
2386 ret = be_fill_queue(cq, phba->params.icds_per_ctrl / 2,
2387 sizeof(struct sol_cqe), cq_vaddress);
2388 if (ret) {
2389 shost_printk(KERN_ERR, phba->shost,
2390 "be_fill_queue Failed for ISCSI CQ \n");
2391 goto create_cq_error;
2392 }
2393
2394 mem->dma = paddr;
2395 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2396 false, 0);
2397 if (ret) {
2398 shost_printk(KERN_ERR, phba->shost,
2399 "beiscsi_cmd_eq_create"
2400 "Failed for ISCSI CQ \n");
2401 goto create_cq_error;
2402 }
2403 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2404 cq->id, eq->id);
2405 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
6733b39a 2406 }
6733b39a 2407 return 0;
bfead3b2
JK		 2408
2409create_cq_error:
2410 for (i = 0; i < phba->num_cpus; i++) {
2411 cq = &phwi_context->be_cq[i];
2412 mem = &cq->dma_mem;
2413 if (mem->va)
2414 pci_free_consistent(phba->pcidev, num_cq_pages
2415 * PAGE_SIZE,
2416 mem->va, mem->dma);
2417 }
2418 return ret;
2419
6733b39a
JK		 2420}
2421
2422static int
2423beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2424 struct hwi_context_memory *phwi_context,
2425 struct hwi_controller *phwi_ctrlr,
2426 unsigned int def_pdu_ring_sz)
2427{
2428 unsigned int idx;
2429 int ret;
2430 struct be_queue_info *dq, *cq;
2431 struct be_dma_mem *mem;
2432 struct be_mem_descriptor *mem_descr;
2433 void *dq_vaddress;
2434
2435 idx = 0;
2436 dq = &phwi_context->be_def_hdrq;
bfead3b2 2437 cq = &phwi_context->be_cq[0];
6733b39a
JK		 2438 mem = &dq->dma_mem;
2439 mem_descr = phba->init_mem;
2440 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2441 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2442 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2443 sizeof(struct phys_addr),
2444 sizeof(struct phys_addr), dq_vaddress);
2445 if (ret) {
2446 shost_printk(KERN_ERR, phba->shost,
2447 "be_fill_queue Failed for DEF PDU HDR\n");
2448 return ret;
2449 }
2450 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2451 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2452 def_pdu_ring_sz,
2453 phba->params.defpdu_hdr_sz);
2454 if (ret) {
2455 shost_printk(KERN_ERR, phba->shost,
2456 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2457 return ret;
2458 }
2459 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2460 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2461 phwi_context->be_def_hdrq.id);
2462 hwi_post_async_buffers(phba, 1);
2463 return 0;
2464}
2465
2466static int
2467beiscsi_create_def_data(struct beiscsi_hba *phba,
2468 struct hwi_context_memory *phwi_context,
2469 struct hwi_controller *phwi_ctrlr,
2470 unsigned int def_pdu_ring_sz)
2471{
2472 unsigned int idx;
2473 int ret;
2474 struct be_queue_info *dataq, *cq;
2475 struct be_dma_mem *mem;
2476 struct be_mem_descriptor *mem_descr;
2477 void *dq_vaddress;
2478
2479 idx = 0;
2480 dataq = &phwi_context->be_def_dataq;
bfead3b2 2481 cq = &phwi_context->be_cq[0];
6733b39a
JK		 2482 mem = &dataq->dma_mem;
2483 mem_descr = phba->init_mem;
2484 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2485 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2486 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2487 sizeof(struct phys_addr),
2488 sizeof(struct phys_addr), dq_vaddress);
2489 if (ret) {
2490 shost_printk(KERN_ERR, phba->shost,
2491 "be_fill_queue Failed for DEF PDU DATA\n");
2492 return ret;
2493 }
2494 mem->dma = mem_descr->mem_array[idx].bus_address.u.a64.address;
2495 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2496 def_pdu_ring_sz,
2497 phba->params.defpdu_data_sz);
2498 if (ret) {
2499 shost_printk(KERN_ERR, phba->shost,
2500 "be_cmd_create_default_pdu_queue Failed"
2501 " for DEF PDU DATA\n");
2502 return ret;
2503 }
2504 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2505 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2506 phwi_context->be_def_dataq.id);
2507 hwi_post_async_buffers(phba, 0);
2508 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED \n");
2509 return 0;
2510}
2511
2512static int
2513beiscsi_post_pages(struct beiscsi_hba *phba)
2514{
2515 struct be_mem_descriptor *mem_descr;
2516 struct mem_array *pm_arr;
2517 unsigned int page_offset, i;
2518 struct be_dma_mem sgl;
2519 int status;
2520
2521 mem_descr = phba->init_mem;
2522 mem_descr += HWI_MEM_SGE;
2523 pm_arr = mem_descr->mem_array;
2524
2525 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2526 phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2527 for (i = 0; i < mem_descr->num_elements; i++) {
2528 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2529 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2530 page_offset,
2531 (pm_arr->size / PAGE_SIZE));
2532 page_offset += pm_arr->size / PAGE_SIZE;
2533 if (status != 0) {
2534 shost_printk(KERN_ERR, phba->shost,
2535 "post sgl failed.\n");
2536 return status;
2537 }
2538 pm_arr++;
2539 }
2540 SE_DEBUG(DBG_LVL_8, "POSTED PAGES \n");
2541 return 0;
2542}
2543
bfead3b2
JK		 2544static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2545{
2546 struct be_dma_mem *mem = &q->dma_mem;
2547 if (mem->va)
2548 pci_free_consistent(phba->pcidev, mem->size,
2549 mem->va, mem->dma);
2550}
2551
2552static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2553 u16 len, u16 entry_size)
2554{
2555 struct be_dma_mem *mem = &q->dma_mem;
2556
2557 memset(q, 0, sizeof(*q));
2558 q->len = len;
2559 q->entry_size = entry_size;
2560 mem->size = len * entry_size;
2561 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2562 if (!mem->va)
2563 return -1;
2564 memset(mem->va, 0, mem->size);
2565 return 0;
2566}
2567
6733b39a
JK		 2568static int
2569beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2570 struct hwi_context_memory *phwi_context,
2571 struct hwi_controller *phwi_ctrlr)
2572{
2573 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2574 u64 pa_addr_lo;
2575 unsigned int idx, num, i;
2576 struct mem_array *pwrb_arr;
2577 void *wrb_vaddr;
2578 struct be_dma_mem sgl;
2579 struct be_mem_descriptor *mem_descr;
2580 int status;
2581
2582 idx = 0;
2583 mem_descr = phba->init_mem;
2584 mem_descr += HWI_MEM_WRB;
2585 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2586 GFP_KERNEL);
2587 if (!pwrb_arr) {
2588 shost_printk(KERN_ERR, phba->shost,
2589 "Memory alloc failed in create wrb ring.\n");
2590 return -ENOMEM;
2591 }
2592 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2593 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2594 num_wrb_rings = mem_descr->mem_array[idx].size /
2595 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2596
2597 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2598 if (num_wrb_rings) {
2599 pwrb_arr[num].virtual_address = wrb_vaddr;
2600 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2601 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2602 sizeof(struct iscsi_wrb);
2603 wrb_vaddr += pwrb_arr[num].size;
2604 pa_addr_lo += pwrb_arr[num].size;
2605 num_wrb_rings--;
2606 } else {
2607 idx++;
2608 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2609 pa_addr_lo = mem_descr->mem_array[idx].\
2610 bus_address.u.a64.address;
2611 num_wrb_rings = mem_descr->mem_array[idx].size /
2612 (phba->params.wrbs_per_cxn *
2613 sizeof(struct iscsi_wrb));
2614 pwrb_arr[num].virtual_address = wrb_vaddr;
2615 pwrb_arr[num].bus_address.u.a64.address\
2616 = pa_addr_lo;
2617 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2618 sizeof(struct iscsi_wrb);
2619 wrb_vaddr += pwrb_arr[num].size;
2620 pa_addr_lo += pwrb_arr[num].size;
2621 num_wrb_rings--;
2622 }
2623 }
2624 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2625 wrb_mem_index = 0;
2626 offset = 0;
2627 size = 0;
2628
2629 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2630 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2631 &phwi_context->be_wrbq[i]);
2632 if (status != 0) {
2633 shost_printk(KERN_ERR, phba->shost,
2634 "wrbq create failed.");
2635 return status;
2636 }
2637 phwi_ctrlr->wrb_context[i].cid = phwi_context->be_wrbq[i].id;
2638 }
2639 kfree(pwrb_arr);
2640 return 0;
2641}
2642
2643static void free_wrb_handles(struct beiscsi_hba *phba)
2644{
2645 unsigned int index;
2646 struct hwi_controller *phwi_ctrlr;
2647 struct hwi_wrb_context *pwrb_context;
2648
2649 phwi_ctrlr = phba->phwi_ctrlr;
2650 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2651 pwrb_context = &phwi_ctrlr->wrb_context[index];
2652 kfree(pwrb_context->pwrb_handle_base);
2653 kfree(pwrb_context->pwrb_handle_basestd);
2654 }
2655}
2656
bfead3b2
JK		 2657static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
2658{
2659 struct be_queue_info *q;
2660 struct be_ctrl_info *ctrl = &phba->ctrl;
2661
2662 q = &phba->ctrl.mcc_obj.q;
2663 if (q->created)
2664 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
2665 be_queue_free(phba, q);
2666
2667 q = &phba->ctrl.mcc_obj.cq;
2668 if (q->created)
2669 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2670 be_queue_free(phba, q);
2671}
2672
6733b39a
JK		 2673static void hwi_cleanup(struct beiscsi_hba *phba)
2674{
2675 struct be_queue_info *q;
2676 struct be_ctrl_info *ctrl = &phba->ctrl;
2677 struct hwi_controller *phwi_ctrlr;
2678 struct hwi_context_memory *phwi_context;
bfead3b2 2679 int i, eq_num;
6733b39a
JK		 2680
2681 phwi_ctrlr = phba->phwi_ctrlr;
2682 phwi_context = phwi_ctrlr->phwi_ctxt;
2683 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2684 q = &phwi_context->be_wrbq[i];
2685 if (q->created)
2686 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
2687 }
6733b39a
JK		 2688 free_wrb_handles(phba);
2689
2690 q = &phwi_context->be_def_hdrq;
2691 if (q->created)
2692 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2693
2694 q = &phwi_context->be_def_dataq;
2695 if (q->created)
2696 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
2697
2698 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
2699
bfead3b2
JK		 2700 for (i = 0; i < (phba->num_cpus); i++) {
2701 q = &phwi_context->be_cq[i];
2702 if (q->created)
2703 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
2704 }
2705 if (phba->msix_enabled)
2706 eq_num = 1;
2707 else
2708 eq_num = 0;
2709 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
2710 q = &phwi_context->be_eq[i].q;
2711 if (q->created)
2712 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
2713 }
2714 be_mcc_queues_destroy(phba);
2715}
6733b39a 2716
bfead3b2
JK		 2717static int be_mcc_queues_create(struct beiscsi_hba *phba,
2718 struct hwi_context_memory *phwi_context)
2719{
2720 struct be_queue_info *q, *cq;
2721 struct be_ctrl_info *ctrl = &phba->ctrl;
2722
2723 /* Alloc MCC compl queue */
2724 cq = &phba->ctrl.mcc_obj.cq;
2725 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
2726 sizeof(struct be_mcc_compl)))
2727 goto err;
2728 /* Ask BE to create MCC compl queue; */
2729 if (phba->msix_enabled) {
2730 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
2731 [phba->num_cpus].q, false, true, 0))
2732 goto mcc_cq_free;
2733 } else {
2734 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
2735 false, true, 0))
2736 goto mcc_cq_free;
2737 }
2738
2739 /* Alloc MCC queue */
2740 q = &phba->ctrl.mcc_obj.q;
2741 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2742 goto mcc_cq_destroy;
2743
2744 /* Ask BE to create MCC queue */
35e66019 2745 if (beiscsi_cmd_mccq_create(phba, q, cq))
bfead3b2
JK		 2746 goto mcc_q_free;
2747
2748 return 0;
2749
2750mcc_q_free:
2751 be_queue_free(phba, q);
2752mcc_cq_destroy:
2753 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
2754mcc_cq_free:
2755 be_queue_free(phba, cq);
2756err:
2757 return -1;
2758}
2759
2760static int find_num_cpus(void)
2761{
2762 int num_cpus = 0;
2763
2764 num_cpus = num_online_cpus();
2765 if (num_cpus >= MAX_CPUS)
2766 num_cpus = MAX_CPUS - 1;
2767
2768 SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", num_cpus);
2769 return num_cpus;
6733b39a
JK		 2770}
2771
2772static int hwi_init_port(struct beiscsi_hba *phba)
2773{
2774 struct hwi_controller *phwi_ctrlr;
2775 struct hwi_context_memory *phwi_context;
2776 unsigned int def_pdu_ring_sz;
2777 struct be_ctrl_info *ctrl = &phba->ctrl;
2778 int status;
2779
2780 def_pdu_ring_sz =
2781 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
2782 phwi_ctrlr = phba->phwi_ctrlr;
6733b39a 2783 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 2784 phwi_context->max_eqd = 0;
2785 phwi_context->min_eqd = 0;
2786 phwi_context->cur_eqd = 64;
6733b39a 2787 be_cmd_fw_initialize(&phba->ctrl);
bfead3b2
JK		 2788
2789 status = beiscsi_create_eqs(phba, phwi_context);
6733b39a
JK		 2790 if (status != 0) {
2791 shost_printk(KERN_ERR, phba->shost, "EQ not created \n");
2792 goto error;
2793 }
2794
bfead3b2
JK		 2795 status = be_mcc_queues_create(phba, phwi_context);
2796 if (status != 0)
2797 goto error;
2798
2799 status = mgmt_check_supported_fw(ctrl, phba);
6733b39a
JK		 2800 if (status != 0) {
2801 shost_printk(KERN_ERR, phba->shost,
2802 "Unsupported fw version \n");
2803 goto error;
2804 }
2805
35e66019
JK		 2806 if (phba->fw_config.iscsi_features == 0x1)
2807 ring_mode = 1;
2808 else
2809 ring_mode = 0;
6733b39a
JK		 2810 status = mgmt_get_fw_config(ctrl, phba);
2811 if (status != 0) {
2812 shost_printk(KERN_ERR, phba->shost,
2813 "Error getting fw config\n");
2814 goto error;
2815 }
2816
bfead3b2 2817 status = beiscsi_create_cqs(phba, phwi_context);
6733b39a
JK		 2818 if (status != 0) {
2819 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
2820 goto error;
2821 }
2822
2823 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
2824 def_pdu_ring_sz);
2825 if (status != 0) {
2826 shost_printk(KERN_ERR, phba->shost,
2827 "Default Header not created\n");
2828 goto error;
2829 }
2830
2831 status = beiscsi_create_def_data(phba, phwi_context,
2832 phwi_ctrlr, def_pdu_ring_sz);
2833 if (status != 0) {
2834 shost_printk(KERN_ERR, phba->shost,
2835 "Default Data not created\n");
2836 goto error;
2837 }
2838
2839 status = beiscsi_post_pages(phba);
2840 if (status != 0) {
2841 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
2842 goto error;
2843 }
2844
2845 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
2846 if (status != 0) {
2847 shost_printk(KERN_ERR, phba->shost,
2848 "WRB Rings not created\n");
2849 goto error;
2850 }
2851
2852 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
2853 return 0;
2854
2855error:
2856 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
2857 hwi_cleanup(phba);
2858 return -ENOMEM;
2859}
2860
6733b39a
JK		 2861static int hwi_init_controller(struct beiscsi_hba *phba)
2862{
2863 struct hwi_controller *phwi_ctrlr;
2864
2865 phwi_ctrlr = phba->phwi_ctrlr;
2866 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
2867 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
2868 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
2869 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p \n",
2870 phwi_ctrlr->phwi_ctxt);
2871 } else {
2872 shost_printk(KERN_ERR, phba->shost,
2873 "HWI_MEM_ADDN_CONTEXT is more than one element."
2874 "Failing to load\n");
2875 return -ENOMEM;
2876 }
2877
2878 iscsi_init_global_templates(phba);
2879 beiscsi_init_wrb_handle(phba);
2880 hwi_init_async_pdu_ctx(phba);
2881 if (hwi_init_port(phba) != 0) {
2882 shost_printk(KERN_ERR, phba->shost,
2883 "hwi_init_controller failed\n");
2884 return -ENOMEM;
2885 }
2886 return 0;
2887}
2888
2889static void beiscsi_free_mem(struct beiscsi_hba *phba)
2890{
2891 struct be_mem_descriptor *mem_descr;
2892 int i, j;
2893
2894 mem_descr = phba->init_mem;
2895 i = 0;
2896 j = 0;
2897 for (i = 0; i < SE_MEM_MAX; i++) {
2898 for (j = mem_descr->num_elements; j > 0; j--) {
2899 pci_free_consistent(phba->pcidev,
2900 mem_descr->mem_array[j - 1].size,
2901 mem_descr->mem_array[j - 1].virtual_address,
2902 mem_descr->mem_array[j - 1].bus_address.
2903 u.a64.address);
2904 }
2905 kfree(mem_descr->mem_array);
2906 mem_descr++;
2907 }
2908 kfree(phba->init_mem);
2909 kfree(phba->phwi_ctrlr);
2910}
2911
2912static int beiscsi_init_controller(struct beiscsi_hba *phba)
2913{
2914 int ret = -ENOMEM;
2915
2916 ret = beiscsi_get_memory(phba);
2917 if (ret < 0) {
2918 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
2919 "Failed in beiscsi_alloc_memory \n");
2920 return ret;
2921 }
2922
2923 ret = hwi_init_controller(phba);
2924 if (ret)
2925 goto free_init;
2926 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
2927 return 0;
2928
2929free_init:
2930 beiscsi_free_mem(phba);
2931 return -ENOMEM;
2932}
2933
2934static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
2935{
2936 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
2937 struct sgl_handle *psgl_handle;
2938 struct iscsi_sge *pfrag;
2939 unsigned int arr_index, i, idx;
2940
2941 phba->io_sgl_hndl_avbl = 0;
2942 phba->eh_sgl_hndl_avbl = 0;
bfead3b2 2943
35e66019
JK		 2944 if (ring_mode) {
2945 phba->sgl_hndl_array = kzalloc(sizeof(struct sgl_handle *) *
2946 phba->params.icds_per_ctrl,
2947 GFP_KERNEL);
2948 if (!phba->sgl_hndl_array) {
2949 shost_printk(KERN_ERR, phba->shost,
2950 "Mem Alloc Failed. Failing to load\n");
2951 return -ENOMEM;
2952 }
2953 }
2954
6733b39a
JK		 2955 mem_descr_sglh = phba->init_mem;
2956 mem_descr_sglh += HWI_MEM_SGLH;
2957 if (1 == mem_descr_sglh->num_elements) {
2958 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2959 phba->params.ios_per_ctrl,
2960 GFP_KERNEL);
2961 if (!phba->io_sgl_hndl_base) {
35e66019
JK		 2962 if (ring_mode)
2963 kfree(phba->sgl_hndl_array);
6733b39a
JK		 2964 shost_printk(KERN_ERR, phba->shost,
2965 "Mem Alloc Failed. Failing to load\n");
2966 return -ENOMEM;
2967 }
2968 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
2969 (phba->params.icds_per_ctrl -
2970 phba->params.ios_per_ctrl),
2971 GFP_KERNEL);
2972 if (!phba->eh_sgl_hndl_base) {
2973 kfree(phba->io_sgl_hndl_base);
2974 shost_printk(KERN_ERR, phba->shost,
2975 "Mem Alloc Failed. Failing to load\n");
2976 return -ENOMEM;
2977 }
2978 } else {
2979 shost_printk(KERN_ERR, phba->shost,
2980 "HWI_MEM_SGLH is more than one element."
2981 "Failing to load\n");
2982 return -ENOMEM;
2983 }
2984
2985 arr_index = 0;
2986 idx = 0;
2987 while (idx < mem_descr_sglh->num_elements) {
2988 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
2989
2990 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
2991 sizeof(struct sgl_handle)); i++) {
2992 if (arr_index < phba->params.ios_per_ctrl) {
2993 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
2994 phba->io_sgl_hndl_avbl++;
2995 arr_index++;
2996 } else {
2997 phba->eh_sgl_hndl_base[arr_index -
2998 phba->params.ios_per_ctrl] =
2999 psgl_handle;
3000 arr_index++;
3001 phba->eh_sgl_hndl_avbl++;
3002 }
3003 psgl_handle++;
3004 }
3005 idx++;
3006 }
3007 SE_DEBUG(DBG_LVL_8,
3008 "phba->io_sgl_hndl_avbl=%d"
3009 "phba->eh_sgl_hndl_avbl=%d \n",
3010 phba->io_sgl_hndl_avbl,
3011 phba->eh_sgl_hndl_avbl);
3012 mem_descr_sg = phba->init_mem;
3013 mem_descr_sg += HWI_MEM_SGE;
3014 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d \n",
3015 mem_descr_sg->num_elements);
3016 arr_index = 0;
3017 idx = 0;
3018 while (idx < mem_descr_sg->num_elements) {
3019 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3020
3021 for (i = 0;
3022 i < (mem_descr_sg->mem_array[idx].size) /
3023 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3024 i++) {
3025 if (arr_index < phba->params.ios_per_ctrl)
3026 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3027 else
3028 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3029 phba->params.ios_per_ctrl];
3030 psgl_handle->pfrag = pfrag;
3031 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3032 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3033 pfrag += phba->params.num_sge_per_io;
3034 psgl_handle->sgl_index =
3035 phba->fw_config.iscsi_cid_start + arr_index++;
3036 }
3037 idx++;
3038 }
3039 phba->io_sgl_free_index = 0;
3040 phba->io_sgl_alloc_index = 0;
3041 phba->eh_sgl_free_index = 0;
3042 phba->eh_sgl_alloc_index = 0;
3043 return 0;
3044}
3045
3046static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3047{
3048 int i, new_cid;
3049
3050 phba->cid_array = kmalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3051 GFP_KERNEL);
3052 if (!phba->cid_array) {
3053 shost_printk(KERN_ERR, phba->shost,
3054 "Failed to allocate memory in "
3055 "hba_setup_cid_tbls\n");
3056 return -ENOMEM;
3057 }
3058 phba->ep_array = kmalloc(sizeof(struct iscsi_endpoint *) *
3059 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3060 if (!phba->ep_array) {
3061 shost_printk(KERN_ERR, phba->shost,
3062 "Failed to allocate memory in "
3063 "hba_setup_cid_tbls \n");
3064 kfree(phba->cid_array);
3065 return -ENOMEM;
3066 }
3067 new_cid = phba->fw_config.iscsi_icd_start;
3068 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3069 phba->cid_array[i] = new_cid;
3070 new_cid += 2;
3071 }
3072 phba->avlbl_cids = phba->params.cxns_per_ctrl;
3073 return 0;
3074}
3075
3076static unsigned char hwi_enable_intr(struct beiscsi_hba *phba)
3077{
3078 struct be_ctrl_info *ctrl = &phba->ctrl;
3079 struct hwi_controller *phwi_ctrlr;
3080 struct hwi_context_memory *phwi_context;
3081 struct be_queue_info *eq;
3082 u8 __iomem *addr;
bfead3b2 3083 u32 reg, i;
6733b39a
JK		 3084 u32 enabled;
3085
3086 phwi_ctrlr = phba->phwi_ctrlr;
3087 phwi_context = phwi_ctrlr->phwi_ctxt;
3088
6733b39a
JK		 3089 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3090 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3091 reg = ioread32(addr);
3092 SE_DEBUG(DBG_LVL_8, "reg =x%08x \n", reg);
3093
3094 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3095 if (!enabled) {
3096 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3097 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr);
3098 iowrite32(reg, addr);
bfead3b2
JK		 3099 for (i = 0; i <= phba->num_cpus; i++) {
3100 eq = &phwi_context->be_eq[i].q;
3101 SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id);
3102 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3103 }
6733b39a
JK		 3104 } else
3105 shost_printk(KERN_WARNING, phba->shost,
3106 "In hwi_enable_intr, Not Enabled \n");
3107 return true;
3108}
3109
3110static void hwi_disable_intr(struct beiscsi_hba *phba)
3111{
3112 struct be_ctrl_info *ctrl = &phba->ctrl;
3113
3114 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3115 u32 reg = ioread32(addr);
3116
3117 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3118 if (enabled) {
3119 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3120 iowrite32(reg, addr);
3121 } else
3122 shost_printk(KERN_WARNING, phba->shost,
3123 "In hwi_disable_intr, Already Disabled \n");
3124}
3125
3126static int beiscsi_init_port(struct beiscsi_hba *phba)
3127{
3128 int ret;
3129
3130 ret = beiscsi_init_controller(phba);
3131 if (ret < 0) {
3132 shost_printk(KERN_ERR, phba->shost,
3133 "beiscsi_dev_probe - Failed in"
3134 "beiscsi_init_controller \n");
3135 return ret;
3136 }
3137 ret = beiscsi_init_sgl_handle(phba);
3138 if (ret < 0) {
3139 shost_printk(KERN_ERR, phba->shost,
3140 "beiscsi_dev_probe - Failed in"
3141 "beiscsi_init_sgl_handle \n");
3142 goto do_cleanup_ctrlr;
3143 }
3144
3145 if (hba_setup_cid_tbls(phba)) {
3146 shost_printk(KERN_ERR, phba->shost,
3147 "Failed in hba_setup_cid_tbls\n");
35e66019
JK		 3148 if (ring_mode)
3149 kfree(phba->sgl_hndl_array);
6733b39a
JK		 3150 kfree(phba->io_sgl_hndl_base);
3151 kfree(phba->eh_sgl_hndl_base);
3152 goto do_cleanup_ctrlr;
3153 }
3154
3155 return ret;
3156
3157do_cleanup_ctrlr:
3158 hwi_cleanup(phba);
3159 return ret;
3160}
3161
3162static void hwi_purge_eq(struct beiscsi_hba *phba)
3163{
3164 struct hwi_controller *phwi_ctrlr;
3165 struct hwi_context_memory *phwi_context;
3166 struct be_queue_info *eq;
3167 struct be_eq_entry *eqe = NULL;
bfead3b2 3168 int i, eq_msix;
6733b39a
JK		 3169
3170 phwi_ctrlr = phba->phwi_ctrlr;
3171 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3172 if (phba->msix_enabled)
3173 eq_msix = 1;
3174 else
3175 eq_msix = 0;
6733b39a 3176
bfead3b2
JK		 3177 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3178 eq = &phwi_context->be_eq[i].q;
6733b39a 3179 eqe = queue_tail_node(eq);
bfead3b2
JK		 3180
3181 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3182 & EQE_VALID_MASK) {
3183 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3184 queue_tail_inc(eq);
3185 eqe = queue_tail_node(eq);
3186 }
6733b39a
JK		 3187 }
3188}
3189
3190static void beiscsi_clean_port(struct beiscsi_hba *phba)
3191{
3192 unsigned char mgmt_status;
3193
3194 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3195 if (mgmt_status)
3196 shost_printk(KERN_WARNING, phba->shost,
3197 "mgmt_epfw_cleanup FAILED \n");
3198 hwi_cleanup(phba);
3199 hwi_purge_eq(phba);
35e66019
JK		 3200 if (ring_mode)
3201 kfree(phba->sgl_hndl_array);
6733b39a
JK		 3202 kfree(phba->io_sgl_hndl_base);
3203 kfree(phba->eh_sgl_hndl_base);
3204 kfree(phba->cid_array);
3205 kfree(phba->ep_array);
3206}
3207
3208void
3209beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3210 struct beiscsi_offload_params *params)
3211{
3212 struct wrb_handle *pwrb_handle;
3213 struct iscsi_target_context_update_wrb *pwrb = NULL;
3214 struct be_mem_descriptor *mem_descr;
3215 struct beiscsi_hba *phba = beiscsi_conn->phba;
3216 u32 doorbell = 0;
3217
3218 /*
3219 * We can always use 0 here because it is reserved by libiscsi for
3220 * login/startup related tasks.
3221 */
3222 pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid, 0);
3223 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3224 memset(pwrb, 0, sizeof(*pwrb));
3225 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3226 max_burst_length, pwrb, params->dw[offsetof
3227 (struct amap_beiscsi_offload_params,
3228 max_burst_length) / 32]);
3229 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3230 max_send_data_segment_length, pwrb,
3231 params->dw[offsetof(struct amap_beiscsi_offload_params,
3232 max_send_data_segment_length) / 32]);
3233 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3234 first_burst_length,
3235 pwrb,
3236 params->dw[offsetof(struct amap_beiscsi_offload_params,
3237 first_burst_length) / 32]);
3238
3239 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3240 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3241 erl) / 32] & OFFLD_PARAMS_ERL));
3242 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3243 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3244 dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3245 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3246 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3247 hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3248 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3249 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3250 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3251 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3252 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3253 imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3254 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3255 pwrb,
3256 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3257 exp_statsn) / 32] + 1));
3258 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3259 0x7);
3260 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3261 pwrb, pwrb_handle->wrb_index);
3262 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3263 pwrb, pwrb_handle->nxt_wrb_index);
3264 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3265 session_state, pwrb, 0);
3266 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3267 pwrb, 1);
3268 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3269 pwrb, 0);
3270 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3271 0);
3272
3273 mem_descr = phba->init_mem;
3274 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3275
3276 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3277 pad_buffer_addr_hi, pwrb,
3278 mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3279 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3280 pad_buffer_addr_lo, pwrb,
3281 mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3282
3283 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3284
3285 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
35e66019
JK		 3286 if (!ring_mode)
3287 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
bfead3b2 3288 << DB_DEF_PDU_WRB_INDEX_SHIFT;
6733b39a
JK		 3289 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3290
3291 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3292}
3293
3294static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3295 int *index, int *age)
3296{
bfead3b2 3297 *index = (int)itt;
6733b39a
JK		 3298 if (age)
3299 *age = conn->session->age;
3300}
3301
3302/**
3303 * beiscsi_alloc_pdu - allocates pdu and related resources
3304 * @task: libiscsi task
3305 * @opcode: opcode of pdu for task
3306 *
3307 * This is called with the session lock held. It will allocate
3308 * the wrb and sgl if needed for the command. And it will prep
3309 * the pdu's itt. beiscsi_parse_pdu will later translate
3310 * the pdu itt to the libiscsi task itt.
3311 */
3312static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3313{
3314 struct beiscsi_io_task *io_task = task->dd_data;
3315 struct iscsi_conn *conn = task->conn;
3316 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3317 struct beiscsi_hba *phba = beiscsi_conn->phba;
3318 struct hwi_wrb_context *pwrb_context;
3319 struct hwi_controller *phwi_ctrlr;
3320 itt_t itt;
2afc95bf
JK		 3321 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3322 dma_addr_t paddr;
6733b39a 3323
2afc95bf
JK		 3324 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3325 GFP_KERNEL, &paddr);
2afc95bf
JK		 3326 if (!io_task->cmd_bhs)
3327 return -ENOMEM;
2afc95bf 3328 io_task->bhs_pa.u.a64.address = paddr;
bfead3b2 3329 io_task->libiscsi_itt = (itt_t)task->itt;
6733b39a
JK		 3330 io_task->pwrb_handle = alloc_wrb_handle(phba,
3331 beiscsi_conn->beiscsi_conn_cid,
3332 task->itt);
6733b39a
JK		 3333 io_task->conn = beiscsi_conn;
3334
3335 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3336 task->hdr_max = sizeof(struct be_cmd_bhs);
3337
3338 if (task->sc) {
3339 spin_lock(&phba->io_sgl_lock);
3340 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3341 spin_unlock(&phba->io_sgl_lock);
2afc95bf
JK		 3342 if (!io_task->psgl_handle)
3343 goto free_hndls;
6733b39a
JK		 3344 } else {
3345 io_task->scsi_cmnd = NULL;
3346 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3347 if (!beiscsi_conn->login_in_progress) {
3348 spin_lock(&phba->mgmt_sgl_lock);
3349 io_task->psgl_handle = (struct sgl_handle *)
3350 alloc_mgmt_sgl_handle(phba);
3351 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3352 if (!io_task->psgl_handle)
3353 goto free_hndls;
3354
6733b39a
JK		 3355 beiscsi_conn->login_in_progress = 1;
3356 beiscsi_conn->plogin_sgl_handle =
3357 io_task->psgl_handle;
3358 } else {
3359 io_task->psgl_handle =
3360 beiscsi_conn->plogin_sgl_handle;
3361 }
3362 } else {
3363 spin_lock(&phba->mgmt_sgl_lock);
3364 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3365 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3366 if (!io_task->psgl_handle)
3367 goto free_hndls;
6733b39a
JK		 3368 }
3369 }
bfead3b2
JK		 3370 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3371 wrb_index << 16) | (unsigned int)
3372 (io_task->psgl_handle->sgl_index));
35e66019
JK		 3373 if (ring_mode) {
3374 phba->sgl_hndl_array[io_task->psgl_handle->sgl_index -
3375 phba->fw_config.iscsi_cid_start] =
3376 io_task->psgl_handle;
3377 io_task->psgl_handle->task = task;
3378 io_task->psgl_handle->cid = beiscsi_conn->beiscsi_conn_cid;
3379 } else
3380 io_task->pwrb_handle->pio_handle = task;
bfead3b2 3381
6733b39a
JK		 3382 io_task->cmd_bhs->iscsi_hdr.itt = itt;
3383 return 0;
2afc95bf
JK		 3384
3385free_hndls:
3386 phwi_ctrlr = phba->phwi_ctrlr;
3387 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
3388 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3389 io_task->pwrb_handle = NULL;
3390 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3391 io_task->bhs_pa.u.a64.address);
3392 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed \n");
3393 return -ENOMEM;
6733b39a
JK		 3394}
3395
3396static void beiscsi_cleanup_task(struct iscsi_task *task)
3397{
3398 struct beiscsi_io_task *io_task = task->dd_data;
3399 struct iscsi_conn *conn = task->conn;
3400 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3401 struct beiscsi_hba *phba = beiscsi_conn->phba;
2afc95bf 3402 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
6733b39a
JK		 3403 struct hwi_wrb_context *pwrb_context;
3404 struct hwi_controller *phwi_ctrlr;
3405
3406 phwi_ctrlr = phba->phwi_ctrlr;
3407 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid];
3408 if (io_task->pwrb_handle) {
3409 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3410 io_task->pwrb_handle = NULL;
3411 }
3412
2afc95bf
JK		 3413 if (io_task->cmd_bhs) {
3414 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3415 io_task->bhs_pa.u.a64.address);
3416 }
3417
6733b39a
JK		 3418 if (task->sc) {
3419 if (io_task->psgl_handle) {
3420 spin_lock(&phba->io_sgl_lock);
3421 free_io_sgl_handle(phba, io_task->psgl_handle);
3422 spin_unlock(&phba->io_sgl_lock);
3423 io_task->psgl_handle = NULL;
3424 }
3425 } else {
3426 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)
3427 return;
3428 if (io_task->psgl_handle) {
3429 spin_lock(&phba->mgmt_sgl_lock);
3430 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3431 spin_unlock(&phba->mgmt_sgl_lock);
3432 io_task->psgl_handle = NULL;
3433 }
3434 }
3435}
3436
3437static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3438 unsigned int num_sg, unsigned int xferlen,
3439 unsigned int writedir)
3440{
3441
3442 struct beiscsi_io_task *io_task = task->dd_data;
3443 struct iscsi_conn *conn = task->conn;
3444 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3445 struct beiscsi_hba *phba = beiscsi_conn->phba;
3446 struct iscsi_wrb *pwrb = NULL;
3447 unsigned int doorbell = 0;
3448
3449 pwrb = io_task->pwrb_handle->pwrb;
6733b39a
JK		 3450 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3451 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3452
3453 if (writedir) {
6733b39a
JK		 3454 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3455 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3456 &io_task->cmd_bhs->iscsi_data_pdu,
3457 (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3458 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3459 &io_task->cmd_bhs->iscsi_data_pdu,
3460 ISCSI_OPCODE_SCSI_DATA_OUT);
3461 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3462 &io_task->cmd_bhs->iscsi_data_pdu, 1);
35e66019
JK		 3463 if (ring_mode)
3464 io_task->psgl_handle->type = INI_WR_CMD;
3465 else
3466 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
bfead3b2 3467 INI_WR_CMD);
6733b39a 3468 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
6733b39a 3469 } else {
35e66019
JK		 3470 if (ring_mode)
3471 io_task->psgl_handle->type = INI_RD_CMD;
3472 else
3473 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
bfead3b2 3474 INI_RD_CMD);
6733b39a
JK		 3475 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3476 }
3477 memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3478 dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3479 io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3480
3481 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3482 cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr.
3483 lun[0]));
3484 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3485 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3486 io_task->pwrb_handle->wrb_index);
3487 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3488 be32_to_cpu(task->cmdsn));
3489 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3490 io_task->psgl_handle->sgl_index);
3491
3492 hwi_write_sgl(pwrb, sg, num_sg, io_task);
3493
3494 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3495 io_task->pwrb_handle->nxt_wrb_index);
3496 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3497
3498 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
35e66019
JK		 3499 if (!ring_mode)
3500 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 3501 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3502 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3503
3504 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3505 return 0;
3506}
3507
3508static int beiscsi_mtask(struct iscsi_task *task)
3509{
3510 struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data;
3511 struct iscsi_conn *conn = task->conn;
3512 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3513 struct beiscsi_hba *phba = beiscsi_conn->phba;
bfead3b2 3514 struct iscsi_session *session;
6733b39a 3515 struct iscsi_wrb *pwrb = NULL;
bfead3b2
JK		 3516 struct hwi_controller *phwi_ctrlr;
3517 struct hwi_wrb_context *pwrb_context;
3518 struct wrb_handle *pwrb_handle;
6733b39a 3519 unsigned int doorbell = 0;
bfead3b2 3520 unsigned int i, cid;
6733b39a
JK		 3521 struct iscsi_task *aborted_task;
3522
bfead3b2 3523 cid = beiscsi_conn->beiscsi_conn_cid;
6733b39a
JK		 3524 pwrb = io_task->pwrb_handle->pwrb;
3525 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
3526 be32_to_cpu(task->cmdsn));
3527 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3528 io_task->pwrb_handle->wrb_index);
3529 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
3530 io_task->psgl_handle->sgl_index);
3531
3532 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
3533 case ISCSI_OP_LOGIN:
35e66019
JK		 3534 if (ring_mode)
3535 io_task->psgl_handle->type = TGT_DM_CMD;
3536 else
3537 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
bfead3b2 3538 TGT_DM_CMD);
6733b39a
JK		 3539 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3540 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
3541 hwi_write_buffer(pwrb, task);
3542 break;
3543 case ISCSI_OP_NOOP_OUT:
35e66019
JK		 3544 if (ring_mode)
3545 io_task->psgl_handle->type = INI_RD_CMD;
3546 else
3547 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
bfead3b2 3548 INI_RD_CMD);
6733b39a
JK		 3549 hwi_write_buffer(pwrb, task);
3550 break;
3551 case ISCSI_OP_TEXT:
35e66019
JK		 3552 if (ring_mode)
3553 io_task->psgl_handle->type = INI_WR_CMD;
3554 else
3555 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
bfead3b2 3556 INI_WR_CMD);
6733b39a
JK		 3557 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3558 hwi_write_buffer(pwrb, task);
3559 break;
3560 case ISCSI_OP_SCSI_TMFUNC:
bfead3b2
JK		 3561 session = conn->session;
3562 i = ((struct iscsi_tm *)task->hdr)->rtt;
3563 phwi_ctrlr = phba->phwi_ctrlr;
3564 pwrb_context = &phwi_ctrlr->wrb_context[cid];
3565 pwrb_handle = pwrb_context->pwrb_handle_basestd[be32_to_cpu(i)
3566 >> 16];
3567 aborted_task = pwrb_handle->pio_handle;
6733b39a
JK		 3568 if (!aborted_task)
3569 return 0;
bfead3b2 3570
6733b39a
JK		 3571 aborted_io_task = aborted_task->dd_data;
3572 if (!aborted_io_task->scsi_cmnd)
3573 return 0;
3574
3575 mgmt_invalidate_icds(phba,
3576 aborted_io_task->psgl_handle->sgl_index,
bfead3b2 3577 cid);
35e66019
JK		 3578 if (ring_mode)
3579 io_task->psgl_handle->type = INI_TMF_CMD;
3580 else
3581 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
bfead3b2 3582 INI_TMF_CMD);
6733b39a
JK		 3583 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
3584 hwi_write_buffer(pwrb, task);
3585 break;
3586 case ISCSI_OP_LOGOUT:
3587 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
35e66019
JK		 3588 if (ring_mode)
3589 io_task->psgl_handle->type = HWH_TYPE_LOGOUT;
3590 else
6733b39a
JK		 3591 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3592 HWH_TYPE_LOGOUT);
3593 hwi_write_buffer(pwrb, task);
3594 break;
3595
3596 default:
3597 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported \n",
3598 task->hdr->opcode & ISCSI_OPCODE_MASK);
3599 return -EINVAL;
3600 }
3601
3602 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
3603 be32_to_cpu(task->data_count));
3604 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
3605 io_task->pwrb_handle->nxt_wrb_index);
3606 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
3607
bfead3b2 3608 doorbell |= cid & DB_WRB_POST_CID_MASK;
35e66019
JK		 3609 if (!ring_mode)
3610 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 3611 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
3612 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3613 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3614 return 0;
3615}
3616
3617static int beiscsi_task_xmit(struct iscsi_task *task)
3618{
3619 struct iscsi_conn *conn = task->conn;
3620 struct beiscsi_io_task *io_task = task->dd_data;
3621 struct scsi_cmnd *sc = task->sc;
3622 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3623 struct scatterlist *sg;
3624 int num_sg;
3625 unsigned int writedir = 0, xferlen = 0;
3626
3627 SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t"
3628 "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid,
3629 task, conn, beiscsi_conn);
3630 if (!sc)
3631 return beiscsi_mtask(task);
3632
3633 io_task->scsi_cmnd = sc;
3634 num_sg = scsi_dma_map(sc);
3635 if (num_sg < 0) {
3636 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
3637 return num_sg;
3638 }
3639 SE_DEBUG(DBG_LVL_4, "xferlen=0x%08x scmd=%p num_sg=%d sernum=%lu\n",
3640 (scsi_bufflen(sc)), sc, num_sg, sc->serial_number);
3641 xferlen = scsi_bufflen(sc);
3642 sg = scsi_sglist(sc);
3643 if (sc->sc_data_direction == DMA_TO_DEVICE) {
3644 writedir = 1;
3645 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x \n",
3646 task->imm_count);
3647 } else
3648 writedir = 0;
3649 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
3650}
3651
bfead3b2 3652
6733b39a
JK		 3653static void beiscsi_remove(struct pci_dev *pcidev)
3654{
3655 struct beiscsi_hba *phba = NULL;
bfead3b2
JK		 3656 struct hwi_controller *phwi_ctrlr;
3657 struct hwi_context_memory *phwi_context;
3658 struct be_eq_obj *pbe_eq;
3659 unsigned int i, msix_vec;
6733b39a
JK		 3660
3661 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
3662 if (!phba) {
3663 dev_err(&pcidev->dev, "beiscsi_remove called with no phba \n");
3664 return;
3665 }
3666
bfead3b2
JK		 3667 phwi_ctrlr = phba->phwi_ctrlr;
3668 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 3669 hwi_disable_intr(phba);
bfead3b2
JK		 3670 if (phba->msix_enabled) {
3671 for (i = 0; i <= phba->num_cpus; i++) {
3672 msix_vec = phba->msix_entries[i].vector;
3673 free_irq(msix_vec, &phwi_context->be_eq[i]);
3674 }
3675 } else
3676 if (phba->pcidev->irq)
3677 free_irq(phba->pcidev->irq, phba);
3678 pci_disable_msix(phba->pcidev);
6733b39a
JK		 3679 destroy_workqueue(phba->wq);
3680 if (blk_iopoll_enabled)
bfead3b2
JK		 3681 for (i = 0; i < phba->num_cpus; i++) {
3682 pbe_eq = &phwi_context->be_eq[i];
3683 blk_iopoll_disable(&pbe_eq->iopoll);
3684 }
6733b39a
JK		 3685
3686 beiscsi_clean_port(phba);
3687 beiscsi_free_mem(phba);
3688 beiscsi_unmap_pci_function(phba);
3689 pci_free_consistent(phba->pcidev,
3690 phba->ctrl.mbox_mem_alloced.size,
3691 phba->ctrl.mbox_mem_alloced.va,
3692 phba->ctrl.mbox_mem_alloced.dma);
3693 iscsi_host_remove(phba->shost);
3694 pci_dev_put(phba->pcidev);
3695 iscsi_host_free(phba->shost);
3696}
3697
bfead3b2
JK		 3698static void beiscsi_msix_enable(struct beiscsi_hba *phba)
3699{
3700 int i, status;
3701
3702 for (i = 0; i <= phba->num_cpus; i++)
3703 phba->msix_entries[i].entry = i;
3704
3705 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
3706 (phba->num_cpus + 1));
3707 if (!status)
3708 phba->msix_enabled = true;
3709
3710 return;
3711}
3712
6733b39a
JK		 3713static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
3714 const struct pci_device_id *id)
3715{
3716 struct beiscsi_hba *phba = NULL;
bfead3b2
JK		 3717 struct hwi_controller *phwi_ctrlr;
3718 struct hwi_context_memory *phwi_context;
3719 struct be_eq_obj *pbe_eq;
3720 int ret, msix_vec, num_cpus, i;
6733b39a
JK		 3721
3722 ret = beiscsi_enable_pci(pcidev);
3723 if (ret < 0) {
3724 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3725 "Failed to enable pci device \n");
3726 return ret;
3727 }
3728
3729 phba = beiscsi_hba_alloc(pcidev);
3730 if (!phba) {
3731 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
3732 " Failed in beiscsi_hba_alloc \n");
3733 goto disable_pci;
3734 }
bfead3b2 3735 SE_DEBUG(DBG_LVL_8, " phba = %p \n", phba);
6733b39a
JK		 3736
3737 pci_set_drvdata(pcidev, phba);
bfead3b2
JK		 3738 if (enable_msix)
3739 num_cpus = find_num_cpus();
3740 else
3741 num_cpus = 1;
3742 phba->num_cpus = num_cpus;
3743 SE_DEBUG(DBG_LVL_8, "num_cpus = %d \n", phba->num_cpus);
3744
3745 if (enable_msix)
3746 beiscsi_msix_enable(phba);
6733b39a
JK		 3747 ret = be_ctrl_init(phba, pcidev);
3748 if (ret) {
3749 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3750 "Failed in be_ctrl_init\n");
3751 goto hba_free;
3752 }
3753
3754 spin_lock_init(&phba->io_sgl_lock);
3755 spin_lock_init(&phba->mgmt_sgl_lock);
3756 spin_lock_init(&phba->isr_lock);
3757 beiscsi_get_params(phba);
3758 ret = beiscsi_init_port(phba);
3759 if (ret < 0) {
3760 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3761 "Failed in beiscsi_init_port\n");
3762 goto free_port;
3763 }
3764
3765 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
3766 phba->shost->host_no);
bfead3b2 3767 phba->wq = create_workqueue(phba->wq_name);
6733b39a
JK		 3768 if (!phba->wq) {
3769 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3770 "Failed to allocate work queue\n");
3771 goto free_twq;
3772 }
3773
3774 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
3775
bfead3b2
JK		 3776 phwi_ctrlr = phba->phwi_ctrlr;
3777 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 3778 if (blk_iopoll_enabled) {
bfead3b2
JK		 3779 for (i = 0; i < phba->num_cpus; i++) {
3780 pbe_eq = &phwi_context->be_eq[i];
3781 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
3782 be_iopoll);
3783 blk_iopoll_enable(&pbe_eq->iopoll);
3784 }
6733b39a 3785 }
6733b39a
JK		 3786 ret = beiscsi_init_irqs(phba);
3787 if (ret < 0) {
3788 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3789 "Failed to beiscsi_init_irqs\n");
3790 goto free_blkenbld;
3791 }
3792 ret = hwi_enable_intr(phba);
3793 if (ret < 0) {
3794 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
3795 "Failed to hwi_enable_intr\n");
3796 goto free_ctrlr;
3797 }
6733b39a
JK		 3798 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED \n\n\n");
3799 return 0;
3800
3801free_ctrlr:
bfead3b2
JK		 3802 if (phba->msix_enabled) {
3803 for (i = 0; i <= phba->num_cpus; i++) {
3804 msix_vec = phba->msix_entries[i].vector;
3805 free_irq(msix_vec, &phwi_context->be_eq[i]);
3806 }
3807 } else
3808 if (phba->pcidev->irq)
3809 free_irq(phba->pcidev->irq, phba);
3810 pci_disable_msix(phba->pcidev);
6733b39a
JK		 3811free_blkenbld:
3812 destroy_workqueue(phba->wq);
3813 if (blk_iopoll_enabled)
bfead3b2
JK		 3814 for (i = 0; i < phba->num_cpus; i++) {
3815 pbe_eq = &phwi_context->be_eq[i];
3816 blk_iopoll_disable(&pbe_eq->iopoll);
3817 }
6733b39a
JK		 3818free_twq:
3819 beiscsi_clean_port(phba);
3820 beiscsi_free_mem(phba);
3821free_port:
3822 pci_free_consistent(phba->pcidev,
3823 phba->ctrl.mbox_mem_alloced.size,
3824 phba->ctrl.mbox_mem_alloced.va,
3825 phba->ctrl.mbox_mem_alloced.dma);
3826 beiscsi_unmap_pci_function(phba);
3827hba_free:
3828 iscsi_host_remove(phba->shost);
3829 pci_dev_put(phba->pcidev);
3830 iscsi_host_free(phba->shost);
3831disable_pci:
3832 pci_disable_device(pcidev);
3833 return ret;
3834}
3835
3836struct iscsi_transport beiscsi_iscsi_transport = {
3837 .owner = THIS_MODULE,
3838 .name = DRV_NAME,
3839 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST |
3840 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
3841 .param_mask = ISCSI_MAX_RECV_DLENGTH |
3842 ISCSI_MAX_XMIT_DLENGTH |
3843 ISCSI_HDRDGST_EN |
3844 ISCSI_DATADGST_EN |
3845 ISCSI_INITIAL_R2T_EN |
3846 ISCSI_MAX_R2T |
3847 ISCSI_IMM_DATA_EN |
3848 ISCSI_FIRST_BURST |
3849 ISCSI_MAX_BURST |
3850 ISCSI_PDU_INORDER_EN |
3851 ISCSI_DATASEQ_INORDER_EN |
3852 ISCSI_ERL |
3853 ISCSI_CONN_PORT |
3854 ISCSI_CONN_ADDRESS |
3855 ISCSI_EXP_STATSN |
3856 ISCSI_PERSISTENT_PORT |
3857 ISCSI_PERSISTENT_ADDRESS |
3858 ISCSI_TARGET_NAME | ISCSI_TPGT |
3859 ISCSI_USERNAME | ISCSI_PASSWORD |
3860 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
3861 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
3fe5ae8b 3862 ISCSI_LU_RESET_TMO | ISCSI_TGT_RESET_TMO |
6733b39a
JK		 3863 ISCSI_PING_TMO | ISCSI_RECV_TMO |
3864 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
3865 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
3866 ISCSI_HOST_INITIATOR_NAME,
3867 .create_session = beiscsi_session_create,
3868 .destroy_session = beiscsi_session_destroy,
3869 .create_conn = beiscsi_conn_create,
3870 .bind_conn = beiscsi_conn_bind,
3871 .destroy_conn = iscsi_conn_teardown,
3872 .set_param = beiscsi_set_param,
3873 .get_conn_param = beiscsi_conn_get_param,
3874 .get_session_param = iscsi_session_get_param,
3875 .get_host_param = beiscsi_get_host_param,
3876 .start_conn = beiscsi_conn_start,
3877 .stop_conn = beiscsi_conn_stop,
3878 .send_pdu = iscsi_conn_send_pdu,
3879 .xmit_task = beiscsi_task_xmit,
3880 .cleanup_task = beiscsi_cleanup_task,
3881 .alloc_pdu = beiscsi_alloc_pdu,
3882 .parse_pdu_itt = beiscsi_parse_pdu,
3883 .get_stats = beiscsi_conn_get_stats,
3884 .ep_connect = beiscsi_ep_connect,
3885 .ep_poll = beiscsi_ep_poll,
3886 .ep_disconnect = beiscsi_ep_disconnect,
3887 .session_recovery_timedout = iscsi_session_recovery_timedout,
3888};
3889
3890static struct pci_driver beiscsi_pci_driver = {
3891 .name = DRV_NAME,
3892 .probe = beiscsi_dev_probe,
3893 .remove = beiscsi_remove,
3894 .id_table = beiscsi_pci_id_table
3895};
3896
bfead3b2 3897
6733b39a
JK		 3898static int __init beiscsi_module_init(void)
3899{
3900 int ret;
3901
3902 beiscsi_scsi_transport =
3903 iscsi_register_transport(&beiscsi_iscsi_transport);
3904 if (!beiscsi_scsi_transport) {
3905 SE_DEBUG(DBG_LVL_1,
3906 "beiscsi_module_init - Unable to register beiscsi"
3907 "transport.\n");
3908 ret = -ENOMEM;
3909 }
3910 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p \n",
3911 &beiscsi_iscsi_transport);
3912
3913 ret = pci_register_driver(&beiscsi_pci_driver);
3914 if (ret) {
3915 SE_DEBUG(DBG_LVL_1,
3916 "beiscsi_module_init - Unable to register"
3917 "beiscsi pci driver.\n");
3918 goto unregister_iscsi_transport;
3919 }
35e66019 3920 ring_mode = 0;
6733b39a
JK		 3921 return 0;
3922
3923unregister_iscsi_transport:
3924 iscsi_unregister_transport(&beiscsi_iscsi_transport);
3925 return ret;
3926}
3927
3928static void __exit beiscsi_module_exit(void)
3929{
3930 pci_unregister_driver(&beiscsi_pci_driver);
3931 iscsi_unregister_transport(&beiscsi_iscsi_transport);
3932}
3933
3934module_init(beiscsi_module_init);
3935module_exit(beiscsi_module_exit);
Linux 6.x block layer and io_uring tree(s)