projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: mpt3sas: Introduce Base function for cloning.
[linux-2.6-block.git] / drivers / scsi / mpt3sas / mpt3sas_base.c
CommitLineData
f92363d1
SR		 1/*
2 * This is the Fusion MPT base driver providing common API layer interface
3 * for access to MPT (Message Passing Technology) firmware.
4 *
5 * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c
a4ffce0d 6 * Copyright (C) 2012-2014 LSI Corporation
a03bd153
SR		 7 * Copyright (C) 2013-2014 Avago Technologies
8 * (mailto: MPT-FusionLinux.pdl@avagotech.com)
f92363d1
SR		 9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version 2
13 * of the License, or (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * NO WARRANTY
21 * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
22 * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
23 * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
24 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
25 * solely responsible for determining the appropriateness of using and
26 * distributing the Program and assumes all risks associated with its
27 * exercise of rights under this Agreement, including but not limited to
28 * the risks and costs of program errors, damage to or loss of data,
29 * programs or equipment, and unavailability or interruption of operations.
30
31 * DISCLAIMER OF LIABILITY
32 * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
33 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
37 * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
38 * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
39
40 * You should have received a copy of the GNU General Public License
41 * along with this program; if not, write to the Free Software
42 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
43 * USA.
44 */
45
f92363d1
SR		 46#include <linux/kernel.h>
47#include <linux/module.h>
48#include <linux/errno.h>
49#include <linux/init.h>
50#include <linux/slab.h>
51#include <linux/types.h>
52#include <linux/pci.h>
53#include <linux/kdev_t.h>
54#include <linux/blkdev.h>
55#include <linux/delay.h>
56#include <linux/interrupt.h>
57#include <linux/dma-mapping.h>
58#include <linux/io.h>
59#include <linux/time.h>
23409bd4 60#include <linux/ktime.h>
f92363d1 61#include <linux/kthread.h>
016d5c35 62#include <asm/page.h> /* To get host page size per arch */
f92363d1
SR		 63#include <linux/aer.h>
64
65
66#include "mpt3sas_base.h"
67
68static MPT_CALLBACK mpt_callbacks[MPT_MAX_CALLBACKS];
69
70
71#define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */
72
73 /* maximum controller queue depth */
74#define MAX_HBA_QUEUE_DEPTH 30000
75#define MAX_CHAIN_DEPTH 100000
76static int max_queue_depth = -1;
77module_param(max_queue_depth, int, 0);
78MODULE_PARM_DESC(max_queue_depth, " max controller queue depth ");
79
80static int max_sgl_entries = -1;
81module_param(max_sgl_entries, int, 0);
82MODULE_PARM_DESC(max_sgl_entries, " max sg entries ");
83
84static int msix_disable = -1;
85module_param(msix_disable, int, 0);
86MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
87
64038301
SPS		 88static int smp_affinity_enable = 1;
89module_param(smp_affinity_enable, int, S_IRUGO);
90MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disbale Default: enable(1)");
91
fb77bb53 92static int max_msix_vectors = -1;
9c500060
SR		 93module_param(max_msix_vectors, int, 0);
94MODULE_PARM_DESC(max_msix_vectors,
fb77bb53 95 " max msix vectors");
f92363d1
SR		 96
97static int mpt3sas_fwfault_debug;
98MODULE_PARM_DESC(mpt3sas_fwfault_debug,
99 " enable detection of firmware fault and halt firmware - (default=0)");
100
9b05c91a 101static int
98c56ad3 102_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
f92363d1
SR		 103
104/**
105 * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
106 *
107 */
108static int
e4dca7b7 109_scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp)
f92363d1
SR		 110{
111 int ret = param_set_int(val, kp);
112 struct MPT3SAS_ADAPTER *ioc;
113
114 if (ret)
115 return ret;
116
08c4d550 117 /* global ioc spinlock to protect controller list on list operations */
f92363d1 118 pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug);
08c4d550 119 spin_lock(&gioc_lock);
f92363d1
SR		 120 list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
121 ioc->fwfault_debug = mpt3sas_fwfault_debug;
08c4d550 122 spin_unlock(&gioc_lock);
f92363d1
SR		 123 return 0;
124}
125module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug,
126 param_get_int, &mpt3sas_fwfault_debug, 0644);
127
182ac784
SPS		 128/**
129 * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region
130 *
131 * @dst_iomem: Pointer to the destination location in BAR0 space.
132 * @src: Pointer to the Source data.
133 * @size: Size of data to be copied.
134 */
135static void
136_base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size)
137{
138 int i;
139 u32 *src_virt_mem = (u32 *)(src);
140
141 for (i = 0; i < size/4; i++)
142 writel((u32)src_virt_mem[i],
143 (void __iomem *)dst_iomem + (i * 4));
144}
145
22ae5a3c
SPS		 146/**
147 * _base_get_chain - Calculates and Returns virtual chain address
148 * for the provided smid in BAR0 space.
149 *
150 * @ioc: per adapter object
151 * @smid: system request message index
152 * @sge_chain_count: Scatter gather chain count.
153 *
154 * @Return: chain address.
155 */
156static inline void __iomem*
157_base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,
158 u8 sge_chain_count)
159{
160 void __iomem *base_chain, *chain_virt;
161 u16 cmd_credit = ioc->facts.RequestCredit + 1;
162
163 base_chain = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +
164 (cmd_credit * ioc->request_sz) +
165 REPLY_FREE_POOL_SIZE;
166 chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *
167 ioc->request_sz) + (sge_chain_count * ioc->request_sz);
168 return chain_virt;
169}
170
171/**
172 * _base_get_chain_phys - Calculates and Returns physical address
173 * in BAR0 for scatter gather chains, for
174 * the provided smid.
175 *
176 * @ioc: per adapter object
177 * @smid: system request message index
178 * @sge_chain_count: Scatter gather chain count.
179 *
180 * @Return - Physical chain address.
181 */
182static inline void *
183_base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,
184 u8 sge_chain_count)
185{
186 void *base_chain_phys, *chain_phys;
187 u16 cmd_credit = ioc->facts.RequestCredit + 1;
188
189 base_chain_phys = (void *)ioc->chip_phys + MPI_FRAME_START_OFFSET +
190 (cmd_credit * ioc->request_sz) +
191 REPLY_FREE_POOL_SIZE;
192 chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *
193 ioc->request_sz) + (sge_chain_count * ioc->request_sz);
194 return chain_phys;
195}
196
197/**
198 * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host
199 * buffer address for the provided smid.
200 * (Each smid can have 64K starts from 17024)
201 *
202 * @ioc: per adapter object
203 * @smid: system request message index
204 *
205 * @Returns - Pointer to buffer location in BAR0.
206 */
207
208static void __iomem *
209_base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
210{
211 u16 cmd_credit = ioc->facts.RequestCredit + 1;
212 // Added extra 1 to reach end of chain.
213 void __iomem *chain_end = _base_get_chain(ioc,
214 cmd_credit + 1,
215 ioc->facts.MaxChainDepth);
216 return chain_end + (smid * 64 * 1024);
217}
218
219/**
220 * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped
221 * Host buffer Physical address for the provided smid.
222 * (Each smid can have 64K starts from 17024)
223 *
224 * @ioc: per adapter object
225 * @smid: system request message index
226 *
227 * @Returns - Pointer to buffer location in BAR0.
228 */
229static void *
230_base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
231{
232 u16 cmd_credit = ioc->facts.RequestCredit + 1;
233 void *chain_end_phys = _base_get_chain_phys(ioc,
234 cmd_credit + 1,
235 ioc->facts.MaxChainDepth);
236 return chain_end_phys + (smid * 64 * 1024);
237}
238
182ac784
SPS		 239/**
240 * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain
241 * lookup list and Provides chain_buffer
242 * address for the matching dma address.
243 * (Each smid can have 64K starts from 17024)
244 *
245 * @ioc: per adapter object
246 * @chain_buffer_dma: Chain buffer dma address.
247 *
248 * @Returns - Pointer to chain buffer. Or Null on Failure.
249 */
250static void *
251_base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
252 dma_addr_t chain_buffer_dma)
253{
254 u16 index;
255
256 for (index = 0; index < ioc->chain_depth; index++) {
257 if (ioc->chain_lookup[index].chain_buffer_dma ==
258 chain_buffer_dma)
259 return ioc->chain_lookup[index].chain_buffer;
260 }
261 pr_info(MPT3SAS_FMT
262 "Provided chain_buffer_dma address is not in the lookup list\n",
263 ioc->name);
264 return NULL;
265}
266
267/**
268 * _clone_sg_entries - MPI EP's scsiio and config requests
269 * are handled here. Base function for
270 * double buffering, before submitting
271 * the requests.
272 *
273 * @ioc: per adapter object.
274 * @mpi_request: mf request pointer.
275 * @smid: system request message index.
276 *
277 * @Returns: Nothing.
278 */
279static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
280 void *mpi_request, u16 smid)
281{
282 Mpi2SGESimple32_t *sgel, *sgel_next;
283 u32 sgl_flags, sge_chain_count = 0;
284 bool is_write = 0;
285 u16 i = 0;
286 void __iomem *buffer_iomem;
287 void *buffer_iomem_phys;
288 void __iomem *buff_ptr;
289 void *buff_ptr_phys;
290 void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO];
291 void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO], *dst_addr_phys;
292 MPI2RequestHeader_t *request_hdr;
293 struct scsi_cmnd *scmd;
294 struct scatterlist *sg_scmd = NULL;
295 int is_scsiio_req = 0;
296
297 request_hdr = (MPI2RequestHeader_t *) mpi_request;
298
299 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
300 Mpi25SCSIIORequest_t *scsiio_request =
301 (Mpi25SCSIIORequest_t *)mpi_request;
302 sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL;
303 is_scsiio_req = 1;
304 } else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) {
305 Mpi2ConfigRequest_t *config_req =
306 (Mpi2ConfigRequest_t *)mpi_request;
307 sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE;
308 } else
309 return;
310
311 /* From smid we can get scsi_cmd, once we have sg_scmd,
312 * we just need to get sg_virt and sg_next to get virual
313 * address associated with sgel->Address.
314 */
315
316 if (is_scsiio_req) {
317 /* Get scsi_cmd using smid */
318 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
319 if (scmd == NULL) {
320 pr_err(MPT3SAS_FMT "scmd is NULL\n", ioc->name);
321 return;
322 }
323
324 /* Get sg_scmd from scmd provided */
325 sg_scmd = scsi_sglist(scmd);
326 }
327
328 /*
329 * 0 - 255 System register
330 * 256 - 4352 MPI Frame. (This is based on maxCredit 32)
331 * 4352 - 4864 Reply_free pool (512 byte is reserved
332 * considering maxCredit 32. Reply need extra
333 * room, for mCPU case kept four times of
334 * maxCredit).
335 * 4864 - 17152 SGE chain element. (32cmd * 3 chain of
336 * 128 byte size = 12288)
337 * 17152 - x Host buffer mapped with smid.
338 * (Each smid can have 64K Max IO.)
339 * BAR0+Last 1K MSIX Addr and Data
340 * Total size in use 2113664 bytes of 4MB BAR0
341 */
342
343 buffer_iomem = _base_get_buffer_bar0(ioc, smid);
344 buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);
345
346 buff_ptr = buffer_iomem;
347 buff_ptr_phys = buffer_iomem_phys;
348
349 if (sgel->FlagsLength &
350 (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
351 is_write = 1;
352
353 for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
354
355 sgl_flags = (sgel->FlagsLength >> MPI2_SGE_FLAGS_SHIFT);
356
357 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
358 case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
359 /*
360 * Helper function which on passing
361 * chain_buffer_dma returns chain_buffer. Get
362 * the virtual address for sgel->Address
363 */
364 sgel_next =
365 _base_get_chain_buffer_dma_to_chain_buffer(ioc,
366 sgel->Address);
367 if (sgel_next == NULL)
368 return;
369 /*
370 * This is coping 128 byte chain
371 * frame (not a host buffer)
372 */
373 dst_chain_addr[sge_chain_count] =
374 _base_get_chain(ioc,
375 smid, sge_chain_count);
376 src_chain_addr[sge_chain_count] =
377 (void *) sgel_next;
378 dst_addr_phys =
379 _base_get_chain_phys(ioc,
380 smid, sge_chain_count);
381 sgel->Address = (dma_addr_t)dst_addr_phys;
382 sgel = sgel_next;
383 sge_chain_count++;
384 break;
385 case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
386 if (is_write) {
387 if (is_scsiio_req) {
388 _base_clone_to_sys_mem(buff_ptr,
389 sg_virt(sg_scmd),
390 (sgel->FlagsLength & 0x00ffffff));
391 sgel->Address =
392 (dma_addr_t)buff_ptr_phys;
393 } else {
394 _base_clone_to_sys_mem(buff_ptr,
395 ioc->config_vaddr,
396 (sgel->FlagsLength & 0x00ffffff));
397 sgel->Address =
398 (dma_addr_t)buff_ptr_phys;
399 }
400 }
401 buff_ptr += (sgel->FlagsLength & 0x00ffffff);
402 buff_ptr_phys += (sgel->FlagsLength & 0x00ffffff);
403 if ((sgel->FlagsLength &
404 (MPI2_SGE_FLAGS_END_OF_BUFFER
405 << MPI2_SGE_FLAGS_SHIFT)))
406 goto eob_clone_chain;
407 else {
408 /*
409 * Every single element in MPT will have
410 * associated sg_next. Better to sanity that
411 * sg_next is not NULL, but it will be a bug
412 * if it is null.
413 */
414 if (is_scsiio_req) {
415 sg_scmd = sg_next(sg_scmd);
416 if (sg_scmd)
417 sgel++;
418 else
419 goto eob_clone_chain;
420 }
421 }
422 break;
423 }
424 }
425
426eob_clone_chain:
427 for (i = 0; i < sge_chain_count; i++) {
428 if (is_scsiio_req)
429 _base_clone_to_sys_mem(dst_chain_addr[i],
430 src_chain_addr[i], ioc->request_sz);
431 }
432}
433
f92363d1
SR		 434/**
435 * mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc
436 * @arg: input argument, used to derive ioc
437 *
438 * Return 0 if controller is removed from pci subsystem.
439 * Return -1 for other case.
440 */
441static int mpt3sas_remove_dead_ioc_func(void *arg)
442{
443 struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;
444 struct pci_dev *pdev;
445
446 if ((ioc == NULL))
447 return -1;
448
449 pdev = ioc->pdev;
450 if ((pdev == NULL))
451 return -1;
64cdb418 452 pci_stop_and_remove_bus_device_locked(pdev);
f92363d1
SR		 453 return 0;
454}
455
456/**
457 * _base_fault_reset_work - workq handling ioc fault conditions
458 * @work: input argument, used to derive ioc
459 * Context: sleep.
460 *
461 * Return nothing.
462 */
463static void
464_base_fault_reset_work(struct work_struct *work)
465{
466 struct MPT3SAS_ADAPTER *ioc =
467 container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work);
468 unsigned long flags;
469 u32 doorbell;
470 int rc;
471 struct task_struct *p;
472
473
474 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
16e179bd 475 if (ioc->shost_recovery || ioc->pci_error_recovery)
f92363d1
SR		 476 goto rearm_timer;
477 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
478
479 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
480 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) {
481 pr_err(MPT3SAS_FMT "SAS host is non-operational !!!!\n",
482 ioc->name);
483
16e179bd
SR		 484 /* It may be possible that EEH recovery can resolve some of
485 * pci bus failure issues rather removing the dead ioc function
486 * by considering controller is in a non-operational state. So
487 * here priority is given to the EEH recovery. If it doesn't
488 * not resolve this issue, mpt3sas driver will consider this
489 * controller to non-operational state and remove the dead ioc
490 * function.
491 */
492 if (ioc->non_operational_loop++ < 5) {
493 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,
494 flags);
495 goto rearm_timer;
496 }
497
f92363d1
SR		 498 /*
499 * Call _scsih_flush_pending_cmds callback so that we flush all
500 * pending commands back to OS. This call is required to aovid
501 * deadlock at block layer. Dead IOC will fail to do diag reset,
502 * and this call is safe since dead ioc will never return any
503 * command back from HW.
504 */
505 ioc->schedule_dead_ioc_flush_running_cmds(ioc);
506 /*
507 * Set remove_host flag early since kernel thread will
508 * take some time to execute.
509 */
510 ioc->remove_host = 1;
511 /*Remove the Dead Host */
512 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,
c84b06a4 513 "%s_dead_ioc_%d", ioc->driver_name, ioc->id);
f92363d1
SR		 514 if (IS_ERR(p))
515 pr_err(MPT3SAS_FMT
516 "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",
517 ioc->name, __func__);
518 else
519 pr_err(MPT3SAS_FMT
520 "%s: Running mpt3sas_dead_ioc thread success !!!!\n",
521 ioc->name, __func__);
522 return; /* don't rearm timer */
523 }
524
16e179bd
SR		 525 ioc->non_operational_loop = 0;
526
f92363d1 527 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
98c56ad3 528 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
f92363d1
SR		 529 pr_warn(MPT3SAS_FMT "%s: hard reset: %s\n", ioc->name,
530 __func__, (rc == 0) ? "success" : "failed");
531 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
532 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
533 mpt3sas_base_fault_info(ioc, doorbell &
534 MPI2_DOORBELL_DATA_MASK);
535 if (rc && (doorbell & MPI2_IOC_STATE_MASK) !=
536 MPI2_IOC_STATE_OPERATIONAL)
537 return; /* don't rearm timer */
538 }
539
540 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
541 rearm_timer:
542 if (ioc->fault_reset_work_q)
543 queue_delayed_work(ioc->fault_reset_work_q,
544 &ioc->fault_reset_work,
545 msecs_to_jiffies(FAULT_POLLING_INTERVAL));
546 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
547}
548
549/**
550 * mpt3sas_base_start_watchdog - start the fault_reset_work_q
551 * @ioc: per adapter object
552 * Context: sleep.
553 *
554 * Return nothing.
555 */
556void
557mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
558{
559 unsigned long flags;
560
561 if (ioc->fault_reset_work_q)
562 return;
563
564 /* initialize fault polling */
565
566 INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
567 snprintf(ioc->fault_reset_work_q_name,
c84b06a4
SR		 568 sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",
569 ioc->driver_name, ioc->id);
f92363d1
SR		 570 ioc->fault_reset_work_q =
571 create_singlethread_workqueue(ioc->fault_reset_work_q_name);
572 if (!ioc->fault_reset_work_q) {
573 pr_err(MPT3SAS_FMT "%s: failed (line=%d)\n",
574 ioc->name, __func__, __LINE__);
575 return;
576 }
577 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
578 if (ioc->fault_reset_work_q)
579 queue_delayed_work(ioc->fault_reset_work_q,
580 &ioc->fault_reset_work,
581 msecs_to_jiffies(FAULT_POLLING_INTERVAL));
582 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
583}
584
585/**
586 * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q
587 * @ioc: per adapter object
588 * Context: sleep.
589 *
590 * Return nothing.
591 */
592void
593mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)
594{
595 unsigned long flags;
596 struct workqueue_struct *wq;
597
598 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
599 wq = ioc->fault_reset_work_q;
600 ioc->fault_reset_work_q = NULL;
601 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
602 if (wq) {
4dc06fd8 603 if (!cancel_delayed_work_sync(&ioc->fault_reset_work))
f92363d1
SR		 604 flush_workqueue(wq);
605 destroy_workqueue(wq);
606 }
607}
608
609/**
610 * mpt3sas_base_fault_info - verbose translation of firmware FAULT code
611 * @ioc: per adapter object
612 * @fault_code: fault code
613 *
614 * Return nothing.
615 */
616void
617mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code)
618{
619 pr_err(MPT3SAS_FMT "fault_state(0x%04x)!\n",
620 ioc->name, fault_code);
621}
622
623/**
624 * mpt3sas_halt_firmware - halt's mpt controller firmware
625 * @ioc: per adapter object
626 *
627 * For debugging timeout related issues. Writing 0xCOFFEE00
628 * to the doorbell register will halt controller firmware. With
629 * the purpose to stop both driver and firmware, the enduser can
630 * obtain a ring buffer from controller UART.
631 */
632void
633mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)
634{
635 u32 doorbell;
636
637 if (!ioc->fwfault_debug)
638 return;
639
640 dump_stack();
641
642 doorbell = readl(&ioc->chip->Doorbell);
643 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
644 mpt3sas_base_fault_info(ioc , doorbell);
645 else {
646 writel(0xC0FFEE00, &ioc->chip->Doorbell);
647 pr_err(MPT3SAS_FMT "Firmware is halted due to command timeout\n",
648 ioc->name);
649 }
650
651 if (ioc->fwfault_debug == 2)
652 for (;;)
653 ;
654 else
655 panic("panic in %s\n", __func__);
656}
657
f92363d1
SR		 658/**
659 * _base_sas_ioc_info - verbose translation of the ioc status
660 * @ioc: per adapter object
661 * @mpi_reply: reply mf payload returned from firmware
662 * @request_hdr: request mf
663 *
664 * Return nothing.
665 */
666static void
667_base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,
668 MPI2RequestHeader_t *request_hdr)
669{
670 u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
671 MPI2_IOCSTATUS_MASK;
672 char *desc = NULL;
673 u16 frame_sz;
674 char *func_str = NULL;
675
676 /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */
677 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
678 request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
679 request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION)
680 return;
681
682 if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
683 return;
684
685 switch (ioc_status) {
686
687/****************************************************************************
688* Common IOCStatus values for all replies
689****************************************************************************/
690
691 case MPI2_IOCSTATUS_INVALID_FUNCTION:
692 desc = "invalid function";
693 break;
694 case MPI2_IOCSTATUS_BUSY:
695 desc = "busy";
696 break;
697 case MPI2_IOCSTATUS_INVALID_SGL:
698 desc = "invalid sgl";
699 break;
700 case MPI2_IOCSTATUS_INTERNAL_ERROR:
701 desc = "internal error";
702 break;
703 case MPI2_IOCSTATUS_INVALID_VPID:
704 desc = "invalid vpid";
705 break;
706 case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
707 desc = "insufficient resources";
708 break;
b130b0d5
SS		 709 case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
710 desc = "insufficient power";
711 break;
f92363d1
SR		 712 case MPI2_IOCSTATUS_INVALID_FIELD:
713 desc = "invalid field";
714 break;
715 case MPI2_IOCSTATUS_INVALID_STATE:
716 desc = "invalid state";
717 break;
718 case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
719 desc = "op state not supported";
720 break;
721
722/****************************************************************************
723* Config IOCStatus values
724****************************************************************************/
725
726 case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION:
727 desc = "config invalid action";
728 break;
729 case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE:
730 desc = "config invalid type";
731 break;
732 case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE:
733 desc = "config invalid page";
734 break;
735 case MPI2_IOCSTATUS_CONFIG_INVALID_DATA:
736 desc = "config invalid data";
737 break;
738 case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS:
739 desc = "config no defaults";
740 break;
741 case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT:
742 desc = "config cant commit";
743 break;
744
745/****************************************************************************
746* SCSI IO Reply
747****************************************************************************/
748
749 case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
750 case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
751 case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
752 case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
753 case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
754 case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
755 case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
756 case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
757 case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
758 case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
759 case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
760 case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
761 break;
762
763/****************************************************************************
764* For use by SCSI Initiator and SCSI Target end-to-end data protection
765****************************************************************************/
766
767 case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
768 desc = "eedp guard error";
769 break;
770 case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
771 desc = "eedp ref tag error";
772 break;
773 case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
774 desc = "eedp app tag error";
775 break;
776
777/****************************************************************************
778* SCSI Target values
779****************************************************************************/
780
781 case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX:
782 desc = "target invalid io index";
783 break;
784 case MPI2_IOCSTATUS_TARGET_ABORTED:
785 desc = "target aborted";
786 break;
787 case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE:
788 desc = "target no conn retryable";
789 break;
790 case MPI2_IOCSTATUS_TARGET_NO_CONNECTION:
791 desc = "target no connection";
792 break;
793 case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH:
794 desc = "target xfer count mismatch";
795 break;
796 case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR:
797 desc = "target data offset error";
798 break;
799 case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA:
800 desc = "target too much write data";
801 break;
802 case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT:
803 desc = "target iu too short";
804 break;
805 case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT:
806 desc = "target ack nak timeout";
807 break;
808 case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED:
809 desc = "target nak received";
810 break;
811
812/****************************************************************************
813* Serial Attached SCSI values
814****************************************************************************/
815
816 case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED:
817 desc = "smp request failed";
818 break;
819 case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN:
820 desc = "smp data overrun";
821 break;
822
823/****************************************************************************
824* Diagnostic Buffer Post / Diagnostic Release values
825****************************************************************************/
826
827 case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED:
828 desc = "diagnostic released";
829 break;
830 default:
831 break;
832 }
833
834 if (!desc)
835 return;
836
837 switch (request_hdr->Function) {
838 case MPI2_FUNCTION_CONFIG:
839 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;
840 func_str = "config_page";
841 break;
842 case MPI2_FUNCTION_SCSI_TASK_MGMT:
843 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t);
844 func_str = "task_mgmt";
845 break;
846 case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
847 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t);
848 func_str = "sas_iounit_ctl";
849 break;
850 case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR:
851 frame_sz = sizeof(Mpi2SepRequest_t);
852 func_str = "enclosure";
853 break;
854 case MPI2_FUNCTION_IOC_INIT:
855 frame_sz = sizeof(Mpi2IOCInitRequest_t);
856 func_str = "ioc_init";
857 break;
858 case MPI2_FUNCTION_PORT_ENABLE:
859 frame_sz = sizeof(Mpi2PortEnableRequest_t);
860 func_str = "port_enable";
861 break;
862 case MPI2_FUNCTION_SMP_PASSTHROUGH:
863 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;
864 func_str = "smp_passthru";
865 break;
aff39e61
SPS		 866 case MPI2_FUNCTION_NVME_ENCAPSULATED:
867 frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) +
868 ioc->sge_size;
869 func_str = "nvme_encapsulated";
870 break;
f92363d1
SR		 871 default:
872 frame_sz = 32;
873 func_str = "unknown";
874 break;
875 }
876
877 pr_warn(MPT3SAS_FMT "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",
878 ioc->name, desc, ioc_status, request_hdr, func_str);
879
880 _debug_dump_mf(request_hdr, frame_sz/4);
881}
882
883/**
884 * _base_display_event_data - verbose translation of firmware asyn events
885 * @ioc: per adapter object
886 * @mpi_reply: reply mf payload returned from firmware
887 *
888 * Return nothing.
889 */
890static void
891_base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
892 Mpi2EventNotificationReply_t *mpi_reply)
893{
894 char *desc = NULL;
895 u16 event;
896
897 if (!(ioc->logging_level & MPT_DEBUG_EVENTS))
898 return;
899
900 event = le16_to_cpu(mpi_reply->Event);
901
902 switch (event) {
903 case MPI2_EVENT_LOG_DATA:
904 desc = "Log Data";
905 break;
906 case MPI2_EVENT_STATE_CHANGE:
907 desc = "Status Change";
908 break;
909 case MPI2_EVENT_HARD_RESET_RECEIVED:
910 desc = "Hard Reset Received";
911 break;
912 case MPI2_EVENT_EVENT_CHANGE:
913 desc = "Event Change";
914 break;
915 case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
916 desc = "Device Status Change";
917 break;
918 case MPI2_EVENT_IR_OPERATION_STATUS:
7786ab6a
SR		 919 if (!ioc->hide_ir_msg)
920 desc = "IR Operation Status";
f92363d1
SR		 921 break;
922 case MPI2_EVENT_SAS_DISCOVERY:
923 {
924 Mpi2EventDataSasDiscovery_t *event_data =
925 (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData;
926 pr_info(MPT3SAS_FMT "Discovery: (%s)", ioc->name,
927 (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED) ?
928 "start" : "stop");
929 if (event_data->DiscoveryStatus)
bbaf61e2 930 pr_cont(" discovery_status(0x%08x)",
f92363d1 931 le32_to_cpu(event_data->DiscoveryStatus));
bbaf61e2 932 pr_cont("\n");
f92363d1
SR		 933 return;
934 }
935 case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
936 desc = "SAS Broadcast Primitive";
937 break;
938 case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
939 desc = "SAS Init Device Status Change";
940 break;
941 case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW:
942 desc = "SAS Init Table Overflow";
943 break;
944 case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
945 desc = "SAS Topology Change List";
946 break;
947 case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
948 desc = "SAS Enclosure Device Status Change";
949 break;
950 case MPI2_EVENT_IR_VOLUME:
7786ab6a
SR		 951 if (!ioc->hide_ir_msg)
952 desc = "IR Volume";
f92363d1
SR		 953 break;
954 case MPI2_EVENT_IR_PHYSICAL_DISK:
7786ab6a
SR		 955 if (!ioc->hide_ir_msg)
956 desc = "IR Physical Disk";
f92363d1
SR		 957 break;
958 case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
7786ab6a
SR		 959 if (!ioc->hide_ir_msg)
960 desc = "IR Configuration Change List";
f92363d1
SR		 961 break;
962 case MPI2_EVENT_LOG_ENTRY_ADDED:
7786ab6a
SR		 963 if (!ioc->hide_ir_msg)
964 desc = "Log Entry Added";
f92363d1 965 break;
2d8ce8c9
SR		 966 case MPI2_EVENT_TEMP_THRESHOLD:
967 desc = "Temperature Threshold";
968 break;
a470a51c 969 case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
b99b1993 970 desc = "Cable Event";
a470a51c 971 break;
4318c734
SPS		 972 case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
973 desc = "PCIE Device Status Change";
974 break;
975 case MPI2_EVENT_PCIE_ENUMERATION:
976 {
977 Mpi26EventDataPCIeEnumeration_t *event_data =
978 (Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData;
979 pr_info(MPT3SAS_FMT "PCIE Enumeration: (%s)", ioc->name,
980 (event_data->ReasonCode ==
981 MPI26_EVENT_PCIE_ENUM_RC_STARTED) ?
982 "start" : "stop");
983 if (event_data->EnumerationStatus)
984 pr_info("enumeration_status(0x%08x)",
985 le32_to_cpu(event_data->EnumerationStatus));
986 pr_info("\n");
987 return;
988 }
989 case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
990 desc = "PCIE Topology Change List";
991 break;
f92363d1
SR		 992 }
993
994 if (!desc)
995 return;
996
997 pr_info(MPT3SAS_FMT "%s\n", ioc->name, desc);
998}
f92363d1
SR		 999
1000/**
1001 * _base_sas_log_info - verbose translation of firmware log info
1002 * @ioc: per adapter object
1003 * @log_info: log info
1004 *
1005 * Return nothing.
1006 */
1007static void
1008_base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info)
1009{
1010 union loginfo_type {
1011 u32 loginfo;
1012 struct {
1013 u32 subcode:16;
1014 u32 code:8;
1015 u32 originator:4;
1016 u32 bus_type:4;
1017 } dw;
1018 };
1019 union loginfo_type sas_loginfo;
1020 char *originator_str = NULL;
1021
1022 sas_loginfo.loginfo = log_info;
1023 if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
1024 return;
1025
1026 /* each nexus loss loginfo */
1027 if (log_info == 0x31170000)
1028 return;
1029
1030 /* eat the loginfos associated with task aborts */
1031 if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==
1032 0x31140000 || log_info == 0x31130000))
1033 return;
1034
1035 switch (sas_loginfo.dw.originator) {
1036 case 0:
1037 originator_str = "IOP";
1038 break;
1039 case 1:
1040 originator_str = "PL";
1041 break;
1042 case 2:
7786ab6a
SR		 1043 if (!ioc->hide_ir_msg)
1044 originator_str = "IR";
1045 else
1046 originator_str = "WarpDrive";
f92363d1
SR		 1047 break;
1048 }
1049
1050 pr_warn(MPT3SAS_FMT
1051 "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",
1052 ioc->name, log_info,
1053 originator_str, sas_loginfo.dw.code,
1054 sas_loginfo.dw.subcode);
1055}
1056
1057/**
1058 * _base_display_reply_info -
1059 * @ioc: per adapter object
1060 * @smid: system request message index
1061 * @msix_index: MSIX table index supplied by the OS
1062 * @reply: reply message frame(lower 32bit addr)
1063 *
1064 * Return nothing.
1065 */
1066static void
1067_base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1068 u32 reply)
1069{
1070 MPI2DefaultReply_t *mpi_reply;
1071 u16 ioc_status;
1072 u32 loginfo = 0;
1073
1074 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1075 if (unlikely(!mpi_reply)) {
1076 pr_err(MPT3SAS_FMT "mpi_reply not valid at %s:%d/%s()!\n",
1077 ioc->name, __FILE__, __LINE__, __func__);
1078 return;
1079 }
1080 ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
af009411 1081
f92363d1
SR		 1082 if ((ioc_status & MPI2_IOCSTATUS_MASK) &&
1083 (ioc->logging_level & MPT_DEBUG_REPLY)) {
1084 _base_sas_ioc_info(ioc , mpi_reply,
1085 mpt3sas_base_get_msg_frame(ioc, smid));
1086 }
af009411 1087
f92363d1
SR		 1088 if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
1089 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo);
1090 _base_sas_log_info(ioc, loginfo);
1091 }
1092
1093 if (ioc_status || loginfo) {
1094 ioc_status &= MPI2_IOCSTATUS_MASK;
1095 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);
1096 }
1097}
1098
1099/**
1100 * mpt3sas_base_done - base internal command completion routine
1101 * @ioc: per adapter object
1102 * @smid: system request message index
1103 * @msix_index: MSIX table index supplied by the OS
1104 * @reply: reply message frame(lower 32bit addr)
1105 *
1106 * Return 1 meaning mf should be freed from _base_interrupt
1107 * 0 means the mf is freed from this function.
1108 */
1109u8
1110mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1111 u32 reply)
1112{
1113 MPI2DefaultReply_t *mpi_reply;
1114
1115 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1116 if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK)
fd0331b3 1117 return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
f92363d1
SR		 1118
1119 if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)
1120 return 1;
1121
1122 ioc->base_cmds.status |= MPT3_CMD_COMPLETE;
1123 if (mpi_reply) {
1124 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;
1125 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
1126 }
1127 ioc->base_cmds.status &= ~MPT3_CMD_PENDING;
1128
1129 complete(&ioc->base_cmds.done);
1130 return 1;
1131}
1132
1133/**
1134 * _base_async_event - main callback handler for firmware asyn events
1135 * @ioc: per adapter object
1136 * @msix_index: MSIX table index supplied by the OS
1137 * @reply: reply message frame(lower 32bit addr)
1138 *
1139 * Return 1 meaning mf should be freed from _base_interrupt
1140 * 0 means the mf is freed from this function.
1141 */
1142static u8
1143_base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
1144{
1145 Mpi2EventNotificationReply_t *mpi_reply;
1146 Mpi2EventAckRequest_t *ack_request;
1147 u16 smid;
fd0331b3 1148 struct _event_ack_list *delayed_event_ack;
f92363d1
SR		 1149
1150 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1151 if (!mpi_reply)
1152 return 1;
1153 if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION)
1154 return 1;
af009411 1155
f92363d1 1156 _base_display_event_data(ioc, mpi_reply);
af009411 1157
f92363d1
SR		 1158 if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED))
1159 goto out;
1160 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
1161 if (!smid) {
fd0331b3
SS		 1162 delayed_event_ack = kzalloc(sizeof(*delayed_event_ack),
1163 GFP_ATOMIC);
1164 if (!delayed_event_ack)
1165 goto out;
1166 INIT_LIST_HEAD(&delayed_event_ack->list);
1167 delayed_event_ack->Event = mpi_reply->Event;
1168 delayed_event_ack->EventContext = mpi_reply->EventContext;
1169 list_add_tail(&delayed_event_ack->list,
1170 &ioc->delayed_event_ack_list);
1171 dewtprintk(ioc, pr_info(MPT3SAS_FMT
1172 "DELAYED: EVENT ACK: event (0x%04x)\n",
1173 ioc->name, le16_to_cpu(mpi_reply->Event)));
f92363d1
SR		 1174 goto out;
1175 }
1176
1177 ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
1178 memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
1179 ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
1180 ack_request->Event = mpi_reply->Event;
1181 ack_request->EventContext = mpi_reply->EventContext;
1182 ack_request->VF_ID = 0; /* TODO */
1183 ack_request->VP_ID = 0;
81c16f83 1184 ioc->put_smid_default(ioc, smid);
f92363d1
SR		 1185
1186 out:
1187
1188 /* scsih callback handler */
1189 mpt3sas_scsih_event_callback(ioc, msix_index, reply);
1190
1191 /* ctl callback handler */
1192 mpt3sas_ctl_event_callback(ioc, msix_index, reply);
1193
1194 return 1;
1195}
1196
61dfb8a5 1197static struct scsiio_tracker *
dbec4c90 1198_get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
12e7c678 1199{
dbec4c90
SPS		 1200 struct scsi_cmnd *cmd;
1201
12e7c678
HR		 1202 if (WARN_ON(!smid) ||
1203 WARN_ON(smid >= ioc->hi_priority_smid))
1204 return NULL;
dbec4c90
SPS		 1205
1206 cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1207 if (cmd)
1208 return scsi_cmd_priv(cmd);
1209
1210 return NULL;
12e7c678
HR		 1211}
1212
f92363d1
SR		 1213/**
1214 * _base_get_cb_idx - obtain the callback index
1215 * @ioc: per adapter object
1216 * @smid: system request message index
1217 *
1218 * Return callback index.
1219 */
1220static u8
1221_base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1222{
1223 int i;
b0cd285e 1224 u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;
ba4494d4 1225 u8 cb_idx = 0xFF;
f92363d1
SR		 1226
1227 if (smid < ioc->hi_priority_smid) {
12e7c678
HR		 1228 struct scsiio_tracker *st;
1229
b0cd285e 1230 if (smid < ctl_smid) {
dbec4c90 1231 st = _get_st_from_smid(ioc, smid);
b0cd285e
HR		 1232 if (st)
1233 cb_idx = st->cb_idx;
1234 } else if (smid == ctl_smid)
1235 cb_idx = ioc->ctl_cb_idx;
f92363d1
SR		 1236 } else if (smid < ioc->internal_smid) {
1237 i = smid - ioc->hi_priority_smid;
1238 cb_idx = ioc->hpr_lookup[i].cb_idx;
1239 } else if (smid <= ioc->hba_queue_depth) {
1240 i = smid - ioc->internal_smid;
1241 cb_idx = ioc->internal_lookup[i].cb_idx;
ba4494d4 1242 }
f92363d1
SR		 1243 return cb_idx;
1244}
1245
1246/**
1247 * _base_mask_interrupts - disable interrupts
1248 * @ioc: per adapter object
1249 *
1250 * Disabling ResetIRQ, Reply and Doorbell Interrupts
1251 *
1252 * Return nothing.
1253 */
1254static void
1255_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1256{
1257 u32 him_register;
1258
1259 ioc->mask_interrupts = 1;
1260 him_register = readl(&ioc->chip->HostInterruptMask);
1261 him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK;
1262 writel(him_register, &ioc->chip->HostInterruptMask);
1263 readl(&ioc->chip->HostInterruptMask);
1264}
1265
1266/**
1267 * _base_unmask_interrupts - enable interrupts
1268 * @ioc: per adapter object
1269 *
1270 * Enabling only Reply Interrupts
1271 *
1272 * Return nothing.
1273 */
1274static void
1275_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1276{
1277 u32 him_register;
1278
1279 him_register = readl(&ioc->chip->HostInterruptMask);
1280 him_register &= ~MPI2_HIM_RIM;
1281 writel(him_register, &ioc->chip->HostInterruptMask);
1282 ioc->mask_interrupts = 0;
1283}
1284
1285union reply_descriptor {
1286 u64 word;
1287 struct {
1288 u32 low;
1289 u32 high;
1290 } u;
1291};
1292
1293/**
1294 * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
1295 * @irq: irq number (not used)
1296 * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
1297 * @r: pt_regs pointer (not used)
1298 *
1299 * Return IRQ_HANDLE if processed, else IRQ_NONE.
1300 */
1301static irqreturn_t
1302_base_interrupt(int irq, void *bus_id)
1303{
1304 struct adapter_reply_queue *reply_q = bus_id;
1305 union reply_descriptor rd;
1306 u32 completed_cmds;
1307 u8 request_desript_type;
1308 u16 smid;
1309 u8 cb_idx;
1310 u32 reply;
1311 u8 msix_index = reply_q->msix_index;
1312 struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1313 Mpi2ReplyDescriptorsUnion_t *rpf;
1314 u8 rc;
1315
1316 if (ioc->mask_interrupts)
1317 return IRQ_NONE;
1318
1319 if (!atomic_add_unless(&reply_q->busy, 1, 1))
1320 return IRQ_NONE;
1321
1322 rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
1323 request_desript_type = rpf->Default.ReplyFlags
1324 & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1325 if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
1326 atomic_dec(&reply_q->busy);
1327 return IRQ_NONE;
1328 }
1329
1330 completed_cmds = 0;
1331 cb_idx = 0xFF;
1332 do {
1333 rd.word = le64_to_cpu(rpf->Words);
1334 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX)
1335 goto out;
1336 reply = 0;
1337 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
1338 if (request_desript_type ==
1339 MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
1340 request_desript_type ==
aff39e61
SPS		 1341 MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
1342 request_desript_type ==
1343 MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) {
f92363d1
SR		 1344 cb_idx = _base_get_cb_idx(ioc, smid);
1345 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1346 (likely(mpt_callbacks[cb_idx] != NULL))) {
1347 rc = mpt_callbacks[cb_idx](ioc, smid,
1348 msix_index, 0);
1349 if (rc)
1350 mpt3sas_base_free_smid(ioc, smid);
1351 }
1352 } else if (request_desript_type ==
1353 MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
1354 reply = le32_to_cpu(
1355 rpf->AddressReply.ReplyFrameAddress);
1356 if (reply > ioc->reply_dma_max_address ||
1357 reply < ioc->reply_dma_min_address)
1358 reply = 0;
1359 if (smid) {
1360 cb_idx = _base_get_cb_idx(ioc, smid);
1361 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1362 (likely(mpt_callbacks[cb_idx] != NULL))) {
1363 rc = mpt_callbacks[cb_idx](ioc, smid,
1364 msix_index, reply);
1365 if (reply)
1366 _base_display_reply_info(ioc,
1367 smid, msix_index, reply);
1368 if (rc)
1369 mpt3sas_base_free_smid(ioc,
1370 smid);
1371 }
1372 } else {
1373 _base_async_event(ioc, msix_index, reply);
1374 }
1375
1376 /* reply free queue handling */
1377 if (reply) {
1378 ioc->reply_free_host_index =
1379 (ioc->reply_free_host_index ==
1380 (ioc->reply_free_queue_depth - 1)) ?
1381 0 : ioc->reply_free_host_index + 1;
1382 ioc->reply_free[ioc->reply_free_host_index] =
1383 cpu_to_le32(reply);
f92363d1
SR		 1384 writel(ioc->reply_free_host_index,
1385 &ioc->chip->ReplyFreeHostIndex);
1386 }
1387 }
1388
1389 rpf->Words = cpu_to_le64(ULLONG_MAX);
1390 reply_q->reply_post_host_index =
1391 (reply_q->reply_post_host_index ==
1392 (ioc->reply_post_queue_depth - 1)) ? 0 :
1393 reply_q->reply_post_host_index + 1;
1394 request_desript_type =
1395 reply_q->reply_post_free[reply_q->reply_post_host_index].
1396 Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1397 completed_cmds++;
6b4c335a
C		 1398 /* Update the reply post host index after continuously
1399 * processing the threshold number of Reply Descriptors.
1400 * So that FW can find enough entries to post the Reply
1401 * Descriptors in the reply descriptor post queue.
1402 */
1403 if (completed_cmds > ioc->hba_queue_depth/3) {
1404 if (ioc->combined_reply_queue) {
1405 writel(reply_q->reply_post_host_index |
1406 ((msix_index & 7) <<
1407 MPI2_RPHI_MSIX_INDEX_SHIFT),
1408 ioc->replyPostRegisterIndex[msix_index/8]);
1409 } else {
1410 writel(reply_q->reply_post_host_index |
1411 (msix_index <<
1412 MPI2_RPHI_MSIX_INDEX_SHIFT),
1413 &ioc->chip->ReplyPostHostIndex);
1414 }
1415 completed_cmds = 1;
1416 }
f92363d1
SR		 1417 if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
1418 goto out;
1419 if (!reply_q->reply_post_host_index)
1420 rpf = reply_q->reply_post_free;
1421 else
1422 rpf++;
1423 } while (1);
1424
1425 out:
1426
1427 if (!completed_cmds) {
1428 atomic_dec(&reply_q->busy);
1429 return IRQ_NONE;
1430 }
1431
7786ab6a
SR		 1432 if (ioc->is_warpdrive) {
1433 writel(reply_q->reply_post_host_index,
1434 ioc->reply_post_host_index[msix_index]);
1435 atomic_dec(&reply_q->busy);
1436 return IRQ_HANDLED;
1437 }
fb77bb53
SR		 1438
1439 /* Update Reply Post Host Index.
1440 * For those HBA's which support combined reply queue feature
1441 * 1. Get the correct Supplemental Reply Post Host Index Register.
1442 * i.e. (msix_index / 8)th entry from Supplemental Reply Post Host
1443 * Index Register address bank i.e replyPostRegisterIndex[],
1444 * 2. Then update this register with new reply host index value
1445 * in ReplyPostIndex field and the MSIxIndex field with
1446 * msix_index value reduced to a value between 0 and 7,
1447 * using a modulo 8 operation. Since each Supplemental Reply Post
1448 * Host Index Register supports 8 MSI-X vectors.
1449 *
1450 * For other HBA's just update the Reply Post Host Index register with
1451 * new reply host index value in ReplyPostIndex Field and msix_index
1452 * value in MSIxIndex field.
1453 */
0bb337c9 1454 if (ioc->combined_reply_queue)
fb77bb53
SR		 1455 writel(reply_q->reply_post_host_index | ((msix_index & 7) <<
1456 MPI2_RPHI_MSIX_INDEX_SHIFT),
1457 ioc->replyPostRegisterIndex[msix_index/8]);
1458 else
1459 writel(reply_q->reply_post_host_index | (msix_index <<
1460 MPI2_RPHI_MSIX_INDEX_SHIFT),
1461 &ioc->chip->ReplyPostHostIndex);
f92363d1
SR		 1462 atomic_dec(&reply_q->busy);
1463 return IRQ_HANDLED;
1464}
1465
1466/**
1467 * _base_is_controller_msix_enabled - is controller support muli-reply queues
1468 * @ioc: per adapter object
1469 *
1470 */
1471static inline int
1472_base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)
1473{
1474 return (ioc->facts.IOCCapabilities &
1475 MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;
1476}
1477
1478/**
5f0dfb7a 1479 * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts
f92363d1 1480 * @ioc: per adapter object
5f0dfb7a 1481 * Context: non ISR conext
f92363d1 1482 *
5f0dfb7a 1483 * Called when a Task Management request has completed.
f92363d1
SR		 1484 *
1485 * Return nothing.
1486 */
1487void
5f0dfb7a 1488mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 1489{
1490 struct adapter_reply_queue *reply_q;
1491
1492 /* If MSIX capability is turned off
1493 * then multi-queues are not enabled
1494 */
1495 if (!_base_is_controller_msix_enabled(ioc))
1496 return;
1497
1498 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
5f0dfb7a
C		 1499 if (ioc->shost_recovery || ioc->remove_host ||
1500 ioc->pci_error_recovery)
f92363d1
SR		 1501 return;
1502 /* TMs are on msix_index == 0 */
1503 if (reply_q->msix_index == 0)
1504 continue;
1d55abc0 1505 synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
f92363d1
SR		 1506 }
1507}
1508
1509/**
1510 * mpt3sas_base_release_callback_handler - clear interrupt callback handler
1511 * @cb_idx: callback index
1512 *
1513 * Return nothing.
1514 */
1515void
1516mpt3sas_base_release_callback_handler(u8 cb_idx)
1517{
1518 mpt_callbacks[cb_idx] = NULL;
1519}
1520
1521/**
1522 * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler
1523 * @cb_func: callback function
1524 *
1525 * Returns cb_func.
1526 */
1527u8
1528mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func)
1529{
1530 u8 cb_idx;
1531
1532 for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--)
1533 if (mpt_callbacks[cb_idx] == NULL)
1534 break;
1535
1536 mpt_callbacks[cb_idx] = cb_func;
1537 return cb_idx;
1538}
1539
1540/**
1541 * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler
1542 *
1543 * Return nothing.
1544 */
1545void
1546mpt3sas_base_initialize_callback_handler(void)
1547{
1548 u8 cb_idx;
1549
1550 for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++)
1551 mpt3sas_base_release_callback_handler(cb_idx);
1552}
1553
1554
1555/**
1556 * _base_build_zero_len_sge - build zero length sg entry
1557 * @ioc: per adapter object
1558 * @paddr: virtual address for SGE
1559 *
1560 * Create a zero length scatter gather entry to insure the IOCs hardware has
1561 * something to use if the target device goes brain dead and tries
1562 * to send data even when none is asked for.
1563 *
1564 * Return nothing.
1565 */
1566static void
1567_base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1568{
1569 u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT |
1570 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST |
1571 MPI2_SGE_FLAGS_SIMPLE_ELEMENT) <<
1572 MPI2_SGE_FLAGS_SHIFT);
1573 ioc->base_add_sg_single(paddr, flags_length, -1);
1574}
1575
1576/**
1577 * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr.
1578 * @paddr: virtual address for SGE
1579 * @flags_length: SGE flags and data transfer length
1580 * @dma_addr: Physical address
1581 *
1582 * Return nothing.
1583 */
1584static void
1585_base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1586{
1587 Mpi2SGESimple32_t *sgel = paddr;
1588
1589 flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING |
1590 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1591 sgel->FlagsLength = cpu_to_le32(flags_length);
1592 sgel->Address = cpu_to_le32(dma_addr);
1593}
1594
1595
1596/**
1597 * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr.
1598 * @paddr: virtual address for SGE
1599 * @flags_length: SGE flags and data transfer length
1600 * @dma_addr: Physical address
1601 *
1602 * Return nothing.
1603 */
1604static void
1605_base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1606{
1607 Mpi2SGESimple64_t *sgel = paddr;
1608
1609 flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
1610 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1611 sgel->FlagsLength = cpu_to_le32(flags_length);
1612 sgel->Address = cpu_to_le64(dma_addr);
1613}
1614
1615/**
1616 * _base_get_chain_buffer_tracker - obtain chain tracker
1617 * @ioc: per adapter object
dbec4c90 1618 * @scmd: SCSI commands of the IO request
f92363d1
SR		 1619 *
1620 * Returns chain tracker(from ioc->free_chain_list)
1621 */
1622static struct chain_tracker *
dbec4c90
SPS		 1623_base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
1624 struct scsi_cmnd *scmd)
f92363d1
SR		 1625{
1626 struct chain_tracker *chain_req;
dbec4c90 1627 struct scsiio_tracker *st = scsi_cmd_priv(scmd);
f92363d1
SR		 1628 unsigned long flags;
1629
1630 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
1631 if (list_empty(&ioc->free_chain_list)) {
1632 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
1633 dfailprintk(ioc, pr_warn(MPT3SAS_FMT
1634 "chain buffers not available\n", ioc->name));
1635 return NULL;
1636 }
1637 chain_req = list_entry(ioc->free_chain_list.next,
1638 struct chain_tracker, tracker_list);
1639 list_del_init(&chain_req->tracker_list);
dbec4c90 1640 list_add_tail(&chain_req->tracker_list, &st->chain_list);
f92363d1
SR		 1641 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
1642 return chain_req;
1643}
1644
1645
1646/**
1647 * _base_build_sg - build generic sg
1648 * @ioc: per adapter object
1649 * @psge: virtual address for SGE
1650 * @data_out_dma: physical address for WRITES
1651 * @data_out_sz: data xfer size for WRITES
1652 * @data_in_dma: physical address for READS
1653 * @data_in_sz: data xfer size for READS
1654 *
1655 * Return nothing.
1656 */
1657static void
1658_base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,
1659 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1660 size_t data_in_sz)
1661{
1662 u32 sgl_flags;
1663
1664 if (!data_out_sz && !data_in_sz) {
1665 _base_build_zero_len_sge(ioc, psge);
1666 return;
1667 }
1668
1669 if (data_out_sz && data_in_sz) {
1670 /* WRITE sgel first */
1671 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1672 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC);
1673 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1674 ioc->base_add_sg_single(psge, sgl_flags |
1675 data_out_sz, data_out_dma);
1676
1677 /* incr sgel */
1678 psge += ioc->sge_size;
1679
1680 /* READ sgel last */
1681 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1682 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1683 MPI2_SGE_FLAGS_END_OF_LIST);
1684 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1685 ioc->base_add_sg_single(psge, sgl_flags |
1686 data_in_sz, data_in_dma);
1687 } else if (data_out_sz) /* WRITE */ {
1688 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1689 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1690 MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC);
1691 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1692 ioc->base_add_sg_single(psge, sgl_flags |
1693 data_out_sz, data_out_dma);
1694 } else if (data_in_sz) /* READ */ {
1695 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1696 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1697 MPI2_SGE_FLAGS_END_OF_LIST);
1698 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1699 ioc->base_add_sg_single(psge, sgl_flags |
1700 data_in_sz, data_in_dma);
1701 }
1702}
1703
aff39e61
SPS		 1704/* IEEE format sgls */
1705
1706/**
1707 * _base_build_nvme_prp - This function is called for NVMe end devices to build
1708 * a native SGL (NVMe PRP). The native SGL is built starting in the first PRP
1709 * entry of the NVMe message (PRP1). If the data buffer is small enough to be
1710 * described entirely using PRP1, then PRP2 is not used. If needed, PRP2 is
1711 * used to describe a larger data buffer. If the data buffer is too large to
1712 * describe using the two PRP entriess inside the NVMe message, then PRP1
1713 * describes the first data memory segment, and PRP2 contains a pointer to a PRP
1714 * list located elsewhere in memory to describe the remaining data memory
1715 * segments. The PRP list will be contiguous.
1716
1717 * The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP
1718 * consists of a list of PRP entries to describe a number of noncontigous
1719 * physical memory segments as a single memory buffer, just as a SGL does. Note
1720 * however, that this function is only used by the IOCTL call, so the memory
1721 * given will be guaranteed to be contiguous. There is no need to translate
1722 * non-contiguous SGL into a PRP in this case. All PRPs will describe
1723 * contiguous space that is one page size each.
1724 *
1725 * Each NVMe message contains two PRP entries. The first (PRP1) either contains
1726 * a PRP list pointer or a PRP element, depending upon the command. PRP2
1727 * contains the second PRP element if the memory being described fits within 2
1728 * PRP entries, or a PRP list pointer if the PRP spans more than two entries.
1729 *
1730 * A PRP list pointer contains the address of a PRP list, structured as a linear
1731 * array of PRP entries. Each PRP entry in this list describes a segment of
1732 * physical memory.
1733 *
1734 * Each 64-bit PRP entry comprises an address and an offset field. The address
1735 * always points at the beginning of a 4KB physical memory page, and the offset
1736 * describes where within that 4KB page the memory segment begins. Only the
1737 * first element in a PRP list may contain a non-zero offest, implying that all
1738 * memory segments following the first begin at the start of a 4KB page.
1739 *
1740 * Each PRP element normally describes 4KB of physical memory, with exceptions
1741 * for the first and last elements in the list. If the memory being described
1742 * by the list begins at a non-zero offset within the first 4KB page, then the
1743 * first PRP element will contain a non-zero offset indicating where the region
1744 * begins within the 4KB page. The last memory segment may end before the end
1745 * of the 4KB segment, depending upon the overall size of the memory being
1746 * described by the PRP list.
1747 *
1748 * Since PRP entries lack any indication of size, the overall data buffer length
1749 * is used to determine where the end of the data memory buffer is located, and
1750 * how many PRP entries are required to describe it.
1751 *
1752 * @ioc: per adapter object
1753 * @smid: system request message index for getting asscociated SGL
1754 * @nvme_encap_request: the NVMe request msg frame pointer
1755 * @data_out_dma: physical address for WRITES
1756 * @data_out_sz: data xfer size for WRITES
1757 * @data_in_dma: physical address for READS
1758 * @data_in_sz: data xfer size for READS
1759 *
1760 * Returns nothing.
1761 */
1762static void
1763_base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,
1764 Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request,
1765 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1766 size_t data_in_sz)
1767{
1768 int prp_size = NVME_PRP_SIZE;
d8335ae2
AB		 1769 __le64 *prp_entry, *prp1_entry, *prp2_entry;
1770 __le64 *prp_page;
1771 dma_addr_t prp_entry_dma, prp_page_dma, dma_addr;
aff39e61
SPS		 1772 u32 offset, entry_len;
1773 u32 page_mask_result, page_mask;
aff39e61
SPS		 1774 size_t length;
1775
1776 /*
1777 * Not all commands require a data transfer. If no data, just return
1778 * without constructing any PRP.
1779 */
1780 if (!data_in_sz && !data_out_sz)
1781 return;
1782 /*
1783 * Set pointers to PRP1 and PRP2, which are in the NVMe command.
1784 * PRP1 is located at a 24 byte offset from the start of the NVMe
1785 * command. Then set the current PRP entry pointer to PRP1.
1786 */
494f401b 1787 prp1_entry = (__le64 *)(nvme_encap_request->NVMe_Command +
aff39e61 1788 NVME_CMD_PRP1_OFFSET);
494f401b 1789 prp2_entry = (__le64 *)(nvme_encap_request->NVMe_Command +
aff39e61
SPS		 1790 NVME_CMD_PRP2_OFFSET);
1791 prp_entry = prp1_entry;
1792 /*
1793 * For the PRP entries, use the specially allocated buffer of
1794 * contiguous memory.
1795 */
494f401b 1796 prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);
d8335ae2 1797 prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
aff39e61
SPS		 1798
1799 /*
1800 * Check if we are within 1 entry of a page boundary we don't
1801 * want our first entry to be a PRP List entry.
1802 */
1803 page_mask = ioc->page_size - 1;
1804 page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask;
1805 if (!page_mask_result) {
1806 /* Bump up to next page boundary. */
494f401b 1807 prp_page = (__le64 *)((u8 *)prp_page + prp_size);
d8335ae2 1808 prp_page_dma = prp_page_dma + prp_size;
aff39e61
SPS		 1809 }
1810
1811 /*
1812 * Set PRP physical pointer, which initially points to the current PRP
1813 * DMA memory page.
1814 */
d8335ae2 1815 prp_entry_dma = prp_page_dma;
aff39e61
SPS		 1816
1817 /* Get physical address and length of the data buffer. */
1818 if (data_in_sz) {
d8335ae2 1819 dma_addr = data_in_dma;
aff39e61
SPS		 1820 length = data_in_sz;
1821 } else {
d8335ae2 1822 dma_addr = data_out_dma;
aff39e61
SPS		 1823 length = data_out_sz;
1824 }
1825
1826 /* Loop while the length is not zero. */
1827 while (length) {
1828 /*
1829 * Check if we need to put a list pointer here if we are at
1830 * page boundary - prp_size (8 bytes).
1831 */
d8335ae2 1832 page_mask_result = (prp_entry_dma + prp_size) & page_mask;
aff39e61
SPS		 1833 if (!page_mask_result) {
1834 /*
1835 * This is the last entry in a PRP List, so we need to
1836 * put a PRP list pointer here. What this does is:
1837 * - bump the current memory pointer to the next
1838 * address, which will be the next full page.
1839 * - set the PRP Entry to point to that page. This
1840 * is now the PRP List pointer.
1841 * - bump the PRP Entry pointer the start of the
1842 * next page. Since all of this PRP memory is
1843 * contiguous, no need to get a new page - it's
1844 * just the next address.
1845 */
d8335ae2
AB		 1846 prp_entry_dma++;
1847 *prp_entry = cpu_to_le64(prp_entry_dma);
aff39e61
SPS		 1848 prp_entry++;
1849 }
1850
1851 /* Need to handle if entry will be part of a page. */
d8335ae2 1852 offset = dma_addr & page_mask;
aff39e61
SPS		 1853 entry_len = ioc->page_size - offset;
1854
1855 if (prp_entry == prp1_entry) {
1856 /*
1857 * Must fill in the first PRP pointer (PRP1) before
1858 * moving on.
1859 */
d8335ae2 1860 *prp1_entry = cpu_to_le64(dma_addr);
aff39e61
SPS		 1861
1862 /*
1863 * Now point to the second PRP entry within the
1864 * command (PRP2).
1865 */
1866 prp_entry = prp2_entry;
1867 } else if (prp_entry == prp2_entry) {
1868 /*
1869 * Should the PRP2 entry be a PRP List pointer or just
1870 * a regular PRP pointer? If there is more than one
1871 * more page of data, must use a PRP List pointer.
1872 */
1873 if (length > ioc->page_size) {
1874 /*
1875 * PRP2 will contain a PRP List pointer because
1876 * more PRP's are needed with this command. The
1877 * list will start at the beginning of the
1878 * contiguous buffer.
1879 */
d8335ae2 1880 *prp2_entry = cpu_to_le64(prp_entry_dma);
aff39e61
SPS		 1881
1882 /*
1883 * The next PRP Entry will be the start of the
1884 * first PRP List.
1885 */
1886 prp_entry = prp_page;
1887 } else {
1888 /*
1889 * After this, the PRP Entries are complete.
1890 * This command uses 2 PRP's and no PRP list.
1891 */
d8335ae2 1892 *prp2_entry = cpu_to_le64(dma_addr);
aff39e61
SPS		 1893 }
1894 } else {
1895 /*
1896 * Put entry in list and bump the addresses.
1897 *
1898 * After PRP1 and PRP2 are filled in, this will fill in
1899 * all remaining PRP entries in a PRP List, one per
1900 * each time through the loop.
1901 */
d8335ae2 1902 *prp_entry = cpu_to_le64(dma_addr);
aff39e61 1903 prp_entry++;
d8335ae2 1904 prp_entry_dma++;
aff39e61
SPS		 1905 }
1906
1907 /*
1908 * Bump the phys address of the command's data buffer by the
1909 * entry_len.
1910 */
d8335ae2 1911 dma_addr += entry_len;
aff39e61
SPS		 1912
1913 /* Decrement length accounting for last partial page. */
1914 if (entry_len > length)
1915 length = 0;
1916 else
1917 length -= entry_len;
1918 }
1919}
1920
016d5c35
SPS		 1921/**
1922 * base_make_prp_nvme -
1923 * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
1924 *
1925 * @ioc: per adapter object
1926 * @scmd: SCSI command from the mid-layer
1927 * @mpi_request: mpi request
1928 * @smid: msg Index
1929 * @sge_count: scatter gather element count.
1930 *
1931 * Returns: true: PRPs are built
1932 * false: IEEE SGLs needs to be built
1933 */
494f401b 1934static void
016d5c35
SPS		 1935base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,
1936 struct scsi_cmnd *scmd,
1937 Mpi25SCSIIORequest_t *mpi_request,
1938 u16 smid, int sge_count)
1939{
d8335ae2 1940 int sge_len, num_prp_in_chain = 0;
016d5c35 1941 Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl;
494f401b 1942 __le64 *curr_buff;
d8335ae2 1943 dma_addr_t msg_dma, sge_addr, offset;
016d5c35
SPS		 1944 u32 page_mask, page_mask_result;
1945 struct scatterlist *sg_scmd;
1946 u32 first_prp_len;
1947 int data_len = scsi_bufflen(scmd);
1948 u32 nvme_pg_size;
1949
1950 nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);
1951 /*
1952 * Nvme has a very convoluted prp format. One prp is required
1953 * for each page or partial page. Driver need to split up OS sg_list
1954 * entries if it is longer than one page or cross a page
1955 * boundary. Driver also have to insert a PRP list pointer entry as
1956 * the last entry in each physical page of the PRP list.
1957 *
1958 * NOTE: The first PRP "entry" is actually placed in the first
1959 * SGL entry in the main message as IEEE 64 format. The 2nd
1960 * entry in the main message is the chain element, and the rest
1961 * of the PRP entries are built in the contiguous pcie buffer.
1962 */
1963 page_mask = nvme_pg_size - 1;
1964
1965 /*
1966 * Native SGL is needed.
1967 * Put a chain element in main message frame that points to the first
1968 * chain buffer.
1969 *
1970 * NOTE: The ChainOffset field must be 0 when using a chain pointer to
1971 * a native SGL.
1972 */
1973
1974 /* Set main message chain element pointer */
1975 main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
1976 /*
1977 * For NVMe the chain element needs to be the 2nd SG entry in the main
1978 * message.
1979 */
1980 main_chain_element = (Mpi25IeeeSgeChain64_t *)
1981 ((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));
1982
1983 /*
1984 * For the PRP entries, use the specially allocated buffer of
1985 * contiguous memory. Normal chain buffers can't be used
1986 * because each chain buffer would need to be the size of an OS
1987 * page (4k).
1988 */
1989 curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);
d8335ae2 1990 msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
016d5c35 1991
d8335ae2 1992 main_chain_element->Address = cpu_to_le64(msg_dma);
016d5c35
SPS		 1993 main_chain_element->NextChainOffset = 0;
1994 main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
1995 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
1996 MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
1997
1998 /* Build first prp, sge need not to be page aligned*/
1999 ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2000 sg_scmd = scsi_sglist(scmd);
2001 sge_addr = sg_dma_address(sg_scmd);
2002 sge_len = sg_dma_len(sg_scmd);
2003
d8335ae2 2004 offset = sge_addr & page_mask;
016d5c35
SPS		 2005 first_prp_len = nvme_pg_size - offset;
2006
2007 ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2008 ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2009
2010 data_len -= first_prp_len;
2011
2012 if (sge_len > first_prp_len) {
2013 sge_addr += first_prp_len;
2014 sge_len -= first_prp_len;
2015 } else if (data_len && (sge_len == first_prp_len)) {
2016 sg_scmd = sg_next(sg_scmd);
2017 sge_addr = sg_dma_address(sg_scmd);
2018 sge_len = sg_dma_len(sg_scmd);
2019 }
2020
2021 for (;;) {
d8335ae2 2022 offset = sge_addr & page_mask;
016d5c35
SPS		 2023
2024 /* Put PRP pointer due to page boundary*/
2025 page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask;
2026 if (unlikely(!page_mask_result)) {
2027 scmd_printk(KERN_NOTICE,
2028 scmd, "page boundary curr_buff: 0x%p\n",
2029 curr_buff);
d8335ae2
AB		 2030 msg_dma += 8;
2031 *curr_buff = cpu_to_le64(msg_dma);
016d5c35
SPS		 2032 curr_buff++;
2033 num_prp_in_chain++;
2034 }
2035
2036 *curr_buff = cpu_to_le64(sge_addr);
2037 curr_buff++;
d8335ae2 2038 msg_dma += 8;
016d5c35
SPS		 2039 num_prp_in_chain++;
2040
2041 sge_addr += nvme_pg_size;
2042 sge_len -= nvme_pg_size;
2043 data_len -= nvme_pg_size;
2044
2045 if (data_len <= 0)
2046 break;
2047
2048 if (sge_len > 0)
2049 continue;
2050
2051 sg_scmd = sg_next(sg_scmd);
2052 sge_addr = sg_dma_address(sg_scmd);
2053 sge_len = sg_dma_len(sg_scmd);
2054 }
2055
2056 main_chain_element->Length =
2057 cpu_to_le32(num_prp_in_chain * sizeof(u64));
2058 return;
2059}
2060
2061static bool
2062base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,
2063 struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count)
2064{
2065 u32 data_length = 0;
2066 struct scatterlist *sg_scmd;
2067 bool build_prp = true;
2068
494f401b 2069 data_length = scsi_bufflen(scmd);
016d5c35
SPS		 2070 sg_scmd = scsi_sglist(scmd);
2071
2072 /* If Datalenth is <= 16K and number of SGE’s entries are <= 2
2073 * we built IEEE SGL
2074 */
2075 if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2))
2076 build_prp = false;
2077
2078 return build_prp;
2079}
2080
2081/**
2082 * _base_check_pcie_native_sgl - This function is called for PCIe end devices to
2083 * determine if the driver needs to build a native SGL. If so, that native
2084 * SGL is built in the special contiguous buffers allocated especially for
2085 * PCIe SGL creation. If the driver will not build a native SGL, return
2086 * TRUE and a normal IEEE SGL will be built. Currently this routine
2087 * supports NVMe.
2088 * @ioc: per adapter object
2089 * @mpi_request: mf request pointer
2090 * @smid: system request message index
2091 * @scmd: scsi command
2092 * @pcie_device: points to the PCIe device's info
2093 *
2094 * Returns 0 if native SGL was built, 1 if no SGL was built
2095 */
2096static int
2097_base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,
2098 Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd,
2099 struct _pcie_device *pcie_device)
2100{
2101 struct scatterlist *sg_scmd;
2102 int sges_left;
2103
2104 /* Get the SG list pointer and info. */
2105 sg_scmd = scsi_sglist(scmd);
2106 sges_left = scsi_dma_map(scmd);
2107 if (sges_left < 0) {
2108 sdev_printk(KERN_ERR, scmd->device,
2109 "scsi_dma_map failed: request for %d bytes!\n",
2110 scsi_bufflen(scmd));
2111 return 1;
2112 }
2113
2114 /* Check if we need to build a native SG list. */
2115 if (base_is_prp_possible(ioc, pcie_device,
2116 scmd, sges_left) == 0) {
2117 /* We built a native SG list, just return. */
2118 goto out;
2119 }
2120
2121 /*
2122 * Build native NVMe PRP.
2123 */
2124 base_make_prp_nvme(ioc, scmd, mpi_request,
2125 smid, sges_left);
2126
2127 return 0;
2128out:
2129 scsi_dma_unmap(scmd);
2130 return 1;
2131}
f92363d1
SR		 2132
2133/**
2134 * _base_add_sg_single_ieee - add sg element for IEEE format
2135 * @paddr: virtual address for SGE
2136 * @flags: SGE flags
2137 * @chain_offset: number of 128 byte elements from start of segment
2138 * @length: data transfer length
2139 * @dma_addr: Physical address
2140 *
2141 * Return nothing.
2142 */
2143static void
2144_base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length,
2145 dma_addr_t dma_addr)
2146{
2147 Mpi25IeeeSgeChain64_t *sgel = paddr;
2148
2149 sgel->Flags = flags;
2150 sgel->NextChainOffset = chain_offset;
2151 sgel->Length = cpu_to_le32(length);
2152 sgel->Address = cpu_to_le64(dma_addr);
2153}
2154
2155/**
2156 * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format
2157 * @ioc: per adapter object
2158 * @paddr: virtual address for SGE
2159 *
2160 * Create a zero length scatter gather entry to insure the IOCs hardware has
2161 * something to use if the target device goes brain dead and tries
2162 * to send data even when none is asked for.
2163 *
2164 * Return nothing.
2165 */
2166static void
2167_base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)
2168{
2169 u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2170 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2171 MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
b130b0d5 2172
f92363d1
SR		 2173 _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1);
2174}
2175
471ef9d4
SR		 2176/**
2177 * _base_build_sg_scmd - main sg creation routine
016d5c35 2178 * pcie_device is unused here!
471ef9d4
SR		 2179 * @ioc: per adapter object
2180 * @scmd: scsi command
2181 * @smid: system request message index
016d5c35 2182 * @unused: unused pcie_device pointer
471ef9d4
SR		 2183 * Context: none.
2184 *
2185 * The main routine that builds scatter gather table from a given
2186 * scsi request sent via the .queuecommand main handler.
2187 *
2188 * Returns 0 success, anything else error
2189 */
2190static int
2191_base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,
016d5c35 2192 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused)
471ef9d4
SR		 2193{
2194 Mpi2SCSIIORequest_t *mpi_request;
2195 dma_addr_t chain_dma;
2196 struct scatterlist *sg_scmd;
2197 void *sg_local, *chain;
2198 u32 chain_offset;
2199 u32 chain_length;
2200 u32 chain_flags;
2201 int sges_left;
2202 u32 sges_in_segment;
2203 u32 sgl_flags;
2204 u32 sgl_flags_last_element;
2205 u32 sgl_flags_end_buffer;
2206 struct chain_tracker *chain_req;
2207
2208 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2209
2210 /* init scatter gather flags */
2211 sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT;
2212 if (scmd->sc_data_direction == DMA_TO_DEVICE)
2213 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
2214 sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT)
2215 << MPI2_SGE_FLAGS_SHIFT;
2216 sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT |
2217 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST)
2218 << MPI2_SGE_FLAGS_SHIFT;
2219 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2220
2221 sg_scmd = scsi_sglist(scmd);
2222 sges_left = scsi_dma_map(scmd);
2223 if (sges_left < 0) {
2224 sdev_printk(KERN_ERR, scmd->device,
2225 "pci_map_sg failed: request for %d bytes!\n",
2226 scsi_bufflen(scmd));
2227 return -ENOMEM;
2228 }
2229
2230 sg_local = &mpi_request->SGL;
2231 sges_in_segment = ioc->max_sges_in_main_message;
2232 if (sges_left <= sges_in_segment)
2233 goto fill_in_last_segment;
2234
2235 mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) +
2236 (sges_in_segment * ioc->sge_size))/4;
2237
2238 /* fill in main message segment when there is a chain following */
2239 while (sges_in_segment) {
2240 if (sges_in_segment == 1)
2241 ioc->base_add_sg_single(sg_local,
2242 sgl_flags_last_element | sg_dma_len(sg_scmd),
2243 sg_dma_address(sg_scmd));
2244 else
2245 ioc->base_add_sg_single(sg_local, sgl_flags |
2246 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2247 sg_scmd = sg_next(sg_scmd);
2248 sg_local += ioc->sge_size;
2249 sges_left--;
2250 sges_in_segment--;
2251 }
2252
2253 /* initializing the chain flags and pointers */
2254 chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT;
dbec4c90 2255 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
471ef9d4
SR		 2256 if (!chain_req)
2257 return -1;
2258 chain = chain_req->chain_buffer;
2259 chain_dma = chain_req->chain_buffer_dma;
2260 do {
2261 sges_in_segment = (sges_left <=
2262 ioc->max_sges_in_chain_message) ? sges_left :
2263 ioc->max_sges_in_chain_message;
2264 chain_offset = (sges_left == sges_in_segment) ?
2265 0 : (sges_in_segment * ioc->sge_size)/4;
2266 chain_length = sges_in_segment * ioc->sge_size;
2267 if (chain_offset) {
2268 chain_offset = chain_offset <<
2269 MPI2_SGE_CHAIN_OFFSET_SHIFT;
2270 chain_length += ioc->sge_size;
2271 }
2272 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |
2273 chain_length, chain_dma);
2274 sg_local = chain;
2275 if (!chain_offset)
2276 goto fill_in_last_segment;
2277
2278 /* fill in chain segments */
2279 while (sges_in_segment) {
2280 if (sges_in_segment == 1)
2281 ioc->base_add_sg_single(sg_local,
2282 sgl_flags_last_element |
2283 sg_dma_len(sg_scmd),
2284 sg_dma_address(sg_scmd));
2285 else
2286 ioc->base_add_sg_single(sg_local, sgl_flags |
2287 sg_dma_len(sg_scmd),
2288 sg_dma_address(sg_scmd));
2289 sg_scmd = sg_next(sg_scmd);
2290 sg_local += ioc->sge_size;
2291 sges_left--;
2292 sges_in_segment--;
2293 }
2294
dbec4c90 2295 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
471ef9d4
SR		 2296 if (!chain_req)
2297 return -1;
2298 chain = chain_req->chain_buffer;
2299 chain_dma = chain_req->chain_buffer_dma;
2300 } while (1);
2301
2302
2303 fill_in_last_segment:
2304
2305 /* fill the last segment */
2306 while (sges_left) {
2307 if (sges_left == 1)
2308 ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |
2309 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2310 else
2311 ioc->base_add_sg_single(sg_local, sgl_flags |
2312 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2313 sg_scmd = sg_next(sg_scmd);
2314 sg_local += ioc->sge_size;
2315 sges_left--;
2316 }
2317
2318 return 0;
2319}
2320
f92363d1
SR		 2321/**
2322 * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format
2323 * @ioc: per adapter object
2324 * @scmd: scsi command
2325 * @smid: system request message index
016d5c35
SPS		 2326 * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be
2327 * constructed on need.
f92363d1
SR		 2328 * Context: none.
2329 *
2330 * The main routine that builds scatter gather table from a given
2331 * scsi request sent via the .queuecommand main handler.
2332 *
2333 * Returns 0 success, anything else error
2334 */
2335static int
2336_base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,
016d5c35 2337 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device)
f92363d1 2338{
016d5c35 2339 Mpi25SCSIIORequest_t *mpi_request;
f92363d1
SR		 2340 dma_addr_t chain_dma;
2341 struct scatterlist *sg_scmd;
2342 void *sg_local, *chain;
2343 u32 chain_offset;
2344 u32 chain_length;
f92363d1
SR		 2345 int sges_left;
2346 u32 sges_in_segment;
2347 u8 simple_sgl_flags;
2348 u8 simple_sgl_flags_last;
2349 u8 chain_sgl_flags;
2350 struct chain_tracker *chain_req;
2351
2352 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2353
2354 /* init scatter gather flags */
2355 simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2356 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2357 simple_sgl_flags_last = simple_sgl_flags |
2358 MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2359 chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2360 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2361
016d5c35
SPS		 2362 /* Check if we need to build a native SG list. */
2363 if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,
2364 smid, scmd, pcie_device) == 0)) {
2365 /* We built a native SG list, just return. */
2366 return 0;
2367 }
2368
f92363d1
SR		 2369 sg_scmd = scsi_sglist(scmd);
2370 sges_left = scsi_dma_map(scmd);
62f5c74c 2371 if (sges_left < 0) {
f92363d1
SR		 2372 sdev_printk(KERN_ERR, scmd->device,
2373 "pci_map_sg failed: request for %d bytes!\n",
2374 scsi_bufflen(scmd));
2375 return -ENOMEM;
2376 }
2377
2378 sg_local = &mpi_request->SGL;
2379 sges_in_segment = (ioc->request_sz -
016d5c35 2380 offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;
f92363d1
SR		 2381 if (sges_left <= sges_in_segment)
2382 goto fill_in_last_segment;
2383
2384 mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) +
016d5c35 2385 (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);
f92363d1
SR		 2386
2387 /* fill in main message segment when there is a chain following */
2388 while (sges_in_segment > 1) {
2389 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2390 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2391 sg_scmd = sg_next(sg_scmd);
2392 sg_local += ioc->sge_size_ieee;
2393 sges_left--;
2394 sges_in_segment--;
2395 }
2396
25ef16d0 2397 /* initializing the pointers */
dbec4c90 2398 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
f92363d1
SR		 2399 if (!chain_req)
2400 return -1;
2401 chain = chain_req->chain_buffer;
2402 chain_dma = chain_req->chain_buffer_dma;
2403 do {
2404 sges_in_segment = (sges_left <=
2405 ioc->max_sges_in_chain_message) ? sges_left :
2406 ioc->max_sges_in_chain_message;
2407 chain_offset = (sges_left == sges_in_segment) ?
2408 0 : sges_in_segment;
2409 chain_length = sges_in_segment * ioc->sge_size_ieee;
2410 if (chain_offset)
2411 chain_length += ioc->sge_size_ieee;
2412 _base_add_sg_single_ieee(sg_local, chain_sgl_flags,
2413 chain_offset, chain_length, chain_dma);
2414
2415 sg_local = chain;
2416 if (!chain_offset)
2417 goto fill_in_last_segment;
2418
2419 /* fill in chain segments */
2420 while (sges_in_segment) {
2421 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2422 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2423 sg_scmd = sg_next(sg_scmd);
2424 sg_local += ioc->sge_size_ieee;
2425 sges_left--;
2426 sges_in_segment--;
2427 }
2428
dbec4c90 2429 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
f92363d1
SR		 2430 if (!chain_req)
2431 return -1;
2432 chain = chain_req->chain_buffer;
2433 chain_dma = chain_req->chain_buffer_dma;
2434 } while (1);
2435
2436
2437 fill_in_last_segment:
2438
2439 /* fill the last segment */
62f5c74c 2440 while (sges_left > 0) {
f92363d1
SR		 2441 if (sges_left == 1)
2442 _base_add_sg_single_ieee(sg_local,
2443 simple_sgl_flags_last, 0, sg_dma_len(sg_scmd),
2444 sg_dma_address(sg_scmd));
2445 else
2446 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2447 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2448 sg_scmd = sg_next(sg_scmd);
2449 sg_local += ioc->sge_size_ieee;
2450 sges_left--;
2451 }
2452
2453 return 0;
2454}
2455
2456/**
2457 * _base_build_sg_ieee - build generic sg for IEEE format
2458 * @ioc: per adapter object
2459 * @psge: virtual address for SGE
2460 * @data_out_dma: physical address for WRITES
2461 * @data_out_sz: data xfer size for WRITES
2462 * @data_in_dma: physical address for READS
2463 * @data_in_sz: data xfer size for READS
2464 *
2465 * Return nothing.
2466 */
2467static void
2468_base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
2469 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2470 size_t data_in_sz)
2471{
2472 u8 sgl_flags;
2473
2474 if (!data_out_sz && !data_in_sz) {
2475 _base_build_zero_len_sge_ieee(ioc, psge);
2476 return;
2477 }
2478
2479 if (data_out_sz && data_in_sz) {
2480 /* WRITE sgel first */
2481 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2482 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2483 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2484 data_out_dma);
2485
2486 /* incr sgel */
2487 psge += ioc->sge_size_ieee;
2488
2489 /* READ sgel last */
2490 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2491 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2492 data_in_dma);
2493 } else if (data_out_sz) /* WRITE */ {
2494 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2495 MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2496 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2497 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2498 data_out_dma);
2499 } else if (data_in_sz) /* READ */ {
2500 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2501 MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2502 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2503 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2504 data_in_dma);
2505 }
2506}
2507
2508#define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10))
2509
2510/**
2511 * _base_config_dma_addressing - set dma addressing
2512 * @ioc: per adapter object
2513 * @pdev: PCI device struct
2514 *
2515 * Returns 0 for success, non-zero for failure.
2516 */
2517static int
2518_base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
2519{
2520 struct sysinfo s;
9b05c91a
SR		 2521 u64 consistent_dma_mask;
2522
0448f019
SPS		 2523 if (ioc->is_mcpu_endpoint)
2524 goto try_32bit;
2525
9b05c91a
SR		 2526 if (ioc->dma_mask)
2527 consistent_dma_mask = DMA_BIT_MASK(64);
2528 else
2529 consistent_dma_mask = DMA_BIT_MASK(32);
f92363d1
SR		 2530
2531 if (sizeof(dma_addr_t) > 4) {
2532 const uint64_t required_mask =
2533 dma_get_required_mask(&pdev->dev);
2534 if ((required_mask > DMA_BIT_MASK(32)) &&
2535 !pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) &&
9b05c91a 2536 !pci_set_consistent_dma_mask(pdev, consistent_dma_mask)) {
f92363d1
SR		 2537 ioc->base_add_sg_single = &_base_add_sg_single_64;
2538 ioc->sge_size = sizeof(Mpi2SGESimple64_t);
9b05c91a 2539 ioc->dma_mask = 64;
f92363d1
SR		 2540 goto out;
2541 }
2542 }
2543
0448f019 2544 try_32bit:
f92363d1
SR		 2545 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))
2546 && !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
2547 ioc->base_add_sg_single = &_base_add_sg_single_32;
2548 ioc->sge_size = sizeof(Mpi2SGESimple32_t);
9b05c91a 2549 ioc->dma_mask = 32;
f92363d1
SR		 2550 } else
2551 return -ENODEV;
2552
2553 out:
2554 si_meminfo(&s);
2555 pr_info(MPT3SAS_FMT
9b05c91a
SR		 2556 "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
2557 ioc->name, ioc->dma_mask, convert_to_kb(s.totalram));
2558
2559 return 0;
2560}
f92363d1 2561
9b05c91a
SR		 2562static int
2563_base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc,
2564 struct pci_dev *pdev)
2565{
2566 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
2567 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2568 return -ENODEV;
2569 }
f92363d1
SR		 2570 return 0;
2571}
2572
2573/**
2574 * _base_check_enable_msix - checks MSIX capabable.
2575 * @ioc: per adapter object
2576 *
2577 * Check to see if card is capable of MSIX, and set number
2578 * of available msix vectors
2579 */
2580static int
2581_base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2582{
2583 int base;
2584 u16 message_control;
2585
42081173
SR		 2586 /* Check whether controller SAS2008 B0 controller,
2587 * if it is SAS2008 B0 controller use IO-APIC instead of MSIX
2588 */
2589 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&
2590 ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {
2591 return -EINVAL;
2592 }
2593
f92363d1
SR		 2594 base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);
2595 if (!base) {
2596 dfailprintk(ioc, pr_info(MPT3SAS_FMT "msix not supported\n",
2597 ioc->name));
2598 return -EINVAL;
2599 }
2600
2601 /* get msix vector count */
42081173
SR		 2602 /* NUMA_IO not supported for older controllers */
2603 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||
2604 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||
2605 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||
2606 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||
2607 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||
2608 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||
2609 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)
2610 ioc->msix_vector_count = 1;
2611 else {
2612 pci_read_config_word(ioc->pdev, base + 2, &message_control);
2613 ioc->msix_vector_count = (message_control & 0x3FF) + 1;
2614 }
f92363d1
SR		 2615 dinitprintk(ioc, pr_info(MPT3SAS_FMT
2616 "msix is supported, vector_count(%d)\n",
2617 ioc->name, ioc->msix_vector_count));
2618 return 0;
2619}
2620
2621/**
2622 * _base_free_irq - free irq
2623 * @ioc: per adapter object
2624 *
2625 * Freeing respective reply_queue from the list.
2626 */
2627static void
2628_base_free_irq(struct MPT3SAS_ADAPTER *ioc)
2629{
2630 struct adapter_reply_queue *reply_q, *next;
2631
2632 if (list_empty(&ioc->reply_queue_list))
2633 return;
2634
2635 list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
2636 list_del(&reply_q->list);
1d55abc0
HR		 2637 free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
2638 reply_q);
f92363d1
SR		 2639 kfree(reply_q);
2640 }
2641}
2642
2643/**
2644 * _base_request_irq - request irq
2645 * @ioc: per adapter object
2646 * @index: msix index into vector table
f92363d1
SR		 2647 *
2648 * Inserting respective reply_queue into the list.
2649 */
2650static int
1d55abc0 2651_base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
f92363d1 2652{
1d55abc0 2653 struct pci_dev *pdev = ioc->pdev;
f92363d1
SR		 2654 struct adapter_reply_queue *reply_q;
2655 int r;
2656
2657 reply_q = kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL);
2658 if (!reply_q) {
2659 pr_err(MPT3SAS_FMT "unable to allocate memory %d!\n",
2660 ioc->name, (int)sizeof(struct adapter_reply_queue));
2661 return -ENOMEM;
2662 }
2663 reply_q->ioc = ioc;
2664 reply_q->msix_index = index;
14b3114d 2665
f92363d1
SR		 2666 atomic_set(&reply_q->busy, 0);
2667 if (ioc->msix_enable)
2668 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
c84b06a4 2669 ioc->driver_name, ioc->id, index);
f92363d1
SR		 2670 else
2671 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
c84b06a4 2672 ioc->driver_name, ioc->id);
1d55abc0
HR		 2673 r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
2674 IRQF_SHARED, reply_q->name, reply_q);
f92363d1
SR		 2675 if (r) {
2676 pr_err(MPT3SAS_FMT "unable to allocate interrupt %d!\n",
1d55abc0 2677 reply_q->name, pci_irq_vector(pdev, index));
da3cec25 2678 kfree(reply_q);
f92363d1
SR		 2679 return -EBUSY;
2680 }
2681
2682 INIT_LIST_HEAD(&reply_q->list);
2683 list_add_tail(&reply_q->list, &ioc->reply_queue_list);
2684 return 0;
2685}
2686
2687/**
2688 * _base_assign_reply_queues - assigning msix index for each cpu
2689 * @ioc: per adapter object
2690 *
2691 * The enduser would need to set the affinity via /proc/irq/#/smp_affinity
2692 *
2693 * It would nice if we could call irq_set_affinity, however it is not
2694 * an exported symbol
2695 */
2696static void
2697_base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
2698{
91b265bf 2699 unsigned int cpu, nr_cpus, nr_msix, index = 0;
14b3114d 2700 struct adapter_reply_queue *reply_q;
f92363d1
SR		 2701
2702 if (!_base_is_controller_msix_enabled(ioc))
2703 return;
2704
2705 memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
2706
91b265bf
MP		 2707 nr_cpus = num_online_cpus();
2708 nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,
2709 ioc->facts.MaxMSIxVectors);
2710 if (!nr_msix)
2711 return;
f92363d1 2712
1d55abc0
HR		 2713 if (smp_affinity_enable) {
2714 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
2715 const cpumask_t *mask = pci_irq_get_affinity(ioc->pdev,
2716 reply_q->msix_index);
2717 if (!mask) {
2718 pr_warn(MPT3SAS_FMT "no affinity for msi %x\n",
2719 ioc->name, reply_q->msix_index);
2720 continue;
2721 }
2722
2723 for_each_cpu(cpu, mask)
2724 ioc->cpu_msix_table[cpu] = reply_q->msix_index;
2725 }
2726 return;
2727 }
91b265bf
MP		 2728 cpu = cpumask_first(cpu_online_mask);
2729
14b3114d
SR		 2730 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
2731
91b265bf
MP		 2732 unsigned int i, group = nr_cpus / nr_msix;
2733
14b3114d
SR		 2734 if (cpu >= nr_cpus)
2735 break;
2736
91b265bf
MP		 2737 if (index < nr_cpus % nr_msix)
2738 group++;
2739
2740 for (i = 0 ; i < group ; i++) {
1d55abc0 2741 ioc->cpu_msix_table[cpu] = reply_q->msix_index;
91b265bf 2742 cpu = cpumask_next(cpu, cpu_online_mask);
f92363d1 2743 }
91b265bf 2744 index++;
14b3114d 2745 }
f92363d1
SR		 2746}
2747
2748/**
2749 * _base_disable_msix - disables msix
2750 * @ioc: per adapter object
2751 *
2752 */
2753static void
2754_base_disable_msix(struct MPT3SAS_ADAPTER *ioc)
2755{
2756 if (!ioc->msix_enable)
2757 return;
2758 pci_disable_msix(ioc->pdev);
2759 ioc->msix_enable = 0;
2760}
2761
2762/**
2763 * _base_enable_msix - enables msix, failback to io_apic
2764 * @ioc: per adapter object
2765 *
2766 */
2767static int
2768_base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2769{
f92363d1 2770 int r;
bb350661 2771 int i, local_max_msix_vectors;
f92363d1 2772 u8 try_msix = 0;
1d55abc0 2773 unsigned int irq_flags = PCI_IRQ_MSIX;
f92363d1 2774
f92363d1
SR		 2775 if (msix_disable == -1 || msix_disable == 0)
2776 try_msix = 1;
2777
2778 if (!try_msix)
2779 goto try_ioapic;
2780
2781 if (_base_check_enable_msix(ioc) != 0)
2782 goto try_ioapic;
2783
2784 ioc->reply_queue_count = min_t(int, ioc->cpu_count,
1d55abc0 2785 ioc->msix_vector_count);
f92363d1 2786
9c500060
SR		 2787 printk(MPT3SAS_FMT "MSI-X vectors supported: %d, no of cores"
2788 ": %d, max_msix_vectors: %d\n", ioc->name, ioc->msix_vector_count,
2789 ioc->cpu_count, max_msix_vectors);
2790
9b05c91a 2791 if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
06f5f976 2792 local_max_msix_vectors = (reset_devices) ? 1 : 8;
bb350661
SPS		 2793 else
2794 local_max_msix_vectors = max_msix_vectors;
9b05c91a 2795
1d55abc0 2796 if (local_max_msix_vectors > 0)
bb350661 2797 ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
9c500060 2798 ioc->reply_queue_count);
1d55abc0 2799 else if (local_max_msix_vectors == 0)
9b05c91a 2800 goto try_ioapic;
9c500060 2801
64038301
SPS		 2802 if (ioc->msix_vector_count < ioc->cpu_count)
2803 smp_affinity_enable = 0;
2804
1d55abc0
HR		 2805 if (smp_affinity_enable)
2806 irq_flags |= PCI_IRQ_AFFINITY;
f92363d1 2807
1d55abc0
HR		 2808 r = pci_alloc_irq_vectors(ioc->pdev, 1, ioc->reply_queue_count,
2809 irq_flags);
2810 if (r < 0) {
f92363d1 2811 dfailprintk(ioc, pr_info(MPT3SAS_FMT
1d55abc0 2812 "pci_alloc_irq_vectors failed (r=%d) !!!\n",
f92363d1 2813 ioc->name, r));
f92363d1
SR		 2814 goto try_ioapic;
2815 }
2816
2817 ioc->msix_enable = 1;
1d55abc0
HR		 2818 ioc->reply_queue_count = r;
2819 for (i = 0; i < ioc->reply_queue_count; i++) {
2820 r = _base_request_irq(ioc, i);
f92363d1
SR		 2821 if (r) {
2822 _base_free_irq(ioc);
2823 _base_disable_msix(ioc);
f92363d1
SR		 2824 goto try_ioapic;
2825 }
2826 }
2827
f92363d1
SR		 2828 return 0;
2829
2830/* failback to io_apic interrupt routing */
2831 try_ioapic:
2832
9b05c91a 2833 ioc->reply_queue_count = 1;
1d55abc0
HR		 2834 r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);
2835 if (r < 0) {
2836 dfailprintk(ioc, pr_info(MPT3SAS_FMT
2837 "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",
2838 ioc->name, r));
2839 } else
2840 r = _base_request_irq(ioc, 0);
f92363d1
SR		 2841
2842 return r;
2843}
2844
580d4e31
SR		 2845/**
2846 * mpt3sas_base_unmap_resources - free controller resources
2847 * @ioc: per adapter object
2848 */
8bbb1cf6 2849static void
580d4e31
SR		 2850mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
2851{
2852 struct pci_dev *pdev = ioc->pdev;
2853
2854 dexitprintk(ioc, printk(MPT3SAS_FMT "%s\n",
2855 ioc->name, __func__));
2856
2857 _base_free_irq(ioc);
2858 _base_disable_msix(ioc);
2859
0bb337c9 2860 if (ioc->combined_reply_queue) {
580d4e31 2861 kfree(ioc->replyPostRegisterIndex);
5f985d88
TH		 2862 ioc->replyPostRegisterIndex = NULL;
2863 }
580d4e31
SR		 2864
2865 if (ioc->chip_phys) {
2866 iounmap(ioc->chip);
2867 ioc->chip_phys = 0;
2868 }
2869
2870 if (pci_is_enabled(pdev)) {
2871 pci_release_selected_regions(ioc->pdev, ioc->bars);
2872 pci_disable_pcie_error_reporting(pdev);
2873 pci_disable_device(pdev);
2874 }
2875}
2876
f92363d1
SR		 2877/**
2878 * mpt3sas_base_map_resources - map in controller resources (io/irq/memap)
2879 * @ioc: per adapter object
2880 *
2881 * Returns 0 for success, non-zero for failure.
2882 */
2883int
2884mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
2885{
2886 struct pci_dev *pdev = ioc->pdev;
2887 u32 memap_sz;
2888 u32 pio_sz;
2889 int i, r = 0;
2890 u64 pio_chip = 0;
2891 u64 chip_phys = 0;
2892 struct adapter_reply_queue *reply_q;
2893
2894 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n",
2895 ioc->name, __func__));
2896
2897 ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
2898 if (pci_enable_device_mem(pdev)) {
2899 pr_warn(MPT3SAS_FMT "pci_enable_device_mem: failed\n",
2900 ioc->name);
cf9bd21a 2901 ioc->bars = 0;
f92363d1
SR		 2902 return -ENODEV;
2903 }
2904
2905
2906 if (pci_request_selected_regions(pdev, ioc->bars,
c84b06a4 2907 ioc->driver_name)) {
f92363d1
SR		 2908 pr_warn(MPT3SAS_FMT "pci_request_selected_regions: failed\n",
2909 ioc->name);
cf9bd21a 2910 ioc->bars = 0;
f92363d1
SR		 2911 r = -ENODEV;
2912 goto out_fail;
2913 }
2914
2915/* AER (Advanced Error Reporting) hooks */
2916 pci_enable_pcie_error_reporting(pdev);
2917
2918 pci_set_master(pdev);
2919
2920
2921 if (_base_config_dma_addressing(ioc, pdev) != 0) {
2922 pr_warn(MPT3SAS_FMT "no suitable DMA mask for %s\n",
2923 ioc->name, pci_name(pdev));
2924 r = -ENODEV;
2925 goto out_fail;
2926 }
2927
5aeeb78a
SR		 2928 for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) &&
2929 (!memap_sz || !pio_sz); i++) {
f92363d1
SR		 2930 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
2931 if (pio_sz)
2932 continue;
2933 pio_chip = (u64)pci_resource_start(pdev, i);
2934 pio_sz = pci_resource_len(pdev, i);
2935 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
2936 if (memap_sz)
2937 continue;
2938 ioc->chip_phys = pci_resource_start(pdev, i);
2939 chip_phys = (u64)ioc->chip_phys;
2940 memap_sz = pci_resource_len(pdev, i);
2941 ioc->chip = ioremap(ioc->chip_phys, memap_sz);
f92363d1
SR		 2942 }
2943 }
2944
5aeeb78a
SR		 2945 if (ioc->chip == NULL) {
2946 pr_err(MPT3SAS_FMT "unable to map adapter memory! "
2947 " or resource not found\n", ioc->name);
2948 r = -EINVAL;
2949 goto out_fail;
2950 }
2951
f92363d1 2952 _base_mask_interrupts(ioc);
9b05c91a 2953
98c56ad3 2954 r = _base_get_ioc_facts(ioc);
9b05c91a
SR		 2955 if (r)
2956 goto out_fail;
2957
2958 if (!ioc->rdpq_array_enable_assigned) {
2959 ioc->rdpq_array_enable = ioc->rdpq_array_capable;
2960 ioc->rdpq_array_enable_assigned = 1;
2961 }
2962
f92363d1
SR		 2963 r = _base_enable_msix(ioc);
2964 if (r)
2965 goto out_fail;
2966
fb77bb53
SR		 2967 /* Use the Combined reply queue feature only for SAS3 C0 & higher
2968 * revision HBAs and also only when reply queue count is greater than 8
2969 */
0bb337c9 2970 if (ioc->combined_reply_queue && ioc->reply_queue_count > 8) {
fb77bb53
SR		 2971 /* Determine the Supplemental Reply Post Host Index Registers
2972 * Addresse. Supplemental Reply Post Host Index Registers
2973 * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
2974 * each register is at offset bytes of
2975 * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
2976 */
2977 ioc->replyPostRegisterIndex = kcalloc(
0bb337c9 2978 ioc->combined_reply_index_count,
fb77bb53
SR		 2979 sizeof(resource_size_t *), GFP_KERNEL);
2980 if (!ioc->replyPostRegisterIndex) {
2981 dfailprintk(ioc, printk(MPT3SAS_FMT
2982 "allocation for reply Post Register Index failed!!!\n",
2983 ioc->name));
2984 r = -ENOMEM;
2985 goto out_fail;
2986 }
2987
0bb337c9 2988 for (i = 0; i < ioc->combined_reply_index_count; i++) {
fb77bb53
SR		 2989 ioc->replyPostRegisterIndex[i] = (resource_size_t *)
2990 ((u8 *)&ioc->chip->Doorbell +
2991 MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
2992 (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
2993 }
2994 } else
0bb337c9 2995 ioc->combined_reply_queue = 0;
fb77bb53 2996
ce7c6c9e
GE		 2997 if (ioc->is_warpdrive) {
2998 ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
2999 &ioc->chip->ReplyPostHostIndex;
3000
3001 for (i = 1; i < ioc->cpu_msix_table_sz; i++)
3002 ioc->reply_post_host_index[i] =
3003 (resource_size_t __iomem *)
3004 ((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
3005 * 4)));
3006 }
3007
f92363d1
SR		 3008 list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
3009 pr_info(MPT3SAS_FMT "%s: IRQ %d\n",
3010 reply_q->name, ((ioc->msix_enable) ? "PCI-MSI-X enabled" :
1d55abc0
HR		 3011 "IO-APIC enabled"),
3012 pci_irq_vector(ioc->pdev, reply_q->msix_index));
f92363d1
SR		 3013
3014 pr_info(MPT3SAS_FMT "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3015 ioc->name, (unsigned long long)chip_phys, ioc->chip, memap_sz);
3016 pr_info(MPT3SAS_FMT "ioport(0x%016llx), size(%d)\n",
3017 ioc->name, (unsigned long long)pio_chip, pio_sz);
3018
3019 /* Save PCI configuration state for recovery from PCI AER/EEH errors */
3020 pci_save_state(pdev);
3021 return 0;
3022
3023 out_fail:
580d4e31 3024 mpt3sas_base_unmap_resources(ioc);
f92363d1
SR		 3025 return r;
3026}
3027
3028/**
3029 * mpt3sas_base_get_msg_frame - obtain request mf pointer
3030 * @ioc: per adapter object
3031 * @smid: system request message index(smid zero is invalid)
3032 *
3033 * Returns virt pointer to message frame.
3034 */
3035void *
3036mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3037{
3038 return (void *)(ioc->request + (smid * ioc->request_sz));
3039}
3040
3041/**
3042 * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr
3043 * @ioc: per adapter object
3044 * @smid: system request message index
3045 *
3046 * Returns virt pointer to sense buffer.
3047 */
3048void *
3049mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3050{
3051 return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));
3052}
3053
3054/**
3055 * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr
3056 * @ioc: per adapter object
3057 * @smid: system request message index
3058 *
3059 * Returns phys pointer to the low 32bit address of the sense buffer.
3060 */
3061__le32
3062mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3063{
3064 return cpu_to_le32(ioc->sense_dma + ((smid - 1) *
3065 SCSI_SENSE_BUFFERSIZE));
3066}
3067
016d5c35
SPS		 3068/**
3069 * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr
3070 * @ioc: per adapter object
3071 * @smid: system request message index
3072 *
3073 * Returns virt pointer to a PCIe SGL.
3074 */
3075void *
3076mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3077{
dbec4c90 3078 return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);
016d5c35
SPS		 3079}
3080
3081/**
3082 * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr
3083 * @ioc: per adapter object
3084 * @smid: system request message index
3085 *
3086 * Returns phys pointer to the address of the PCIe buffer.
3087 */
d8335ae2 3088dma_addr_t
016d5c35
SPS		 3089mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3090{
dbec4c90 3091 return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;
016d5c35
SPS		 3092}
3093
f92363d1
SR		 3094/**
3095 * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address
3096 * @ioc: per adapter object
3097 * @phys_addr: lower 32 physical addr of the reply
3098 *
3099 * Converts 32bit lower physical addr into a virt address.
3100 */
3101void *
3102mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
3103{
3104 if (!phys_addr)
3105 return NULL;
3106 return ioc->reply + (phys_addr - (u32)ioc->reply_dma);
3107}
3108
03d1fb3a
SS		 3109static inline u8
3110_base_get_msix_index(struct MPT3SAS_ADAPTER *ioc)
3111{
3112 return ioc->cpu_msix_table[raw_smp_processor_id()];
3113}
3114
f92363d1
SR		 3115/**
3116 * mpt3sas_base_get_smid - obtain a free smid from internal queue
3117 * @ioc: per adapter object
3118 * @cb_idx: callback index
3119 *
3120 * Returns smid (zero is invalid)
3121 */
3122u16
3123mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3124{
3125 unsigned long flags;
3126 struct request_tracker *request;
3127 u16 smid;
3128
3129 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3130 if (list_empty(&ioc->internal_free_list)) {
3131 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3132 pr_err(MPT3SAS_FMT "%s: smid not available\n",
3133 ioc->name, __func__);
3134 return 0;
3135 }
3136
3137 request = list_entry(ioc->internal_free_list.next,
3138 struct request_tracker, tracker_list);
3139 request->cb_idx = cb_idx;
3140 smid = request->smid;
3141 list_del(&request->tracker_list);
3142 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3143 return smid;
3144}
3145
3146/**
3147 * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue
3148 * @ioc: per adapter object
3149 * @cb_idx: callback index
3150 * @scmd: pointer to scsi command object
3151 *
3152 * Returns smid (zero is invalid)
3153 */
3154u16
3155mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
3156 struct scsi_cmnd *scmd)
3157{
dbec4c90
SPS		 3158 struct scsiio_tracker *request = scsi_cmd_priv(scmd);
3159 unsigned int tag = scmd->request->tag;
f92363d1
SR		 3160 u16 smid;
3161
dbec4c90 3162 smid = tag + 1;
f92363d1 3163 request->cb_idx = cb_idx;
03d1fb3a 3164 request->msix_io = _base_get_msix_index(ioc);
dbec4c90
SPS		 3165 request->smid = smid;
3166 INIT_LIST_HEAD(&request->chain_list);
f92363d1
SR		 3167 return smid;
3168}
3169
3170/**
3171 * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue
3172 * @ioc: per adapter object
3173 * @cb_idx: callback index
3174 *
3175 * Returns smid (zero is invalid)
3176 */
3177u16
3178mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3179{
3180 unsigned long flags;
3181 struct request_tracker *request;
3182 u16 smid;
3183
3184 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3185 if (list_empty(&ioc->hpr_free_list)) {
3186 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3187 return 0;
3188 }
3189
3190 request = list_entry(ioc->hpr_free_list.next,
3191 struct request_tracker, tracker_list);
3192 request->cb_idx = cb_idx;
3193 smid = request->smid;
3194 list_del(&request->tracker_list);
3195 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3196 return smid;
3197}
3198
6a2d4618
HR		 3199static void
3200_base_recovery_check(struct MPT3SAS_ADAPTER *ioc)
3201{
3202 /*
3203 * See _wait_for_commands_to_complete() call with regards to this code.
3204 */
3205 if (ioc->shost_recovery && ioc->pending_io_count) {
272e253c
HR		 3206 ioc->pending_io_count = atomic_read(&ioc->shost->host_busy);
3207 if (ioc->pending_io_count == 0)
6a2d4618 3208 wake_up(&ioc->reset_wq);
6a2d4618
HR		 3209 }
3210}
3211
dbec4c90
SPS		 3212void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
3213 struct scsiio_tracker *st)
3214{
3215 if (WARN_ON(st->smid == 0))
3216 return;
3217 st->cb_idx = 0xFF;
3218 st->direct_io = 0;
3219 if (!list_empty(&st->chain_list)) {
3220 unsigned long flags;
3221
3222 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3223 list_splice_init(&st->chain_list, &ioc->free_chain_list);
3224 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3225 }
3226}
3227
f92363d1
SR		 3228/**
3229 * mpt3sas_base_free_smid - put smid back on free_list
3230 * @ioc: per adapter object
3231 * @smid: system request message index
3232 *
3233 * Return nothing.
3234 */
3235void
3236mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3237{
3238 unsigned long flags;
3239 int i;
f92363d1 3240
f92363d1 3241 if (smid < ioc->hi_priority_smid) {
dbec4c90 3242 struct scsiio_tracker *st;
f92363d1 3243
dbec4c90
SPS		 3244 st = _get_st_from_smid(ioc, smid);
3245 if (!st) {
3246 _base_recovery_check(ioc);
3247 return;
3248 }
3249 mpt3sas_base_clear_st(ioc, st);
6a2d4618 3250 _base_recovery_check(ioc);
f92363d1 3251 return;
dbec4c90
SPS		 3252 }
3253
3254 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3255 if (smid < ioc->internal_smid) {
f92363d1
SR		 3256 /* hi-priority */
3257 i = smid - ioc->hi_priority_smid;
3258 ioc->hpr_lookup[i].cb_idx = 0xFF;
3259 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);
3260 } else if (smid <= ioc->hba_queue_depth) {
3261 /* internal queue */
3262 i = smid - ioc->internal_smid;
3263 ioc->internal_lookup[i].cb_idx = 0xFF;
3264 list_add(&ioc->internal_lookup[i].tracker_list,
3265 &ioc->internal_free_list);
3266 }
3267 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3268}
3269
3270/**
3271 * _base_writeq - 64 bit write to MMIO
3272 * @ioc: per adapter object
3273 * @b: data payload
3274 * @addr: address in MMIO space
3275 * @writeq_lock: spin lock
3276 *
3277 * Glue for handling an atomic 64 bit word to MMIO. This special handling takes
3278 * care of 32 bit environment where its not quarenteed to send the entire word
3279 * in one transfer.
3280 */
3281#if defined(writeq) && defined(CONFIG_64BIT)
3282static inline void
3283_base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3284{
3285 writeq(cpu_to_le64(b), addr);
3286}
3287#else
3288static inline void
3289_base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3290{
3291 unsigned long flags;
3292 __u64 data_out = cpu_to_le64(b);
3293
3294 spin_lock_irqsave(writeq_lock, flags);
3295 writel((u32)(data_out), addr);
3296 writel((u32)(data_out >> 32), (addr + 4));
3297 spin_unlock_irqrestore(writeq_lock, flags);
3298}
3299#endif
3300
f92363d1 3301/**
81c16f83 3302 * _base_put_smid_scsi_io - send SCSI_IO request to firmware
f92363d1
SR		 3303 * @ioc: per adapter object
3304 * @smid: system request message index
3305 * @handle: device handle
3306 *
3307 * Return nothing.
3308 */
81c16f83
SPS		 3309static void
3310_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
f92363d1
SR		 3311{
3312 Mpi2RequestDescriptorUnion_t descriptor;
3313 u64 *request = (u64 *)&descriptor;
3314
3315
3316 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3317 descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
3318 descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3319 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3320 descriptor.SCSIIO.LMID = 0;
3321 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3322 &ioc->scsi_lookup_lock);
3323}
3324
3325/**
81c16f83 3326 * _base_put_smid_fast_path - send fast path request to firmware
f92363d1
SR		 3327 * @ioc: per adapter object
3328 * @smid: system request message index
3329 * @handle: device handle
3330 *
3331 * Return nothing.
3332 */
81c16f83
SPS		 3333static void
3334_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
f92363d1
SR		 3335 u16 handle)
3336{
3337 Mpi2RequestDescriptorUnion_t descriptor;
3338 u64 *request = (u64 *)&descriptor;
3339
3340 descriptor.SCSIIO.RequestFlags =
3341 MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
3342 descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
3343 descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3344 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3345 descriptor.SCSIIO.LMID = 0;
3346 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3347 &ioc->scsi_lookup_lock);
3348}
3349
3350/**
81c16f83 3351 * _base_put_smid_hi_priority - send Task Management request to firmware
f92363d1
SR		 3352 * @ioc: per adapter object
3353 * @smid: system request message index
03d1fb3a 3354 * @msix_task: msix_task will be same as msix of IO incase of task abort else 0.
f92363d1
SR		 3355 * Return nothing.
3356 */
81c16f83
SPS		 3357static void
3358_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
03d1fb3a 3359 u16 msix_task)
f92363d1
SR		 3360{
3361 Mpi2RequestDescriptorUnion_t descriptor;
3362 u64 *request = (u64 *)&descriptor;
3363
3364 descriptor.HighPriority.RequestFlags =
3365 MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
03d1fb3a 3366 descriptor.HighPriority.MSIxIndex = msix_task;
f92363d1
SR		 3367 descriptor.HighPriority.SMID = cpu_to_le16(smid);
3368 descriptor.HighPriority.LMID = 0;
3369 descriptor.HighPriority.Reserved1 = 0;
3370 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3371 &ioc->scsi_lookup_lock);
3372}
3373
aff39e61
SPS		 3374/**
3375 * _base_put_smid_nvme_encap - send NVMe encapsulated request to
3376 * firmware
3377 * @ioc: per adapter object
3378 * @smid: system request message index
3379 *
3380 * Return nothing.
3381 */
3382static void
3383_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3384{
3385 Mpi2RequestDescriptorUnion_t descriptor;
3386 u64 *request = (u64 *)&descriptor;
3387
3388 descriptor.Default.RequestFlags =
3389 MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
3390 descriptor.Default.MSIxIndex = _base_get_msix_index(ioc);
3391 descriptor.Default.SMID = cpu_to_le16(smid);
3392 descriptor.Default.LMID = 0;
3393 descriptor.Default.DescriptorTypeDependent = 0;
3394 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3395 &ioc->scsi_lookup_lock);
3396}
3397
f92363d1 3398/**
81c16f83 3399 * _base_put_smid_default - Default, primarily used for config pages
f92363d1
SR		 3400 * @ioc: per adapter object
3401 * @smid: system request message index
3402 *
3403 * Return nothing.
3404 */
81c16f83
SPS		 3405static void
3406_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
f92363d1
SR		 3407{
3408 Mpi2RequestDescriptorUnion_t descriptor;
3409 u64 *request = (u64 *)&descriptor;
3410
3411 descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3412 descriptor.Default.MSIxIndex = _base_get_msix_index(ioc);
3413 descriptor.Default.SMID = cpu_to_le16(smid);
3414 descriptor.Default.LMID = 0;
3415 descriptor.Default.DescriptorTypeDependent = 0;
3416 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3417 &ioc->scsi_lookup_lock);
3418}
3419
81c16f83
SPS		 3420/**
3421* _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using
3422* Atomic Request Descriptor
3423* @ioc: per adapter object
3424* @smid: system request message index
3425* @handle: device handle, unused in this function, for function type match
3426*
3427* Return nothing.
3428*/
3429static void
3430_base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3431 u16 handle)
3432{
3433 Mpi26AtomicRequestDescriptor_t descriptor;
3434 u32 *request = (u32 *)&descriptor;
3435
3436 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3437 descriptor.MSIxIndex = _base_get_msix_index(ioc);
3438 descriptor.SMID = cpu_to_le16(smid);
3439
3440 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
3441}
3442
3443/**
3444 * _base_put_smid_fast_path_atomic - send fast path request to firmware
3445 * using Atomic Request Descriptor
3446 * @ioc: per adapter object
3447 * @smid: system request message index
3448 * @handle: device handle, unused in this function, for function type match
3449 * Return nothing
3450 */
3451static void
3452_base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3453 u16 handle)
3454{
3455 Mpi26AtomicRequestDescriptor_t descriptor;
3456 u32 *request = (u32 *)&descriptor;
3457
3458 descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
3459 descriptor.MSIxIndex = _base_get_msix_index(ioc);
3460 descriptor.SMID = cpu_to_le16(smid);
3461
3462 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
3463}
3464
3465/**
3466 * _base_put_smid_hi_priority_atomic - send Task Management request to
3467 * firmware using Atomic Request Descriptor
3468 * @ioc: per adapter object
3469 * @smid: system request message index
3470 * @msix_task: msix_task will be same as msix of IO incase of task abort else 0
3471 *
3472 * Return nothing.
3473 */
3474static void
3475_base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3476 u16 msix_task)
3477{
3478 Mpi26AtomicRequestDescriptor_t descriptor;
3479 u32 *request = (u32 *)&descriptor;
3480
3481 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3482 descriptor.MSIxIndex = msix_task;
3483 descriptor.SMID = cpu_to_le16(smid);
3484
3485 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
3486}
3487
aff39e61
SPS		 3488/**
3489 * _base_put_smid_nvme_encap_atomic - send NVMe encapsulated request to
3490 * firmware using Atomic Request Descriptor
3491 * @ioc: per adapter object
3492 * @smid: system request message index
3493 *
3494 * Return nothing.
3495 */
3496static void
3497_base_put_smid_nvme_encap_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3498{
3499 Mpi26AtomicRequestDescriptor_t descriptor;
3500 u32 *request = (u32 *)&descriptor;
3501
3502 descriptor.RequestFlags = MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
3503 descriptor.MSIxIndex = _base_get_msix_index(ioc);
3504 descriptor.SMID = cpu_to_le16(smid);
3505
3506 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
3507}
3508
81c16f83
SPS		 3509/**
3510 * _base_put_smid_default - Default, primarily used for config pages
182ac784 3511 * use Atomic Request Descriptor
81c16f83
SPS		 3512 * @ioc: per adapter object
3513 * @smid: system request message index
3514 *
3515 * Return nothing.
3516 */
3517static void
3518_base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3519{
3520 Mpi26AtomicRequestDescriptor_t descriptor;
3521 u32 *request = (u32 *)&descriptor;
3522
3523 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3524 descriptor.MSIxIndex = _base_get_msix_index(ioc);
3525 descriptor.SMID = cpu_to_le16(smid);
3526
3527 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
3528}
3529
1117b31a 3530/**
989e43c7 3531 * _base_display_OEMs_branding - Display branding string
1117b31a
SR		 3532 * @ioc: per adapter object
3533 *
3534 * Return nothing.
3535 */
3536static void
989e43c7 3537_base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
1117b31a
SR		 3538{
3539 if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
3540 return;
3541
989e43c7
SR		 3542 switch (ioc->pdev->subsystem_vendor) {
3543 case PCI_VENDOR_ID_INTEL:
3544 switch (ioc->pdev->device) {
3545 case MPI2_MFGPAGE_DEVID_SAS2008:
3546 switch (ioc->pdev->subsystem_device) {
3547 case MPT2SAS_INTEL_RMS2LL080_SSDID:
3548 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3549 MPT2SAS_INTEL_RMS2LL080_BRANDING);
3550 break;
3551 case MPT2SAS_INTEL_RMS2LL040_SSDID:
3552 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3553 MPT2SAS_INTEL_RMS2LL040_BRANDING);
3554 break;
3555 case MPT2SAS_INTEL_SSD910_SSDID:
3556 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3557 MPT2SAS_INTEL_SSD910_BRANDING);
3558 break;
3559 default:
3560 pr_info(MPT3SAS_FMT
3561 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3562 ioc->name, ioc->pdev->subsystem_device);
3563 break;
3564 }
3565 case MPI2_MFGPAGE_DEVID_SAS2308_2:
3566 switch (ioc->pdev->subsystem_device) {
3567 case MPT2SAS_INTEL_RS25GB008_SSDID:
3568 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3569 MPT2SAS_INTEL_RS25GB008_BRANDING);
3570 break;
3571 case MPT2SAS_INTEL_RMS25JB080_SSDID:
3572 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3573 MPT2SAS_INTEL_RMS25JB080_BRANDING);
3574 break;
3575 case MPT2SAS_INTEL_RMS25JB040_SSDID:
3576 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3577 MPT2SAS_INTEL_RMS25JB040_BRANDING);
3578 break;
3579 case MPT2SAS_INTEL_RMS25KB080_SSDID:
3580 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3581 MPT2SAS_INTEL_RMS25KB080_BRANDING);
3582 break;
3583 case MPT2SAS_INTEL_RMS25KB040_SSDID:
3584 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3585 MPT2SAS_INTEL_RMS25KB040_BRANDING);
3586 break;
3587 case MPT2SAS_INTEL_RMS25LB040_SSDID:
3588 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3589 MPT2SAS_INTEL_RMS25LB040_BRANDING);
3590 break;
3591 case MPT2SAS_INTEL_RMS25LB080_SSDID:
3592 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3593 MPT2SAS_INTEL_RMS25LB080_BRANDING);
3594 break;
3595 default:
3596 pr_info(MPT3SAS_FMT
3597 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3598 ioc->name, ioc->pdev->subsystem_device);
3599 break;
3600 }
3601 case MPI25_MFGPAGE_DEVID_SAS3008:
3602 switch (ioc->pdev->subsystem_device) {
3603 case MPT3SAS_INTEL_RMS3JC080_SSDID:
3604 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3605 MPT3SAS_INTEL_RMS3JC080_BRANDING);
3606 break;
3607
3608 case MPT3SAS_INTEL_RS3GC008_SSDID:
3609 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3610 MPT3SAS_INTEL_RS3GC008_BRANDING);
3611 break;
3612 case MPT3SAS_INTEL_RS3FC044_SSDID:
3613 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3614 MPT3SAS_INTEL_RS3FC044_BRANDING);
3615 break;
3616 case MPT3SAS_INTEL_RS3UC080_SSDID:
3617 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3618 MPT3SAS_INTEL_RS3UC080_BRANDING);
3619 break;
3620 default:
3621 pr_info(MPT3SAS_FMT
3622 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3623 ioc->name, ioc->pdev->subsystem_device);
3624 break;
3625 }
1117b31a
SR		 3626 break;
3627 default:
3628 pr_info(MPT3SAS_FMT
989e43c7
SR		 3629 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3630 ioc->name, ioc->pdev->subsystem_device);
1117b31a
SR		 3631 break;
3632 }
3633 break;
989e43c7
SR		 3634 case PCI_VENDOR_ID_DELL:
3635 switch (ioc->pdev->device) {
3636 case MPI2_MFGPAGE_DEVID_SAS2008:
3637 switch (ioc->pdev->subsystem_device) {
3638 case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID:
3639 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3640 MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING);
3641 break;
3642 case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID:
3643 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3644 MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING);
3645 break;
3646 case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID:
3647 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3648 MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING);
3649 break;
3650 case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID:
3651 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3652 MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING);
3653 break;
3654 case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID:
3655 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3656 MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING);
3657 break;
3658 case MPT2SAS_DELL_PERC_H200_SSDID:
3659 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3660 MPT2SAS_DELL_PERC_H200_BRANDING);
3661 break;
3662 case MPT2SAS_DELL_6GBPS_SAS_SSDID:
3663 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3664 MPT2SAS_DELL_6GBPS_SAS_BRANDING);
3665 break;
3666 default:
3667 pr_info(MPT3SAS_FMT
3668 "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",
3669 ioc->name, ioc->pdev->subsystem_device);
3670 break;
3671 }
3672 break;
3673 case MPI25_MFGPAGE_DEVID_SAS3008:
3674 switch (ioc->pdev->subsystem_device) {
3675 case MPT3SAS_DELL_12G_HBA_SSDID:
3676 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3677 MPT3SAS_DELL_12G_HBA_BRANDING);
3678 break;
3679 default:
3680 pr_info(MPT3SAS_FMT
3681 "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",
3682 ioc->name, ioc->pdev->subsystem_device);
3683 break;
3684 }
fb84dfc4
SR		 3685 break;
3686 default:
3687 pr_info(MPT3SAS_FMT
989e43c7 3688 "Dell HBA: Subsystem ID: 0x%X\n", ioc->name,
fb84dfc4
SR		 3689 ioc->pdev->subsystem_device);
3690 break;
3691 }
3692 break;
989e43c7
SR		 3693 case PCI_VENDOR_ID_CISCO:
3694 switch (ioc->pdev->device) {
3695 case MPI25_MFGPAGE_DEVID_SAS3008:
3696 switch (ioc->pdev->subsystem_device) {
3697 case MPT3SAS_CISCO_12G_8E_HBA_SSDID:
3698 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3699 MPT3SAS_CISCO_12G_8E_HBA_BRANDING);
3700 break;
3701 case MPT3SAS_CISCO_12G_8I_HBA_SSDID:
3702 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3703 MPT3SAS_CISCO_12G_8I_HBA_BRANDING);
3704 break;
3705 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
3706 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3707 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
3708 break;
3709 default:
3710 pr_info(MPT3SAS_FMT
3711 "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
3712 ioc->name, ioc->pdev->subsystem_device);
3713 break;
3714 }
d8eb4a47 3715 break;
989e43c7
SR		 3716 case MPI25_MFGPAGE_DEVID_SAS3108_1:
3717 switch (ioc->pdev->subsystem_device) {
3718 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
3719 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
d8eb4a47 3720 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
989e43c7
SR		 3721 break;
3722 case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID:
3723 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3724 MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING
3725 );
3726 break;
3727 default:
3728 pr_info(MPT3SAS_FMT
3729 "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
3730 ioc->name, ioc->pdev->subsystem_device);
3731 break;
3732 }
38e4141e
SR		 3733 break;
3734 default:
3735 pr_info(MPT3SAS_FMT
989e43c7
SR		 3736 "Cisco SAS HBA: Subsystem ID: 0x%X\n",
3737 ioc->name, ioc->pdev->subsystem_device);
38e4141e
SR		 3738 break;
3739 }
3740 break;
989e43c7
SR		 3741 case MPT2SAS_HP_3PAR_SSVID:
3742 switch (ioc->pdev->device) {
3743 case MPI2_MFGPAGE_DEVID_SAS2004:
3744 switch (ioc->pdev->subsystem_device) {
3745 case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID:
3746 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3747 MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING);
3748 break;
3749 default:
3750 pr_info(MPT3SAS_FMT
3751 "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
3752 ioc->name, ioc->pdev->subsystem_device);
3753 break;
3754 }
3755 case MPI2_MFGPAGE_DEVID_SAS2308_2:
3756 switch (ioc->pdev->subsystem_device) {
3757 case MPT2SAS_HP_2_4_INTERNAL_SSDID:
3758 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3759 MPT2SAS_HP_2_4_INTERNAL_BRANDING);
3760 break;
3761 case MPT2SAS_HP_2_4_EXTERNAL_SSDID:
3762 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3763 MPT2SAS_HP_2_4_EXTERNAL_BRANDING);
3764 break;
3765 case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID:
3766 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3767 MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING);
3768 break;
3769 case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID:
3770 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3771 MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING);
3772 break;
3773 default:
3774 pr_info(MPT3SAS_FMT
3775 "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
3776 ioc->name, ioc->pdev->subsystem_device);
3777 break;
3778 }
d8eb4a47
SR		 3779 default:
3780 pr_info(MPT3SAS_FMT
989e43c7
SR		 3781 "HP SAS HBA: Subsystem ID: 0x%X\n",
3782 ioc->name, ioc->pdev->subsystem_device);
d8eb4a47
SR		 3783 break;
3784 }
38e4141e 3785 default:
38e4141e
SR		 3786 break;
3787 }
3788}
fb84dfc4 3789
f92363d1
SR		 3790/**
3791 * _base_display_ioc_capabilities - Disply IOC's capabilities.
3792 * @ioc: per adapter object
3793 *
3794 * Return nothing.
3795 */
3796static void
3797_base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
3798{
3799 int i = 0;
3800 char desc[16];
3801 u32 iounit_pg1_flags;
3802 u32 bios_version;
3803
3804 bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
3805 strncpy(desc, ioc->manu_pg0.ChipName, 16);
3806 pr_info(MPT3SAS_FMT "%s: FWVersion(%02d.%02d.%02d.%02d), "\
3807 "ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
3808 ioc->name, desc,
3809 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
3810 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
3811 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
3812 ioc->facts.FWVersion.Word & 0x000000FF,
3813 ioc->pdev->revision,
3814 (bios_version & 0xFF000000) >> 24,
3815 (bios_version & 0x00FF0000) >> 16,
3816 (bios_version & 0x0000FF00) >> 8,
3817 bios_version & 0x000000FF);
3818
989e43c7 3819 _base_display_OEMs_branding(ioc);
1117b31a 3820
016d5c35
SPS		 3821 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
3822 pr_info("%sNVMe", i ? "," : "");
3823 i++;
3824 }
3825
f92363d1
SR		 3826 pr_info(MPT3SAS_FMT "Protocol=(", ioc->name);
3827
3828 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {
3829 pr_info("Initiator");
3830 i++;
3831 }
3832
3833 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {
3834 pr_info("%sTarget", i ? "," : "");
3835 i++;
3836 }
3837
3838 i = 0;
3839 pr_info("), ");
3840 pr_info("Capabilities=(");
3841
7786ab6a
SR		 3842 if (!ioc->hide_ir_msg) {
3843 if (ioc->facts.IOCCapabilities &
f92363d1
SR		 3844 MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) {
3845 pr_info("Raid");
3846 i++;
7786ab6a 3847 }
f92363d1
SR		 3848 }
3849
3850 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {
3851 pr_info("%sTLR", i ? "," : "");
3852 i++;
3853 }
3854
3855 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {
3856 pr_info("%sMulticast", i ? "," : "");
3857 i++;
3858 }
3859
3860 if (ioc->facts.IOCCapabilities &
3861 MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) {
3862 pr_info("%sBIDI Target", i ? "," : "");
3863 i++;
3864 }
3865
3866 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {
3867 pr_info("%sEEDP", i ? "," : "");
3868 i++;
3869 }
3870
3871 if (ioc->facts.IOCCapabilities &
3872 MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) {
3873 pr_info("%sSnapshot Buffer", i ? "," : "");
3874 i++;
3875 }
3876
3877 if (ioc->facts.IOCCapabilities &
3878 MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) {
3879 pr_info("%sDiag Trace Buffer", i ? "," : "");
3880 i++;
3881 }
3882
3883 if (ioc->facts.IOCCapabilities &
3884 MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) {
3885 pr_info("%sDiag Extended Buffer", i ? "," : "");
3886 i++;
3887 }
3888
3889 if (ioc->facts.IOCCapabilities &
3890 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) {
3891 pr_info("%sTask Set Full", i ? "," : "");
3892 i++;
3893 }
3894
3895 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
3896 if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) {
3897 pr_info("%sNCQ", i ? "," : "");
3898 i++;
3899 }
3900
3901 pr_info(")\n");
3902}
3903
3904/**
3905 * mpt3sas_base_update_missing_delay - change the missing delay timers
3906 * @ioc: per adapter object
3907 * @device_missing_delay: amount of time till device is reported missing
3908 * @io_missing_delay: interval IO is returned when there is a missing device
3909 *
3910 * Return nothing.
3911 *
3912 * Passed on the command line, this function will modify the device missing
3913 * delay, as well as the io missing delay. This should be called at driver
3914 * load time.
3915 */
3916void
3917mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
3918 u16 device_missing_delay, u8 io_missing_delay)
3919{
3920 u16 dmd, dmd_new, dmd_orignal;
3921 u8 io_missing_delay_original;
3922 u16 sz;
3923 Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
3924 Mpi2ConfigReply_t mpi_reply;
3925 u8 num_phys = 0;
3926 u16 ioc_status;
3927
3928 mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
3929 if (!num_phys)
3930 return;
3931
3932 sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
3933 sizeof(Mpi2SasIOUnit1PhyData_t));
3934 sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
3935 if (!sas_iounit_pg1) {
3936 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3937 ioc->name, __FILE__, __LINE__, __func__);
3938 goto out;
3939 }
3940 if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
3941 sas_iounit_pg1, sz))) {
3942 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3943 ioc->name, __FILE__, __LINE__, __func__);
3944 goto out;
3945 }
3946 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
3947 MPI2_IOCSTATUS_MASK;
3948 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
3949 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3950 ioc->name, __FILE__, __LINE__, __func__);
3951 goto out;
3952 }
3953
3954 /* device missing delay */
3955 dmd = sas_iounit_pg1->ReportDeviceMissingDelay;
3956 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
3957 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
3958 else
3959 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
3960 dmd_orignal = dmd;
3961 if (device_missing_delay > 0x7F) {
3962 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 :
3963 device_missing_delay;
3964 dmd = dmd / 16;
3965 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16;
3966 } else
3967 dmd = device_missing_delay;
3968 sas_iounit_pg1->ReportDeviceMissingDelay = dmd;
3969
3970 /* io missing delay */
3971 io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay;
3972 sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay;
3973
3974 if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,
3975 sz)) {
3976 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
3977 dmd_new = (dmd &
3978 MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
3979 else
3980 dmd_new =
3981 dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
3982 pr_info(MPT3SAS_FMT "device_missing_delay: old(%d), new(%d)\n",
3983 ioc->name, dmd_orignal, dmd_new);
3984 pr_info(MPT3SAS_FMT "ioc_missing_delay: old(%d), new(%d)\n",
3985 ioc->name, io_missing_delay_original,
3986 io_missing_delay);
3987 ioc->device_missing_delay = dmd_new;
3988 ioc->io_missing_delay = io_missing_delay;
3989 }
3990
3991out:
3992 kfree(sas_iounit_pg1);
3993}
3994/**
3995 * _base_static_config_pages - static start of day config pages
3996 * @ioc: per adapter object
3997 *
3998 * Return nothing.
3999 */
4000static void
4001_base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
4002{
4003 Mpi2ConfigReply_t mpi_reply;
4004 u32 iounit_pg1_flags;
4005
4006 mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
4007 if (ioc->ir_firmware)
4008 mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
4009 &ioc->manu_pg10);
4010
4011 /*
4012 * Ensure correct T10 PI operation if vendor left EEDPTagMode
4013 * flag unset in NVDATA.
4014 */
4015 mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11);
4016 if (ioc->manu_pg11.EEDPTagMode == 0) {
4017 pr_err("%s: overriding NVDATA EEDPTagMode setting\n",
4018 ioc->name);
4019 ioc->manu_pg11.EEDPTagMode &= ~0x3;
4020 ioc->manu_pg11.EEDPTagMode |= 0x1;
4021 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
4022 &ioc->manu_pg11);
4023 }
4024
4025 mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
4026 mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
4027 mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);
4028 mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);
4029 mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
2d8ce8c9 4030 mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
f92363d1
SR		 4031 _base_display_ioc_capabilities(ioc);
4032
4033 /*
4034 * Enable task_set_full handling in iounit_pg1 when the
4035 * facts capabilities indicate that its supported.
4036 */
4037 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4038 if ((ioc->facts.IOCCapabilities &
4039 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING))
4040 iounit_pg1_flags &=
4041 ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4042 else
4043 iounit_pg1_flags |=
4044 MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4045 ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);
4046 mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
2d8ce8c9
SR		 4047
4048 if (ioc->iounit_pg8.NumSensors)
4049 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
f92363d1
SR		 4050}
4051
4052/**
4053 * _base_release_memory_pools - release memory
4054 * @ioc: per adapter object
4055 *
4056 * Free memory allocated from _base_allocate_memory_pools.
4057 *
4058 * Return nothing.
4059 */
4060static void
4061_base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
4062{
9b05c91a
SR		 4063 int i = 0;
4064 struct reply_post_struct *rps;
f92363d1
SR		 4065
4066 dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4067 __func__));
4068
4069 if (ioc->request) {
4070 pci_free_consistent(ioc->pdev, ioc->request_dma_sz,
4071 ioc->request, ioc->request_dma);
4072 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4073 "request_pool(0x%p): free\n",
4074 ioc->name, ioc->request));
4075 ioc->request = NULL;
4076 }
4077
4078 if (ioc->sense) {
e9d98418
RP		 4079 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
4080 dma_pool_destroy(ioc->sense_dma_pool);
f92363d1
SR		 4081 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4082 "sense_pool(0x%p): free\n",
4083 ioc->name, ioc->sense));
4084 ioc->sense = NULL;
4085 }
4086
4087 if (ioc->reply) {
e9d98418
RP		 4088 dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
4089 dma_pool_destroy(ioc->reply_dma_pool);
f92363d1
SR		 4090 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4091 "reply_pool(0x%p): free\n",
4092 ioc->name, ioc->reply));
4093 ioc->reply = NULL;
4094 }
4095
4096 if (ioc->reply_free) {
e9d98418 4097 dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
f92363d1 4098 ioc->reply_free_dma);
e9d98418 4099 dma_pool_destroy(ioc->reply_free_dma_pool);
f92363d1
SR		 4100 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4101 "reply_free_pool(0x%p): free\n",
4102 ioc->name, ioc->reply_free));
4103 ioc->reply_free = NULL;
4104 }
4105
9b05c91a
SR		 4106 if (ioc->reply_post) {
4107 do {
4108 rps = &ioc->reply_post[i];
4109 if (rps->reply_post_free) {
e9d98418 4110 dma_pool_free(
9b05c91a
SR		 4111 ioc->reply_post_free_dma_pool,
4112 rps->reply_post_free,
4113 rps->reply_post_free_dma);
4114 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4115 "reply_post_free_pool(0x%p): free\n",
4116 ioc->name, rps->reply_post_free));
4117 rps->reply_post_free = NULL;
4118 }
4119 } while (ioc->rdpq_array_enable &&
4120 (++i < ioc->reply_queue_count));
4121
e9d98418 4122 dma_pool_destroy(ioc->reply_post_free_dma_pool);
9b05c91a 4123 kfree(ioc->reply_post);
f92363d1
SR		 4124 }
4125
016d5c35
SPS		 4126 if (ioc->pcie_sgl_dma_pool) {
4127 for (i = 0; i < ioc->scsiio_depth; i++) {
dbec4c90
SPS		 4128 dma_pool_free(ioc->pcie_sgl_dma_pool,
4129 ioc->pcie_sg_lookup[i].pcie_sgl,
4130 ioc->pcie_sg_lookup[i].pcie_sgl_dma);
016d5c35
SPS		 4131 }
4132 if (ioc->pcie_sgl_dma_pool)
13a06405 4133 dma_pool_destroy(ioc->pcie_sgl_dma_pool);
016d5c35
SPS		 4134 }
4135
f92363d1
SR		 4136 if (ioc->config_page) {
4137 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4138 "config_page(0x%p): free\n", ioc->name,
4139 ioc->config_page));
4140 pci_free_consistent(ioc->pdev, ioc->config_page_sz,
4141 ioc->config_page, ioc->config_page_dma);
4142 }
4143
f92363d1
SR		 4144 kfree(ioc->hpr_lookup);
4145 kfree(ioc->internal_lookup);
4146 if (ioc->chain_lookup) {
4147 for (i = 0; i < ioc->chain_depth; i++) {
4148 if (ioc->chain_lookup[i].chain_buffer)
e9d98418 4149 dma_pool_free(ioc->chain_dma_pool,
f92363d1
SR		 4150 ioc->chain_lookup[i].chain_buffer,
4151 ioc->chain_lookup[i].chain_buffer_dma);
4152 }
e9d98418 4153 dma_pool_destroy(ioc->chain_dma_pool);
f92363d1
SR		 4154 free_pages((ulong)ioc->chain_lookup, ioc->chain_pages);
4155 ioc->chain_lookup = NULL;
4156 }
4157}
4158
4159/**
4160 * _base_allocate_memory_pools - allocate start of day memory pools
4161 * @ioc: per adapter object
f92363d1
SR		 4162 *
4163 * Returns 0 success, anything else error
4164 */
4165static int
98c56ad3 4166_base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 4167{
4168 struct mpt3sas_facts *facts;
4169 u16 max_sge_elements;
4170 u16 chains_needed_per_io;
4171 u32 sz, total_sz, reply_post_free_sz;
4172 u32 retry_sz;
016d5c35 4173 u16 max_request_credit, nvme_blocks_needed;
f92363d1
SR		 4174 unsigned short sg_tablesize;
4175 u16 sge_size;
4176 int i;
4177
4178 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4179 __func__));
4180
4181
4182 retry_sz = 0;
4183 facts = &ioc->facts;
4184
4185 /* command line tunables for max sgl entries */
4186 if (max_sgl_entries != -1)
4187 sg_tablesize = max_sgl_entries;
471ef9d4
SR		 4188 else {
4189 if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
4190 sg_tablesize = MPT2SAS_SG_DEPTH;
4191 else
4192 sg_tablesize = MPT3SAS_SG_DEPTH;
4193 }
f92363d1 4194
06f5f976
SR		 4195 /* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */
4196 if (reset_devices)
4197 sg_tablesize = min_t(unsigned short, sg_tablesize,
4198 MPT_KDUMP_MIN_PHYS_SEGMENTS);
4199
0448f019
SPS		 4200 if (ioc->is_mcpu_endpoint)
4201 ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
4202 else {
4203 if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS)
4204 sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
4205 else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) {
4206 sg_tablesize = min_t(unsigned short, sg_tablesize,
4207 SG_MAX_SEGMENTS);
4208 pr_warn(MPT3SAS_FMT
4209 "sg_tablesize(%u) is bigger than kernel "
4210 "defined SG_CHUNK_SIZE(%u)\n", ioc->name,
4211 sg_tablesize, MPT_MAX_PHYS_SEGMENTS);
4212 }
4213 ioc->shost->sg_tablesize = sg_tablesize;
ad666a0f 4214 }
f92363d1 4215
fd0331b3
SS		 4216 ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),
4217 (facts->RequestCredit / 4));
4218 if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {
4219 if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT +
4220 INTERNAL_SCSIIO_CMDS_COUNT)) {
4221 pr_err(MPT3SAS_FMT "IOC doesn't have enough Request \
4222 Credits, it has just %d number of credits\n",
4223 ioc->name, facts->RequestCredit);
4224 return -ENOMEM;
4225 }
4226 ioc->internal_depth = 10;
4227 }
4228
4229 ioc->hi_priority_depth = ioc->internal_depth - (5);
f92363d1
SR		 4230 /* command line tunables for max controller queue depth */
4231 if (max_queue_depth != -1 && max_queue_depth != 0) {
4232 max_request_credit = min_t(u16, max_queue_depth +
fd0331b3 4233 ioc->internal_depth, facts->RequestCredit);
f92363d1
SR		 4234 if (max_request_credit > MAX_HBA_QUEUE_DEPTH)
4235 max_request_credit = MAX_HBA_QUEUE_DEPTH;
06f5f976
SR		 4236 } else if (reset_devices)
4237 max_request_credit = min_t(u16, facts->RequestCredit,
4238 (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));
4239 else
f92363d1
SR		 4240 max_request_credit = min_t(u16, facts->RequestCredit,
4241 MAX_HBA_QUEUE_DEPTH);
4242
fd0331b3
SS		 4243 /* Firmware maintains additional facts->HighPriorityCredit number of
4244 * credits for HiPriprity Request messages, so hba queue depth will be
4245 * sum of max_request_credit and high priority queue depth.
4246 */
4247 ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;
f92363d1
SR		 4248
4249 /* request frame size */
4250 ioc->request_sz = facts->IOCRequestFrameSize * 4;
4251
4252 /* reply frame size */
4253 ioc->reply_sz = facts->ReplyFrameSize * 4;
4254
ebb3024e
SS		 4255 /* chain segment size */
4256 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
4257 if (facts->IOCMaxChainSegmentSize)
4258 ioc->chain_segment_sz =
4259 facts->IOCMaxChainSegmentSize *
4260 MAX_CHAIN_ELEMT_SZ;
4261 else
4262 /* set to 128 bytes size if IOCMaxChainSegmentSize is zero */
4263 ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *
4264 MAX_CHAIN_ELEMT_SZ;
4265 } else
4266 ioc->chain_segment_sz = ioc->request_sz;
4267
f92363d1
SR		 4268 /* calculate the max scatter element size */
4269 sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);
4270
4271 retry_allocation:
4272 total_sz = 0;
4273 /* calculate number of sg elements left over in the 1st frame */
4274 max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -
4275 sizeof(Mpi2SGEIOUnion_t)) + sge_size);
4276 ioc->max_sges_in_main_message = max_sge_elements/sge_size;
4277
4278 /* now do the same for a chain buffer */
ebb3024e 4279 max_sge_elements = ioc->chain_segment_sz - sge_size;
f92363d1
SR		 4280 ioc->max_sges_in_chain_message = max_sge_elements/sge_size;
4281
4282 /*
4283 * MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE
4284 */
4285 chains_needed_per_io = ((ioc->shost->sg_tablesize -
4286 ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)
4287 + 1;
4288 if (chains_needed_per_io > facts->MaxChainDepth) {
4289 chains_needed_per_io = facts->MaxChainDepth;
4290 ioc->shost->sg_tablesize = min_t(u16,
4291 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message
4292 * chains_needed_per_io), ioc->shost->sg_tablesize);
4293 }
4294 ioc->chains_needed_per_io = chains_needed_per_io;
4295
4296 /* reply free queue sizing - taking into account for 64 FW events */
4297 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
4298
0448f019
SPS		 4299 /* mCPU manage single counters for simplicity */
4300 if (ioc->is_mcpu_endpoint)
4301 ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;
4302 else {
4303 /* calculate reply descriptor post queue depth */
4304 ioc->reply_post_queue_depth = ioc->hba_queue_depth +
4305 ioc->reply_free_queue_depth + 1;
4306 /* align the reply post queue on the next 16 count boundary */
4307 if (ioc->reply_post_queue_depth % 16)
4308 ioc->reply_post_queue_depth += 16 -
4309 (ioc->reply_post_queue_depth % 16);
4310 }
f92363d1 4311
f92363d1
SR		 4312 if (ioc->reply_post_queue_depth >
4313 facts->MaxReplyDescriptorPostQueueDepth) {
4314 ioc->reply_post_queue_depth =
4315 facts->MaxReplyDescriptorPostQueueDepth -
4316 (facts->MaxReplyDescriptorPostQueueDepth % 16);
4317 ioc->hba_queue_depth =
4318 ((ioc->reply_post_queue_depth - 64) / 2) - 1;
4319 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
4320 }
4321
4322 dinitprintk(ioc, pr_info(MPT3SAS_FMT "scatter gather: " \
4323 "sge_in_main_msg(%d), sge_per_chain(%d), sge_per_io(%d), "
4324 "chains_per_io(%d)\n", ioc->name, ioc->max_sges_in_main_message,
4325 ioc->max_sges_in_chain_message, ioc->shost->sg_tablesize,
4326 ioc->chains_needed_per_io));
4327
9b05c91a
SR		 4328 /* reply post queue, 16 byte align */
4329 reply_post_free_sz = ioc->reply_post_queue_depth *
4330 sizeof(Mpi2DefaultReplyDescriptor_t);
4331
4332 sz = reply_post_free_sz;
4333 if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
4334 sz *= ioc->reply_queue_count;
4335
4336 ioc->reply_post = kcalloc((ioc->rdpq_array_enable) ?
4337 (ioc->reply_queue_count):1,
4338 sizeof(struct reply_post_struct), GFP_KERNEL);
4339
4340 if (!ioc->reply_post) {
4341 pr_err(MPT3SAS_FMT "reply_post_free pool: kcalloc failed\n",
4342 ioc->name);
4343 goto out;
4344 }
e9d98418
RP		 4345 ioc->reply_post_free_dma_pool = dma_pool_create("reply_post_free pool",
4346 &ioc->pdev->dev, sz, 16, 0);
9b05c91a
SR		 4347 if (!ioc->reply_post_free_dma_pool) {
4348 pr_err(MPT3SAS_FMT
e9d98418 4349 "reply_post_free pool: dma_pool_create failed\n",
9b05c91a
SR		 4350 ioc->name);
4351 goto out;
4352 }
4353 i = 0;
4354 do {
4355 ioc->reply_post[i].reply_post_free =
e9d98418 4356 dma_pool_alloc(ioc->reply_post_free_dma_pool,
9b05c91a
SR		 4357 GFP_KERNEL,
4358 &ioc->reply_post[i].reply_post_free_dma);
4359 if (!ioc->reply_post[i].reply_post_free) {
4360 pr_err(MPT3SAS_FMT
e9d98418 4361 "reply_post_free pool: dma_pool_alloc failed\n",
9b05c91a
SR		 4362 ioc->name);
4363 goto out;
4364 }
4365 memset(ioc->reply_post[i].reply_post_free, 0, sz);
4366 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4367 "reply post free pool (0x%p): depth(%d),"
4368 "element_size(%d), pool_size(%d kB)\n", ioc->name,
4369 ioc->reply_post[i].reply_post_free,
4370 ioc->reply_post_queue_depth, 8, sz/1024));
4371 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4372 "reply_post_free_dma = (0x%llx)\n", ioc->name,
4373 (unsigned long long)
4374 ioc->reply_post[i].reply_post_free_dma));
4375 total_sz += sz;
4376 } while (ioc->rdpq_array_enable && (++i < ioc->reply_queue_count));
4377
4378 if (ioc->dma_mask == 64) {
4379 if (_base_change_consistent_dma_mask(ioc, ioc->pdev) != 0) {
4380 pr_warn(MPT3SAS_FMT
4381 "no suitable consistent DMA mask for %s\n",
4382 ioc->name, pci_name(ioc->pdev));
4383 goto out;
4384 }
4385 }
4386
f92363d1
SR		 4387 ioc->scsiio_depth = ioc->hba_queue_depth -
4388 ioc->hi_priority_depth - ioc->internal_depth;
4389
4390 /* set the scsi host can_queue depth
4391 * with some internal commands that could be outstanding
4392 */
fd0331b3 4393 ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;
f92363d1
SR		 4394 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4395 "scsi host: can_queue depth (%d)\n",
4396 ioc->name, ioc->shost->can_queue));
4397
4398
4399 /* contiguous pool for request and chains, 16 byte align, one extra "
4400 * "frame for smid=0
4401 */
4402 ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;
4403 sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);
4404
4405 /* hi-priority queue */
4406 sz += (ioc->hi_priority_depth * ioc->request_sz);
4407
4408 /* internal queue */
4409 sz += (ioc->internal_depth * ioc->request_sz);
4410
4411 ioc->request_dma_sz = sz;
4412 ioc->request = pci_alloc_consistent(ioc->pdev, sz, &ioc->request_dma);
4413 if (!ioc->request) {
4414 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
4415 "failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
4416 "total(%d kB)\n", ioc->name, ioc->hba_queue_depth,
4417 ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
4418 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)
4419 goto out;
fd0331b3
SS		 4420 retry_sz = 64;
4421 ioc->hba_queue_depth -= retry_sz;
8ff045c9 4422 _base_release_memory_pools(ioc);
f92363d1
SR		 4423 goto retry_allocation;
4424 }
4425
4426 if (retry_sz)
4427 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
4428 "succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
4429 "total(%d kb)\n", ioc->name, ioc->hba_queue_depth,
4430 ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
4431
4432 /* hi-priority queue */
4433 ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *
4434 ioc->request_sz);
4435 ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *
4436 ioc->request_sz);
4437
4438 /* internal queue */
4439 ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *
4440 ioc->request_sz);
4441 ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *
4442 ioc->request_sz);
4443
4444 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4445 "request pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
4446 ioc->name, ioc->request, ioc->hba_queue_depth, ioc->request_sz,
4447 (ioc->hba_queue_depth * ioc->request_sz)/1024));
4448
4449 dinitprintk(ioc, pr_info(MPT3SAS_FMT "request pool: dma(0x%llx)\n",
4450 ioc->name, (unsigned long long) ioc->request_dma));
4451 total_sz += sz;
4452
f92363d1
SR		 4453 dinitprintk(ioc, pr_info(MPT3SAS_FMT "scsiio(0x%p): depth(%d)\n",
4454 ioc->name, ioc->request, ioc->scsiio_depth));
4455
4456 ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
4457 sz = ioc->chain_depth * sizeof(struct chain_tracker);
4458 ioc->chain_pages = get_order(sz);
4459 ioc->chain_lookup = (struct chain_tracker *)__get_free_pages(
4460 GFP_KERNEL, ioc->chain_pages);
4461 if (!ioc->chain_lookup) {
4462 pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages failed\n",
4463 ioc->name);
4464 goto out;
4465 }
e9d98418 4466 ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
ebb3024e 4467 ioc->chain_segment_sz, 16, 0);
f92363d1 4468 if (!ioc->chain_dma_pool) {
e9d98418 4469 pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
f92363d1
SR		 4470 ioc->name);
4471 goto out;
4472 }
4473 for (i = 0; i < ioc->chain_depth; i++) {
e9d98418 4474 ioc->chain_lookup[i].chain_buffer = dma_pool_alloc(
f92363d1
SR		 4475 ioc->chain_dma_pool , GFP_KERNEL,
4476 &ioc->chain_lookup[i].chain_buffer_dma);
4477 if (!ioc->chain_lookup[i].chain_buffer) {
4478 ioc->chain_depth = i;
4479 goto chain_done;
4480 }
ebb3024e 4481 total_sz += ioc->chain_segment_sz;
f92363d1
SR		 4482 }
4483 chain_done:
4484 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4485 "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
ebb3024e
SS		 4486 ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
4487 ((ioc->chain_depth * ioc->chain_segment_sz))/1024));
f92363d1
SR		 4488
4489 /* initialize hi-priority queue smid's */
4490 ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
4491 sizeof(struct request_tracker), GFP_KERNEL);
4492 if (!ioc->hpr_lookup) {
4493 pr_err(MPT3SAS_FMT "hpr_lookup: kcalloc failed\n",
4494 ioc->name);
4495 goto out;
4496 }
4497 ioc->hi_priority_smid = ioc->scsiio_depth + 1;
4498 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4499 "hi_priority(0x%p): depth(%d), start smid(%d)\n",
4500 ioc->name, ioc->hi_priority,
4501 ioc->hi_priority_depth, ioc->hi_priority_smid));
4502
4503 /* initialize internal queue smid's */
4504 ioc->internal_lookup = kcalloc(ioc->internal_depth,
4505 sizeof(struct request_tracker), GFP_KERNEL);
4506 if (!ioc->internal_lookup) {
4507 pr_err(MPT3SAS_FMT "internal_lookup: kcalloc failed\n",
4508 ioc->name);
4509 goto out;
4510 }
4511 ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;
4512 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4513 "internal(0x%p): depth(%d), start smid(%d)\n",
4514 ioc->name, ioc->internal,
4515 ioc->internal_depth, ioc->internal_smid));
016d5c35
SPS		 4516 /*
4517 * The number of NVMe page sized blocks needed is:
4518 * (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1
4519 * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry
4520 * that is placed in the main message frame. 8 is the size of each PRP
4521 * entry or PRP list pointer entry. 8 is subtracted from page_size
4522 * because of the PRP list pointer entry at the end of a page, so this
4523 * is not counted as a PRP entry. The 1 added page is a round up.
4524 *
4525 * To avoid allocation failures due to the amount of memory that could
4526 * be required for NVMe PRP's, only each set of NVMe blocks will be
4527 * contiguous, so a new set is allocated for each possible I/O.
4528 */
4529 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
4530 nvme_blocks_needed =
4531 (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
4532 nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);
4533 nvme_blocks_needed++;
4534
dbec4c90
SPS		 4535 sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;
4536 ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);
4537 if (!ioc->pcie_sg_lookup) {
4538 pr_info(MPT3SAS_FMT
4539 "PCIe SGL lookup: kzalloc failed\n", ioc->name);
4540 goto out;
4541 }
016d5c35
SPS		 4542 sz = nvme_blocks_needed * ioc->page_size;
4543 ioc->pcie_sgl_dma_pool =
13a06405 4544 dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, 16, 0);
016d5c35
SPS		 4545 if (!ioc->pcie_sgl_dma_pool) {
4546 pr_info(MPT3SAS_FMT
13a06405 4547 "PCIe SGL pool: dma_pool_create failed\n",
016d5c35
SPS		 4548 ioc->name);
4549 goto out;
4550 }
4551 for (i = 0; i < ioc->scsiio_depth; i++) {
dbec4c90
SPS		 4552 ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc(
4553 ioc->pcie_sgl_dma_pool, GFP_KERNEL,
4554 &ioc->pcie_sg_lookup[i].pcie_sgl_dma);
4555 if (!ioc->pcie_sg_lookup[i].pcie_sgl) {
016d5c35 4556 pr_info(MPT3SAS_FMT
13a06405 4557 "PCIe SGL pool: dma_pool_alloc failed\n",
016d5c35
SPS		 4558 ioc->name);
4559 goto out;
4560 }
4561 }
f92363d1 4562
016d5c35
SPS		 4563 dinitprintk(ioc, pr_info(MPT3SAS_FMT "PCIe sgl pool depth(%d), "
4564 "element_size(%d), pool_size(%d kB)\n", ioc->name,
4565 ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
4566 total_sz += sz * ioc->scsiio_depth;
4567 }
f92363d1
SR		 4568 /* sense buffers, 4 byte align */
4569 sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
e9d98418
RP		 4570 ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
4571 4, 0);
f92363d1 4572 if (!ioc->sense_dma_pool) {
e9d98418 4573 pr_err(MPT3SAS_FMT "sense pool: dma_pool_create failed\n",
f92363d1
SR		 4574 ioc->name);
4575 goto out;
4576 }
e9d98418 4577 ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
f92363d1
SR		 4578 &ioc->sense_dma);
4579 if (!ioc->sense) {
e9d98418 4580 pr_err(MPT3SAS_FMT "sense pool: dma_pool_alloc failed\n",
f92363d1
SR		 4581 ioc->name);
4582 goto out;
4583 }
4584 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4585 "sense pool(0x%p): depth(%d), element_size(%d), pool_size"
4586 "(%d kB)\n", ioc->name, ioc->sense, ioc->scsiio_depth,
4587 SCSI_SENSE_BUFFERSIZE, sz/1024));
4588 dinitprintk(ioc, pr_info(MPT3SAS_FMT "sense_dma(0x%llx)\n",
4589 ioc->name, (unsigned long long)ioc->sense_dma));
4590 total_sz += sz;
4591
4592 /* reply pool, 4 byte align */
4593 sz = ioc->reply_free_queue_depth * ioc->reply_sz;
e9d98418
RP		 4594 ioc->reply_dma_pool = dma_pool_create("reply pool", &ioc->pdev->dev, sz,
4595 4, 0);
f92363d1 4596 if (!ioc->reply_dma_pool) {
e9d98418 4597 pr_err(MPT3SAS_FMT "reply pool: dma_pool_create failed\n",
f92363d1
SR		 4598 ioc->name);
4599 goto out;
4600 }
e9d98418 4601 ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,
f92363d1
SR		 4602 &ioc->reply_dma);
4603 if (!ioc->reply) {
e9d98418 4604 pr_err(MPT3SAS_FMT "reply pool: dma_pool_alloc failed\n",
f92363d1
SR		 4605 ioc->name);
4606 goto out;
4607 }
4608 ioc->reply_dma_min_address = (u32)(ioc->reply_dma);
4609 ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;
4610 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4611 "reply pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
4612 ioc->name, ioc->reply,
4613 ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024));
4614 dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_dma(0x%llx)\n",
4615 ioc->name, (unsigned long long)ioc->reply_dma));
4616 total_sz += sz;
4617
4618 /* reply free queue, 16 byte align */
4619 sz = ioc->reply_free_queue_depth * 4;
e9d98418
RP		 4620 ioc->reply_free_dma_pool = dma_pool_create("reply_free pool",
4621 &ioc->pdev->dev, sz, 16, 0);
f92363d1 4622 if (!ioc->reply_free_dma_pool) {
e9d98418 4623 pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_create failed\n",
f92363d1
SR		 4624 ioc->name);
4625 goto out;
4626 }
e9d98418 4627 ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, GFP_KERNEL,
f92363d1
SR		 4628 &ioc->reply_free_dma);
4629 if (!ioc->reply_free) {
e9d98418 4630 pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_alloc failed\n",
f92363d1
SR		 4631 ioc->name);
4632 goto out;
4633 }
4634 memset(ioc->reply_free, 0, sz);
4635 dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_free pool(0x%p): " \
4636 "depth(%d), element_size(%d), pool_size(%d kB)\n", ioc->name,
4637 ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));
4638 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4639 "reply_free_dma (0x%llx)\n",
4640 ioc->name, (unsigned long long)ioc->reply_free_dma));
4641 total_sz += sz;
4642
f92363d1
SR		 4643 ioc->config_page_sz = 512;
4644 ioc->config_page = pci_alloc_consistent(ioc->pdev,
4645 ioc->config_page_sz, &ioc->config_page_dma);
4646 if (!ioc->config_page) {
4647 pr_err(MPT3SAS_FMT
e9d98418 4648 "config page: dma_pool_alloc failed\n",
f92363d1
SR		 4649 ioc->name);
4650 goto out;
4651 }
4652 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4653 "config page(0x%p): size(%d)\n",
4654 ioc->name, ioc->config_page, ioc->config_page_sz));
4655 dinitprintk(ioc, pr_info(MPT3SAS_FMT "config_page_dma(0x%llx)\n",
4656 ioc->name, (unsigned long long)ioc->config_page_dma));
4657 total_sz += ioc->config_page_sz;
4658
4659 pr_info(MPT3SAS_FMT "Allocated physical memory: size(%d kB)\n",
4660 ioc->name, total_sz/1024);
4661 pr_info(MPT3SAS_FMT
4662 "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",
4663 ioc->name, ioc->shost->can_queue, facts->RequestCredit);
4664 pr_info(MPT3SAS_FMT "Scatter Gather Elements per IO(%d)\n",
4665 ioc->name, ioc->shost->sg_tablesize);
4666 return 0;
4667
4668 out:
4669 return -ENOMEM;
4670}
4671
4672/**
4673 * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter.
4674 * @ioc: Pointer to MPT_ADAPTER structure
4675 * @cooked: Request raw or cooked IOC state
4676 *
4677 * Returns all IOC Doorbell register bits if cooked==0, else just the
4678 * Doorbell bits in MPI_IOC_STATE_MASK.
4679 */
4680u32
4681mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
4682{
4683 u32 s, sc;
4684
4685 s = readl(&ioc->chip->Doorbell);
4686 sc = s & MPI2_IOC_STATE_MASK;
4687 return cooked ? sc : s;
4688}
4689
4690/**
4691 * _base_wait_on_iocstate - waiting on a particular ioc state
4692 * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
4693 * @timeout: timeout in second
f92363d1
SR		 4694 *
4695 * Returns 0 for success, non-zero for failure.
4696 */
4697static int
98c56ad3 4698_base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)
f92363d1
SR		 4699{
4700 u32 count, cntdn;
4701 u32 current_state;
4702
4703 count = 0;
98c56ad3 4704 cntdn = 1000 * timeout;
f92363d1
SR		 4705 do {
4706 current_state = mpt3sas_base_get_iocstate(ioc, 1);
4707 if (current_state == ioc_state)
4708 return 0;
4709 if (count && current_state == MPI2_IOC_STATE_FAULT)
4710 break;
98c56ad3
CO		 4711
4712 usleep_range(1000, 1500);
f92363d1
SR		 4713 count++;
4714 } while (--cntdn);
4715
4716 return current_state;
4717}
4718
4719/**
4720 * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
4721 * a write to the doorbell)
4722 * @ioc: per adapter object
4723 * @timeout: timeout in second
f92363d1
SR		 4724 *
4725 * Returns 0 for success, non-zero for failure.
4726 *
4727 * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
4728 */
4dc8c808 4729static int
98c56ad3 4730_base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
4dc8c808 4731
f92363d1 4732static int
98c56ad3 4733_base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
f92363d1
SR		 4734{
4735 u32 cntdn, count;
4736 u32 int_status;
4737
4738 count = 0;
98c56ad3 4739 cntdn = 1000 * timeout;
f92363d1
SR		 4740 do {
4741 int_status = readl(&ioc->chip->HostInterruptStatus);
4742 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
4743 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4744 "%s: successful count(%d), timeout(%d)\n",
4745 ioc->name, __func__, count, timeout));
4746 return 0;
4747 }
98c56ad3
CO		 4748
4749 usleep_range(1000, 1500);
4750 count++;
4751 } while (--cntdn);
4752
4753 pr_err(MPT3SAS_FMT
4754 "%s: failed due to timeout count(%d), int_status(%x)!\n",
4755 ioc->name, __func__, count, int_status);
4756 return -EFAULT;
4757}
4758
4759static int
4760_base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
4761{
4762 u32 cntdn, count;
4763 u32 int_status;
4764
4765 count = 0;
4766 cntdn = 2000 * timeout;
4767 do {
4768 int_status = readl(&ioc->chip->HostInterruptStatus);
4769 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
4770 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4771 "%s: successful count(%d), timeout(%d)\n",
4772 ioc->name, __func__, count, timeout));
4773 return 0;
4774 }
4775
4776 udelay(500);
f92363d1
SR		 4777 count++;
4778 } while (--cntdn);
4779
4780 pr_err(MPT3SAS_FMT
4781 "%s: failed due to timeout count(%d), int_status(%x)!\n",
4782 ioc->name, __func__, count, int_status);
4783 return -EFAULT;
98c56ad3 4784
f92363d1
SR		 4785}
4786
4787/**
4788 * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
4789 * @ioc: per adapter object
4790 * @timeout: timeout in second
f92363d1
SR		 4791 *
4792 * Returns 0 for success, non-zero for failure.
4793 *
4794 * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
4795 * doorbell.
4796 */
4797static int
98c56ad3 4798_base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)
f92363d1
SR		 4799{
4800 u32 cntdn, count;
4801 u32 int_status;
4802 u32 doorbell;
4803
4804 count = 0;
98c56ad3 4805 cntdn = 1000 * timeout;
f92363d1
SR		 4806 do {
4807 int_status = readl(&ioc->chip->HostInterruptStatus);
4808 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
4809 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4810 "%s: successful count(%d), timeout(%d)\n",
4811 ioc->name, __func__, count, timeout));
4812 return 0;
4813 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
4814 doorbell = readl(&ioc->chip->Doorbell);
4815 if ((doorbell & MPI2_IOC_STATE_MASK) ==
4816 MPI2_IOC_STATE_FAULT) {
4817 mpt3sas_base_fault_info(ioc , doorbell);
4818 return -EFAULT;
4819 }
4820 } else if (int_status == 0xFFFFFFFF)
4821 goto out;
4822
98c56ad3 4823 usleep_range(1000, 1500);
f92363d1
SR		 4824 count++;
4825 } while (--cntdn);
4826
4827 out:
4828 pr_err(MPT3SAS_FMT
4829 "%s: failed due to timeout count(%d), int_status(%x)!\n",
4830 ioc->name, __func__, count, int_status);
4831 return -EFAULT;
4832}
4833
4834/**
4835 * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
4836 * @ioc: per adapter object
4837 * @timeout: timeout in second
f92363d1
SR		 4838 *
4839 * Returns 0 for success, non-zero for failure.
4840 *
4841 */
4842static int
98c56ad3 4843_base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)
f92363d1
SR		 4844{
4845 u32 cntdn, count;
4846 u32 doorbell_reg;
4847
4848 count = 0;
98c56ad3 4849 cntdn = 1000 * timeout;
f92363d1
SR		 4850 do {
4851 doorbell_reg = readl(&ioc->chip->Doorbell);
4852 if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
4853 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4854 "%s: successful count(%d), timeout(%d)\n",
4855 ioc->name, __func__, count, timeout));
4856 return 0;
4857 }
98c56ad3
CO		 4858
4859 usleep_range(1000, 1500);
f92363d1
SR		 4860 count++;
4861 } while (--cntdn);
4862
4863 pr_err(MPT3SAS_FMT
4864 "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",
4865 ioc->name, __func__, count, doorbell_reg);
4866 return -EFAULT;
4867}
4868
4869/**
4870 * _base_send_ioc_reset - send doorbell reset
4871 * @ioc: per adapter object
4872 * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
4873 * @timeout: timeout in second
f92363d1
SR		 4874 *
4875 * Returns 0 for success, non-zero for failure.
4876 */
4877static int
98c56ad3 4878_base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)
f92363d1
SR		 4879{
4880 u32 ioc_state;
4881 int r = 0;
4882
4883 if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) {
4884 pr_err(MPT3SAS_FMT "%s: unknown reset_type\n",
4885 ioc->name, __func__);
4886 return -EFAULT;
4887 }
4888
4889 if (!(ioc->facts.IOCCapabilities &
4890 MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY))
4891 return -EFAULT;
4892
4893 pr_info(MPT3SAS_FMT "sending message unit reset !!\n", ioc->name);
4894
4895 writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
4896 &ioc->chip->Doorbell);
98c56ad3 4897 if ((_base_wait_for_doorbell_ack(ioc, 15))) {
f92363d1
SR		 4898 r = -EFAULT;
4899 goto out;
4900 }
98c56ad3 4901 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
f92363d1
SR		 4902 if (ioc_state) {
4903 pr_err(MPT3SAS_FMT
4904 "%s: failed going to ready state (ioc_state=0x%x)\n",
4905 ioc->name, __func__, ioc_state);
4906 r = -EFAULT;
4907 goto out;
4908 }
4909 out:
4910 pr_info(MPT3SAS_FMT "message unit reset: %s\n",
4911 ioc->name, ((r == 0) ? "SUCCESS" : "FAILED"));
4912 return r;
4913}
4914
4915/**
4916 * _base_handshake_req_reply_wait - send request thru doorbell interface
4917 * @ioc: per adapter object
4918 * @request_bytes: request length
4919 * @request: pointer having request payload
4920 * @reply_bytes: reply length
4921 * @reply: pointer to reply payload
4922 * @timeout: timeout in second
f92363d1
SR		 4923 *
4924 * Returns 0 for success, non-zero for failure.
4925 */
4926static int
4927_base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
98c56ad3 4928 u32 *request, int reply_bytes, u16 *reply, int timeout)
f92363d1
SR		 4929{
4930 MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
4931 int i;
4932 u8 failed;
f92363d1
SR		 4933 __le32 *mfp;
4934
4935 /* make sure doorbell is not in use */
4936 if ((readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {
4937 pr_err(MPT3SAS_FMT
4938 "doorbell is in use (line=%d)\n",
4939 ioc->name, __LINE__);
4940 return -EFAULT;
4941 }
4942
4943 /* clear pending doorbell interrupts from previous state changes */
4944 if (readl(&ioc->chip->HostInterruptStatus) &
4945 MPI2_HIS_IOC2SYS_DB_STATUS)
4946 writel(0, &ioc->chip->HostInterruptStatus);
4947
4948 /* send message to ioc */
4949 writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) |
4950 ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
4951 &ioc->chip->Doorbell);
4952
98c56ad3 4953 if ((_base_spin_on_doorbell_int(ioc, 5))) {
f92363d1
SR		 4954 pr_err(MPT3SAS_FMT
4955 "doorbell handshake int failed (line=%d)\n",
4956 ioc->name, __LINE__);
4957 return -EFAULT;
4958 }
4959 writel(0, &ioc->chip->HostInterruptStatus);
4960
98c56ad3 4961 if ((_base_wait_for_doorbell_ack(ioc, 5))) {
f92363d1
SR		 4962 pr_err(MPT3SAS_FMT
4963 "doorbell handshake ack failed (line=%d)\n",
4964 ioc->name, __LINE__);
4965 return -EFAULT;
4966 }
4967
4968 /* send message 32-bits at a time */
4969 for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
4970 writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
98c56ad3 4971 if ((_base_wait_for_doorbell_ack(ioc, 5)))
f92363d1
SR		 4972 failed = 1;
4973 }
4974
4975 if (failed) {
4976 pr_err(MPT3SAS_FMT
4977 "doorbell handshake sending request failed (line=%d)\n",
4978 ioc->name, __LINE__);
4979 return -EFAULT;
4980 }
4981
4982 /* now wait for the reply */
98c56ad3 4983 if ((_base_wait_for_doorbell_int(ioc, timeout))) {
f92363d1
SR		 4984 pr_err(MPT3SAS_FMT
4985 "doorbell handshake int failed (line=%d)\n",
4986 ioc->name, __LINE__);
4987 return -EFAULT;
4988 }
4989
4990 /* read the first two 16-bits, it gives the total length of the reply */
4991 reply[0] = le16_to_cpu(readl(&ioc->chip->Doorbell)
4992 & MPI2_DOORBELL_DATA_MASK);
4993 writel(0, &ioc->chip->HostInterruptStatus);
98c56ad3 4994 if ((_base_wait_for_doorbell_int(ioc, 5))) {
f92363d1
SR		 4995 pr_err(MPT3SAS_FMT
4996 "doorbell handshake int failed (line=%d)\n",
4997 ioc->name, __LINE__);
4998 return -EFAULT;
4999 }
5000 reply[1] = le16_to_cpu(readl(&ioc->chip->Doorbell)
5001 & MPI2_DOORBELL_DATA_MASK);
5002 writel(0, &ioc->chip->HostInterruptStatus);
5003
5004 for (i = 2; i < default_reply->MsgLength * 2; i++) {
98c56ad3 5005 if ((_base_wait_for_doorbell_int(ioc, 5))) {
f92363d1
SR		 5006 pr_err(MPT3SAS_FMT
5007 "doorbell handshake int failed (line=%d)\n",
5008 ioc->name, __LINE__);
5009 return -EFAULT;
5010 }
5011 if (i >= reply_bytes/2) /* overflow case */
8bbb1cf6 5012 readl(&ioc->chip->Doorbell);
f92363d1
SR		 5013 else
5014 reply[i] = le16_to_cpu(readl(&ioc->chip->Doorbell)
5015 & MPI2_DOORBELL_DATA_MASK);
5016 writel(0, &ioc->chip->HostInterruptStatus);
5017 }
5018
98c56ad3
CO		 5019 _base_wait_for_doorbell_int(ioc, 5);
5020 if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {
f92363d1
SR		 5021 dhsprintk(ioc, pr_info(MPT3SAS_FMT
5022 "doorbell is in use (line=%d)\n", ioc->name, __LINE__));
5023 }
5024 writel(0, &ioc->chip->HostInterruptStatus);
5025
5026 if (ioc->logging_level & MPT_DEBUG_INIT) {
5027 mfp = (__le32 *)reply;
5028 pr_info("\toffset:data\n");
5029 for (i = 0; i < reply_bytes/4; i++)
5030 pr_info("\t[0x%02x]:%08x\n", i*4,
5031 le32_to_cpu(mfp[i]));
5032 }
5033 return 0;
5034}
5035
5036/**
5037 * mpt3sas_base_sas_iounit_control - send sas iounit control to FW
5038 * @ioc: per adapter object
5039 * @mpi_reply: the reply payload from FW
5040 * @mpi_request: the request payload sent to FW
5041 *
5042 * The SAS IO Unit Control Request message allows the host to perform low-level
5043 * operations, such as resets on the PHYs of the IO Unit, also allows the host
5044 * to obtain the IOC assigned device handles for a device if it has other
5045 * identifying information about the device, in addition allows the host to
5046 * remove IOC resources associated with the device.
5047 *
5048 * Returns 0 for success, non-zero for failure.
5049 */
5050int
5051mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
5052 Mpi2SasIoUnitControlReply_t *mpi_reply,
5053 Mpi2SasIoUnitControlRequest_t *mpi_request)
5054{
5055 u16 smid;
5056 u32 ioc_state;
eb44552b 5057 bool issue_reset = false;
f92363d1
SR		 5058 int rc;
5059 void *request;
5060 u16 wait_state_count;
5061
5062 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5063 __func__));
5064
5065 mutex_lock(&ioc->base_cmds.mutex);
5066
5067 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
5068 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
5069 ioc->name, __func__);
5070 rc = -EAGAIN;
5071 goto out;
5072 }
5073
5074 wait_state_count = 0;
5075 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5076 while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
5077 if (wait_state_count++ == 10) {
5078 pr_err(MPT3SAS_FMT
5079 "%s: failed due to ioc not operational\n",
5080 ioc->name, __func__);
5081 rc = -EFAULT;
5082 goto out;
5083 }
5084 ssleep(1);
5085 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5086 pr_info(MPT3SAS_FMT
5087 "%s: waiting for operational state(count=%d)\n",
5088 ioc->name, __func__, wait_state_count);
5089 }
5090
5091 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5092 if (!smid) {
5093 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5094 ioc->name, __func__);
5095 rc = -EAGAIN;
5096 goto out;
5097 }
5098
5099 rc = 0;
5100 ioc->base_cmds.status = MPT3_CMD_PENDING;
5101 request = mpt3sas_base_get_msg_frame(ioc, smid);
5102 ioc->base_cmds.smid = smid;
5103 memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t));
5104 if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
5105 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
5106 ioc->ioc_link_reset_in_progress = 1;
5107 init_completion(&ioc->base_cmds.done);
81c16f83 5108 ioc->put_smid_default(ioc, smid);
8bbb1cf6 5109 wait_for_completion_timeout(&ioc->base_cmds.done,
f92363d1
SR		 5110 msecs_to_jiffies(10000));
5111 if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
5112 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
5113 ioc->ioc_link_reset_in_progress)
5114 ioc->ioc_link_reset_in_progress = 0;
5115 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5116 pr_err(MPT3SAS_FMT "%s: timeout\n",
5117 ioc->name, __func__);
5118 _debug_dump_mf(mpi_request,
5119 sizeof(Mpi2SasIoUnitControlRequest_t)/4);
5120 if (!(ioc->base_cmds.status & MPT3_CMD_RESET))
eb44552b 5121 issue_reset = true;
f92363d1
SR		 5122 goto issue_host_reset;
5123 }
5124 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
5125 memcpy(mpi_reply, ioc->base_cmds.reply,
5126 sizeof(Mpi2SasIoUnitControlReply_t));
5127 else
5128 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t));
5129 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5130 goto out;
5131
5132 issue_host_reset:
5133 if (issue_reset)
98c56ad3 5134 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
f92363d1
SR		 5135 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5136 rc = -EFAULT;
5137 out:
5138 mutex_unlock(&ioc->base_cmds.mutex);
5139 return rc;
5140}
5141
5142/**
5143 * mpt3sas_base_scsi_enclosure_processor - sending request to sep device
5144 * @ioc: per adapter object
5145 * @mpi_reply: the reply payload from FW
5146 * @mpi_request: the request payload sent to FW
5147 *
5148 * The SCSI Enclosure Processor request message causes the IOC to
5149 * communicate with SES devices to control LED status signals.
5150 *
5151 * Returns 0 for success, non-zero for failure.
5152 */
5153int
5154mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
5155 Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request)
5156{
5157 u16 smid;
5158 u32 ioc_state;
eb44552b 5159 bool issue_reset = false;
f92363d1
SR		 5160 int rc;
5161 void *request;
5162 u16 wait_state_count;
5163
5164 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5165 __func__));
5166
5167 mutex_lock(&ioc->base_cmds.mutex);
5168
5169 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
5170 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
5171 ioc->name, __func__);
5172 rc = -EAGAIN;
5173 goto out;
5174 }
5175
5176 wait_state_count = 0;
5177 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5178 while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
5179 if (wait_state_count++ == 10) {
5180 pr_err(MPT3SAS_FMT
5181 "%s: failed due to ioc not operational\n",
5182 ioc->name, __func__);
5183 rc = -EFAULT;
5184 goto out;
5185 }
5186 ssleep(1);
5187 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5188 pr_info(MPT3SAS_FMT
5189 "%s: waiting for operational state(count=%d)\n",
5190 ioc->name,
5191 __func__, wait_state_count);
5192 }
5193
5194 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5195 if (!smid) {
5196 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5197 ioc->name, __func__);
5198 rc = -EAGAIN;
5199 goto out;
5200 }
5201
5202 rc = 0;
5203 ioc->base_cmds.status = MPT3_CMD_PENDING;
5204 request = mpt3sas_base_get_msg_frame(ioc, smid);
5205 ioc->base_cmds.smid = smid;
5206 memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
5207 init_completion(&ioc->base_cmds.done);
81c16f83 5208 ioc->put_smid_default(ioc, smid);
8bbb1cf6 5209 wait_for_completion_timeout(&ioc->base_cmds.done,
f92363d1
SR		 5210 msecs_to_jiffies(10000));
5211 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5212 pr_err(MPT3SAS_FMT "%s: timeout\n",
5213 ioc->name, __func__);
5214 _debug_dump_mf(mpi_request,
5215 sizeof(Mpi2SepRequest_t)/4);
5216 if (!(ioc->base_cmds.status & MPT3_CMD_RESET))
eb44552b 5217 issue_reset = false;
f92363d1
SR		 5218 goto issue_host_reset;
5219 }
5220 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
5221 memcpy(mpi_reply, ioc->base_cmds.reply,
5222 sizeof(Mpi2SepReply_t));
5223 else
5224 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t));
5225 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5226 goto out;
5227
5228 issue_host_reset:
5229 if (issue_reset)
98c56ad3 5230 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
f92363d1
SR		 5231 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5232 rc = -EFAULT;
5233 out:
5234 mutex_unlock(&ioc->base_cmds.mutex);
5235 return rc;
5236}
5237
5238/**
5239 * _base_get_port_facts - obtain port facts reply and save in ioc
5240 * @ioc: per adapter object
f92363d1
SR		 5241 *
5242 * Returns 0 for success, non-zero for failure.
5243 */
5244static int
98c56ad3 5245_base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)
f92363d1
SR		 5246{
5247 Mpi2PortFactsRequest_t mpi_request;
5248 Mpi2PortFactsReply_t mpi_reply;
5249 struct mpt3sas_port_facts *pfacts;
5250 int mpi_reply_sz, mpi_request_sz, r;
5251
5252 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5253 __func__));
5254
5255 mpi_reply_sz = sizeof(Mpi2PortFactsReply_t);
5256 mpi_request_sz = sizeof(Mpi2PortFactsRequest_t);
5257 memset(&mpi_request, 0, mpi_request_sz);
5258 mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
5259 mpi_request.PortNumber = port;
5260 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
98c56ad3 5261 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
f92363d1
SR		 5262
5263 if (r != 0) {
5264 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5265 ioc->name, __func__, r);
5266 return r;
5267 }
5268
5269 pfacts = &ioc->pfacts[port];
5270 memset(pfacts, 0, sizeof(struct mpt3sas_port_facts));
5271 pfacts->PortNumber = mpi_reply.PortNumber;
5272 pfacts->VP_ID = mpi_reply.VP_ID;
5273 pfacts->VF_ID = mpi_reply.VF_ID;
5274 pfacts->MaxPostedCmdBuffers =
5275 le16_to_cpu(mpi_reply.MaxPostedCmdBuffers);
5276
5277 return 0;
5278}
5279
4dc8c808
SR		 5280/**
5281 * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
5282 * @ioc: per adapter object
5283 * @timeout:
4dc8c808
SR		 5284 *
5285 * Returns 0 for success, non-zero for failure.
5286 */
5287static int
98c56ad3 5288_base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)
4dc8c808
SR		 5289{
5290 u32 ioc_state;
5291 int rc;
5292
5293 dinitprintk(ioc, printk(MPT3SAS_FMT "%s\n", ioc->name,
5294 __func__));
5295
5296 if (ioc->pci_error_recovery) {
5297 dfailprintk(ioc, printk(MPT3SAS_FMT
5298 "%s: host in pci error recovery\n", ioc->name, __func__));
5299 return -EFAULT;
5300 }
5301
5302 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
5303 dhsprintk(ioc, printk(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
5304 ioc->name, __func__, ioc_state));
5305
5306 if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) ||
5307 (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
5308 return 0;
5309
5310 if (ioc_state & MPI2_DOORBELL_USED) {
5311 dhsprintk(ioc, printk(MPT3SAS_FMT
5312 "unexpected doorbell active!\n", ioc->name));
5313 goto issue_diag_reset;
5314 }
5315
5316 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
5317 mpt3sas_base_fault_info(ioc, ioc_state &
5318 MPI2_DOORBELL_DATA_MASK);
5319 goto issue_diag_reset;
5320 }
5321
98c56ad3 5322 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
4dc8c808
SR		 5323 if (ioc_state) {
5324 dfailprintk(ioc, printk(MPT3SAS_FMT
5325 "%s: failed going to ready state (ioc_state=0x%x)\n",
5326 ioc->name, __func__, ioc_state));
5327 return -EFAULT;
5328 }
5329
5330 issue_diag_reset:
98c56ad3 5331 rc = _base_diag_reset(ioc);
4dc8c808
SR		 5332 return rc;
5333}
5334
f92363d1
SR		 5335/**
5336 * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
5337 * @ioc: per adapter object
f92363d1
SR		 5338 *
5339 * Returns 0 for success, non-zero for failure.
5340 */
5341static int
98c56ad3 5342_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5343{
5344 Mpi2IOCFactsRequest_t mpi_request;
5345 Mpi2IOCFactsReply_t mpi_reply;
5346 struct mpt3sas_facts *facts;
5347 int mpi_reply_sz, mpi_request_sz, r;
5348
5349 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5350 __func__));
5351
98c56ad3 5352 r = _base_wait_for_iocstate(ioc, 10);
4dc8c808
SR		 5353 if (r) {
5354 dfailprintk(ioc, printk(MPT3SAS_FMT
5355 "%s: failed getting to correct state\n",
5356 ioc->name, __func__));
5357 return r;
5358 }
f92363d1
SR		 5359 mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t);
5360 mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t);
5361 memset(&mpi_request, 0, mpi_request_sz);
5362 mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
5363 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
98c56ad3 5364 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
f92363d1
SR		 5365
5366 if (r != 0) {
5367 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5368 ioc->name, __func__, r);
5369 return r;
5370 }
5371
5372 facts = &ioc->facts;
5373 memset(facts, 0, sizeof(struct mpt3sas_facts));
5374 facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion);
5375 facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion);
5376 facts->VP_ID = mpi_reply.VP_ID;
5377 facts->VF_ID = mpi_reply.VF_ID;
5378 facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions);
5379 facts->MaxChainDepth = mpi_reply.MaxChainDepth;
5380 facts->WhoInit = mpi_reply.WhoInit;
5381 facts->NumberOfPorts = mpi_reply.NumberOfPorts;
5382 facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors;
5383 facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit);
5384 facts->MaxReplyDescriptorPostQueueDepth =
5385 le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth);
5386 facts->ProductID = le16_to_cpu(mpi_reply.ProductID);
5387 facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities);
5388 if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID))
5389 ioc->ir_firmware = 1;
9b05c91a 5390 if ((facts->IOCCapabilities &
06f5f976 5391 MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices))
9b05c91a 5392 ioc->rdpq_array_capable = 1;
81c16f83
SPS		 5393 if (facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ)
5394 ioc->atomic_desc_capable = 1;
f92363d1
SR		 5395 facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
5396 facts->IOCRequestFrameSize =
5397 le16_to_cpu(mpi_reply.IOCRequestFrameSize);
ebb3024e
SS		 5398 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
5399 facts->IOCMaxChainSegmentSize =
5400 le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize);
5401 }
f92363d1
SR		 5402 facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators);
5403 facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets);
5404 ioc->shost->max_id = -1;
5405 facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders);
5406 facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures);
5407 facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags);
5408 facts->HighPriorityCredit =
5409 le16_to_cpu(mpi_reply.HighPriorityCredit);
5410 facts->ReplyFrameSize = mpi_reply.ReplyFrameSize;
5411 facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle);
016d5c35
SPS		 5412 facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize;
5413
5414 /*
5415 * Get the Page Size from IOC Facts. If it's 0, default to 4k.
5416 */
5417 ioc->page_size = 1 << facts->CurrentHostPageSize;
5418 if (ioc->page_size == 1) {
5419 pr_info(MPT3SAS_FMT "CurrentHostPageSize is 0: Setting "
5420 "default host page size to 4k\n", ioc->name);
5421 ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;
5422 }
5423 dinitprintk(ioc, pr_info(MPT3SAS_FMT "CurrentHostPageSize(%d)\n",
5424 ioc->name, facts->CurrentHostPageSize));
f92363d1
SR		 5425
5426 dinitprintk(ioc, pr_info(MPT3SAS_FMT
5427 "hba queue depth(%d), max chains per io(%d)\n",
5428 ioc->name, facts->RequestCredit,
5429 facts->MaxChainDepth));
5430 dinitprintk(ioc, pr_info(MPT3SAS_FMT
5431 "request frame size(%d), reply frame size(%d)\n", ioc->name,
5432 facts->IOCRequestFrameSize * 4, facts->ReplyFrameSize * 4));
5433 return 0;
5434}
5435
5436/**
5437 * _base_send_ioc_init - send ioc_init to firmware
5438 * @ioc: per adapter object
f92363d1
SR		 5439 *
5440 * Returns 0 for success, non-zero for failure.
5441 */
5442static int
98c56ad3 5443_base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5444{
5445 Mpi2IOCInitRequest_t mpi_request;
5446 Mpi2IOCInitReply_t mpi_reply;
9b05c91a 5447 int i, r = 0;
23409bd4 5448 ktime_t current_time;
f92363d1 5449 u16 ioc_status;
9b05c91a
SR		 5450 u32 reply_post_free_array_sz = 0;
5451 Mpi2IOCInitRDPQArrayEntry *reply_post_free_array = NULL;
5452 dma_addr_t reply_post_free_array_dma;
f92363d1
SR		 5453
5454 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5455 __func__));
5456
5457 memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t));
5458 mpi_request.Function = MPI2_FUNCTION_IOC_INIT;
5459 mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
5460 mpi_request.VF_ID = 0; /* TODO */
5461 mpi_request.VP_ID = 0;
d357e84d 5462 mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);
f92363d1 5463 mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
016d5c35 5464 mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K;
f92363d1
SR		 5465
5466 if (_base_is_controller_msix_enabled(ioc))
5467 mpi_request.HostMSIxVectors = ioc->reply_queue_count;
5468 mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);
5469 mpi_request.ReplyDescriptorPostQueueDepth =
5470 cpu_to_le16(ioc->reply_post_queue_depth);
5471 mpi_request.ReplyFreeQueueDepth =
5472 cpu_to_le16(ioc->reply_free_queue_depth);
5473
5474 mpi_request.SenseBufferAddressHigh =
5475 cpu_to_le32((u64)ioc->sense_dma >> 32);
5476 mpi_request.SystemReplyAddressHigh =
5477 cpu_to_le32((u64)ioc->reply_dma >> 32);
5478 mpi_request.SystemRequestFrameBaseAddress =
5479 cpu_to_le64((u64)ioc->request_dma);
5480 mpi_request.ReplyFreeQueueAddress =
5481 cpu_to_le64((u64)ioc->reply_free_dma);
f92363d1 5482
9b05c91a
SR		 5483 if (ioc->rdpq_array_enable) {
5484 reply_post_free_array_sz = ioc->reply_queue_count *
5485 sizeof(Mpi2IOCInitRDPQArrayEntry);
5486 reply_post_free_array = pci_alloc_consistent(ioc->pdev,
5487 reply_post_free_array_sz, &reply_post_free_array_dma);
5488 if (!reply_post_free_array) {
5489 pr_err(MPT3SAS_FMT
5490 "reply_post_free_array: pci_alloc_consistent failed\n",
5491 ioc->name);
5492 r = -ENOMEM;
5493 goto out;
5494 }
5495 memset(reply_post_free_array, 0, reply_post_free_array_sz);
5496 for (i = 0; i < ioc->reply_queue_count; i++)
5497 reply_post_free_array[i].RDPQBaseAddress =
5498 cpu_to_le64(
5499 (u64)ioc->reply_post[i].reply_post_free_dma);
5500 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
5501 mpi_request.ReplyDescriptorPostQueueAddress =
5502 cpu_to_le64((u64)reply_post_free_array_dma);
5503 } else {
5504 mpi_request.ReplyDescriptorPostQueueAddress =
5505 cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
5506 }
f92363d1
SR		 5507
5508 /* This time stamp specifies number of milliseconds
5509 * since epoch ~ midnight January 1, 1970.
5510 */
23409bd4
TR		 5511 current_time = ktime_get_real();
5512 mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time));
f92363d1
SR		 5513
5514 if (ioc->logging_level & MPT_DEBUG_INIT) {
5515 __le32 *mfp;
5516 int i;
5517
5518 mfp = (__le32 *)&mpi_request;
5519 pr_info("\toffset:data\n");
5520 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++)
5521 pr_info("\t[0x%02x]:%08x\n", i*4,
5522 le32_to_cpu(mfp[i]));
5523 }
5524
5525 r = _base_handshake_req_reply_wait(ioc,
5526 sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
98c56ad3 5527 sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 10);
f92363d1
SR		 5528
5529 if (r != 0) {
5530 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5531 ioc->name, __func__, r);
9b05c91a 5532 goto out;
f92363d1
SR		 5533 }
5534
5535 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
5536 if (ioc_status != MPI2_IOCSTATUS_SUCCESS ||
5537 mpi_reply.IOCLogInfo) {
5538 pr_err(MPT3SAS_FMT "%s: failed\n", ioc->name, __func__);
5539 r = -EIO;
5540 }
5541
9b05c91a
SR		 5542out:
5543 if (reply_post_free_array)
5544 pci_free_consistent(ioc->pdev, reply_post_free_array_sz,
5545 reply_post_free_array,
5546 reply_post_free_array_dma);
5547 return r;
f92363d1
SR		 5548}
5549
5550/**
5551 * mpt3sas_port_enable_done - command completion routine for port enable
5552 * @ioc: per adapter object
5553 * @smid: system request message index
5554 * @msix_index: MSIX table index supplied by the OS
5555 * @reply: reply message frame(lower 32bit addr)
5556 *
5557 * Return 1 meaning mf should be freed from _base_interrupt
5558 * 0 means the mf is freed from this function.
5559 */
5560u8
5561mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
5562 u32 reply)
5563{
5564 MPI2DefaultReply_t *mpi_reply;
5565 u16 ioc_status;
5566
5567 if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)
5568 return 1;
5569
5570 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
5571 if (!mpi_reply)
5572 return 1;
5573
5574 if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE)
5575 return 1;
5576
5577 ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;
5578 ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;
5579 ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;
5580 memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
5581 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
5582 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5583 ioc->port_enable_failed = 1;
5584
5585 if (ioc->is_driver_loading) {
5586 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
5587 mpt3sas_port_enable_complete(ioc);
5588 return 1;
5589 } else {
5590 ioc->start_scan_failed = ioc_status;
5591 ioc->start_scan = 0;
5592 return 1;
5593 }
5594 }
5595 complete(&ioc->port_enable_cmds.done);
5596 return 1;
5597}
5598
5599/**
5600 * _base_send_port_enable - send port_enable(discovery stuff) to firmware
5601 * @ioc: per adapter object
f92363d1
SR		 5602 *
5603 * Returns 0 for success, non-zero for failure.
5604 */
5605static int
98c56ad3 5606_base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5607{
5608 Mpi2PortEnableRequest_t *mpi_request;
5609 Mpi2PortEnableReply_t *mpi_reply;
f92363d1
SR		 5610 int r = 0;
5611 u16 smid;
5612 u16 ioc_status;
5613
5614 pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
5615
5616 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5617 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5618 ioc->name, __func__);
5619 return -EAGAIN;
5620 }
5621
5622 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
5623 if (!smid) {
5624 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5625 ioc->name, __func__);
5626 return -EAGAIN;
5627 }
5628
5629 ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
5630 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5631 ioc->port_enable_cmds.smid = smid;
5632 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
5633 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
5634
5635 init_completion(&ioc->port_enable_cmds.done);
81c16f83 5636 ioc->put_smid_default(ioc, smid);
8bbb1cf6 5637 wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
f92363d1
SR		 5638 if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
5639 pr_err(MPT3SAS_FMT "%s: timeout\n",
5640 ioc->name, __func__);
5641 _debug_dump_mf(mpi_request,
5642 sizeof(Mpi2PortEnableRequest_t)/4);
5643 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)
5644 r = -EFAULT;
5645 else
5646 r = -ETIME;
5647 goto out;
5648 }
5649
5650 mpi_reply = ioc->port_enable_cmds.reply;
5651 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
5652 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
5653 pr_err(MPT3SAS_FMT "%s: failed with (ioc_status=0x%08x)\n",
5654 ioc->name, __func__, ioc_status);
5655 r = -EFAULT;
5656 goto out;
5657 }
5658
5659 out:
5660 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
5661 pr_info(MPT3SAS_FMT "port enable: %s\n", ioc->name, ((r == 0) ?
5662 "SUCCESS" : "FAILED"));
5663 return r;
5664}
5665
5666/**
5667 * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply)
5668 * @ioc: per adapter object
5669 *
5670 * Returns 0 for success, non-zero for failure.
5671 */
5672int
5673mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
5674{
5675 Mpi2PortEnableRequest_t *mpi_request;
5676 u16 smid;
5677
5678 pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
5679
5680 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5681 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5682 ioc->name, __func__);
5683 return -EAGAIN;
5684 }
5685
5686 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
5687 if (!smid) {
5688 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5689 ioc->name, __func__);
5690 return -EAGAIN;
5691 }
5692
5693 ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
5694 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5695 ioc->port_enable_cmds.smid = smid;
5696 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
5697 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
5698
81c16f83 5699 ioc->put_smid_default(ioc, smid);
f92363d1
SR		 5700 return 0;
5701}
5702
5703/**
5704 * _base_determine_wait_on_discovery - desposition
5705 * @ioc: per adapter object
5706 *
5707 * Decide whether to wait on discovery to complete. Used to either
5708 * locate boot device, or report volumes ahead of physical devices.
5709 *
5710 * Returns 1 for wait, 0 for don't wait
5711 */
5712static int
5713_base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)
5714{
5715 /* We wait for discovery to complete if IR firmware is loaded.
5716 * The sas topology events arrive before PD events, so we need time to
5717 * turn on the bit in ioc->pd_handles to indicate PD
5718 * Also, it maybe required to report Volumes ahead of physical
5719 * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set.
5720 */
5721 if (ioc->ir_firmware)
5722 return 1;
5723
5724 /* if no Bios, then we don't need to wait */
5725 if (!ioc->bios_pg3.BiosVersion)
5726 return 0;
5727
5728 /* Bios is present, then we drop down here.
5729 *
5730 * If there any entries in the Bios Page 2, then we wait
5731 * for discovery to complete.
5732 */
5733
5734 /* Current Boot Device */
5735 if ((ioc->bios_pg2.CurrentBootDeviceForm &
5736 MPI2_BIOSPAGE2_FORM_MASK) ==
5737 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
5738 /* Request Boot Device */
5739 (ioc->bios_pg2.ReqBootDeviceForm &
5740 MPI2_BIOSPAGE2_FORM_MASK) ==
5741 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
5742 /* Alternate Request Boot Device */
5743 (ioc->bios_pg2.ReqAltBootDeviceForm &
5744 MPI2_BIOSPAGE2_FORM_MASK) ==
5745 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED)
5746 return 0;
5747
5748 return 1;
5749}
5750
5751/**
5752 * _base_unmask_events - turn on notification for this event
5753 * @ioc: per adapter object
5754 * @event: firmware event
5755 *
5756 * The mask is stored in ioc->event_masks.
5757 */
5758static void
5759_base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
5760{
5761 u32 desired_event;
5762
5763 if (event >= 128)
5764 return;
5765
5766 desired_event = (1 << (event % 32));
5767
5768 if (event < 32)
5769 ioc->event_masks[0] &= ~desired_event;
5770 else if (event < 64)
5771 ioc->event_masks[1] &= ~desired_event;
5772 else if (event < 96)
5773 ioc->event_masks[2] &= ~desired_event;
5774 else if (event < 128)
5775 ioc->event_masks[3] &= ~desired_event;
5776}
5777
5778/**
5779 * _base_event_notification - send event notification
5780 * @ioc: per adapter object
f92363d1
SR		 5781 *
5782 * Returns 0 for success, non-zero for failure.
5783 */
5784static int
98c56ad3 5785_base_event_notification(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5786{
5787 Mpi2EventNotificationRequest_t *mpi_request;
f92363d1
SR		 5788 u16 smid;
5789 int r = 0;
5790 int i;
5791
5792 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5793 __func__));
5794
5795 if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
5796 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5797 ioc->name, __func__);
5798 return -EAGAIN;
5799 }
5800
5801 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5802 if (!smid) {
5803 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5804 ioc->name, __func__);
5805 return -EAGAIN;
5806 }
5807 ioc->base_cmds.status = MPT3_CMD_PENDING;
5808 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5809 ioc->base_cmds.smid = smid;
5810 memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t));
5811 mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
5812 mpi_request->VF_ID = 0; /* TODO */
5813 mpi_request->VP_ID = 0;
5814 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
5815 mpi_request->EventMasks[i] =
5816 cpu_to_le32(ioc->event_masks[i]);
5817 init_completion(&ioc->base_cmds.done);
81c16f83 5818 ioc->put_smid_default(ioc, smid);
8bbb1cf6 5819 wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
f92363d1
SR		 5820 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5821 pr_err(MPT3SAS_FMT "%s: timeout\n",
5822 ioc->name, __func__);
5823 _debug_dump_mf(mpi_request,
5824 sizeof(Mpi2EventNotificationRequest_t)/4);
5825 if (ioc->base_cmds.status & MPT3_CMD_RESET)
5826 r = -EFAULT;
5827 else
5828 r = -ETIME;
5829 } else
5830 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s: complete\n",
5831 ioc->name, __func__));
5832 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5833 return r;
5834}
5835
5836/**
5837 * mpt3sas_base_validate_event_type - validating event types
5838 * @ioc: per adapter object
5839 * @event: firmware event
5840 *
5841 * This will turn on firmware event notification when application
5842 * ask for that event. We don't mask events that are already enabled.
5843 */
5844void
5845mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
5846{
5847 int i, j;
5848 u32 event_mask, desired_event;
5849 u8 send_update_to_fw;
5850
5851 for (i = 0, send_update_to_fw = 0; i <
5852 MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) {
5853 event_mask = ~event_type[i];
5854 desired_event = 1;
5855 for (j = 0; j < 32; j++) {
5856 if (!(event_mask & desired_event) &&
5857 (ioc->event_masks[i] & desired_event)) {
5858 ioc->event_masks[i] &= ~desired_event;
5859 send_update_to_fw = 1;
5860 }
5861 desired_event = (desired_event << 1);
5862 }
5863 }
5864
5865 if (!send_update_to_fw)
5866 return;
5867
5868 mutex_lock(&ioc->base_cmds.mutex);
98c56ad3 5869 _base_event_notification(ioc);
f92363d1
SR		 5870 mutex_unlock(&ioc->base_cmds.mutex);
5871}
5872
5873/**
5874 * _base_diag_reset - the "big hammer" start of day reset
5875 * @ioc: per adapter object
f92363d1
SR		 5876 *
5877 * Returns 0 for success, non-zero for failure.
5878 */
5879static int
98c56ad3 5880_base_diag_reset(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5881{
5882 u32 host_diagnostic;
5883 u32 ioc_state;
5884 u32 count;
5885 u32 hcb_size;
5886
5887 pr_info(MPT3SAS_FMT "sending diag reset !!\n", ioc->name);
5888
5889 drsprintk(ioc, pr_info(MPT3SAS_FMT "clear interrupts\n",
5890 ioc->name));
5891
5892 count = 0;
5893 do {
5894 /* Write magic sequence to WriteSequence register
5895 * Loop until in diagnostic mode
5896 */
5897 drsprintk(ioc, pr_info(MPT3SAS_FMT
5898 "write magic sequence\n", ioc->name));
5899 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
5900 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);
5901 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);
5902 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);
5903 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);
5904 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);
5905 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
5906
5907 /* wait 100 msec */
98c56ad3 5908 msleep(100);
f92363d1
SR		 5909
5910 if (count++ > 20)
5911 goto out;
5912
5913 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
5914 drsprintk(ioc, pr_info(MPT3SAS_FMT
5915 "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",
5916 ioc->name, count, host_diagnostic));
5917
5918 } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0);
5919
5920 hcb_size = readl(&ioc->chip->HCBSize);
5921
5922 drsprintk(ioc, pr_info(MPT3SAS_FMT "diag reset: issued\n",
5923 ioc->name));
5924 writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER,
5925 &ioc->chip->HostDiagnostic);
5926
b453ff84 5927 /*This delay allows the chip PCIe hardware time to finish reset tasks*/
98c56ad3 5928 msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
f92363d1 5929
b453ff84
SR		 5930 /* Approximately 300 second max wait */
5931 for (count = 0; count < (300000000 /
5932 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) {
f92363d1
SR		 5933
5934 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
5935
5936 if (host_diagnostic == 0xFFFFFFFF)
5937 goto out;
5938 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
5939 break;
5940
98c56ad3 5941 msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000);
f92363d1
SR		 5942 }
5943
5944 if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
5945
5946 drsprintk(ioc, pr_info(MPT3SAS_FMT
5947 "restart the adapter assuming the HCB Address points to good F/W\n",
5948 ioc->name));
5949 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK;
5950 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW;
5951 writel(host_diagnostic, &ioc->chip->HostDiagnostic);
5952
5953 drsprintk(ioc, pr_info(MPT3SAS_FMT
5954 "re-enable the HCDW\n", ioc->name));
5955 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE,
5956 &ioc->chip->HCBSize);
5957 }
5958
5959 drsprintk(ioc, pr_info(MPT3SAS_FMT "restart the adapter\n",
5960 ioc->name));
5961 writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET,
5962 &ioc->chip->HostDiagnostic);
5963
5964 drsprintk(ioc, pr_info(MPT3SAS_FMT
5965 "disable writes to the diagnostic register\n", ioc->name));
5966 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
5967
5968 drsprintk(ioc, pr_info(MPT3SAS_FMT
5969 "Wait for FW to go to the READY state\n", ioc->name));
98c56ad3 5970 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);
f92363d1
SR		 5971 if (ioc_state) {
5972 pr_err(MPT3SAS_FMT
5973 "%s: failed going to ready state (ioc_state=0x%x)\n",
5974 ioc->name, __func__, ioc_state);
5975 goto out;
5976 }
5977
5978 pr_info(MPT3SAS_FMT "diag reset: SUCCESS\n", ioc->name);
5979 return 0;
5980
5981 out:
5982 pr_err(MPT3SAS_FMT "diag reset: FAILED\n", ioc->name);
5983 return -EFAULT;
5984}
5985
5986/**
5987 * _base_make_ioc_ready - put controller in READY state
5988 * @ioc: per adapter object
f92363d1
SR		 5989 * @type: FORCE_BIG_HAMMER or SOFT_RESET
5990 *
5991 * Returns 0 for success, non-zero for failure.
5992 */
5993static int
98c56ad3 5994_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)
f92363d1
SR		 5995{
5996 u32 ioc_state;
5997 int rc;
5998 int count;
5999
6000 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6001 __func__));
6002
6003 if (ioc->pci_error_recovery)
6004 return 0;
6005
6006 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6007 dhsprintk(ioc, pr_info(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
6008 ioc->name, __func__, ioc_state));
6009
6010 /* if in RESET state, it should move to READY state shortly */
6011 count = 0;
6012 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
6013 while ((ioc_state & MPI2_IOC_STATE_MASK) !=
6014 MPI2_IOC_STATE_READY) {
6015 if (count++ == 10) {
6016 pr_err(MPT3SAS_FMT
6017 "%s: failed going to ready state (ioc_state=0x%x)\n",
6018 ioc->name, __func__, ioc_state);
6019 return -EFAULT;
6020 }
98c56ad3 6021 ssleep(1);
f92363d1
SR		 6022 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6023 }
6024 }
6025
6026 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY)
6027 return 0;
6028
6029 if (ioc_state & MPI2_DOORBELL_USED) {
6030 dhsprintk(ioc, pr_info(MPT3SAS_FMT
6031 "unexpected doorbell active!\n",
6032 ioc->name));
6033 goto issue_diag_reset;
6034 }
6035
6036 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
6037 mpt3sas_base_fault_info(ioc, ioc_state &
6038 MPI2_DOORBELL_DATA_MASK);
6039 goto issue_diag_reset;
6040 }
6041
6042 if (type == FORCE_BIG_HAMMER)
6043 goto issue_diag_reset;
6044
6045 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
6046 if (!(_base_send_ioc_reset(ioc,
98c56ad3 6047 MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) {
f92363d1
SR		 6048 return 0;
6049 }
6050
6051 issue_diag_reset:
98c56ad3 6052 rc = _base_diag_reset(ioc);
f92363d1
SR		 6053 return rc;
6054}
6055
6056/**
6057 * _base_make_ioc_operational - put controller in OPERATIONAL state
6058 * @ioc: per adapter object
f92363d1
SR		 6059 *
6060 * Returns 0 for success, non-zero for failure.
6061 */
6062static int
98c56ad3 6063_base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
f92363d1 6064{
5ec8a175 6065 int r, i, index;
f92363d1
SR		 6066 unsigned long flags;
6067 u32 reply_address;
6068 u16 smid;
6069 struct _tr_list *delayed_tr, *delayed_tr_next;
fd0331b3
SS		 6070 struct _sc_list *delayed_sc, *delayed_sc_next;
6071 struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next;
7786ab6a 6072 u8 hide_flag;
f92363d1 6073 struct adapter_reply_queue *reply_q;
5ec8a175 6074 Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig;
f92363d1
SR		 6075
6076 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6077 __func__));
6078
6079 /* clean the delayed target reset list */
6080 list_for_each_entry_safe(delayed_tr, delayed_tr_next,
6081 &ioc->delayed_tr_list, list) {
6082 list_del(&delayed_tr->list);
6083 kfree(delayed_tr);
6084 }
6085
6086
6087 list_for_each_entry_safe(delayed_tr, delayed_tr_next,
6088 &ioc->delayed_tr_volume_list, list) {
6089 list_del(&delayed_tr->list);
6090 kfree(delayed_tr);
6091 }
6092
fd0331b3
SS		 6093 list_for_each_entry_safe(delayed_sc, delayed_sc_next,
6094 &ioc->delayed_sc_list, list) {
6095 list_del(&delayed_sc->list);
6096 kfree(delayed_sc);
6097 }
6098
6099 list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next,
6100 &ioc->delayed_event_ack_list, list) {
6101 list_del(&delayed_event_ack->list);
6102 kfree(delayed_event_ack);
6103 }
6104
f92363d1 6105 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
f92363d1
SR		 6106
6107 /* hi-priority queue */
6108 INIT_LIST_HEAD(&ioc->hpr_free_list);
6109 smid = ioc->hi_priority_smid;
6110 for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {
6111 ioc->hpr_lookup[i].cb_idx = 0xFF;
6112 ioc->hpr_lookup[i].smid = smid;
6113 list_add_tail(&ioc->hpr_lookup[i].tracker_list,
6114 &ioc->hpr_free_list);
6115 }
6116
6117 /* internal queue */
6118 INIT_LIST_HEAD(&ioc->internal_free_list);
6119 smid = ioc->internal_smid;
6120 for (i = 0; i < ioc->internal_depth; i++, smid++) {
6121 ioc->internal_lookup[i].cb_idx = 0xFF;
6122 ioc->internal_lookup[i].smid = smid;
6123 list_add_tail(&ioc->internal_lookup[i].tracker_list,
6124 &ioc->internal_free_list);
6125 }
6126
6127 /* chain pool */
6128 INIT_LIST_HEAD(&ioc->free_chain_list);
6129 for (i = 0; i < ioc->chain_depth; i++)
6130 list_add_tail(&ioc->chain_lookup[i].tracker_list,
6131 &ioc->free_chain_list);
6132
6133 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
6134
6135 /* initialize Reply Free Queue */
6136 for (i = 0, reply_address = (u32)ioc->reply_dma ;
6137 i < ioc->reply_free_queue_depth ; i++, reply_address +=
6138 ioc->reply_sz)
6139 ioc->reply_free[i] = cpu_to_le32(reply_address);
6140
6141 /* initialize reply queues */
6142 if (ioc->is_driver_loading)
6143 _base_assign_reply_queues(ioc);
6144
6145 /* initialize Reply Post Free Queue */
5ec8a175
CO		 6146 index = 0;
6147 reply_post_free_contig = ioc->reply_post[0].reply_post_free;
f92363d1 6148 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
5ec8a175
CO		 6149 /*
6150 * If RDPQ is enabled, switch to the next allocation.
6151 * Otherwise advance within the contiguous region.
6152 */
6153 if (ioc->rdpq_array_enable) {
6154 reply_q->reply_post_free =
6155 ioc->reply_post[index++].reply_post_free;
6156 } else {
6157 reply_q->reply_post_free = reply_post_free_contig;
6158 reply_post_free_contig += ioc->reply_post_queue_depth;
6159 }
6160
f92363d1 6161 reply_q->reply_post_host_index = 0;
f92363d1
SR		 6162 for (i = 0; i < ioc->reply_post_queue_depth; i++)
6163 reply_q->reply_post_free[i].Words =
6164 cpu_to_le64(ULLONG_MAX);
6165 if (!_base_is_controller_msix_enabled(ioc))
6166 goto skip_init_reply_post_free_queue;
f92363d1
SR		 6167 }
6168 skip_init_reply_post_free_queue:
6169
98c56ad3 6170 r = _base_send_ioc_init(ioc);
f92363d1
SR		 6171 if (r)
6172 return r;
6173
6174 /* initialize reply free host index */
6175 ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;
6176 writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);
6177
6178 /* initialize reply post host index */
6179 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
0bb337c9 6180 if (ioc->combined_reply_queue)
fb77bb53
SR		 6181 writel((reply_q->msix_index & 7)<<
6182 MPI2_RPHI_MSIX_INDEX_SHIFT,
6183 ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
6184 else
6185 writel(reply_q->msix_index <<
6186 MPI2_RPHI_MSIX_INDEX_SHIFT,
6187 &ioc->chip->ReplyPostHostIndex);
6188
f92363d1
SR		 6189 if (!_base_is_controller_msix_enabled(ioc))
6190 goto skip_init_reply_post_host_index;
6191 }
6192
6193 skip_init_reply_post_host_index:
6194
6195 _base_unmask_interrupts(ioc);
98c56ad3 6196 r = _base_event_notification(ioc);
f92363d1
SR		 6197 if (r)
6198 return r;
6199
98c56ad3 6200 _base_static_config_pages(ioc);
f92363d1
SR		 6201
6202 if (ioc->is_driver_loading) {
7786ab6a
SR		 6203
6204 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
6205 == 0x80) {
6206 hide_flag = (u8) (
6207 le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &
6208 MFG_PAGE10_HIDE_SSDS_MASK);
6209 if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK)
6210 ioc->mfg_pg10_hide_flag = hide_flag;
6211 }
6212
f92363d1
SR		 6213 ioc->wait_for_discovery_to_complete =
6214 _base_determine_wait_on_discovery(ioc);
6215
6216 return r; /* scan_start and scan_finished support */
6217 }
6218
98c56ad3 6219 r = _base_send_port_enable(ioc);
f92363d1
SR		 6220 if (r)
6221 return r;
6222
6223 return r;
6224}
6225
6226/**
6227 * mpt3sas_base_free_resources - free resources controller resources
6228 * @ioc: per adapter object
6229 *
6230 * Return nothing.
6231 */
6232void
6233mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
6234{
f92363d1
SR		 6235 dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6236 __func__));
6237
08c4d550
SR		 6238 /* synchronizing freeing resource with pci_access_mutex lock */
6239 mutex_lock(&ioc->pci_access_mutex);
cf9bd21a
JL		 6240 if (ioc->chip_phys && ioc->chip) {
6241 _base_mask_interrupts(ioc);
6242 ioc->shost_recovery = 1;
98c56ad3 6243 _base_make_ioc_ready(ioc, SOFT_RESET);
cf9bd21a
JL		 6244 ioc->shost_recovery = 0;
6245 }
6246
580d4e31 6247 mpt3sas_base_unmap_resources(ioc);
08c4d550 6248 mutex_unlock(&ioc->pci_access_mutex);
f92363d1
SR		 6249 return;
6250}
6251
6252/**
6253 * mpt3sas_base_attach - attach controller instance
6254 * @ioc: per adapter object
6255 *
6256 * Returns 0 for success, non-zero for failure.
6257 */
6258int
6259mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
6260{
6261 int r, i;
6262 int cpu_id, last_cpu_id = 0;
6263
6264 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6265 __func__));
6266
6267 /* setup cpu_msix_table */
6268 ioc->cpu_count = num_online_cpus();
6269 for_each_online_cpu(cpu_id)
6270 last_cpu_id = cpu_id;
6271 ioc->cpu_msix_table_sz = last_cpu_id + 1;
6272 ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);
6273 ioc->reply_queue_count = 1;
6274 if (!ioc->cpu_msix_table) {
6275 dfailprintk(ioc, pr_info(MPT3SAS_FMT
6276 "allocation for cpu_msix_table failed!!!\n",
6277 ioc->name));
6278 r = -ENOMEM;
6279 goto out_free_resources;
6280 }
6281
7786ab6a
SR		 6282 if (ioc->is_warpdrive) {
6283 ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,
6284 sizeof(resource_size_t *), GFP_KERNEL);
6285 if (!ioc->reply_post_host_index) {
6286 dfailprintk(ioc, pr_info(MPT3SAS_FMT "allocation "
1d55abc0
HR		 6287 "for reply_post_host_index failed!!!\n",
6288 ioc->name));
7786ab6a
SR		 6289 r = -ENOMEM;
6290 goto out_free_resources;
6291 }
6292 }
6293
9b05c91a
SR		 6294 ioc->rdpq_array_enable_assigned = 0;
6295 ioc->dma_mask = 0;
f92363d1
SR		 6296 r = mpt3sas_base_map_resources(ioc);
6297 if (r)
6298 goto out_free_resources;
6299
f92363d1 6300 pci_set_drvdata(ioc->pdev, ioc->shost);
98c56ad3 6301 r = _base_get_ioc_facts(ioc);
f92363d1
SR		 6302 if (r)
6303 goto out_free_resources;
6304
471ef9d4
SR		 6305 switch (ioc->hba_mpi_version_belonged) {
6306 case MPI2_VERSION:
6307 ioc->build_sg_scmd = &_base_build_sg_scmd;
6308 ioc->build_sg = &_base_build_sg;
6309 ioc->build_zero_len_sge = &_base_build_zero_len_sge;
6310 break;
6311 case MPI25_VERSION:
b130b0d5 6312 case MPI26_VERSION:
471ef9d4
SR		 6313 /*
6314 * In SAS3.0,
6315 * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and
6316 * Target Status - all require the IEEE formated scatter gather
6317 * elements.
6318 */
6319 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;
6320 ioc->build_sg = &_base_build_sg_ieee;
aff39e61 6321 ioc->build_nvme_prp = &_base_build_nvme_prp;
471ef9d4
SR		 6322 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
6323 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
81c16f83 6324
471ef9d4
SR		 6325 break;
6326 }
f92363d1 6327
81c16f83
SPS		 6328 if (ioc->atomic_desc_capable) {
6329 ioc->put_smid_default = &_base_put_smid_default_atomic;
6330 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic;
6331 ioc->put_smid_fast_path = &_base_put_smid_fast_path_atomic;
6332 ioc->put_smid_hi_priority = &_base_put_smid_hi_priority_atomic;
aff39e61 6333 ioc->put_smid_nvme_encap = &_base_put_smid_nvme_encap_atomic;
81c16f83
SPS		 6334 } else {
6335 ioc->put_smid_default = &_base_put_smid_default;
6336 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;
6337 ioc->put_smid_fast_path = &_base_put_smid_fast_path;
6338 ioc->put_smid_hi_priority = &_base_put_smid_hi_priority;
aff39e61 6339 ioc->put_smid_nvme_encap = &_base_put_smid_nvme_encap;
81c16f83
SPS		 6340 }
6341
6342
f92363d1
SR		 6343 /*
6344 * These function pointers for other requests that don't
6345 * the require IEEE scatter gather elements.
6346 *
6347 * For example Configuration Pages and SAS IOUNIT Control don't.
6348 */
6349 ioc->build_sg_mpi = &_base_build_sg;
6350 ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
6351
98c56ad3 6352 r = _base_make_ioc_ready(ioc, SOFT_RESET);
f92363d1
SR		 6353 if (r)
6354 goto out_free_resources;
6355
6356 ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,
6357 sizeof(struct mpt3sas_port_facts), GFP_KERNEL);
6358 if (!ioc->pfacts) {
6359 r = -ENOMEM;
6360 goto out_free_resources;
6361 }
6362
6363 for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
98c56ad3 6364 r = _base_get_port_facts(ioc, i);
f92363d1
SR		 6365 if (r)
6366 goto out_free_resources;
6367 }
6368
98c56ad3 6369 r = _base_allocate_memory_pools(ioc);
f92363d1
SR		 6370 if (r)
6371 goto out_free_resources;
6372
6373 init_waitqueue_head(&ioc->reset_wq);
6374
6375 /* allocate memory pd handle bitmask list */
6376 ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
6377 if (ioc->facts.MaxDevHandle % 8)
6378 ioc->pd_handles_sz++;
6379 ioc->pd_handles = kzalloc(ioc->pd_handles_sz,
6380 GFP_KERNEL);
6381 if (!ioc->pd_handles) {
6382 r = -ENOMEM;
6383 goto out_free_resources;
6384 }
6385 ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,
6386 GFP_KERNEL);
6387 if (!ioc->blocking_handles) {
6388 r = -ENOMEM;
6389 goto out_free_resources;
6390 }
6391
c696f7b8
SPS		 6392 /* allocate memory for pending OS device add list */
6393 ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);
6394 if (ioc->facts.MaxDevHandle % 8)
6395 ioc->pend_os_device_add_sz++;
6396 ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,
6397 GFP_KERNEL);
6398 if (!ioc->pend_os_device_add)
6399 goto out_free_resources;
6400
6401 ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;
6402 ioc->device_remove_in_progress =
6403 kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);
6404 if (!ioc->device_remove_in_progress)
6405 goto out_free_resources;
6406
f92363d1
SR		 6407 ioc->fwfault_debug = mpt3sas_fwfault_debug;
6408
6409 /* base internal command bits */
6410 mutex_init(&ioc->base_cmds.mutex);
6411 ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6412 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6413
6414 /* port_enable command bits */
6415 ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6416 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
6417
6418 /* transport internal command bits */
6419 ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6420 ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
6421 mutex_init(&ioc->transport_cmds.mutex);
6422
6423 /* scsih internal command bits */
6424 ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6425 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
6426 mutex_init(&ioc->scsih_cmds.mutex);
6427
6428 /* task management internal command bits */
6429 ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6430 ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
6431 mutex_init(&ioc->tm_cmds.mutex);
6432
6433 /* config page internal command bits */
6434 ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6435 ioc->config_cmds.status = MPT3_CMD_NOT_USED;
6436 mutex_init(&ioc->config_cmds.mutex);
6437
6438 /* ctl module internal command bits */
6439 ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6440 ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
6441 ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
6442 mutex_init(&ioc->ctl_cmds.mutex);
6443
a5dd7efd
CJ		 6444 if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||
6445 !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||
6446 !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||
6447 !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {
f92363d1
SR		 6448 r = -ENOMEM;
6449 goto out_free_resources;
6450 }
6451
6452 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
6453 ioc->event_masks[i] = -1;
6454
6455 /* here we enable the events we care about */
6456 _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);
6457 _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
6458 _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
6459 _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
6460 _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
6461 _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
6462 _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);
6463 _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);
6464 _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);
6465 _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
2d8ce8c9 6466 _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
b99b1993 6467 _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
4318c734
SPS		 6468 if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
6469 if (ioc->is_gen35_ioc) {
6470 _base_unmask_events(ioc,
6471 MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
6472 _base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);
6473 _base_unmask_events(ioc,
6474 MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
6475 }
6476 }
98c56ad3 6477 r = _base_make_ioc_operational(ioc);
f92363d1
SR		 6478 if (r)
6479 goto out_free_resources;
6480
16e179bd 6481 ioc->non_operational_loop = 0;
459325c4 6482 ioc->got_task_abort_from_ioctl = 0;
f92363d1
SR		 6483 return 0;
6484
6485 out_free_resources:
6486
6487 ioc->remove_host = 1;
6488
6489 mpt3sas_base_free_resources(ioc);
6490 _base_release_memory_pools(ioc);
6491 pci_set_drvdata(ioc->pdev, NULL);
6492 kfree(ioc->cpu_msix_table);
7786ab6a
SR		 6493 if (ioc->is_warpdrive)
6494 kfree(ioc->reply_post_host_index);
f92363d1
SR		 6495 kfree(ioc->pd_handles);
6496 kfree(ioc->blocking_handles);
c696f7b8
SPS		 6497 kfree(ioc->device_remove_in_progress);
6498 kfree(ioc->pend_os_device_add);
f92363d1
SR		 6499 kfree(ioc->tm_cmds.reply);
6500 kfree(ioc->transport_cmds.reply);
6501 kfree(ioc->scsih_cmds.reply);
6502 kfree(ioc->config_cmds.reply);
6503 kfree(ioc->base_cmds.reply);
6504 kfree(ioc->port_enable_cmds.reply);
6505 kfree(ioc->ctl_cmds.reply);
6506 kfree(ioc->ctl_cmds.sense);
6507 kfree(ioc->pfacts);
6508 ioc->ctl_cmds.reply = NULL;
6509 ioc->base_cmds.reply = NULL;
6510 ioc->tm_cmds.reply = NULL;
6511 ioc->scsih_cmds.reply = NULL;
6512 ioc->transport_cmds.reply = NULL;
6513 ioc->config_cmds.reply = NULL;
6514 ioc->pfacts = NULL;
6515 return r;
6516}
6517
6518
6519/**
6520 * mpt3sas_base_detach - remove controller instance
6521 * @ioc: per adapter object
6522 *
6523 * Return nothing.
6524 */
6525void
6526mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
6527{
6528 dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6529 __func__));
6530
6531 mpt3sas_base_stop_watchdog(ioc);
6532 mpt3sas_base_free_resources(ioc);
6533 _base_release_memory_pools(ioc);
6534 pci_set_drvdata(ioc->pdev, NULL);
6535 kfree(ioc->cpu_msix_table);
7786ab6a
SR		 6536 if (ioc->is_warpdrive)
6537 kfree(ioc->reply_post_host_index);
f92363d1
SR		 6538 kfree(ioc->pd_handles);
6539 kfree(ioc->blocking_handles);
c696f7b8
SPS		 6540 kfree(ioc->device_remove_in_progress);
6541 kfree(ioc->pend_os_device_add);
f92363d1
SR		 6542 kfree(ioc->pfacts);
6543 kfree(ioc->ctl_cmds.reply);
6544 kfree(ioc->ctl_cmds.sense);
6545 kfree(ioc->base_cmds.reply);
6546 kfree(ioc->port_enable_cmds.reply);
6547 kfree(ioc->tm_cmds.reply);
6548 kfree(ioc->transport_cmds.reply);
6549 kfree(ioc->scsih_cmds.reply);
6550 kfree(ioc->config_cmds.reply);
6551}
6552
6553/**
6554 * _base_reset_handler - reset callback handler (for base)
6555 * @ioc: per adapter object
6556 * @reset_phase: phase
6557 *
6558 * The handler for doing any required cleanup or initialization.
6559 *
6560 * The reset phase can be MPT3_IOC_PRE_RESET, MPT3_IOC_AFTER_RESET,
6561 * MPT3_IOC_DONE_RESET
6562 *
6563 * Return nothing.
6564 */
6565static void
6566_base_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
6567{
6568 mpt3sas_scsih_reset_handler(ioc, reset_phase);
6569 mpt3sas_ctl_reset_handler(ioc, reset_phase);
6570 switch (reset_phase) {
6571 case MPT3_IOC_PRE_RESET:
6572 dtmprintk(ioc, pr_info(MPT3SAS_FMT
6573 "%s: MPT3_IOC_PRE_RESET\n", ioc->name, __func__));
6574 break;
6575 case MPT3_IOC_AFTER_RESET:
6576 dtmprintk(ioc, pr_info(MPT3SAS_FMT
6577 "%s: MPT3_IOC_AFTER_RESET\n", ioc->name, __func__));
6578 if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {
6579 ioc->transport_cmds.status |= MPT3_CMD_RESET;
6580 mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);
6581 complete(&ioc->transport_cmds.done);
6582 }
6583 if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
6584 ioc->base_cmds.status |= MPT3_CMD_RESET;
6585 mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);
6586 complete(&ioc->base_cmds.done);
6587 }
6588 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
6589 ioc->port_enable_failed = 1;
6590 ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
6591 mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);
6592 if (ioc->is_driver_loading) {
6593 ioc->start_scan_failed =
6594 MPI2_IOCSTATUS_INTERNAL_ERROR;
6595 ioc->start_scan = 0;
6596 ioc->port_enable_cmds.status =
6597 MPT3_CMD_NOT_USED;
6598 } else
6599 complete(&ioc->port_enable_cmds.done);
6600 }
6601 if (ioc->config_cmds.status & MPT3_CMD_PENDING) {
6602 ioc->config_cmds.status |= MPT3_CMD_RESET;
6603 mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);
6604 ioc->config_cmds.smid = USHRT_MAX;
6605 complete(&ioc->config_cmds.done);
6606 }
6607 break;
6608 case MPT3_IOC_DONE_RESET:
6609 dtmprintk(ioc, pr_info(MPT3SAS_FMT
6610 "%s: MPT3_IOC_DONE_RESET\n", ioc->name, __func__));
6611 break;
6612 }
6613}
6614
6615/**
6616 * _wait_for_commands_to_complete - reset controller
6617 * @ioc: Pointer to MPT_ADAPTER structure
f92363d1 6618 *
272e253c 6619 * This function is waiting 10s for all pending commands to complete
f92363d1
SR		 6620 * prior to putting controller in reset.
6621 */
6622static void
98c56ad3 6623_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 6624{
6625 u32 ioc_state;
f92363d1
SR		 6626
6627 ioc->pending_io_count = 0;
f92363d1
SR		 6628
6629 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6630 if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
6631 return;
6632
6633 /* pending command count */
272e253c 6634 ioc->pending_io_count = atomic_read(&ioc->shost->host_busy);
f92363d1
SR		 6635
6636 if (!ioc->pending_io_count)
6637 return;
6638
6639 /* wait for pending commands to complete */
6640 wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);
6641}
6642
6643/**
6644 * mpt3sas_base_hard_reset_handler - reset controller
6645 * @ioc: Pointer to MPT_ADAPTER structure
f92363d1
SR		 6646 * @type: FORCE_BIG_HAMMER or SOFT_RESET
6647 *
6648 * Returns 0 for success, non-zero for failure.
6649 */
6650int
98c56ad3 6651mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
f92363d1
SR		 6652 enum reset_type type)
6653{
6654 int r;
6655 unsigned long flags;
6656 u32 ioc_state;
6657 u8 is_fault = 0, is_trigger = 0;
6658
6659 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: enter\n", ioc->name,
6660 __func__));
6661
6662 if (ioc->pci_error_recovery) {
6663 pr_err(MPT3SAS_FMT "%s: pci error recovery reset\n",
6664 ioc->name, __func__);
6665 r = 0;
6666 goto out_unlocked;
6667 }
6668
6669 if (mpt3sas_fwfault_debug)
6670 mpt3sas_halt_firmware(ioc);
6671
f92363d1
SR		 6672 /* wait for an active reset in progress to complete */
6673 if (!mutex_trylock(&ioc->reset_in_progress_mutex)) {
6674 do {
6675 ssleep(1);
6676 } while (ioc->shost_recovery == 1);
6677 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
6678 __func__));
6679 return ioc->ioc_reset_in_progress_status;
6680 }
6681
6682 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6683 ioc->shost_recovery = 1;
6684 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6685
6686 if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
6687 MPT3_DIAG_BUFFER_IS_REGISTERED) &&
6688 (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
6689 MPT3_DIAG_BUFFER_IS_RELEASED))) {
6690 is_trigger = 1;
6691 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6692 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
6693 is_fault = 1;
6694 }
6695 _base_reset_handler(ioc, MPT3_IOC_PRE_RESET);
98c56ad3 6696 _wait_for_commands_to_complete(ioc);
f92363d1 6697 _base_mask_interrupts(ioc);
98c56ad3 6698 r = _base_make_ioc_ready(ioc, type);
f92363d1
SR		 6699 if (r)
6700 goto out;
6701 _base_reset_handler(ioc, MPT3_IOC_AFTER_RESET);
6702
6703 /* If this hard reset is called while port enable is active, then
6704 * there is no reason to call make_ioc_operational
6705 */
6706 if (ioc->is_driver_loading && ioc->port_enable_failed) {
6707 ioc->remove_host = 1;
6708 r = -EFAULT;
6709 goto out;
6710 }
98c56ad3 6711 r = _base_get_ioc_facts(ioc);
f92363d1
SR		 6712 if (r)
6713 goto out;
9b05c91a
SR		 6714
6715 if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)
6716 panic("%s: Issue occurred with flashing controller firmware."
6717 "Please reboot the system and ensure that the correct"
6718 " firmware version is running\n", ioc->name);
6719
98c56ad3 6720 r = _base_make_ioc_operational(ioc);
f92363d1
SR		 6721 if (!r)
6722 _base_reset_handler(ioc, MPT3_IOC_DONE_RESET);
6723
6724 out:
6725 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: %s\n",
6726 ioc->name, __func__, ((r == 0) ? "SUCCESS" : "FAILED")));
6727
6728 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6729 ioc->ioc_reset_in_progress_status = r;
6730 ioc->shost_recovery = 0;
6731 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6732 ioc->ioc_reset_count++;
6733 mutex_unlock(&ioc->reset_in_progress_mutex);
6734
6735 out_unlocked:
6736 if ((r == 0) && is_trigger) {
6737 if (is_fault)
6738 mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);
6739 else
6740 mpt3sas_trigger_master(ioc,
6741 MASTER_TRIGGER_ADAPTER_RESET);
6742 }
6743 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
6744 __func__));
6745 return r;
6746}
Linux 6.x block layer and io_uring tree(s)