projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
scsi: mpt3sas: Pre-allocate RDPQ Array at driver boot time.
[linux-2.6-block.git] / drivers / scsi / mpt3sas / mpt3sas_base.c
CommitLineData
f92363d1
SR		 1/*
2 * This is the Fusion MPT base driver providing common API layer interface
3 * for access to MPT (Message Passing Technology) firmware.
4 *
5 * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c
a4ffce0d 6 * Copyright (C) 2012-2014 LSI Corporation
a03bd153
SR		 7 * Copyright (C) 2013-2014 Avago Technologies
8 * (mailto: MPT-FusionLinux.pdl@avagotech.com)
f92363d1
SR		 9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version 2
13 * of the License, or (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * NO WARRANTY
21 * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
22 * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
23 * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
24 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
25 * solely responsible for determining the appropriateness of using and
26 * distributing the Program and assumes all risks associated with its
27 * exercise of rights under this Agreement, including but not limited to
28 * the risks and costs of program errors, damage to or loss of data,
29 * programs or equipment, and unavailability or interruption of operations.
30
31 * DISCLAIMER OF LIABILITY
32 * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
33 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
37 * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
38 * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
39
40 * You should have received a copy of the GNU General Public License
41 * along with this program; if not, write to the Free Software
42 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
43 * USA.
44 */
45
f92363d1
SR		 46#include <linux/kernel.h>
47#include <linux/module.h>
48#include <linux/errno.h>
49#include <linux/init.h>
50#include <linux/slab.h>
51#include <linux/types.h>
52#include <linux/pci.h>
53#include <linux/kdev_t.h>
54#include <linux/blkdev.h>
55#include <linux/delay.h>
56#include <linux/interrupt.h>
57#include <linux/dma-mapping.h>
58#include <linux/io.h>
59#include <linux/time.h>
23409bd4 60#include <linux/ktime.h>
f92363d1 61#include <linux/kthread.h>
016d5c35 62#include <asm/page.h> /* To get host page size per arch */
f92363d1
SR		 63#include <linux/aer.h>
64
65
66#include "mpt3sas_base.h"
67
68static MPT_CALLBACK mpt_callbacks[MPT_MAX_CALLBACKS];
69
70
71#define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */
72
73 /* maximum controller queue depth */
74#define MAX_HBA_QUEUE_DEPTH 30000
75#define MAX_CHAIN_DEPTH 100000
76static int max_queue_depth = -1;
77module_param(max_queue_depth, int, 0);
78MODULE_PARM_DESC(max_queue_depth, " max controller queue depth ");
79
80static int max_sgl_entries = -1;
81module_param(max_sgl_entries, int, 0);
82MODULE_PARM_DESC(max_sgl_entries, " max sg entries ");
83
84static int msix_disable = -1;
85module_param(msix_disable, int, 0);
86MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
87
64038301
SPS		 88static int smp_affinity_enable = 1;
89module_param(smp_affinity_enable, int, S_IRUGO);
23b389c2 90MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
64038301 91
fb77bb53 92static int max_msix_vectors = -1;
9c500060
SR		 93module_param(max_msix_vectors, int, 0);
94MODULE_PARM_DESC(max_msix_vectors,
fb77bb53 95 " max msix vectors");
f92363d1
SR		 96
97static int mpt3sas_fwfault_debug;
98MODULE_PARM_DESC(mpt3sas_fwfault_debug,
99 " enable detection of firmware fault and halt firmware - (default=0)");
100
9b05c91a 101static int
98c56ad3 102_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
f92363d1
SR		 103
104/**
105 * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
106 *
107 */
108static int
e4dca7b7 109_scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp)
f92363d1
SR		 110{
111 int ret = param_set_int(val, kp);
112 struct MPT3SAS_ADAPTER *ioc;
113
114 if (ret)
115 return ret;
116
08c4d550 117 /* global ioc spinlock to protect controller list on list operations */
f92363d1 118 pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug);
08c4d550 119 spin_lock(&gioc_lock);
f92363d1
SR		 120 list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
121 ioc->fwfault_debug = mpt3sas_fwfault_debug;
08c4d550 122 spin_unlock(&gioc_lock);
f92363d1
SR		 123 return 0;
124}
125module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug,
126 param_get_int, &mpt3sas_fwfault_debug, 0644);
127
b4472d71
SPS		 128/**
129 * _base_clone_reply_to_sys_mem - copies reply to reply free iomem
130 * in BAR0 space.
131 *
132 * @ioc: per adapter object
133 * @reply: reply message frame(lower 32bit addr)
134 * @index: System request message index.
135 *
136 * @Returns - Nothing
137 */
138static void
139_base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply,
140 u32 index)
141{
142 /*
143 * 256 is offset within sys register.
144 * 256 offset MPI frame starts. Max MPI frame supported is 32.
145 * 32 * 128 = 4K. From here, Clone of reply free for mcpu starts
146 */
147 u16 cmd_credit = ioc->facts.RequestCredit + 1;
148 void __iomem *reply_free_iomem = (void __iomem *)ioc->chip +
149 MPI_FRAME_START_OFFSET +
150 (cmd_credit * ioc->request_sz) + (index * sizeof(u32));
151
152 writel(reply, reply_free_iomem);
153}
154
e5747439
SPS		 155/**
156 * _base_clone_mpi_to_sys_mem - Writes/copies MPI frames
157 * to system/BAR0 region.
158 *
159 * @dst_iomem: Pointer to the destinaltion location in BAR0 space.
160 * @src: Pointer to the Source data.
161 * @size: Size of data to be copied.
162 */
163static void
164_base_clone_mpi_to_sys_mem(void *dst_iomem, void *src, u32 size)
165{
166 int i;
167 u32 *src_virt_mem = (u32 *)src;
168
169 for (i = 0; i < size/4; i++)
170 writel((u32)src_virt_mem[i],
171 (void __iomem *)dst_iomem + (i * 4));
172}
173
182ac784
SPS		 174/**
175 * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region
176 *
177 * @dst_iomem: Pointer to the destination location in BAR0 space.
178 * @src: Pointer to the Source data.
179 * @size: Size of data to be copied.
180 */
181static void
182_base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size)
183{
184 int i;
185 u32 *src_virt_mem = (u32 *)(src);
186
187 for (i = 0; i < size/4; i++)
188 writel((u32)src_virt_mem[i],
189 (void __iomem *)dst_iomem + (i * 4));
190}
191
22ae5a3c
SPS		 192/**
193 * _base_get_chain - Calculates and Returns virtual chain address
194 * for the provided smid in BAR0 space.
195 *
196 * @ioc: per adapter object
197 * @smid: system request message index
198 * @sge_chain_count: Scatter gather chain count.
199 *
200 * @Return: chain address.
201 */
202static inline void __iomem*
203_base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,
204 u8 sge_chain_count)
205{
206 void __iomem *base_chain, *chain_virt;
207 u16 cmd_credit = ioc->facts.RequestCredit + 1;
208
209 base_chain = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +
210 (cmd_credit * ioc->request_sz) +
211 REPLY_FREE_POOL_SIZE;
212 chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *
213 ioc->request_sz) + (sge_chain_count * ioc->request_sz);
214 return chain_virt;
215}
216
217/**
218 * _base_get_chain_phys - Calculates and Returns physical address
219 * in BAR0 for scatter gather chains, for
220 * the provided smid.
221 *
222 * @ioc: per adapter object
223 * @smid: system request message index
224 * @sge_chain_count: Scatter gather chain count.
225 *
226 * @Return - Physical chain address.
227 */
6f9e09fd 228static inline phys_addr_t
22ae5a3c
SPS		 229_base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,
230 u8 sge_chain_count)
231{
6f9e09fd 232 phys_addr_t base_chain_phys, chain_phys;
22ae5a3c
SPS		 233 u16 cmd_credit = ioc->facts.RequestCredit + 1;
234
6f9e09fd 235 base_chain_phys = ioc->chip_phys + MPI_FRAME_START_OFFSET +
22ae5a3c
SPS		 236 (cmd_credit * ioc->request_sz) +
237 REPLY_FREE_POOL_SIZE;
238 chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *
239 ioc->request_sz) + (sge_chain_count * ioc->request_sz);
240 return chain_phys;
241}
242
243/**
244 * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host
245 * buffer address for the provided smid.
246 * (Each smid can have 64K starts from 17024)
247 *
248 * @ioc: per adapter object
249 * @smid: system request message index
250 *
251 * @Returns - Pointer to buffer location in BAR0.
252 */
253
254static void __iomem *
255_base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
256{
257 u16 cmd_credit = ioc->facts.RequestCredit + 1;
258 // Added extra 1 to reach end of chain.
259 void __iomem *chain_end = _base_get_chain(ioc,
260 cmd_credit + 1,
261 ioc->facts.MaxChainDepth);
262 return chain_end + (smid * 64 * 1024);
263}
264
265/**
266 * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped
267 * Host buffer Physical address for the provided smid.
268 * (Each smid can have 64K starts from 17024)
269 *
270 * @ioc: per adapter object
271 * @smid: system request message index
272 *
273 * @Returns - Pointer to buffer location in BAR0.
274 */
6f9e09fd 275static phys_addr_t
22ae5a3c
SPS		 276_base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
277{
278 u16 cmd_credit = ioc->facts.RequestCredit + 1;
6f9e09fd 279 phys_addr_t chain_end_phys = _base_get_chain_phys(ioc,
22ae5a3c
SPS		 280 cmd_credit + 1,
281 ioc->facts.MaxChainDepth);
282 return chain_end_phys + (smid * 64 * 1024);
283}
284
182ac784
SPS		 285/**
286 * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain
287 * lookup list and Provides chain_buffer
288 * address for the matching dma address.
289 * (Each smid can have 64K starts from 17024)
290 *
291 * @ioc: per adapter object
292 * @chain_buffer_dma: Chain buffer dma address.
293 *
294 * @Returns - Pointer to chain buffer. Or Null on Failure.
295 */
296static void *
297_base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
298 dma_addr_t chain_buffer_dma)
299{
300 u16 index;
301
302 for (index = 0; index < ioc->chain_depth; index++) {
303 if (ioc->chain_lookup[index].chain_buffer_dma ==
304 chain_buffer_dma)
305 return ioc->chain_lookup[index].chain_buffer;
306 }
307 pr_info(MPT3SAS_FMT
308 "Provided chain_buffer_dma address is not in the lookup list\n",
309 ioc->name);
310 return NULL;
311}
312
313/**
314 * _clone_sg_entries - MPI EP's scsiio and config requests
315 * are handled here. Base function for
316 * double buffering, before submitting
317 * the requests.
318 *
319 * @ioc: per adapter object.
320 * @mpi_request: mf request pointer.
321 * @smid: system request message index.
322 *
323 * @Returns: Nothing.
324 */
325static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
326 void *mpi_request, u16 smid)
327{
328 Mpi2SGESimple32_t *sgel, *sgel_next;
329 u32 sgl_flags, sge_chain_count = 0;
330 bool is_write = 0;
331 u16 i = 0;
332 void __iomem *buffer_iomem;
6f9e09fd 333 phys_addr_t buffer_iomem_phys;
182ac784 334 void __iomem *buff_ptr;
6f9e09fd 335 phys_addr_t buff_ptr_phys;
182ac784 336 void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO];
6f9e09fd
AB		 337 void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO];
338 phys_addr_t dst_addr_phys;
182ac784
SPS		 339 MPI2RequestHeader_t *request_hdr;
340 struct scsi_cmnd *scmd;
341 struct scatterlist *sg_scmd = NULL;
342 int is_scsiio_req = 0;
343
344 request_hdr = (MPI2RequestHeader_t *) mpi_request;
345
346 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
347 Mpi25SCSIIORequest_t *scsiio_request =
348 (Mpi25SCSIIORequest_t *)mpi_request;
349 sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL;
350 is_scsiio_req = 1;
351 } else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) {
352 Mpi2ConfigRequest_t *config_req =
353 (Mpi2ConfigRequest_t *)mpi_request;
354 sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE;
355 } else
356 return;
357
358 /* From smid we can get scsi_cmd, once we have sg_scmd,
359 * we just need to get sg_virt and sg_next to get virual
360 * address associated with sgel->Address.
361 */
362
363 if (is_scsiio_req) {
364 /* Get scsi_cmd using smid */
365 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
366 if (scmd == NULL) {
367 pr_err(MPT3SAS_FMT "scmd is NULL\n", ioc->name);
368 return;
369 }
370
371 /* Get sg_scmd from scmd provided */
372 sg_scmd = scsi_sglist(scmd);
373 }
374
375 /*
376 * 0 - 255 System register
377 * 256 - 4352 MPI Frame. (This is based on maxCredit 32)
378 * 4352 - 4864 Reply_free pool (512 byte is reserved
379 * considering maxCredit 32. Reply need extra
380 * room, for mCPU case kept four times of
381 * maxCredit).
382 * 4864 - 17152 SGE chain element. (32cmd * 3 chain of
383 * 128 byte size = 12288)
384 * 17152 - x Host buffer mapped with smid.
385 * (Each smid can have 64K Max IO.)
386 * BAR0+Last 1K MSIX Addr and Data
387 * Total size in use 2113664 bytes of 4MB BAR0
388 */
389
390 buffer_iomem = _base_get_buffer_bar0(ioc, smid);
391 buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);
392
393 buff_ptr = buffer_iomem;
394 buff_ptr_phys = buffer_iomem_phys;
6f9e09fd 395 WARN_ON(buff_ptr_phys > U32_MAX);
182ac784 396
cf6bf971 397 if (le32_to_cpu(sgel->FlagsLength) &
182ac784
SPS		 398 (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
399 is_write = 1;
400
401 for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
402
cf6bf971
C		 403 sgl_flags =
404 (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT);
182ac784
SPS		 405
406 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
407 case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
408 /*
409 * Helper function which on passing
410 * chain_buffer_dma returns chain_buffer. Get
411 * the virtual address for sgel->Address
412 */
413 sgel_next =
414 _base_get_chain_buffer_dma_to_chain_buffer(ioc,
cf6bf971 415 le32_to_cpu(sgel->Address));
182ac784
SPS		 416 if (sgel_next == NULL)
417 return;
418 /*
419 * This is coping 128 byte chain
420 * frame (not a host buffer)
421 */
422 dst_chain_addr[sge_chain_count] =
423 _base_get_chain(ioc,
424 smid, sge_chain_count);
425 src_chain_addr[sge_chain_count] =
426 (void *) sgel_next;
6f9e09fd 427 dst_addr_phys = _base_get_chain_phys(ioc,
182ac784 428 smid, sge_chain_count);
6f9e09fd 429 WARN_ON(dst_addr_phys > U32_MAX);
cf6bf971
C		 430 sgel->Address =
431 cpu_to_le32(lower_32_bits(dst_addr_phys));
182ac784
SPS		 432 sgel = sgel_next;
433 sge_chain_count++;
434 break;
435 case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
436 if (is_write) {
437 if (is_scsiio_req) {
438 _base_clone_to_sys_mem(buff_ptr,
439 sg_virt(sg_scmd),
cf6bf971
C		 440 (le32_to_cpu(sgel->FlagsLength) &
441 0x00ffffff));
6f9e09fd
AB		 442 /*
443 * FIXME: this relies on a a zero
444 * PCI mem_offset.
445 */
cf6bf971
C		 446 sgel->Address =
447 cpu_to_le32((u32)buff_ptr_phys);
182ac784
SPS		 448 } else {
449 _base_clone_to_sys_mem(buff_ptr,
450 ioc->config_vaddr,
cf6bf971
C		 451 (le32_to_cpu(sgel->FlagsLength) &
452 0x00ffffff));
453 sgel->Address =
454 cpu_to_le32((u32)buff_ptr_phys);
182ac784
SPS		 455 }
456 }
cf6bf971
C		 457 buff_ptr += (le32_to_cpu(sgel->FlagsLength) &
458 0x00ffffff);
459 buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) &
460 0x00ffffff);
461 if ((le32_to_cpu(sgel->FlagsLength) &
182ac784
SPS		 462 (MPI2_SGE_FLAGS_END_OF_BUFFER
463 << MPI2_SGE_FLAGS_SHIFT)))
464 goto eob_clone_chain;
465 else {
466 /*
467 * Every single element in MPT will have
468 * associated sg_next. Better to sanity that
469 * sg_next is not NULL, but it will be a bug
470 * if it is null.
471 */
472 if (is_scsiio_req) {
473 sg_scmd = sg_next(sg_scmd);
474 if (sg_scmd)
475 sgel++;
476 else
477 goto eob_clone_chain;
478 }
479 }
480 break;
481 }
482 }
483
484eob_clone_chain:
485 for (i = 0; i < sge_chain_count; i++) {
486 if (is_scsiio_req)
487 _base_clone_to_sys_mem(dst_chain_addr[i],
488 src_chain_addr[i], ioc->request_sz);
489 }
490}
491
f92363d1
SR		 492/**
493 * mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc
494 * @arg: input argument, used to derive ioc
495 *
496 * Return 0 if controller is removed from pci subsystem.
497 * Return -1 for other case.
498 */
499static int mpt3sas_remove_dead_ioc_func(void *arg)
500{
501 struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;
502 struct pci_dev *pdev;
503
504 if ((ioc == NULL))
505 return -1;
506
507 pdev = ioc->pdev;
508 if ((pdev == NULL))
509 return -1;
64cdb418 510 pci_stop_and_remove_bus_device_locked(pdev);
f92363d1
SR		 511 return 0;
512}
513
514/**
515 * _base_fault_reset_work - workq handling ioc fault conditions
516 * @work: input argument, used to derive ioc
517 * Context: sleep.
518 *
519 * Return nothing.
520 */
521static void
522_base_fault_reset_work(struct work_struct *work)
523{
524 struct MPT3SAS_ADAPTER *ioc =
525 container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work);
526 unsigned long flags;
527 u32 doorbell;
528 int rc;
529 struct task_struct *p;
530
531
532 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
16e179bd 533 if (ioc->shost_recovery || ioc->pci_error_recovery)
f92363d1
SR		 534 goto rearm_timer;
535 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
536
537 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
538 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) {
539 pr_err(MPT3SAS_FMT "SAS host is non-operational !!!!\n",
540 ioc->name);
541
16e179bd
SR		 542 /* It may be possible that EEH recovery can resolve some of
543 * pci bus failure issues rather removing the dead ioc function
544 * by considering controller is in a non-operational state. So
545 * here priority is given to the EEH recovery. If it doesn't
546 * not resolve this issue, mpt3sas driver will consider this
547 * controller to non-operational state and remove the dead ioc
548 * function.
549 */
550 if (ioc->non_operational_loop++ < 5) {
551 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,
552 flags);
553 goto rearm_timer;
554 }
555
f92363d1
SR		 556 /*
557 * Call _scsih_flush_pending_cmds callback so that we flush all
558 * pending commands back to OS. This call is required to aovid
559 * deadlock at block layer. Dead IOC will fail to do diag reset,
560 * and this call is safe since dead ioc will never return any
561 * command back from HW.
562 */
563 ioc->schedule_dead_ioc_flush_running_cmds(ioc);
564 /*
565 * Set remove_host flag early since kernel thread will
566 * take some time to execute.
567 */
568 ioc->remove_host = 1;
569 /*Remove the Dead Host */
570 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,
c84b06a4 571 "%s_dead_ioc_%d", ioc->driver_name, ioc->id);
f92363d1
SR		 572 if (IS_ERR(p))
573 pr_err(MPT3SAS_FMT
574 "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",
575 ioc->name, __func__);
576 else
577 pr_err(MPT3SAS_FMT
578 "%s: Running mpt3sas_dead_ioc thread success !!!!\n",
579 ioc->name, __func__);
580 return; /* don't rearm timer */
581 }
582
16e179bd
SR		 583 ioc->non_operational_loop = 0;
584
f92363d1 585 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
98c56ad3 586 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
f92363d1
SR		 587 pr_warn(MPT3SAS_FMT "%s: hard reset: %s\n", ioc->name,
588 __func__, (rc == 0) ? "success" : "failed");
589 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
590 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
591 mpt3sas_base_fault_info(ioc, doorbell &
592 MPI2_DOORBELL_DATA_MASK);
593 if (rc && (doorbell & MPI2_IOC_STATE_MASK) !=
594 MPI2_IOC_STATE_OPERATIONAL)
595 return; /* don't rearm timer */
596 }
597
598 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
599 rearm_timer:
600 if (ioc->fault_reset_work_q)
601 queue_delayed_work(ioc->fault_reset_work_q,
602 &ioc->fault_reset_work,
603 msecs_to_jiffies(FAULT_POLLING_INTERVAL));
604 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
605}
606
607/**
608 * mpt3sas_base_start_watchdog - start the fault_reset_work_q
609 * @ioc: per adapter object
610 * Context: sleep.
611 *
612 * Return nothing.
613 */
614void
615mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
616{
617 unsigned long flags;
618
619 if (ioc->fault_reset_work_q)
620 return;
621
622 /* initialize fault polling */
623
624 INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
625 snprintf(ioc->fault_reset_work_q_name,
c84b06a4
SR		 626 sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",
627 ioc->driver_name, ioc->id);
f92363d1
SR		 628 ioc->fault_reset_work_q =
629 create_singlethread_workqueue(ioc->fault_reset_work_q_name);
630 if (!ioc->fault_reset_work_q) {
631 pr_err(MPT3SAS_FMT "%s: failed (line=%d)\n",
632 ioc->name, __func__, __LINE__);
633 return;
634 }
635 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
636 if (ioc->fault_reset_work_q)
637 queue_delayed_work(ioc->fault_reset_work_q,
638 &ioc->fault_reset_work,
639 msecs_to_jiffies(FAULT_POLLING_INTERVAL));
640 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
641}
642
643/**
644 * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q
645 * @ioc: per adapter object
646 * Context: sleep.
647 *
648 * Return nothing.
649 */
650void
651mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)
652{
653 unsigned long flags;
654 struct workqueue_struct *wq;
655
656 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
657 wq = ioc->fault_reset_work_q;
658 ioc->fault_reset_work_q = NULL;
659 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
660 if (wq) {
4dc06fd8 661 if (!cancel_delayed_work_sync(&ioc->fault_reset_work))
f92363d1
SR		 662 flush_workqueue(wq);
663 destroy_workqueue(wq);
664 }
665}
666
667/**
668 * mpt3sas_base_fault_info - verbose translation of firmware FAULT code
669 * @ioc: per adapter object
670 * @fault_code: fault code
671 *
672 * Return nothing.
673 */
674void
675mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code)
676{
677 pr_err(MPT3SAS_FMT "fault_state(0x%04x)!\n",
678 ioc->name, fault_code);
679}
680
681/**
682 * mpt3sas_halt_firmware - halt's mpt controller firmware
683 * @ioc: per adapter object
684 *
685 * For debugging timeout related issues. Writing 0xCOFFEE00
686 * to the doorbell register will halt controller firmware. With
687 * the purpose to stop both driver and firmware, the enduser can
688 * obtain a ring buffer from controller UART.
689 */
690void
691mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)
692{
693 u32 doorbell;
694
695 if (!ioc->fwfault_debug)
696 return;
697
698 dump_stack();
699
700 doorbell = readl(&ioc->chip->Doorbell);
701 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
702 mpt3sas_base_fault_info(ioc , doorbell);
703 else {
704 writel(0xC0FFEE00, &ioc->chip->Doorbell);
705 pr_err(MPT3SAS_FMT "Firmware is halted due to command timeout\n",
706 ioc->name);
707 }
708
709 if (ioc->fwfault_debug == 2)
710 for (;;)
711 ;
712 else
713 panic("panic in %s\n", __func__);
714}
715
f92363d1
SR		 716/**
717 * _base_sas_ioc_info - verbose translation of the ioc status
718 * @ioc: per adapter object
719 * @mpi_reply: reply mf payload returned from firmware
720 * @request_hdr: request mf
721 *
722 * Return nothing.
723 */
724static void
725_base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,
726 MPI2RequestHeader_t *request_hdr)
727{
728 u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
729 MPI2_IOCSTATUS_MASK;
730 char *desc = NULL;
731 u16 frame_sz;
732 char *func_str = NULL;
733
734 /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */
735 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
736 request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
737 request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION)
738 return;
739
740 if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
741 return;
742
743 switch (ioc_status) {
744
745/****************************************************************************
746* Common IOCStatus values for all replies
747****************************************************************************/
748
749 case MPI2_IOCSTATUS_INVALID_FUNCTION:
750 desc = "invalid function";
751 break;
752 case MPI2_IOCSTATUS_BUSY:
753 desc = "busy";
754 break;
755 case MPI2_IOCSTATUS_INVALID_SGL:
756 desc = "invalid sgl";
757 break;
758 case MPI2_IOCSTATUS_INTERNAL_ERROR:
759 desc = "internal error";
760 break;
761 case MPI2_IOCSTATUS_INVALID_VPID:
762 desc = "invalid vpid";
763 break;
764 case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
765 desc = "insufficient resources";
766 break;
b130b0d5
SS		 767 case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
768 desc = "insufficient power";
769 break;
f92363d1
SR		 770 case MPI2_IOCSTATUS_INVALID_FIELD:
771 desc = "invalid field";
772 break;
773 case MPI2_IOCSTATUS_INVALID_STATE:
774 desc = "invalid state";
775 break;
776 case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
777 desc = "op state not supported";
778 break;
779
780/****************************************************************************
781* Config IOCStatus values
782****************************************************************************/
783
784 case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION:
785 desc = "config invalid action";
786 break;
787 case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE:
788 desc = "config invalid type";
789 break;
790 case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE:
791 desc = "config invalid page";
792 break;
793 case MPI2_IOCSTATUS_CONFIG_INVALID_DATA:
794 desc = "config invalid data";
795 break;
796 case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS:
797 desc = "config no defaults";
798 break;
799 case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT:
800 desc = "config cant commit";
801 break;
802
803/****************************************************************************
804* SCSI IO Reply
805****************************************************************************/
806
807 case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
808 case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
809 case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
810 case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
811 case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
812 case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
813 case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
814 case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
815 case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
816 case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
817 case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
818 case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
819 break;
820
821/****************************************************************************
822* For use by SCSI Initiator and SCSI Target end-to-end data protection
823****************************************************************************/
824
825 case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
826 desc = "eedp guard error";
827 break;
828 case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
829 desc = "eedp ref tag error";
830 break;
831 case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
832 desc = "eedp app tag error";
833 break;
834
835/****************************************************************************
836* SCSI Target values
837****************************************************************************/
838
839 case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX:
840 desc = "target invalid io index";
841 break;
842 case MPI2_IOCSTATUS_TARGET_ABORTED:
843 desc = "target aborted";
844 break;
845 case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE:
846 desc = "target no conn retryable";
847 break;
848 case MPI2_IOCSTATUS_TARGET_NO_CONNECTION:
849 desc = "target no connection";
850 break;
851 case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH:
852 desc = "target xfer count mismatch";
853 break;
854 case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR:
855 desc = "target data offset error";
856 break;
857 case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA:
858 desc = "target too much write data";
859 break;
860 case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT:
861 desc = "target iu too short";
862 break;
863 case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT:
864 desc = "target ack nak timeout";
865 break;
866 case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED:
867 desc = "target nak received";
868 break;
869
870/****************************************************************************
871* Serial Attached SCSI values
872****************************************************************************/
873
874 case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED:
875 desc = "smp request failed";
876 break;
877 case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN:
878 desc = "smp data overrun";
879 break;
880
881/****************************************************************************
882* Diagnostic Buffer Post / Diagnostic Release values
883****************************************************************************/
884
885 case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED:
886 desc = "diagnostic released";
887 break;
888 default:
889 break;
890 }
891
892 if (!desc)
893 return;
894
895 switch (request_hdr->Function) {
896 case MPI2_FUNCTION_CONFIG:
897 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;
898 func_str = "config_page";
899 break;
900 case MPI2_FUNCTION_SCSI_TASK_MGMT:
901 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t);
902 func_str = "task_mgmt";
903 break;
904 case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
905 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t);
906 func_str = "sas_iounit_ctl";
907 break;
908 case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR:
909 frame_sz = sizeof(Mpi2SepRequest_t);
910 func_str = "enclosure";
911 break;
912 case MPI2_FUNCTION_IOC_INIT:
913 frame_sz = sizeof(Mpi2IOCInitRequest_t);
914 func_str = "ioc_init";
915 break;
916 case MPI2_FUNCTION_PORT_ENABLE:
917 frame_sz = sizeof(Mpi2PortEnableRequest_t);
918 func_str = "port_enable";
919 break;
920 case MPI2_FUNCTION_SMP_PASSTHROUGH:
921 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;
922 func_str = "smp_passthru";
923 break;
aff39e61
SPS		 924 case MPI2_FUNCTION_NVME_ENCAPSULATED:
925 frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) +
926 ioc->sge_size;
927 func_str = "nvme_encapsulated";
928 break;
f92363d1
SR		 929 default:
930 frame_sz = 32;
931 func_str = "unknown";
932 break;
933 }
934
935 pr_warn(MPT3SAS_FMT "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",
936 ioc->name, desc, ioc_status, request_hdr, func_str);
937
938 _debug_dump_mf(request_hdr, frame_sz/4);
939}
940
941/**
942 * _base_display_event_data - verbose translation of firmware asyn events
943 * @ioc: per adapter object
944 * @mpi_reply: reply mf payload returned from firmware
945 *
946 * Return nothing.
947 */
948static void
949_base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
950 Mpi2EventNotificationReply_t *mpi_reply)
951{
952 char *desc = NULL;
953 u16 event;
954
955 if (!(ioc->logging_level & MPT_DEBUG_EVENTS))
956 return;
957
958 event = le16_to_cpu(mpi_reply->Event);
959
960 switch (event) {
961 case MPI2_EVENT_LOG_DATA:
962 desc = "Log Data";
963 break;
964 case MPI2_EVENT_STATE_CHANGE:
965 desc = "Status Change";
966 break;
967 case MPI2_EVENT_HARD_RESET_RECEIVED:
968 desc = "Hard Reset Received";
969 break;
970 case MPI2_EVENT_EVENT_CHANGE:
971 desc = "Event Change";
972 break;
973 case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
974 desc = "Device Status Change";
975 break;
976 case MPI2_EVENT_IR_OPERATION_STATUS:
7786ab6a
SR		 977 if (!ioc->hide_ir_msg)
978 desc = "IR Operation Status";
f92363d1
SR		 979 break;
980 case MPI2_EVENT_SAS_DISCOVERY:
981 {
982 Mpi2EventDataSasDiscovery_t *event_data =
983 (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData;
984 pr_info(MPT3SAS_FMT "Discovery: (%s)", ioc->name,
985 (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED) ?
986 "start" : "stop");
987 if (event_data->DiscoveryStatus)
bbaf61e2 988 pr_cont(" discovery_status(0x%08x)",
f92363d1 989 le32_to_cpu(event_data->DiscoveryStatus));
bbaf61e2 990 pr_cont("\n");
f92363d1
SR		 991 return;
992 }
993 case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
994 desc = "SAS Broadcast Primitive";
995 break;
996 case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
997 desc = "SAS Init Device Status Change";
998 break;
999 case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW:
1000 desc = "SAS Init Table Overflow";
1001 break;
1002 case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
1003 desc = "SAS Topology Change List";
1004 break;
1005 case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
1006 desc = "SAS Enclosure Device Status Change";
1007 break;
1008 case MPI2_EVENT_IR_VOLUME:
7786ab6a
SR		 1009 if (!ioc->hide_ir_msg)
1010 desc = "IR Volume";
f92363d1
SR		 1011 break;
1012 case MPI2_EVENT_IR_PHYSICAL_DISK:
7786ab6a
SR		 1013 if (!ioc->hide_ir_msg)
1014 desc = "IR Physical Disk";
f92363d1
SR		 1015 break;
1016 case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
7786ab6a
SR		 1017 if (!ioc->hide_ir_msg)
1018 desc = "IR Configuration Change List";
f92363d1
SR		 1019 break;
1020 case MPI2_EVENT_LOG_ENTRY_ADDED:
7786ab6a
SR		 1021 if (!ioc->hide_ir_msg)
1022 desc = "Log Entry Added";
f92363d1 1023 break;
2d8ce8c9
SR		 1024 case MPI2_EVENT_TEMP_THRESHOLD:
1025 desc = "Temperature Threshold";
1026 break;
a470a51c 1027 case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
b99b1993 1028 desc = "Cable Event";
a470a51c 1029 break;
4318c734
SPS		 1030 case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
1031 desc = "PCIE Device Status Change";
1032 break;
1033 case MPI2_EVENT_PCIE_ENUMERATION:
1034 {
1035 Mpi26EventDataPCIeEnumeration_t *event_data =
1036 (Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData;
1037 pr_info(MPT3SAS_FMT "PCIE Enumeration: (%s)", ioc->name,
1038 (event_data->ReasonCode ==
1039 MPI26_EVENT_PCIE_ENUM_RC_STARTED) ?
1040 "start" : "stop");
1041 if (event_data->EnumerationStatus)
1042 pr_info("enumeration_status(0x%08x)",
1043 le32_to_cpu(event_data->EnumerationStatus));
1044 pr_info("\n");
1045 return;
1046 }
1047 case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
1048 desc = "PCIE Topology Change List";
1049 break;
f92363d1
SR		 1050 }
1051
1052 if (!desc)
1053 return;
1054
1055 pr_info(MPT3SAS_FMT "%s\n", ioc->name, desc);
1056}
f92363d1
SR		 1057
1058/**
1059 * _base_sas_log_info - verbose translation of firmware log info
1060 * @ioc: per adapter object
1061 * @log_info: log info
1062 *
1063 * Return nothing.
1064 */
1065static void
1066_base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info)
1067{
1068 union loginfo_type {
1069 u32 loginfo;
1070 struct {
1071 u32 subcode:16;
1072 u32 code:8;
1073 u32 originator:4;
1074 u32 bus_type:4;
1075 } dw;
1076 };
1077 union loginfo_type sas_loginfo;
1078 char *originator_str = NULL;
1079
1080 sas_loginfo.loginfo = log_info;
1081 if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
1082 return;
1083
1084 /* each nexus loss loginfo */
1085 if (log_info == 0x31170000)
1086 return;
1087
1088 /* eat the loginfos associated with task aborts */
1089 if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==
1090 0x31140000 || log_info == 0x31130000))
1091 return;
1092
1093 switch (sas_loginfo.dw.originator) {
1094 case 0:
1095 originator_str = "IOP";
1096 break;
1097 case 1:
1098 originator_str = "PL";
1099 break;
1100 case 2:
7786ab6a
SR		 1101 if (!ioc->hide_ir_msg)
1102 originator_str = "IR";
1103 else
1104 originator_str = "WarpDrive";
f92363d1
SR		 1105 break;
1106 }
1107
1108 pr_warn(MPT3SAS_FMT
1109 "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",
1110 ioc->name, log_info,
1111 originator_str, sas_loginfo.dw.code,
1112 sas_loginfo.dw.subcode);
1113}
1114
1115/**
1116 * _base_display_reply_info -
1117 * @ioc: per adapter object
1118 * @smid: system request message index
1119 * @msix_index: MSIX table index supplied by the OS
1120 * @reply: reply message frame(lower 32bit addr)
1121 *
1122 * Return nothing.
1123 */
1124static void
1125_base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1126 u32 reply)
1127{
1128 MPI2DefaultReply_t *mpi_reply;
1129 u16 ioc_status;
1130 u32 loginfo = 0;
1131
1132 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1133 if (unlikely(!mpi_reply)) {
1134 pr_err(MPT3SAS_FMT "mpi_reply not valid at %s:%d/%s()!\n",
1135 ioc->name, __FILE__, __LINE__, __func__);
1136 return;
1137 }
1138 ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
af009411 1139
f92363d1
SR		 1140 if ((ioc_status & MPI2_IOCSTATUS_MASK) &&
1141 (ioc->logging_level & MPT_DEBUG_REPLY)) {
1142 _base_sas_ioc_info(ioc , mpi_reply,
1143 mpt3sas_base_get_msg_frame(ioc, smid));
1144 }
af009411 1145
f92363d1
SR		 1146 if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
1147 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo);
1148 _base_sas_log_info(ioc, loginfo);
1149 }
1150
1151 if (ioc_status || loginfo) {
1152 ioc_status &= MPI2_IOCSTATUS_MASK;
1153 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);
1154 }
1155}
1156
1157/**
1158 * mpt3sas_base_done - base internal command completion routine
1159 * @ioc: per adapter object
1160 * @smid: system request message index
1161 * @msix_index: MSIX table index supplied by the OS
1162 * @reply: reply message frame(lower 32bit addr)
1163 *
1164 * Return 1 meaning mf should be freed from _base_interrupt
1165 * 0 means the mf is freed from this function.
1166 */
1167u8
1168mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1169 u32 reply)
1170{
1171 MPI2DefaultReply_t *mpi_reply;
1172
1173 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1174 if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK)
fd0331b3 1175 return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
f92363d1
SR		 1176
1177 if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)
1178 return 1;
1179
1180 ioc->base_cmds.status |= MPT3_CMD_COMPLETE;
1181 if (mpi_reply) {
1182 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;
1183 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
1184 }
1185 ioc->base_cmds.status &= ~MPT3_CMD_PENDING;
1186
1187 complete(&ioc->base_cmds.done);
1188 return 1;
1189}
1190
1191/**
1192 * _base_async_event - main callback handler for firmware asyn events
1193 * @ioc: per adapter object
1194 * @msix_index: MSIX table index supplied by the OS
1195 * @reply: reply message frame(lower 32bit addr)
1196 *
1197 * Return 1 meaning mf should be freed from _base_interrupt
1198 * 0 means the mf is freed from this function.
1199 */
1200static u8
1201_base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
1202{
1203 Mpi2EventNotificationReply_t *mpi_reply;
1204 Mpi2EventAckRequest_t *ack_request;
1205 u16 smid;
fd0331b3 1206 struct _event_ack_list *delayed_event_ack;
f92363d1
SR		 1207
1208 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1209 if (!mpi_reply)
1210 return 1;
1211 if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION)
1212 return 1;
af009411 1213
f92363d1 1214 _base_display_event_data(ioc, mpi_reply);
af009411 1215
f92363d1
SR		 1216 if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED))
1217 goto out;
1218 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
1219 if (!smid) {
fd0331b3
SS		 1220 delayed_event_ack = kzalloc(sizeof(*delayed_event_ack),
1221 GFP_ATOMIC);
1222 if (!delayed_event_ack)
1223 goto out;
1224 INIT_LIST_HEAD(&delayed_event_ack->list);
1225 delayed_event_ack->Event = mpi_reply->Event;
1226 delayed_event_ack->EventContext = mpi_reply->EventContext;
1227 list_add_tail(&delayed_event_ack->list,
1228 &ioc->delayed_event_ack_list);
1229 dewtprintk(ioc, pr_info(MPT3SAS_FMT
1230 "DELAYED: EVENT ACK: event (0x%04x)\n",
1231 ioc->name, le16_to_cpu(mpi_reply->Event)));
f92363d1
SR		 1232 goto out;
1233 }
1234
1235 ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
1236 memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
1237 ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
1238 ack_request->Event = mpi_reply->Event;
1239 ack_request->EventContext = mpi_reply->EventContext;
1240 ack_request->VF_ID = 0; /* TODO */
1241 ack_request->VP_ID = 0;
40114bde 1242 mpt3sas_base_put_smid_default(ioc, smid);
f92363d1
SR		 1243
1244 out:
1245
1246 /* scsih callback handler */
1247 mpt3sas_scsih_event_callback(ioc, msix_index, reply);
1248
1249 /* ctl callback handler */
1250 mpt3sas_ctl_event_callback(ioc, msix_index, reply);
1251
1252 return 1;
1253}
1254
61dfb8a5 1255static struct scsiio_tracker *
dbec4c90 1256_get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
12e7c678 1257{
dbec4c90
SPS		 1258 struct scsi_cmnd *cmd;
1259
12e7c678
HR		 1260 if (WARN_ON(!smid) ||
1261 WARN_ON(smid >= ioc->hi_priority_smid))
1262 return NULL;
dbec4c90
SPS		 1263
1264 cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1265 if (cmd)
1266 return scsi_cmd_priv(cmd);
1267
1268 return NULL;
12e7c678
HR		 1269}
1270
f92363d1
SR		 1271/**
1272 * _base_get_cb_idx - obtain the callback index
1273 * @ioc: per adapter object
1274 * @smid: system request message index
1275 *
1276 * Return callback index.
1277 */
1278static u8
1279_base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1280{
1281 int i;
b0cd285e 1282 u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;
ba4494d4 1283 u8 cb_idx = 0xFF;
f92363d1
SR		 1284
1285 if (smid < ioc->hi_priority_smid) {
12e7c678
HR		 1286 struct scsiio_tracker *st;
1287
b0cd285e 1288 if (smid < ctl_smid) {
dbec4c90 1289 st = _get_st_from_smid(ioc, smid);
b0cd285e
HR		 1290 if (st)
1291 cb_idx = st->cb_idx;
1292 } else if (smid == ctl_smid)
1293 cb_idx = ioc->ctl_cb_idx;
f92363d1
SR		 1294 } else if (smid < ioc->internal_smid) {
1295 i = smid - ioc->hi_priority_smid;
1296 cb_idx = ioc->hpr_lookup[i].cb_idx;
1297 } else if (smid <= ioc->hba_queue_depth) {
1298 i = smid - ioc->internal_smid;
1299 cb_idx = ioc->internal_lookup[i].cb_idx;
ba4494d4 1300 }
f92363d1
SR		 1301 return cb_idx;
1302}
1303
1304/**
1305 * _base_mask_interrupts - disable interrupts
1306 * @ioc: per adapter object
1307 *
1308 * Disabling ResetIRQ, Reply and Doorbell Interrupts
1309 *
1310 * Return nothing.
1311 */
1312static void
1313_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1314{
1315 u32 him_register;
1316
1317 ioc->mask_interrupts = 1;
1318 him_register = readl(&ioc->chip->HostInterruptMask);
1319 him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK;
1320 writel(him_register, &ioc->chip->HostInterruptMask);
1321 readl(&ioc->chip->HostInterruptMask);
1322}
1323
1324/**
1325 * _base_unmask_interrupts - enable interrupts
1326 * @ioc: per adapter object
1327 *
1328 * Enabling only Reply Interrupts
1329 *
1330 * Return nothing.
1331 */
1332static void
1333_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1334{
1335 u32 him_register;
1336
1337 him_register = readl(&ioc->chip->HostInterruptMask);
1338 him_register &= ~MPI2_HIM_RIM;
1339 writel(him_register, &ioc->chip->HostInterruptMask);
1340 ioc->mask_interrupts = 0;
1341}
1342
1343union reply_descriptor {
1344 u64 word;
1345 struct {
1346 u32 low;
1347 u32 high;
1348 } u;
1349};
1350
1351/**
1352 * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
1353 * @irq: irq number (not used)
1354 * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
1355 * @r: pt_regs pointer (not used)
1356 *
1357 * Return IRQ_HANDLE if processed, else IRQ_NONE.
1358 */
1359static irqreturn_t
1360_base_interrupt(int irq, void *bus_id)
1361{
1362 struct adapter_reply_queue *reply_q = bus_id;
1363 union reply_descriptor rd;
1364 u32 completed_cmds;
1365 u8 request_desript_type;
1366 u16 smid;
1367 u8 cb_idx;
1368 u32 reply;
1369 u8 msix_index = reply_q->msix_index;
1370 struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1371 Mpi2ReplyDescriptorsUnion_t *rpf;
1372 u8 rc;
1373
1374 if (ioc->mask_interrupts)
1375 return IRQ_NONE;
1376
1377 if (!atomic_add_unless(&reply_q->busy, 1, 1))
1378 return IRQ_NONE;
1379
1380 rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
1381 request_desript_type = rpf->Default.ReplyFlags
1382 & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1383 if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
1384 atomic_dec(&reply_q->busy);
1385 return IRQ_NONE;
1386 }
1387
1388 completed_cmds = 0;
1389 cb_idx = 0xFF;
1390 do {
1391 rd.word = le64_to_cpu(rpf->Words);
1392 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX)
1393 goto out;
1394 reply = 0;
1395 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
1396 if (request_desript_type ==
1397 MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
1398 request_desript_type ==
aff39e61
SPS		 1399 MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
1400 request_desript_type ==
1401 MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) {
f92363d1
SR		 1402 cb_idx = _base_get_cb_idx(ioc, smid);
1403 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1404 (likely(mpt_callbacks[cb_idx] != NULL))) {
1405 rc = mpt_callbacks[cb_idx](ioc, smid,
1406 msix_index, 0);
1407 if (rc)
1408 mpt3sas_base_free_smid(ioc, smid);
1409 }
1410 } else if (request_desript_type ==
1411 MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
1412 reply = le32_to_cpu(
1413 rpf->AddressReply.ReplyFrameAddress);
1414 if (reply > ioc->reply_dma_max_address ||
1415 reply < ioc->reply_dma_min_address)
1416 reply = 0;
1417 if (smid) {
1418 cb_idx = _base_get_cb_idx(ioc, smid);
1419 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1420 (likely(mpt_callbacks[cb_idx] != NULL))) {
1421 rc = mpt_callbacks[cb_idx](ioc, smid,
1422 msix_index, reply);
1423 if (reply)
1424 _base_display_reply_info(ioc,
1425 smid, msix_index, reply);
1426 if (rc)
1427 mpt3sas_base_free_smid(ioc,
1428 smid);
1429 }
1430 } else {
1431 _base_async_event(ioc, msix_index, reply);
1432 }
1433
1434 /* reply free queue handling */
1435 if (reply) {
1436 ioc->reply_free_host_index =
1437 (ioc->reply_free_host_index ==
1438 (ioc->reply_free_queue_depth - 1)) ?
1439 0 : ioc->reply_free_host_index + 1;
1440 ioc->reply_free[ioc->reply_free_host_index] =
1441 cpu_to_le32(reply);
b4472d71
SPS		 1442 if (ioc->is_mcpu_endpoint)
1443 _base_clone_reply_to_sys_mem(ioc,
cf6bf971 1444 reply,
b4472d71 1445 ioc->reply_free_host_index);
f92363d1
SR		 1446 writel(ioc->reply_free_host_index,
1447 &ioc->chip->ReplyFreeHostIndex);
1448 }
1449 }
1450
1451 rpf->Words = cpu_to_le64(ULLONG_MAX);
1452 reply_q->reply_post_host_index =
1453 (reply_q->reply_post_host_index ==
1454 (ioc->reply_post_queue_depth - 1)) ? 0 :
1455 reply_q->reply_post_host_index + 1;
1456 request_desript_type =
1457 reply_q->reply_post_free[reply_q->reply_post_host_index].
1458 Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1459 completed_cmds++;
6b4c335a
C		 1460 /* Update the reply post host index after continuously
1461 * processing the threshold number of Reply Descriptors.
1462 * So that FW can find enough entries to post the Reply
1463 * Descriptors in the reply descriptor post queue.
1464 */
1465 if (completed_cmds > ioc->hba_queue_depth/3) {
1466 if (ioc->combined_reply_queue) {
1467 writel(reply_q->reply_post_host_index |
1468 ((msix_index & 7) <<
1469 MPI2_RPHI_MSIX_INDEX_SHIFT),
1470 ioc->replyPostRegisterIndex[msix_index/8]);
1471 } else {
1472 writel(reply_q->reply_post_host_index |
1473 (msix_index <<
1474 MPI2_RPHI_MSIX_INDEX_SHIFT),
1475 &ioc->chip->ReplyPostHostIndex);
1476 }
1477 completed_cmds = 1;
1478 }
f92363d1
SR		 1479 if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
1480 goto out;
1481 if (!reply_q->reply_post_host_index)
1482 rpf = reply_q->reply_post_free;
1483 else
1484 rpf++;
1485 } while (1);
1486
1487 out:
1488
1489 if (!completed_cmds) {
1490 atomic_dec(&reply_q->busy);
1491 return IRQ_NONE;
1492 }
1493
7786ab6a
SR		 1494 if (ioc->is_warpdrive) {
1495 writel(reply_q->reply_post_host_index,
1496 ioc->reply_post_host_index[msix_index]);
1497 atomic_dec(&reply_q->busy);
1498 return IRQ_HANDLED;
1499 }
fb77bb53
SR		 1500
1501 /* Update Reply Post Host Index.
1502 * For those HBA's which support combined reply queue feature
1503 * 1. Get the correct Supplemental Reply Post Host Index Register.
1504 * i.e. (msix_index / 8)th entry from Supplemental Reply Post Host
1505 * Index Register address bank i.e replyPostRegisterIndex[],
1506 * 2. Then update this register with new reply host index value
1507 * in ReplyPostIndex field and the MSIxIndex field with
1508 * msix_index value reduced to a value between 0 and 7,
1509 * using a modulo 8 operation. Since each Supplemental Reply Post
1510 * Host Index Register supports 8 MSI-X vectors.
1511 *
1512 * For other HBA's just update the Reply Post Host Index register with
1513 * new reply host index value in ReplyPostIndex Field and msix_index
1514 * value in MSIxIndex field.
1515 */
0bb337c9 1516 if (ioc->combined_reply_queue)
fb77bb53
SR		 1517 writel(reply_q->reply_post_host_index | ((msix_index & 7) <<
1518 MPI2_RPHI_MSIX_INDEX_SHIFT),
1519 ioc->replyPostRegisterIndex[msix_index/8]);
1520 else
1521 writel(reply_q->reply_post_host_index | (msix_index <<
1522 MPI2_RPHI_MSIX_INDEX_SHIFT),
1523 &ioc->chip->ReplyPostHostIndex);
f92363d1
SR		 1524 atomic_dec(&reply_q->busy);
1525 return IRQ_HANDLED;
1526}
1527
1528/**
1529 * _base_is_controller_msix_enabled - is controller support muli-reply queues
1530 * @ioc: per adapter object
1531 *
1532 */
1533static inline int
1534_base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)
1535{
1536 return (ioc->facts.IOCCapabilities &
1537 MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;
1538}
1539
1540/**
5f0dfb7a 1541 * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts
f92363d1 1542 * @ioc: per adapter object
5f0dfb7a 1543 * Context: non ISR conext
f92363d1 1544 *
5f0dfb7a 1545 * Called when a Task Management request has completed.
f92363d1
SR		 1546 *
1547 * Return nothing.
1548 */
1549void
5f0dfb7a 1550mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 1551{
1552 struct adapter_reply_queue *reply_q;
1553
1554 /* If MSIX capability is turned off
1555 * then multi-queues are not enabled
1556 */
1557 if (!_base_is_controller_msix_enabled(ioc))
1558 return;
1559
1560 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
5f0dfb7a
C		 1561 if (ioc->shost_recovery || ioc->remove_host ||
1562 ioc->pci_error_recovery)
f92363d1
SR		 1563 return;
1564 /* TMs are on msix_index == 0 */
1565 if (reply_q->msix_index == 0)
1566 continue;
1d55abc0 1567 synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
f92363d1
SR		 1568 }
1569}
1570
1571/**
1572 * mpt3sas_base_release_callback_handler - clear interrupt callback handler
1573 * @cb_idx: callback index
1574 *
1575 * Return nothing.
1576 */
1577void
1578mpt3sas_base_release_callback_handler(u8 cb_idx)
1579{
1580 mpt_callbacks[cb_idx] = NULL;
1581}
1582
1583/**
1584 * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler
1585 * @cb_func: callback function
1586 *
1587 * Returns cb_func.
1588 */
1589u8
1590mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func)
1591{
1592 u8 cb_idx;
1593
1594 for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--)
1595 if (mpt_callbacks[cb_idx] == NULL)
1596 break;
1597
1598 mpt_callbacks[cb_idx] = cb_func;
1599 return cb_idx;
1600}
1601
1602/**
1603 * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler
1604 *
1605 * Return nothing.
1606 */
1607void
1608mpt3sas_base_initialize_callback_handler(void)
1609{
1610 u8 cb_idx;
1611
1612 for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++)
1613 mpt3sas_base_release_callback_handler(cb_idx);
1614}
1615
1616
1617/**
1618 * _base_build_zero_len_sge - build zero length sg entry
1619 * @ioc: per adapter object
1620 * @paddr: virtual address for SGE
1621 *
1622 * Create a zero length scatter gather entry to insure the IOCs hardware has
1623 * something to use if the target device goes brain dead and tries
1624 * to send data even when none is asked for.
1625 *
1626 * Return nothing.
1627 */
1628static void
1629_base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1630{
1631 u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT |
1632 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST |
1633 MPI2_SGE_FLAGS_SIMPLE_ELEMENT) <<
1634 MPI2_SGE_FLAGS_SHIFT);
1635 ioc->base_add_sg_single(paddr, flags_length, -1);
1636}
1637
1638/**
1639 * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr.
1640 * @paddr: virtual address for SGE
1641 * @flags_length: SGE flags and data transfer length
1642 * @dma_addr: Physical address
1643 *
1644 * Return nothing.
1645 */
1646static void
1647_base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1648{
1649 Mpi2SGESimple32_t *sgel = paddr;
1650
1651 flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING |
1652 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1653 sgel->FlagsLength = cpu_to_le32(flags_length);
1654 sgel->Address = cpu_to_le32(dma_addr);
1655}
1656
1657
1658/**
1659 * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr.
1660 * @paddr: virtual address for SGE
1661 * @flags_length: SGE flags and data transfer length
1662 * @dma_addr: Physical address
1663 *
1664 * Return nothing.
1665 */
1666static void
1667_base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1668{
1669 Mpi2SGESimple64_t *sgel = paddr;
1670
1671 flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
1672 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1673 sgel->FlagsLength = cpu_to_le32(flags_length);
1674 sgel->Address = cpu_to_le64(dma_addr);
1675}
1676
1677/**
1678 * _base_get_chain_buffer_tracker - obtain chain tracker
1679 * @ioc: per adapter object
dbec4c90 1680 * @scmd: SCSI commands of the IO request
f92363d1
SR		 1681 *
1682 * Returns chain tracker(from ioc->free_chain_list)
1683 */
1684static struct chain_tracker *
dbec4c90
SPS		 1685_base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
1686 struct scsi_cmnd *scmd)
f92363d1
SR		 1687{
1688 struct chain_tracker *chain_req;
dbec4c90 1689 struct scsiio_tracker *st = scsi_cmd_priv(scmd);
f92363d1
SR		 1690 unsigned long flags;
1691
1692 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
1693 if (list_empty(&ioc->free_chain_list)) {
1694 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
1695 dfailprintk(ioc, pr_warn(MPT3SAS_FMT
1696 "chain buffers not available\n", ioc->name));
1697 return NULL;
1698 }
1699 chain_req = list_entry(ioc->free_chain_list.next,
1700 struct chain_tracker, tracker_list);
1701 list_del_init(&chain_req->tracker_list);
dbec4c90 1702 list_add_tail(&chain_req->tracker_list, &st->chain_list);
f92363d1
SR		 1703 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
1704 return chain_req;
1705}
1706
1707
1708/**
1709 * _base_build_sg - build generic sg
1710 * @ioc: per adapter object
1711 * @psge: virtual address for SGE
1712 * @data_out_dma: physical address for WRITES
1713 * @data_out_sz: data xfer size for WRITES
1714 * @data_in_dma: physical address for READS
1715 * @data_in_sz: data xfer size for READS
1716 *
1717 * Return nothing.
1718 */
1719static void
1720_base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,
1721 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1722 size_t data_in_sz)
1723{
1724 u32 sgl_flags;
1725
1726 if (!data_out_sz && !data_in_sz) {
1727 _base_build_zero_len_sge(ioc, psge);
1728 return;
1729 }
1730
1731 if (data_out_sz && data_in_sz) {
1732 /* WRITE sgel first */
1733 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1734 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC);
1735 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1736 ioc->base_add_sg_single(psge, sgl_flags |
1737 data_out_sz, data_out_dma);
1738
1739 /* incr sgel */
1740 psge += ioc->sge_size;
1741
1742 /* READ sgel last */
1743 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1744 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1745 MPI2_SGE_FLAGS_END_OF_LIST);
1746 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1747 ioc->base_add_sg_single(psge, sgl_flags |
1748 data_in_sz, data_in_dma);
1749 } else if (data_out_sz) /* WRITE */ {
1750 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1751 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1752 MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC);
1753 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1754 ioc->base_add_sg_single(psge, sgl_flags |
1755 data_out_sz, data_out_dma);
1756 } else if (data_in_sz) /* READ */ {
1757 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1758 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1759 MPI2_SGE_FLAGS_END_OF_LIST);
1760 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1761 ioc->base_add_sg_single(psge, sgl_flags |
1762 data_in_sz, data_in_dma);
1763 }
1764}
1765
aff39e61
SPS		 1766/* IEEE format sgls */
1767
1768/**
1769 * _base_build_nvme_prp - This function is called for NVMe end devices to build
1770 * a native SGL (NVMe PRP). The native SGL is built starting in the first PRP
1771 * entry of the NVMe message (PRP1). If the data buffer is small enough to be
1772 * described entirely using PRP1, then PRP2 is not used. If needed, PRP2 is
1773 * used to describe a larger data buffer. If the data buffer is too large to
1774 * describe using the two PRP entriess inside the NVMe message, then PRP1
1775 * describes the first data memory segment, and PRP2 contains a pointer to a PRP
1776 * list located elsewhere in memory to describe the remaining data memory
1777 * segments. The PRP list will be contiguous.
1778
1779 * The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP
1780 * consists of a list of PRP entries to describe a number of noncontigous
1781 * physical memory segments as a single memory buffer, just as a SGL does. Note
1782 * however, that this function is only used by the IOCTL call, so the memory
1783 * given will be guaranteed to be contiguous. There is no need to translate
1784 * non-contiguous SGL into a PRP in this case. All PRPs will describe
1785 * contiguous space that is one page size each.
1786 *
1787 * Each NVMe message contains two PRP entries. The first (PRP1) either contains
1788 * a PRP list pointer or a PRP element, depending upon the command. PRP2
1789 * contains the second PRP element if the memory being described fits within 2
1790 * PRP entries, or a PRP list pointer if the PRP spans more than two entries.
1791 *
1792 * A PRP list pointer contains the address of a PRP list, structured as a linear
1793 * array of PRP entries. Each PRP entry in this list describes a segment of
1794 * physical memory.
1795 *
1796 * Each 64-bit PRP entry comprises an address and an offset field. The address
1797 * always points at the beginning of a 4KB physical memory page, and the offset
1798 * describes where within that 4KB page the memory segment begins. Only the
1799 * first element in a PRP list may contain a non-zero offest, implying that all
1800 * memory segments following the first begin at the start of a 4KB page.
1801 *
1802 * Each PRP element normally describes 4KB of physical memory, with exceptions
1803 * for the first and last elements in the list. If the memory being described
1804 * by the list begins at a non-zero offset within the first 4KB page, then the
1805 * first PRP element will contain a non-zero offset indicating where the region
1806 * begins within the 4KB page. The last memory segment may end before the end
1807 * of the 4KB segment, depending upon the overall size of the memory being
1808 * described by the PRP list.
1809 *
1810 * Since PRP entries lack any indication of size, the overall data buffer length
1811 * is used to determine where the end of the data memory buffer is located, and
1812 * how many PRP entries are required to describe it.
1813 *
1814 * @ioc: per adapter object
1815 * @smid: system request message index for getting asscociated SGL
1816 * @nvme_encap_request: the NVMe request msg frame pointer
1817 * @data_out_dma: physical address for WRITES
1818 * @data_out_sz: data xfer size for WRITES
1819 * @data_in_dma: physical address for READS
1820 * @data_in_sz: data xfer size for READS
1821 *
1822 * Returns nothing.
1823 */
1824static void
1825_base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,
1826 Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request,
1827 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1828 size_t data_in_sz)
1829{
1830 int prp_size = NVME_PRP_SIZE;
d8335ae2
AB		 1831 __le64 *prp_entry, *prp1_entry, *prp2_entry;
1832 __le64 *prp_page;
1833 dma_addr_t prp_entry_dma, prp_page_dma, dma_addr;
aff39e61
SPS		 1834 u32 offset, entry_len;
1835 u32 page_mask_result, page_mask;
aff39e61
SPS		 1836 size_t length;
1837
1838 /*
1839 * Not all commands require a data transfer. If no data, just return
1840 * without constructing any PRP.
1841 */
1842 if (!data_in_sz && !data_out_sz)
1843 return;
1844 /*
1845 * Set pointers to PRP1 and PRP2, which are in the NVMe command.
1846 * PRP1 is located at a 24 byte offset from the start of the NVMe
1847 * command. Then set the current PRP entry pointer to PRP1.
1848 */
494f401b 1849 prp1_entry = (__le64 *)(nvme_encap_request->NVMe_Command +
aff39e61 1850 NVME_CMD_PRP1_OFFSET);
494f401b 1851 prp2_entry = (__le64 *)(nvme_encap_request->NVMe_Command +
aff39e61
SPS		 1852 NVME_CMD_PRP2_OFFSET);
1853 prp_entry = prp1_entry;
1854 /*
1855 * For the PRP entries, use the specially allocated buffer of
1856 * contiguous memory.
1857 */
494f401b 1858 prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);
d8335ae2 1859 prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
aff39e61
SPS		 1860
1861 /*
1862 * Check if we are within 1 entry of a page boundary we don't
1863 * want our first entry to be a PRP List entry.
1864 */
1865 page_mask = ioc->page_size - 1;
1866 page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask;
1867 if (!page_mask_result) {
1868 /* Bump up to next page boundary. */
494f401b 1869 prp_page = (__le64 *)((u8 *)prp_page + prp_size);
d8335ae2 1870 prp_page_dma = prp_page_dma + prp_size;
aff39e61
SPS		 1871 }
1872
1873 /*
1874 * Set PRP physical pointer, which initially points to the current PRP
1875 * DMA memory page.
1876 */
d8335ae2 1877 prp_entry_dma = prp_page_dma;
aff39e61
SPS		 1878
1879 /* Get physical address and length of the data buffer. */
1880 if (data_in_sz) {
d8335ae2 1881 dma_addr = data_in_dma;
aff39e61
SPS		 1882 length = data_in_sz;
1883 } else {
d8335ae2 1884 dma_addr = data_out_dma;
aff39e61
SPS		 1885 length = data_out_sz;
1886 }
1887
1888 /* Loop while the length is not zero. */
1889 while (length) {
1890 /*
1891 * Check if we need to put a list pointer here if we are at
1892 * page boundary - prp_size (8 bytes).
1893 */
d8335ae2 1894 page_mask_result = (prp_entry_dma + prp_size) & page_mask;
aff39e61
SPS		 1895 if (!page_mask_result) {
1896 /*
1897 * This is the last entry in a PRP List, so we need to
1898 * put a PRP list pointer here. What this does is:
1899 * - bump the current memory pointer to the next
1900 * address, which will be the next full page.
1901 * - set the PRP Entry to point to that page. This
1902 * is now the PRP List pointer.
1903 * - bump the PRP Entry pointer the start of the
1904 * next page. Since all of this PRP memory is
1905 * contiguous, no need to get a new page - it's
1906 * just the next address.
1907 */
d8335ae2
AB		 1908 prp_entry_dma++;
1909 *prp_entry = cpu_to_le64(prp_entry_dma);
aff39e61
SPS		 1910 prp_entry++;
1911 }
1912
1913 /* Need to handle if entry will be part of a page. */
d8335ae2 1914 offset = dma_addr & page_mask;
aff39e61
SPS		 1915 entry_len = ioc->page_size - offset;
1916
1917 if (prp_entry == prp1_entry) {
1918 /*
1919 * Must fill in the first PRP pointer (PRP1) before
1920 * moving on.
1921 */
d8335ae2 1922 *prp1_entry = cpu_to_le64(dma_addr);
aff39e61
SPS		 1923
1924 /*
1925 * Now point to the second PRP entry within the
1926 * command (PRP2).
1927 */
1928 prp_entry = prp2_entry;
1929 } else if (prp_entry == prp2_entry) {
1930 /*
1931 * Should the PRP2 entry be a PRP List pointer or just
1932 * a regular PRP pointer? If there is more than one
1933 * more page of data, must use a PRP List pointer.
1934 */
1935 if (length > ioc->page_size) {
1936 /*
1937 * PRP2 will contain a PRP List pointer because
1938 * more PRP's are needed with this command. The
1939 * list will start at the beginning of the
1940 * contiguous buffer.
1941 */
d8335ae2 1942 *prp2_entry = cpu_to_le64(prp_entry_dma);
aff39e61
SPS		 1943
1944 /*
1945 * The next PRP Entry will be the start of the
1946 * first PRP List.
1947 */
1948 prp_entry = prp_page;
1949 } else {
1950 /*
1951 * After this, the PRP Entries are complete.
1952 * This command uses 2 PRP's and no PRP list.
1953 */
d8335ae2 1954 *prp2_entry = cpu_to_le64(dma_addr);
aff39e61
SPS		 1955 }
1956 } else {
1957 /*
1958 * Put entry in list and bump the addresses.
1959 *
1960 * After PRP1 and PRP2 are filled in, this will fill in
1961 * all remaining PRP entries in a PRP List, one per
1962 * each time through the loop.
1963 */
d8335ae2 1964 *prp_entry = cpu_to_le64(dma_addr);
aff39e61 1965 prp_entry++;
d8335ae2 1966 prp_entry_dma++;
aff39e61
SPS		 1967 }
1968
1969 /*
1970 * Bump the phys address of the command's data buffer by the
1971 * entry_len.
1972 */
d8335ae2 1973 dma_addr += entry_len;
aff39e61
SPS		 1974
1975 /* Decrement length accounting for last partial page. */
1976 if (entry_len > length)
1977 length = 0;
1978 else
1979 length -= entry_len;
1980 }
1981}
1982
016d5c35
SPS		 1983/**
1984 * base_make_prp_nvme -
1985 * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
1986 *
1987 * @ioc: per adapter object
1988 * @scmd: SCSI command from the mid-layer
1989 * @mpi_request: mpi request
1990 * @smid: msg Index
1991 * @sge_count: scatter gather element count.
1992 *
1993 * Returns: true: PRPs are built
1994 * false: IEEE SGLs needs to be built
1995 */
494f401b 1996static void
016d5c35
SPS		 1997base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,
1998 struct scsi_cmnd *scmd,
1999 Mpi25SCSIIORequest_t *mpi_request,
2000 u16 smid, int sge_count)
2001{
d8335ae2 2002 int sge_len, num_prp_in_chain = 0;
016d5c35 2003 Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl;
494f401b 2004 __le64 *curr_buff;
d8335ae2 2005 dma_addr_t msg_dma, sge_addr, offset;
016d5c35
SPS		 2006 u32 page_mask, page_mask_result;
2007 struct scatterlist *sg_scmd;
2008 u32 first_prp_len;
2009 int data_len = scsi_bufflen(scmd);
2010 u32 nvme_pg_size;
2011
2012 nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);
2013 /*
2014 * Nvme has a very convoluted prp format. One prp is required
2015 * for each page or partial page. Driver need to split up OS sg_list
2016 * entries if it is longer than one page or cross a page
2017 * boundary. Driver also have to insert a PRP list pointer entry as
2018 * the last entry in each physical page of the PRP list.
2019 *
2020 * NOTE: The first PRP "entry" is actually placed in the first
2021 * SGL entry in the main message as IEEE 64 format. The 2nd
2022 * entry in the main message is the chain element, and the rest
2023 * of the PRP entries are built in the contiguous pcie buffer.
2024 */
2025 page_mask = nvme_pg_size - 1;
2026
2027 /*
2028 * Native SGL is needed.
2029 * Put a chain element in main message frame that points to the first
2030 * chain buffer.
2031 *
2032 * NOTE: The ChainOffset field must be 0 when using a chain pointer to
2033 * a native SGL.
2034 */
2035
2036 /* Set main message chain element pointer */
2037 main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2038 /*
2039 * For NVMe the chain element needs to be the 2nd SG entry in the main
2040 * message.
2041 */
2042 main_chain_element = (Mpi25IeeeSgeChain64_t *)
2043 ((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));
2044
2045 /*
2046 * For the PRP entries, use the specially allocated buffer of
2047 * contiguous memory. Normal chain buffers can't be used
2048 * because each chain buffer would need to be the size of an OS
2049 * page (4k).
2050 */
2051 curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);
d8335ae2 2052 msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
016d5c35 2053
d8335ae2 2054 main_chain_element->Address = cpu_to_le64(msg_dma);
016d5c35
SPS		 2055 main_chain_element->NextChainOffset = 0;
2056 main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2057 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2058 MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2059
2060 /* Build first prp, sge need not to be page aligned*/
2061 ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2062 sg_scmd = scsi_sglist(scmd);
2063 sge_addr = sg_dma_address(sg_scmd);
2064 sge_len = sg_dma_len(sg_scmd);
2065
d8335ae2 2066 offset = sge_addr & page_mask;
016d5c35
SPS		 2067 first_prp_len = nvme_pg_size - offset;
2068
2069 ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2070 ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2071
2072 data_len -= first_prp_len;
2073
2074 if (sge_len > first_prp_len) {
2075 sge_addr += first_prp_len;
2076 sge_len -= first_prp_len;
2077 } else if (data_len && (sge_len == first_prp_len)) {
2078 sg_scmd = sg_next(sg_scmd);
2079 sge_addr = sg_dma_address(sg_scmd);
2080 sge_len = sg_dma_len(sg_scmd);
2081 }
2082
2083 for (;;) {
d8335ae2 2084 offset = sge_addr & page_mask;
016d5c35
SPS		 2085
2086 /* Put PRP pointer due to page boundary*/
2087 page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask;
2088 if (unlikely(!page_mask_result)) {
2089 scmd_printk(KERN_NOTICE,
2090 scmd, "page boundary curr_buff: 0x%p\n",
2091 curr_buff);
d8335ae2
AB		 2092 msg_dma += 8;
2093 *curr_buff = cpu_to_le64(msg_dma);
016d5c35
SPS		 2094 curr_buff++;
2095 num_prp_in_chain++;
2096 }
2097
2098 *curr_buff = cpu_to_le64(sge_addr);
2099 curr_buff++;
d8335ae2 2100 msg_dma += 8;
016d5c35
SPS		 2101 num_prp_in_chain++;
2102
2103 sge_addr += nvme_pg_size;
2104 sge_len -= nvme_pg_size;
2105 data_len -= nvme_pg_size;
2106
2107 if (data_len <= 0)
2108 break;
2109
2110 if (sge_len > 0)
2111 continue;
2112
2113 sg_scmd = sg_next(sg_scmd);
2114 sge_addr = sg_dma_address(sg_scmd);
2115 sge_len = sg_dma_len(sg_scmd);
2116 }
2117
2118 main_chain_element->Length =
2119 cpu_to_le32(num_prp_in_chain * sizeof(u64));
2120 return;
2121}
2122
2123static bool
2124base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,
2125 struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count)
2126{
2127 u32 data_length = 0;
2128 struct scatterlist *sg_scmd;
2129 bool build_prp = true;
2130
494f401b 2131 data_length = scsi_bufflen(scmd);
016d5c35
SPS		 2132 sg_scmd = scsi_sglist(scmd);
2133
2134 /* If Datalenth is <= 16K and number of SGE’s entries are <= 2
2135 * we built IEEE SGL
2136 */
2137 if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2))
2138 build_prp = false;
2139
2140 return build_prp;
2141}
2142
2143/**
2144 * _base_check_pcie_native_sgl - This function is called for PCIe end devices to
2145 * determine if the driver needs to build a native SGL. If so, that native
2146 * SGL is built in the special contiguous buffers allocated especially for
2147 * PCIe SGL creation. If the driver will not build a native SGL, return
2148 * TRUE and a normal IEEE SGL will be built. Currently this routine
2149 * supports NVMe.
2150 * @ioc: per adapter object
2151 * @mpi_request: mf request pointer
2152 * @smid: system request message index
2153 * @scmd: scsi command
2154 * @pcie_device: points to the PCIe device's info
2155 *
2156 * Returns 0 if native SGL was built, 1 if no SGL was built
2157 */
2158static int
2159_base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,
2160 Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd,
2161 struct _pcie_device *pcie_device)
2162{
2163 struct scatterlist *sg_scmd;
2164 int sges_left;
2165
2166 /* Get the SG list pointer and info. */
2167 sg_scmd = scsi_sglist(scmd);
2168 sges_left = scsi_dma_map(scmd);
2169 if (sges_left < 0) {
2170 sdev_printk(KERN_ERR, scmd->device,
2171 "scsi_dma_map failed: request for %d bytes!\n",
2172 scsi_bufflen(scmd));
2173 return 1;
2174 }
2175
2176 /* Check if we need to build a native SG list. */
2177 if (base_is_prp_possible(ioc, pcie_device,
2178 scmd, sges_left) == 0) {
2179 /* We built a native SG list, just return. */
2180 goto out;
2181 }
2182
2183 /*
2184 * Build native NVMe PRP.
2185 */
2186 base_make_prp_nvme(ioc, scmd, mpi_request,
2187 smid, sges_left);
2188
2189 return 0;
2190out:
2191 scsi_dma_unmap(scmd);
2192 return 1;
2193}
f92363d1
SR		 2194
2195/**
2196 * _base_add_sg_single_ieee - add sg element for IEEE format
2197 * @paddr: virtual address for SGE
2198 * @flags: SGE flags
2199 * @chain_offset: number of 128 byte elements from start of segment
2200 * @length: data transfer length
2201 * @dma_addr: Physical address
2202 *
2203 * Return nothing.
2204 */
2205static void
2206_base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length,
2207 dma_addr_t dma_addr)
2208{
2209 Mpi25IeeeSgeChain64_t *sgel = paddr;
2210
2211 sgel->Flags = flags;
2212 sgel->NextChainOffset = chain_offset;
2213 sgel->Length = cpu_to_le32(length);
2214 sgel->Address = cpu_to_le64(dma_addr);
2215}
2216
2217/**
2218 * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format
2219 * @ioc: per adapter object
2220 * @paddr: virtual address for SGE
2221 *
2222 * Create a zero length scatter gather entry to insure the IOCs hardware has
2223 * something to use if the target device goes brain dead and tries
2224 * to send data even when none is asked for.
2225 *
2226 * Return nothing.
2227 */
2228static void
2229_base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)
2230{
2231 u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2232 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2233 MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
b130b0d5 2234
f92363d1
SR		 2235 _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1);
2236}
2237
471ef9d4
SR		 2238/**
2239 * _base_build_sg_scmd - main sg creation routine
016d5c35 2240 * pcie_device is unused here!
471ef9d4
SR		 2241 * @ioc: per adapter object
2242 * @scmd: scsi command
2243 * @smid: system request message index
016d5c35 2244 * @unused: unused pcie_device pointer
471ef9d4
SR		 2245 * Context: none.
2246 *
2247 * The main routine that builds scatter gather table from a given
2248 * scsi request sent via the .queuecommand main handler.
2249 *
2250 * Returns 0 success, anything else error
2251 */
2252static int
2253_base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,
016d5c35 2254 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused)
471ef9d4
SR		 2255{
2256 Mpi2SCSIIORequest_t *mpi_request;
2257 dma_addr_t chain_dma;
2258 struct scatterlist *sg_scmd;
2259 void *sg_local, *chain;
2260 u32 chain_offset;
2261 u32 chain_length;
2262 u32 chain_flags;
2263 int sges_left;
2264 u32 sges_in_segment;
2265 u32 sgl_flags;
2266 u32 sgl_flags_last_element;
2267 u32 sgl_flags_end_buffer;
2268 struct chain_tracker *chain_req;
2269
2270 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2271
2272 /* init scatter gather flags */
2273 sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT;
2274 if (scmd->sc_data_direction == DMA_TO_DEVICE)
2275 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
2276 sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT)
2277 << MPI2_SGE_FLAGS_SHIFT;
2278 sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT |
2279 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST)
2280 << MPI2_SGE_FLAGS_SHIFT;
2281 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2282
2283 sg_scmd = scsi_sglist(scmd);
2284 sges_left = scsi_dma_map(scmd);
2285 if (sges_left < 0) {
2286 sdev_printk(KERN_ERR, scmd->device,
2287 "pci_map_sg failed: request for %d bytes!\n",
2288 scsi_bufflen(scmd));
2289 return -ENOMEM;
2290 }
2291
2292 sg_local = &mpi_request->SGL;
2293 sges_in_segment = ioc->max_sges_in_main_message;
2294 if (sges_left <= sges_in_segment)
2295 goto fill_in_last_segment;
2296
2297 mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) +
2298 (sges_in_segment * ioc->sge_size))/4;
2299
2300 /* fill in main message segment when there is a chain following */
2301 while (sges_in_segment) {
2302 if (sges_in_segment == 1)
2303 ioc->base_add_sg_single(sg_local,
2304 sgl_flags_last_element | sg_dma_len(sg_scmd),
2305 sg_dma_address(sg_scmd));
2306 else
2307 ioc->base_add_sg_single(sg_local, sgl_flags |
2308 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2309 sg_scmd = sg_next(sg_scmd);
2310 sg_local += ioc->sge_size;
2311 sges_left--;
2312 sges_in_segment--;
2313 }
2314
2315 /* initializing the chain flags and pointers */
2316 chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT;
dbec4c90 2317 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
471ef9d4
SR		 2318 if (!chain_req)
2319 return -1;
2320 chain = chain_req->chain_buffer;
2321 chain_dma = chain_req->chain_buffer_dma;
2322 do {
2323 sges_in_segment = (sges_left <=
2324 ioc->max_sges_in_chain_message) ? sges_left :
2325 ioc->max_sges_in_chain_message;
2326 chain_offset = (sges_left == sges_in_segment) ?
2327 0 : (sges_in_segment * ioc->sge_size)/4;
2328 chain_length = sges_in_segment * ioc->sge_size;
2329 if (chain_offset) {
2330 chain_offset = chain_offset <<
2331 MPI2_SGE_CHAIN_OFFSET_SHIFT;
2332 chain_length += ioc->sge_size;
2333 }
2334 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |
2335 chain_length, chain_dma);
2336 sg_local = chain;
2337 if (!chain_offset)
2338 goto fill_in_last_segment;
2339
2340 /* fill in chain segments */
2341 while (sges_in_segment) {
2342 if (sges_in_segment == 1)
2343 ioc->base_add_sg_single(sg_local,
2344 sgl_flags_last_element |
2345 sg_dma_len(sg_scmd),
2346 sg_dma_address(sg_scmd));
2347 else
2348 ioc->base_add_sg_single(sg_local, sgl_flags |
2349 sg_dma_len(sg_scmd),
2350 sg_dma_address(sg_scmd));
2351 sg_scmd = sg_next(sg_scmd);
2352 sg_local += ioc->sge_size;
2353 sges_left--;
2354 sges_in_segment--;
2355 }
2356
dbec4c90 2357 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
471ef9d4
SR		 2358 if (!chain_req)
2359 return -1;
2360 chain = chain_req->chain_buffer;
2361 chain_dma = chain_req->chain_buffer_dma;
2362 } while (1);
2363
2364
2365 fill_in_last_segment:
2366
2367 /* fill the last segment */
2368 while (sges_left) {
2369 if (sges_left == 1)
2370 ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |
2371 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2372 else
2373 ioc->base_add_sg_single(sg_local, sgl_flags |
2374 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2375 sg_scmd = sg_next(sg_scmd);
2376 sg_local += ioc->sge_size;
2377 sges_left--;
2378 }
2379
2380 return 0;
2381}
2382
f92363d1
SR		 2383/**
2384 * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format
2385 * @ioc: per adapter object
2386 * @scmd: scsi command
2387 * @smid: system request message index
016d5c35
SPS		 2388 * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be
2389 * constructed on need.
f92363d1
SR		 2390 * Context: none.
2391 *
2392 * The main routine that builds scatter gather table from a given
2393 * scsi request sent via the .queuecommand main handler.
2394 *
2395 * Returns 0 success, anything else error
2396 */
2397static int
2398_base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,
016d5c35 2399 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device)
f92363d1 2400{
016d5c35 2401 Mpi25SCSIIORequest_t *mpi_request;
f92363d1
SR		 2402 dma_addr_t chain_dma;
2403 struct scatterlist *sg_scmd;
2404 void *sg_local, *chain;
2405 u32 chain_offset;
2406 u32 chain_length;
f92363d1
SR		 2407 int sges_left;
2408 u32 sges_in_segment;
2409 u8 simple_sgl_flags;
2410 u8 simple_sgl_flags_last;
2411 u8 chain_sgl_flags;
2412 struct chain_tracker *chain_req;
2413
2414 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2415
2416 /* init scatter gather flags */
2417 simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2418 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2419 simple_sgl_flags_last = simple_sgl_flags |
2420 MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2421 chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2422 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2423
016d5c35
SPS		 2424 /* Check if we need to build a native SG list. */
2425 if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,
2426 smid, scmd, pcie_device) == 0)) {
2427 /* We built a native SG list, just return. */
2428 return 0;
2429 }
2430
f92363d1
SR		 2431 sg_scmd = scsi_sglist(scmd);
2432 sges_left = scsi_dma_map(scmd);
62f5c74c 2433 if (sges_left < 0) {
f92363d1
SR		 2434 sdev_printk(KERN_ERR, scmd->device,
2435 "pci_map_sg failed: request for %d bytes!\n",
2436 scsi_bufflen(scmd));
2437 return -ENOMEM;
2438 }
2439
2440 sg_local = &mpi_request->SGL;
2441 sges_in_segment = (ioc->request_sz -
016d5c35 2442 offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;
f92363d1
SR		 2443 if (sges_left <= sges_in_segment)
2444 goto fill_in_last_segment;
2445
2446 mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) +
016d5c35 2447 (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);
f92363d1
SR		 2448
2449 /* fill in main message segment when there is a chain following */
2450 while (sges_in_segment > 1) {
2451 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2452 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2453 sg_scmd = sg_next(sg_scmd);
2454 sg_local += ioc->sge_size_ieee;
2455 sges_left--;
2456 sges_in_segment--;
2457 }
2458
25ef16d0 2459 /* initializing the pointers */
dbec4c90 2460 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
f92363d1
SR		 2461 if (!chain_req)
2462 return -1;
2463 chain = chain_req->chain_buffer;
2464 chain_dma = chain_req->chain_buffer_dma;
2465 do {
2466 sges_in_segment = (sges_left <=
2467 ioc->max_sges_in_chain_message) ? sges_left :
2468 ioc->max_sges_in_chain_message;
2469 chain_offset = (sges_left == sges_in_segment) ?
2470 0 : sges_in_segment;
2471 chain_length = sges_in_segment * ioc->sge_size_ieee;
2472 if (chain_offset)
2473 chain_length += ioc->sge_size_ieee;
2474 _base_add_sg_single_ieee(sg_local, chain_sgl_flags,
2475 chain_offset, chain_length, chain_dma);
2476
2477 sg_local = chain;
2478 if (!chain_offset)
2479 goto fill_in_last_segment;
2480
2481 /* fill in chain segments */
2482 while (sges_in_segment) {
2483 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2484 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2485 sg_scmd = sg_next(sg_scmd);
2486 sg_local += ioc->sge_size_ieee;
2487 sges_left--;
2488 sges_in_segment--;
2489 }
2490
dbec4c90 2491 chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
f92363d1
SR		 2492 if (!chain_req)
2493 return -1;
2494 chain = chain_req->chain_buffer;
2495 chain_dma = chain_req->chain_buffer_dma;
2496 } while (1);
2497
2498
2499 fill_in_last_segment:
2500
2501 /* fill the last segment */
62f5c74c 2502 while (sges_left > 0) {
f92363d1
SR		 2503 if (sges_left == 1)
2504 _base_add_sg_single_ieee(sg_local,
2505 simple_sgl_flags_last, 0, sg_dma_len(sg_scmd),
2506 sg_dma_address(sg_scmd));
2507 else
2508 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2509 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2510 sg_scmd = sg_next(sg_scmd);
2511 sg_local += ioc->sge_size_ieee;
2512 sges_left--;
2513 }
2514
2515 return 0;
2516}
2517
2518/**
2519 * _base_build_sg_ieee - build generic sg for IEEE format
2520 * @ioc: per adapter object
2521 * @psge: virtual address for SGE
2522 * @data_out_dma: physical address for WRITES
2523 * @data_out_sz: data xfer size for WRITES
2524 * @data_in_dma: physical address for READS
2525 * @data_in_sz: data xfer size for READS
2526 *
2527 * Return nothing.
2528 */
2529static void
2530_base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
2531 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2532 size_t data_in_sz)
2533{
2534 u8 sgl_flags;
2535
2536 if (!data_out_sz && !data_in_sz) {
2537 _base_build_zero_len_sge_ieee(ioc, psge);
2538 return;
2539 }
2540
2541 if (data_out_sz && data_in_sz) {
2542 /* WRITE sgel first */
2543 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2544 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2545 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2546 data_out_dma);
2547
2548 /* incr sgel */
2549 psge += ioc->sge_size_ieee;
2550
2551 /* READ sgel last */
2552 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2553 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2554 data_in_dma);
2555 } else if (data_out_sz) /* WRITE */ {
2556 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2557 MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2558 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2559 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2560 data_out_dma);
2561 } else if (data_in_sz) /* READ */ {
2562 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2563 MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2564 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2565 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2566 data_in_dma);
2567 }
2568}
2569
2570#define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10))
2571
2572/**
2573 * _base_config_dma_addressing - set dma addressing
2574 * @ioc: per adapter object
2575 * @pdev: PCI device struct
2576 *
2577 * Returns 0 for success, non-zero for failure.
2578 */
2579static int
2580_base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
2581{
2582 struct sysinfo s;
9b05c91a
SR		 2583 u64 consistent_dma_mask;
2584
0448f019
SPS		 2585 if (ioc->is_mcpu_endpoint)
2586 goto try_32bit;
2587
9b05c91a
SR		 2588 if (ioc->dma_mask)
2589 consistent_dma_mask = DMA_BIT_MASK(64);
2590 else
2591 consistent_dma_mask = DMA_BIT_MASK(32);
f92363d1
SR		 2592
2593 if (sizeof(dma_addr_t) > 4) {
2594 const uint64_t required_mask =
2595 dma_get_required_mask(&pdev->dev);
2596 if ((required_mask > DMA_BIT_MASK(32)) &&
2597 !pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) &&
9b05c91a 2598 !pci_set_consistent_dma_mask(pdev, consistent_dma_mask)) {
f92363d1
SR		 2599 ioc->base_add_sg_single = &_base_add_sg_single_64;
2600 ioc->sge_size = sizeof(Mpi2SGESimple64_t);
9b05c91a 2601 ioc->dma_mask = 64;
f92363d1
SR		 2602 goto out;
2603 }
2604 }
2605
0448f019 2606 try_32bit:
f92363d1
SR		 2607 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))
2608 && !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
2609 ioc->base_add_sg_single = &_base_add_sg_single_32;
2610 ioc->sge_size = sizeof(Mpi2SGESimple32_t);
9b05c91a 2611 ioc->dma_mask = 32;
f92363d1
SR		 2612 } else
2613 return -ENODEV;
2614
2615 out:
2616 si_meminfo(&s);
2617 pr_info(MPT3SAS_FMT
9b05c91a
SR		 2618 "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
2619 ioc->name, ioc->dma_mask, convert_to_kb(s.totalram));
2620
2621 return 0;
2622}
f92363d1 2623
9b05c91a
SR		 2624static int
2625_base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc,
2626 struct pci_dev *pdev)
2627{
2628 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
2629 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2630 return -ENODEV;
2631 }
f92363d1
SR		 2632 return 0;
2633}
2634
2635/**
2636 * _base_check_enable_msix - checks MSIX capabable.
2637 * @ioc: per adapter object
2638 *
2639 * Check to see if card is capable of MSIX, and set number
2640 * of available msix vectors
2641 */
2642static int
2643_base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2644{
2645 int base;
2646 u16 message_control;
2647
42081173
SR		 2648 /* Check whether controller SAS2008 B0 controller,
2649 * if it is SAS2008 B0 controller use IO-APIC instead of MSIX
2650 */
2651 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&
2652 ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {
2653 return -EINVAL;
2654 }
2655
f92363d1
SR		 2656 base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);
2657 if (!base) {
2658 dfailprintk(ioc, pr_info(MPT3SAS_FMT "msix not supported\n",
2659 ioc->name));
2660 return -EINVAL;
2661 }
2662
2663 /* get msix vector count */
42081173
SR		 2664 /* NUMA_IO not supported for older controllers */
2665 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||
2666 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||
2667 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||
2668 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||
2669 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||
2670 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||
2671 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)
2672 ioc->msix_vector_count = 1;
2673 else {
2674 pci_read_config_word(ioc->pdev, base + 2, &message_control);
2675 ioc->msix_vector_count = (message_control & 0x3FF) + 1;
2676 }
f92363d1
SR		 2677 dinitprintk(ioc, pr_info(MPT3SAS_FMT
2678 "msix is supported, vector_count(%d)\n",
2679 ioc->name, ioc->msix_vector_count));
2680 return 0;
2681}
2682
2683/**
2684 * _base_free_irq - free irq
2685 * @ioc: per adapter object
2686 *
2687 * Freeing respective reply_queue from the list.
2688 */
2689static void
2690_base_free_irq(struct MPT3SAS_ADAPTER *ioc)
2691{
2692 struct adapter_reply_queue *reply_q, *next;
2693
2694 if (list_empty(&ioc->reply_queue_list))
2695 return;
2696
2697 list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
2698 list_del(&reply_q->list);
1d55abc0
HR		 2699 free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
2700 reply_q);
f92363d1
SR		 2701 kfree(reply_q);
2702 }
2703}
2704
2705/**
2706 * _base_request_irq - request irq
2707 * @ioc: per adapter object
2708 * @index: msix index into vector table
f92363d1
SR		 2709 *
2710 * Inserting respective reply_queue into the list.
2711 */
2712static int
1d55abc0 2713_base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
f92363d1 2714{
1d55abc0 2715 struct pci_dev *pdev = ioc->pdev;
f92363d1
SR		 2716 struct adapter_reply_queue *reply_q;
2717 int r;
2718
2719 reply_q = kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL);
2720 if (!reply_q) {
2721 pr_err(MPT3SAS_FMT "unable to allocate memory %d!\n",
2722 ioc->name, (int)sizeof(struct adapter_reply_queue));
2723 return -ENOMEM;
2724 }
2725 reply_q->ioc = ioc;
2726 reply_q->msix_index = index;
14b3114d 2727
f92363d1
SR		 2728 atomic_set(&reply_q->busy, 0);
2729 if (ioc->msix_enable)
2730 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
c84b06a4 2731 ioc->driver_name, ioc->id, index);
f92363d1
SR		 2732 else
2733 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
c84b06a4 2734 ioc->driver_name, ioc->id);
1d55abc0
HR		 2735 r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
2736 IRQF_SHARED, reply_q->name, reply_q);
f92363d1
SR		 2737 if (r) {
2738 pr_err(MPT3SAS_FMT "unable to allocate interrupt %d!\n",
1d55abc0 2739 reply_q->name, pci_irq_vector(pdev, index));
da3cec25 2740 kfree(reply_q);
f92363d1
SR		 2741 return -EBUSY;
2742 }
2743
2744 INIT_LIST_HEAD(&reply_q->list);
2745 list_add_tail(&reply_q->list, &ioc->reply_queue_list);
2746 return 0;
2747}
2748
2749/**
2750 * _base_assign_reply_queues - assigning msix index for each cpu
2751 * @ioc: per adapter object
2752 *
2753 * The enduser would need to set the affinity via /proc/irq/#/smp_affinity
2754 *
2755 * It would nice if we could call irq_set_affinity, however it is not
2756 * an exported symbol
2757 */
2758static void
2759_base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
2760{
91b265bf 2761 unsigned int cpu, nr_cpus, nr_msix, index = 0;
14b3114d 2762 struct adapter_reply_queue *reply_q;
f92363d1
SR		 2763
2764 if (!_base_is_controller_msix_enabled(ioc))
2765 return;
2766
2767 memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
2768
91b265bf
MP		 2769 nr_cpus = num_online_cpus();
2770 nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,
2771 ioc->facts.MaxMSIxVectors);
2772 if (!nr_msix)
2773 return;
f92363d1 2774
1d55abc0
HR		 2775 if (smp_affinity_enable) {
2776 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
2777 const cpumask_t *mask = pci_irq_get_affinity(ioc->pdev,
2778 reply_q->msix_index);
2779 if (!mask) {
2780 pr_warn(MPT3SAS_FMT "no affinity for msi %x\n",
2781 ioc->name, reply_q->msix_index);
2782 continue;
2783 }
2784
4a8842de
TH		 2785 for_each_cpu_and(cpu, mask, cpu_online_mask) {
2786 if (cpu >= ioc->cpu_msix_table_sz)
2787 break;
1d55abc0 2788 ioc->cpu_msix_table[cpu] = reply_q->msix_index;
4a8842de 2789 }
1d55abc0
HR		 2790 }
2791 return;
2792 }
91b265bf
MP		 2793 cpu = cpumask_first(cpu_online_mask);
2794
14b3114d
SR		 2795 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
2796
91b265bf
MP		 2797 unsigned int i, group = nr_cpus / nr_msix;
2798
14b3114d
SR		 2799 if (cpu >= nr_cpus)
2800 break;
2801
91b265bf
MP		 2802 if (index < nr_cpus % nr_msix)
2803 group++;
2804
2805 for (i = 0 ; i < group ; i++) {
1d55abc0 2806 ioc->cpu_msix_table[cpu] = reply_q->msix_index;
91b265bf 2807 cpu = cpumask_next(cpu, cpu_online_mask);
f92363d1 2808 }
91b265bf 2809 index++;
14b3114d 2810 }
f92363d1
SR		 2811}
2812
2813/**
2814 * _base_disable_msix - disables msix
2815 * @ioc: per adapter object
2816 *
2817 */
2818static void
2819_base_disable_msix(struct MPT3SAS_ADAPTER *ioc)
2820{
2821 if (!ioc->msix_enable)
2822 return;
2823 pci_disable_msix(ioc->pdev);
2824 ioc->msix_enable = 0;
2825}
2826
2827/**
2828 * _base_enable_msix - enables msix, failback to io_apic
2829 * @ioc: per adapter object
2830 *
2831 */
2832static int
2833_base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2834{
f92363d1 2835 int r;
bb350661 2836 int i, local_max_msix_vectors;
f92363d1 2837 u8 try_msix = 0;
1d55abc0 2838 unsigned int irq_flags = PCI_IRQ_MSIX;
f92363d1 2839
f92363d1
SR		 2840 if (msix_disable == -1 || msix_disable == 0)
2841 try_msix = 1;
2842
2843 if (!try_msix)
2844 goto try_ioapic;
2845
2846 if (_base_check_enable_msix(ioc) != 0)
2847 goto try_ioapic;
2848
2849 ioc->reply_queue_count = min_t(int, ioc->cpu_count,
1d55abc0 2850 ioc->msix_vector_count);
f92363d1 2851
9c500060
SR		 2852 printk(MPT3SAS_FMT "MSI-X vectors supported: %d, no of cores"
2853 ": %d, max_msix_vectors: %d\n", ioc->name, ioc->msix_vector_count,
2854 ioc->cpu_count, max_msix_vectors);
2855
9b05c91a 2856 if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
06f5f976 2857 local_max_msix_vectors = (reset_devices) ? 1 : 8;
bb350661
SPS		 2858 else
2859 local_max_msix_vectors = max_msix_vectors;
9b05c91a 2860
1d55abc0 2861 if (local_max_msix_vectors > 0)
bb350661 2862 ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
9c500060 2863 ioc->reply_queue_count);
1d55abc0 2864 else if (local_max_msix_vectors == 0)
9b05c91a 2865 goto try_ioapic;
9c500060 2866
64038301
SPS		 2867 if (ioc->msix_vector_count < ioc->cpu_count)
2868 smp_affinity_enable = 0;
2869
1d55abc0
HR		 2870 if (smp_affinity_enable)
2871 irq_flags |= PCI_IRQ_AFFINITY;
f92363d1 2872
1d55abc0
HR		 2873 r = pci_alloc_irq_vectors(ioc->pdev, 1, ioc->reply_queue_count,
2874 irq_flags);
2875 if (r < 0) {
f92363d1 2876 dfailprintk(ioc, pr_info(MPT3SAS_FMT
1d55abc0 2877 "pci_alloc_irq_vectors failed (r=%d) !!!\n",
f92363d1 2878 ioc->name, r));
f92363d1
SR		 2879 goto try_ioapic;
2880 }
2881
2882 ioc->msix_enable = 1;
1d55abc0
HR		 2883 ioc->reply_queue_count = r;
2884 for (i = 0; i < ioc->reply_queue_count; i++) {
2885 r = _base_request_irq(ioc, i);
f92363d1
SR		 2886 if (r) {
2887 _base_free_irq(ioc);
2888 _base_disable_msix(ioc);
f92363d1
SR		 2889 goto try_ioapic;
2890 }
2891 }
2892
f92363d1
SR		 2893 return 0;
2894
2895/* failback to io_apic interrupt routing */
2896 try_ioapic:
2897
9b05c91a 2898 ioc->reply_queue_count = 1;
1d55abc0
HR		 2899 r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);
2900 if (r < 0) {
2901 dfailprintk(ioc, pr_info(MPT3SAS_FMT
2902 "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",
2903 ioc->name, r));
2904 } else
2905 r = _base_request_irq(ioc, 0);
f92363d1
SR		 2906
2907 return r;
2908}
2909
580d4e31
SR		 2910/**
2911 * mpt3sas_base_unmap_resources - free controller resources
2912 * @ioc: per adapter object
2913 */
8bbb1cf6 2914static void
580d4e31
SR		 2915mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
2916{
2917 struct pci_dev *pdev = ioc->pdev;
2918
2919 dexitprintk(ioc, printk(MPT3SAS_FMT "%s\n",
2920 ioc->name, __func__));
2921
2922 _base_free_irq(ioc);
2923 _base_disable_msix(ioc);
2924
0bb337c9 2925 if (ioc->combined_reply_queue) {
580d4e31 2926 kfree(ioc->replyPostRegisterIndex);
5f985d88
TH		 2927 ioc->replyPostRegisterIndex = NULL;
2928 }
580d4e31
SR		 2929
2930 if (ioc->chip_phys) {
2931 iounmap(ioc->chip);
2932 ioc->chip_phys = 0;
2933 }
2934
2935 if (pci_is_enabled(pdev)) {
2936 pci_release_selected_regions(ioc->pdev, ioc->bars);
2937 pci_disable_pcie_error_reporting(pdev);
2938 pci_disable_device(pdev);
2939 }
2940}
2941
f92363d1
SR		 2942/**
2943 * mpt3sas_base_map_resources - map in controller resources (io/irq/memap)
2944 * @ioc: per adapter object
2945 *
2946 * Returns 0 for success, non-zero for failure.
2947 */
2948int
2949mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
2950{
2951 struct pci_dev *pdev = ioc->pdev;
2952 u32 memap_sz;
2953 u32 pio_sz;
2954 int i, r = 0;
2955 u64 pio_chip = 0;
6f9e09fd 2956 phys_addr_t chip_phys = 0;
f92363d1
SR		 2957 struct adapter_reply_queue *reply_q;
2958
2959 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n",
2960 ioc->name, __func__));
2961
2962 ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
2963 if (pci_enable_device_mem(pdev)) {
2964 pr_warn(MPT3SAS_FMT "pci_enable_device_mem: failed\n",
2965 ioc->name);
cf9bd21a 2966 ioc->bars = 0;
f92363d1
SR		 2967 return -ENODEV;
2968 }
2969
2970
2971 if (pci_request_selected_regions(pdev, ioc->bars,
c84b06a4 2972 ioc->driver_name)) {
f92363d1
SR		 2973 pr_warn(MPT3SAS_FMT "pci_request_selected_regions: failed\n",
2974 ioc->name);
cf9bd21a 2975 ioc->bars = 0;
f92363d1
SR		 2976 r = -ENODEV;
2977 goto out_fail;
2978 }
2979
2980/* AER (Advanced Error Reporting) hooks */
2981 pci_enable_pcie_error_reporting(pdev);
2982
2983 pci_set_master(pdev);
2984
2985
2986 if (_base_config_dma_addressing(ioc, pdev) != 0) {
2987 pr_warn(MPT3SAS_FMT "no suitable DMA mask for %s\n",
2988 ioc->name, pci_name(pdev));
2989 r = -ENODEV;
2990 goto out_fail;
2991 }
2992
5aeeb78a
SR		 2993 for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) &&
2994 (!memap_sz || !pio_sz); i++) {
f92363d1
SR		 2995 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
2996 if (pio_sz)
2997 continue;
2998 pio_chip = (u64)pci_resource_start(pdev, i);
2999 pio_sz = pci_resource_len(pdev, i);
3000 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3001 if (memap_sz)
3002 continue;
3003 ioc->chip_phys = pci_resource_start(pdev, i);
6f9e09fd 3004 chip_phys = ioc->chip_phys;
f92363d1
SR		 3005 memap_sz = pci_resource_len(pdev, i);
3006 ioc->chip = ioremap(ioc->chip_phys, memap_sz);
f92363d1
SR		 3007 }
3008 }
3009
5aeeb78a
SR		 3010 if (ioc->chip == NULL) {
3011 pr_err(MPT3SAS_FMT "unable to map adapter memory! "
3012 " or resource not found\n", ioc->name);
3013 r = -EINVAL;
3014 goto out_fail;
3015 }
3016
f92363d1 3017 _base_mask_interrupts(ioc);
9b05c91a 3018
98c56ad3 3019 r = _base_get_ioc_facts(ioc);
9b05c91a
SR		 3020 if (r)
3021 goto out_fail;
3022
3023 if (!ioc->rdpq_array_enable_assigned) {
3024 ioc->rdpq_array_enable = ioc->rdpq_array_capable;
3025 ioc->rdpq_array_enable_assigned = 1;
3026 }
3027
f92363d1
SR		 3028 r = _base_enable_msix(ioc);
3029 if (r)
3030 goto out_fail;
3031
fb77bb53
SR		 3032 /* Use the Combined reply queue feature only for SAS3 C0 & higher
3033 * revision HBAs and also only when reply queue count is greater than 8
3034 */
0bb337c9 3035 if (ioc->combined_reply_queue && ioc->reply_queue_count > 8) {
fb77bb53
SR		 3036 /* Determine the Supplemental Reply Post Host Index Registers
3037 * Addresse. Supplemental Reply Post Host Index Registers
3038 * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
3039 * each register is at offset bytes of
3040 * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
3041 */
3042 ioc->replyPostRegisterIndex = kcalloc(
0bb337c9 3043 ioc->combined_reply_index_count,
fb77bb53
SR		 3044 sizeof(resource_size_t *), GFP_KERNEL);
3045 if (!ioc->replyPostRegisterIndex) {
3046 dfailprintk(ioc, printk(MPT3SAS_FMT
3047 "allocation for reply Post Register Index failed!!!\n",
3048 ioc->name));
3049 r = -ENOMEM;
3050 goto out_fail;
3051 }
3052
0bb337c9 3053 for (i = 0; i < ioc->combined_reply_index_count; i++) {
fb77bb53 3054 ioc->replyPostRegisterIndex[i] = (resource_size_t *)
cf6bf971 3055 ((u8 __force *)&ioc->chip->Doorbell +
fb77bb53
SR		 3056 MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
3057 (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
3058 }
3059 } else
0bb337c9 3060 ioc->combined_reply_queue = 0;
fb77bb53 3061
ce7c6c9e
GE		 3062 if (ioc->is_warpdrive) {
3063 ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
3064 &ioc->chip->ReplyPostHostIndex;
3065
3066 for (i = 1; i < ioc->cpu_msix_table_sz; i++)
3067 ioc->reply_post_host_index[i] =
3068 (resource_size_t __iomem *)
3069 ((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
3070 * 4)));
3071 }
3072
f92363d1
SR		 3073 list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
3074 pr_info(MPT3SAS_FMT "%s: IRQ %d\n",
3075 reply_q->name, ((ioc->msix_enable) ? "PCI-MSI-X enabled" :
1d55abc0
HR		 3076 "IO-APIC enabled"),
3077 pci_irq_vector(ioc->pdev, reply_q->msix_index));
f92363d1 3078
6f9e09fd
AB		 3079 pr_info(MPT3SAS_FMT "iomem(%pap), mapped(0x%p), size(%d)\n",
3080 ioc->name, &chip_phys, ioc->chip, memap_sz);
f92363d1
SR		 3081 pr_info(MPT3SAS_FMT "ioport(0x%016llx), size(%d)\n",
3082 ioc->name, (unsigned long long)pio_chip, pio_sz);
3083
3084 /* Save PCI configuration state for recovery from PCI AER/EEH errors */
3085 pci_save_state(pdev);
3086 return 0;
3087
3088 out_fail:
580d4e31 3089 mpt3sas_base_unmap_resources(ioc);
f92363d1
SR		 3090 return r;
3091}
3092
3093/**
3094 * mpt3sas_base_get_msg_frame - obtain request mf pointer
3095 * @ioc: per adapter object
3096 * @smid: system request message index(smid zero is invalid)
3097 *
3098 * Returns virt pointer to message frame.
3099 */
3100void *
3101mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3102{
3103 return (void *)(ioc->request + (smid * ioc->request_sz));
3104}
3105
3106/**
3107 * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr
3108 * @ioc: per adapter object
3109 * @smid: system request message index
3110 *
3111 * Returns virt pointer to sense buffer.
3112 */
3113void *
3114mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3115{
3116 return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));
3117}
3118
3119/**
3120 * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr
3121 * @ioc: per adapter object
3122 * @smid: system request message index
3123 *
3124 * Returns phys pointer to the low 32bit address of the sense buffer.
3125 */
3126__le32
3127mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3128{
3129 return cpu_to_le32(ioc->sense_dma + ((smid - 1) *
3130 SCSI_SENSE_BUFFERSIZE));
3131}
3132
016d5c35
SPS		 3133/**
3134 * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr
3135 * @ioc: per adapter object
3136 * @smid: system request message index
3137 *
3138 * Returns virt pointer to a PCIe SGL.
3139 */
3140void *
3141mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3142{
dbec4c90 3143 return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);
016d5c35
SPS		 3144}
3145
3146/**
3147 * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr
3148 * @ioc: per adapter object
3149 * @smid: system request message index
3150 *
3151 * Returns phys pointer to the address of the PCIe buffer.
3152 */
d8335ae2 3153dma_addr_t
016d5c35
SPS		 3154mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3155{
dbec4c90 3156 return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;
016d5c35
SPS		 3157}
3158
f92363d1
SR		 3159/**
3160 * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address
3161 * @ioc: per adapter object
3162 * @phys_addr: lower 32 physical addr of the reply
3163 *
3164 * Converts 32bit lower physical addr into a virt address.
3165 */
3166void *
3167mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
3168{
3169 if (!phys_addr)
3170 return NULL;
3171 return ioc->reply + (phys_addr - (u32)ioc->reply_dma);
3172}
3173
03d1fb3a
SS		 3174static inline u8
3175_base_get_msix_index(struct MPT3SAS_ADAPTER *ioc)
3176{
3177 return ioc->cpu_msix_table[raw_smp_processor_id()];
3178}
3179
f92363d1
SR		 3180/**
3181 * mpt3sas_base_get_smid - obtain a free smid from internal queue
3182 * @ioc: per adapter object
3183 * @cb_idx: callback index
3184 *
3185 * Returns smid (zero is invalid)
3186 */
3187u16
3188mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3189{
3190 unsigned long flags;
3191 struct request_tracker *request;
3192 u16 smid;
3193
3194 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3195 if (list_empty(&ioc->internal_free_list)) {
3196 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3197 pr_err(MPT3SAS_FMT "%s: smid not available\n",
3198 ioc->name, __func__);
3199 return 0;
3200 }
3201
3202 request = list_entry(ioc->internal_free_list.next,
3203 struct request_tracker, tracker_list);
3204 request->cb_idx = cb_idx;
3205 smid = request->smid;
3206 list_del(&request->tracker_list);
3207 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3208 return smid;
3209}
3210
3211/**
3212 * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue
3213 * @ioc: per adapter object
3214 * @cb_idx: callback index
3215 * @scmd: pointer to scsi command object
3216 *
3217 * Returns smid (zero is invalid)
3218 */
3219u16
3220mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
3221 struct scsi_cmnd *scmd)
3222{
dbec4c90
SPS		 3223 struct scsiio_tracker *request = scsi_cmd_priv(scmd);
3224 unsigned int tag = scmd->request->tag;
f92363d1
SR		 3225 u16 smid;
3226
dbec4c90 3227 smid = tag + 1;
f92363d1 3228 request->cb_idx = cb_idx;
03d1fb3a 3229 request->msix_io = _base_get_msix_index(ioc);
dbec4c90
SPS		 3230 request->smid = smid;
3231 INIT_LIST_HEAD(&request->chain_list);
f92363d1
SR		 3232 return smid;
3233}
3234
3235/**
3236 * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue
3237 * @ioc: per adapter object
3238 * @cb_idx: callback index
3239 *
3240 * Returns smid (zero is invalid)
3241 */
3242u16
3243mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3244{
3245 unsigned long flags;
3246 struct request_tracker *request;
3247 u16 smid;
3248
3249 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3250 if (list_empty(&ioc->hpr_free_list)) {
3251 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3252 return 0;
3253 }
3254
3255 request = list_entry(ioc->hpr_free_list.next,
3256 struct request_tracker, tracker_list);
3257 request->cb_idx = cb_idx;
3258 smid = request->smid;
3259 list_del(&request->tracker_list);
3260 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3261 return smid;
3262}
3263
6a2d4618
HR		 3264static void
3265_base_recovery_check(struct MPT3SAS_ADAPTER *ioc)
3266{
3267 /*
3268 * See _wait_for_commands_to_complete() call with regards to this code.
3269 */
3270 if (ioc->shost_recovery && ioc->pending_io_count) {
272e253c
HR		 3271 ioc->pending_io_count = atomic_read(&ioc->shost->host_busy);
3272 if (ioc->pending_io_count == 0)
6a2d4618 3273 wake_up(&ioc->reset_wq);
6a2d4618
HR		 3274 }
3275}
3276
dbec4c90
SPS		 3277void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
3278 struct scsiio_tracker *st)
3279{
3280 if (WARN_ON(st->smid == 0))
3281 return;
3282 st->cb_idx = 0xFF;
3283 st->direct_io = 0;
3284 if (!list_empty(&st->chain_list)) {
3285 unsigned long flags;
3286
3287 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3288 list_splice_init(&st->chain_list, &ioc->free_chain_list);
3289 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3290 }
3291}
3292
f92363d1
SR		 3293/**
3294 * mpt3sas_base_free_smid - put smid back on free_list
3295 * @ioc: per adapter object
3296 * @smid: system request message index
3297 *
3298 * Return nothing.
3299 */
3300void
3301mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3302{
3303 unsigned long flags;
3304 int i;
f92363d1 3305
f92363d1 3306 if (smid < ioc->hi_priority_smid) {
dbec4c90 3307 struct scsiio_tracker *st;
f92363d1 3308
dbec4c90
SPS		 3309 st = _get_st_from_smid(ioc, smid);
3310 if (!st) {
3311 _base_recovery_check(ioc);
3312 return;
3313 }
3314 mpt3sas_base_clear_st(ioc, st);
6a2d4618 3315 _base_recovery_check(ioc);
f92363d1 3316 return;
dbec4c90
SPS		 3317 }
3318
3319 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3320 if (smid < ioc->internal_smid) {
f92363d1
SR		 3321 /* hi-priority */
3322 i = smid - ioc->hi_priority_smid;
3323 ioc->hpr_lookup[i].cb_idx = 0xFF;
3324 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);
3325 } else if (smid <= ioc->hba_queue_depth) {
3326 /* internal queue */
3327 i = smid - ioc->internal_smid;
3328 ioc->internal_lookup[i].cb_idx = 0xFF;
3329 list_add(&ioc->internal_lookup[i].tracker_list,
3330 &ioc->internal_free_list);
3331 }
3332 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3333}
3334
e5747439
SPS		 3335/**
3336 * _base_mpi_ep_writeq - 32 bit write to MMIO
3337 * @b: data payload
3338 * @addr: address in MMIO space
3339 * @writeq_lock: spin lock
3340 *
3341 * This special handling for MPI EP to take care of 32 bit
3342 * environment where its not quarenteed to send the entire word
3343 * in one transfer.
3344 */
3345static inline void
3346_base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
3347 spinlock_t *writeq_lock)
3348{
3349 unsigned long flags;
cf6bf971 3350 __u64 data_out = b;
e5747439
SPS		 3351
3352 spin_lock_irqsave(writeq_lock, flags);
3353 writel((u32)(data_out), addr);
3354 writel((u32)(data_out >> 32), (addr + 4));
3355 spin_unlock_irqrestore(writeq_lock, flags);
3356}
3357
f92363d1
SR		 3358/**
3359 * _base_writeq - 64 bit write to MMIO
3360 * @ioc: per adapter object
3361 * @b: data payload
3362 * @addr: address in MMIO space
3363 * @writeq_lock: spin lock
3364 *
3365 * Glue for handling an atomic 64 bit word to MMIO. This special handling takes
3366 * care of 32 bit environment where its not quarenteed to send the entire word
3367 * in one transfer.
3368 */
3369#if defined(writeq) && defined(CONFIG_64BIT)
3370static inline void
3371_base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3372{
cf6bf971 3373 writeq(b, addr);
f92363d1
SR		 3374}
3375#else
3376static inline void
3377_base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3378{
e5747439 3379 _base_mpi_ep_writeq(b, addr, writeq_lock);
f92363d1
SR		 3380}
3381#endif
3382
e5747439
SPS		 3383/**
3384 * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware
3385 * @ioc: per adapter object
3386 * @smid: system request message index
3387 * @handle: device handle
3388 *
3389 * Return nothing.
3390 */
3391static void
3392_base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
3393{
3394 Mpi2RequestDescriptorUnion_t descriptor;
3395 u64 *request = (u64 *)&descriptor;
3396 void *mpi_req_iomem;
3397 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3398
3399 _clone_sg_entries(ioc, (void *) mfp, smid);
cf6bf971 3400 mpi_req_iomem = (void __force *)ioc->chip +
e5747439
SPS		 3401 MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3402 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3403 ioc->request_sz);
3404 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3405 descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
3406 descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3407 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3408 descriptor.SCSIIO.LMID = 0;
3409 _base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3410 &ioc->scsi_lookup_lock);
3411}
3412
f92363d1 3413/**
81c16f83 3414 * _base_put_smid_scsi_io - send SCSI_IO request to firmware
f92363d1
SR		 3415 * @ioc: per adapter object
3416 * @smid: system request message index
3417 * @handle: device handle
3418 *
3419 * Return nothing.
3420 */
81c16f83
SPS		 3421static void
3422_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
f92363d1
SR		 3423{
3424 Mpi2RequestDescriptorUnion_t descriptor;
3425 u64 *request = (u64 *)&descriptor;
3426
3427
3428 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3429 descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
3430 descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3431 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3432 descriptor.SCSIIO.LMID = 0;
3433 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3434 &ioc->scsi_lookup_lock);
3435}
3436
3437/**
40114bde 3438 * mpt3sas_base_put_smid_fast_path - send fast path request to firmware
f92363d1
SR		 3439 * @ioc: per adapter object
3440 * @smid: system request message index
3441 * @handle: device handle
3442 *
3443 * Return nothing.
3444 */
40114bde
SP		 3445void
3446mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
f92363d1
SR		 3447 u16 handle)
3448{
3449 Mpi2RequestDescriptorUnion_t descriptor;
3450 u64 *request = (u64 *)&descriptor;
3451
3452 descriptor.SCSIIO.RequestFlags =
3453 MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
3454 descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
3455 descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3456 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3457 descriptor.SCSIIO.LMID = 0;
3458 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3459 &ioc->scsi_lookup_lock);
3460}
3461
3462/**
40114bde 3463 * mpt3sas_base_put_smid_hi_priority - send Task Management request to firmware
f92363d1
SR		 3464 * @ioc: per adapter object
3465 * @smid: system request message index
03d1fb3a 3466 * @msix_task: msix_task will be same as msix of IO incase of task abort else 0.
f92363d1
SR		 3467 * Return nothing.
3468 */
40114bde
SP		 3469void
3470mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
03d1fb3a 3471 u16 msix_task)
f92363d1
SR		 3472{
3473 Mpi2RequestDescriptorUnion_t descriptor;
e5747439
SPS		 3474 void *mpi_req_iomem;
3475 u64 *request;
3476
3477 if (ioc->is_mcpu_endpoint) {
3478 MPI2RequestHeader_t *request_hdr;
3479
3480 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3481
3482 request_hdr = (MPI2RequestHeader_t *)mfp;
3483 /* TBD 256 is offset within sys register. */
cf6bf971
C		 3484 mpi_req_iomem = (void __force *)ioc->chip
3485 + MPI_FRAME_START_OFFSET
e5747439
SPS		 3486 + (smid * ioc->request_sz);
3487 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3488 ioc->request_sz);
3489 }
3490
3491 request = (u64 *)&descriptor;
f92363d1
SR		 3492
3493 descriptor.HighPriority.RequestFlags =
3494 MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
03d1fb3a 3495 descriptor.HighPriority.MSIxIndex = msix_task;
f92363d1
SR		 3496 descriptor.HighPriority.SMID = cpu_to_le16(smid);
3497 descriptor.HighPriority.LMID = 0;
3498 descriptor.HighPriority.Reserved1 = 0;
e5747439
SPS		 3499 if (ioc->is_mcpu_endpoint)
3500 _base_mpi_ep_writeq(*request,
3501 &ioc->chip->RequestDescriptorPostLow,
3502 &ioc->scsi_lookup_lock);
3503 else
3504 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3505 &ioc->scsi_lookup_lock);
f92363d1
SR		 3506}
3507
aff39e61 3508/**
40114bde 3509 * mpt3sas_base_put_smid_nvme_encap - send NVMe encapsulated request to
aff39e61
SPS		 3510 * firmware
3511 * @ioc: per adapter object
3512 * @smid: system request message index
3513 *
3514 * Return nothing.
3515 */
40114bde
SP		 3516void
3517mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)
aff39e61
SPS		 3518{
3519 Mpi2RequestDescriptorUnion_t descriptor;
3520 u64 *request = (u64 *)&descriptor;
3521
3522 descriptor.Default.RequestFlags =
3523 MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
3524 descriptor.Default.MSIxIndex = _base_get_msix_index(ioc);
3525 descriptor.Default.SMID = cpu_to_le16(smid);
3526 descriptor.Default.LMID = 0;
3527 descriptor.Default.DescriptorTypeDependent = 0;
3528 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3529 &ioc->scsi_lookup_lock);
3530}
3531
f92363d1 3532/**
40114bde 3533 * mpt3sas_base_put_smid_default - Default, primarily used for config pages
f92363d1
SR		 3534 * @ioc: per adapter object
3535 * @smid: system request message index
3536 *
3537 * Return nothing.
3538 */
40114bde
SP		 3539void
3540mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
f92363d1
SR		 3541{
3542 Mpi2RequestDescriptorUnion_t descriptor;
e5747439
SPS		 3543 void *mpi_req_iomem;
3544 u64 *request;
3545 MPI2RequestHeader_t *request_hdr;
3546
3547 if (ioc->is_mcpu_endpoint) {
3548 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3549
3550 request_hdr = (MPI2RequestHeader_t *)mfp;
f92363d1 3551
e5747439
SPS		 3552 _clone_sg_entries(ioc, (void *) mfp, smid);
3553 /* TBD 256 is offset within sys register */
cf6bf971 3554 mpi_req_iomem = (void __force *)ioc->chip +
e5747439
SPS		 3555 MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3556 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3557 ioc->request_sz);
3558 }
3559 request = (u64 *)&descriptor;
f92363d1
SR		 3560 descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3561 descriptor.Default.MSIxIndex = _base_get_msix_index(ioc);
3562 descriptor.Default.SMID = cpu_to_le16(smid);
3563 descriptor.Default.LMID = 0;
3564 descriptor.Default.DescriptorTypeDependent = 0;
e5747439
SPS		 3565 if (ioc->is_mcpu_endpoint)
3566 _base_mpi_ep_writeq(*request,
3567 &ioc->chip->RequestDescriptorPostLow,
3568 &ioc->scsi_lookup_lock);
3569 else
3570 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3571 &ioc->scsi_lookup_lock);
f92363d1
SR		 3572}
3573
1117b31a 3574/**
989e43c7 3575 * _base_display_OEMs_branding - Display branding string
1117b31a
SR		 3576 * @ioc: per adapter object
3577 *
3578 * Return nothing.
3579 */
3580static void
989e43c7 3581_base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
1117b31a
SR		 3582{
3583 if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
3584 return;
3585
989e43c7
SR		 3586 switch (ioc->pdev->subsystem_vendor) {
3587 case PCI_VENDOR_ID_INTEL:
3588 switch (ioc->pdev->device) {
3589 case MPI2_MFGPAGE_DEVID_SAS2008:
3590 switch (ioc->pdev->subsystem_device) {
3591 case MPT2SAS_INTEL_RMS2LL080_SSDID:
3592 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3593 MPT2SAS_INTEL_RMS2LL080_BRANDING);
3594 break;
3595 case MPT2SAS_INTEL_RMS2LL040_SSDID:
3596 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3597 MPT2SAS_INTEL_RMS2LL040_BRANDING);
3598 break;
3599 case MPT2SAS_INTEL_SSD910_SSDID:
3600 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3601 MPT2SAS_INTEL_SSD910_BRANDING);
3602 break;
3603 default:
3604 pr_info(MPT3SAS_FMT
3605 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3606 ioc->name, ioc->pdev->subsystem_device);
3607 break;
3608 }
3609 case MPI2_MFGPAGE_DEVID_SAS2308_2:
3610 switch (ioc->pdev->subsystem_device) {
3611 case MPT2SAS_INTEL_RS25GB008_SSDID:
3612 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3613 MPT2SAS_INTEL_RS25GB008_BRANDING);
3614 break;
3615 case MPT2SAS_INTEL_RMS25JB080_SSDID:
3616 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3617 MPT2SAS_INTEL_RMS25JB080_BRANDING);
3618 break;
3619 case MPT2SAS_INTEL_RMS25JB040_SSDID:
3620 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3621 MPT2SAS_INTEL_RMS25JB040_BRANDING);
3622 break;
3623 case MPT2SAS_INTEL_RMS25KB080_SSDID:
3624 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3625 MPT2SAS_INTEL_RMS25KB080_BRANDING);
3626 break;
3627 case MPT2SAS_INTEL_RMS25KB040_SSDID:
3628 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3629 MPT2SAS_INTEL_RMS25KB040_BRANDING);
3630 break;
3631 case MPT2SAS_INTEL_RMS25LB040_SSDID:
3632 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3633 MPT2SAS_INTEL_RMS25LB040_BRANDING);
3634 break;
3635 case MPT2SAS_INTEL_RMS25LB080_SSDID:
3636 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3637 MPT2SAS_INTEL_RMS25LB080_BRANDING);
3638 break;
3639 default:
3640 pr_info(MPT3SAS_FMT
3641 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3642 ioc->name, ioc->pdev->subsystem_device);
3643 break;
3644 }
3645 case MPI25_MFGPAGE_DEVID_SAS3008:
3646 switch (ioc->pdev->subsystem_device) {
3647 case MPT3SAS_INTEL_RMS3JC080_SSDID:
3648 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3649 MPT3SAS_INTEL_RMS3JC080_BRANDING);
3650 break;
3651
3652 case MPT3SAS_INTEL_RS3GC008_SSDID:
3653 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3654 MPT3SAS_INTEL_RS3GC008_BRANDING);
3655 break;
3656 case MPT3SAS_INTEL_RS3FC044_SSDID:
3657 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3658 MPT3SAS_INTEL_RS3FC044_BRANDING);
3659 break;
3660 case MPT3SAS_INTEL_RS3UC080_SSDID:
3661 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3662 MPT3SAS_INTEL_RS3UC080_BRANDING);
3663 break;
3664 default:
3665 pr_info(MPT3SAS_FMT
3666 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3667 ioc->name, ioc->pdev->subsystem_device);
3668 break;
3669 }
1117b31a
SR		 3670 break;
3671 default:
3672 pr_info(MPT3SAS_FMT
989e43c7
SR		 3673 "Intel(R) Controller: Subsystem ID: 0x%X\n",
3674 ioc->name, ioc->pdev->subsystem_device);
1117b31a
SR		 3675 break;
3676 }
3677 break;
989e43c7
SR		 3678 case PCI_VENDOR_ID_DELL:
3679 switch (ioc->pdev->device) {
3680 case MPI2_MFGPAGE_DEVID_SAS2008:
3681 switch (ioc->pdev->subsystem_device) {
3682 case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID:
3683 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3684 MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING);
3685 break;
3686 case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID:
3687 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3688 MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING);
3689 break;
3690 case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID:
3691 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3692 MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING);
3693 break;
3694 case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID:
3695 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3696 MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING);
3697 break;
3698 case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID:
3699 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3700 MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING);
3701 break;
3702 case MPT2SAS_DELL_PERC_H200_SSDID:
3703 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3704 MPT2SAS_DELL_PERC_H200_BRANDING);
3705 break;
3706 case MPT2SAS_DELL_6GBPS_SAS_SSDID:
3707 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3708 MPT2SAS_DELL_6GBPS_SAS_BRANDING);
3709 break;
3710 default:
3711 pr_info(MPT3SAS_FMT
3712 "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",
3713 ioc->name, ioc->pdev->subsystem_device);
3714 break;
3715 }
3716 break;
3717 case MPI25_MFGPAGE_DEVID_SAS3008:
3718 switch (ioc->pdev->subsystem_device) {
3719 case MPT3SAS_DELL_12G_HBA_SSDID:
3720 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3721 MPT3SAS_DELL_12G_HBA_BRANDING);
3722 break;
3723 default:
3724 pr_info(MPT3SAS_FMT
3725 "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",
3726 ioc->name, ioc->pdev->subsystem_device);
3727 break;
3728 }
fb84dfc4
SR		 3729 break;
3730 default:
3731 pr_info(MPT3SAS_FMT
989e43c7 3732 "Dell HBA: Subsystem ID: 0x%X\n", ioc->name,
fb84dfc4
SR		 3733 ioc->pdev->subsystem_device);
3734 break;
3735 }
3736 break;
989e43c7
SR		 3737 case PCI_VENDOR_ID_CISCO:
3738 switch (ioc->pdev->device) {
3739 case MPI25_MFGPAGE_DEVID_SAS3008:
3740 switch (ioc->pdev->subsystem_device) {
3741 case MPT3SAS_CISCO_12G_8E_HBA_SSDID:
3742 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3743 MPT3SAS_CISCO_12G_8E_HBA_BRANDING);
3744 break;
3745 case MPT3SAS_CISCO_12G_8I_HBA_SSDID:
3746 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3747 MPT3SAS_CISCO_12G_8I_HBA_BRANDING);
3748 break;
3749 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
3750 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3751 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
3752 break;
3753 default:
3754 pr_info(MPT3SAS_FMT
3755 "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
3756 ioc->name, ioc->pdev->subsystem_device);
3757 break;
3758 }
d8eb4a47 3759 break;
989e43c7
SR		 3760 case MPI25_MFGPAGE_DEVID_SAS3108_1:
3761 switch (ioc->pdev->subsystem_device) {
3762 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
3763 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
d8eb4a47 3764 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
989e43c7
SR		 3765 break;
3766 case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID:
3767 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3768 MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING
3769 );
3770 break;
3771 default:
3772 pr_info(MPT3SAS_FMT
3773 "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
3774 ioc->name, ioc->pdev->subsystem_device);
3775 break;
3776 }
38e4141e
SR		 3777 break;
3778 default:
3779 pr_info(MPT3SAS_FMT
989e43c7
SR		 3780 "Cisco SAS HBA: Subsystem ID: 0x%X\n",
3781 ioc->name, ioc->pdev->subsystem_device);
38e4141e
SR		 3782 break;
3783 }
3784 break;
989e43c7
SR		 3785 case MPT2SAS_HP_3PAR_SSVID:
3786 switch (ioc->pdev->device) {
3787 case MPI2_MFGPAGE_DEVID_SAS2004:
3788 switch (ioc->pdev->subsystem_device) {
3789 case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID:
3790 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3791 MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING);
3792 break;
3793 default:
3794 pr_info(MPT3SAS_FMT
3795 "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
3796 ioc->name, ioc->pdev->subsystem_device);
3797 break;
3798 }
3799 case MPI2_MFGPAGE_DEVID_SAS2308_2:
3800 switch (ioc->pdev->subsystem_device) {
3801 case MPT2SAS_HP_2_4_INTERNAL_SSDID:
3802 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3803 MPT2SAS_HP_2_4_INTERNAL_BRANDING);
3804 break;
3805 case MPT2SAS_HP_2_4_EXTERNAL_SSDID:
3806 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3807 MPT2SAS_HP_2_4_EXTERNAL_BRANDING);
3808 break;
3809 case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID:
3810 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3811 MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING);
3812 break;
3813 case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID:
3814 pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3815 MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING);
3816 break;
3817 default:
3818 pr_info(MPT3SAS_FMT
3819 "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
3820 ioc->name, ioc->pdev->subsystem_device);
3821 break;
3822 }
d8eb4a47
SR		 3823 default:
3824 pr_info(MPT3SAS_FMT
989e43c7
SR		 3825 "HP SAS HBA: Subsystem ID: 0x%X\n",
3826 ioc->name, ioc->pdev->subsystem_device);
d8eb4a47
SR		 3827 break;
3828 }
38e4141e 3829 default:
38e4141e
SR		 3830 break;
3831 }
3832}
fb84dfc4 3833
f92363d1
SR		 3834/**
3835 * _base_display_ioc_capabilities - Disply IOC's capabilities.
3836 * @ioc: per adapter object
3837 *
3838 * Return nothing.
3839 */
3840static void
3841_base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
3842{
3843 int i = 0;
3844 char desc[16];
3845 u32 iounit_pg1_flags;
3846 u32 bios_version;
3847
3848 bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
3849 strncpy(desc, ioc->manu_pg0.ChipName, 16);
3850 pr_info(MPT3SAS_FMT "%s: FWVersion(%02d.%02d.%02d.%02d), "\
3851 "ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
3852 ioc->name, desc,
3853 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
3854 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
3855 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
3856 ioc->facts.FWVersion.Word & 0x000000FF,
3857 ioc->pdev->revision,
3858 (bios_version & 0xFF000000) >> 24,
3859 (bios_version & 0x00FF0000) >> 16,
3860 (bios_version & 0x0000FF00) >> 8,
3861 bios_version & 0x000000FF);
3862
989e43c7 3863 _base_display_OEMs_branding(ioc);
1117b31a 3864
016d5c35
SPS		 3865 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
3866 pr_info("%sNVMe", i ? "," : "");
3867 i++;
3868 }
3869
f92363d1
SR		 3870 pr_info(MPT3SAS_FMT "Protocol=(", ioc->name);
3871
3872 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {
3873 pr_info("Initiator");
3874 i++;
3875 }
3876
3877 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {
3878 pr_info("%sTarget", i ? "," : "");
3879 i++;
3880 }
3881
3882 i = 0;
3883 pr_info("), ");
3884 pr_info("Capabilities=(");
3885
7786ab6a
SR		 3886 if (!ioc->hide_ir_msg) {
3887 if (ioc->facts.IOCCapabilities &
f92363d1
SR		 3888 MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) {
3889 pr_info("Raid");
3890 i++;
7786ab6a 3891 }
f92363d1
SR		 3892 }
3893
3894 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {
3895 pr_info("%sTLR", i ? "," : "");
3896 i++;
3897 }
3898
3899 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {
3900 pr_info("%sMulticast", i ? "," : "");
3901 i++;
3902 }
3903
3904 if (ioc->facts.IOCCapabilities &
3905 MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) {
3906 pr_info("%sBIDI Target", i ? "," : "");
3907 i++;
3908 }
3909
3910 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {
3911 pr_info("%sEEDP", i ? "," : "");
3912 i++;
3913 }
3914
3915 if (ioc->facts.IOCCapabilities &
3916 MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) {
3917 pr_info("%sSnapshot Buffer", i ? "," : "");
3918 i++;
3919 }
3920
3921 if (ioc->facts.IOCCapabilities &
3922 MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) {
3923 pr_info("%sDiag Trace Buffer", i ? "," : "");
3924 i++;
3925 }
3926
3927 if (ioc->facts.IOCCapabilities &
3928 MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) {
3929 pr_info("%sDiag Extended Buffer", i ? "," : "");
3930 i++;
3931 }
3932
3933 if (ioc->facts.IOCCapabilities &
3934 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) {
3935 pr_info("%sTask Set Full", i ? "," : "");
3936 i++;
3937 }
3938
3939 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
3940 if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) {
3941 pr_info("%sNCQ", i ? "," : "");
3942 i++;
3943 }
3944
3945 pr_info(")\n");
3946}
3947
3948/**
3949 * mpt3sas_base_update_missing_delay - change the missing delay timers
3950 * @ioc: per adapter object
3951 * @device_missing_delay: amount of time till device is reported missing
3952 * @io_missing_delay: interval IO is returned when there is a missing device
3953 *
3954 * Return nothing.
3955 *
3956 * Passed on the command line, this function will modify the device missing
3957 * delay, as well as the io missing delay. This should be called at driver
3958 * load time.
3959 */
3960void
3961mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
3962 u16 device_missing_delay, u8 io_missing_delay)
3963{
3964 u16 dmd, dmd_new, dmd_orignal;
3965 u8 io_missing_delay_original;
3966 u16 sz;
3967 Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
3968 Mpi2ConfigReply_t mpi_reply;
3969 u8 num_phys = 0;
3970 u16 ioc_status;
3971
3972 mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
3973 if (!num_phys)
3974 return;
3975
3976 sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
3977 sizeof(Mpi2SasIOUnit1PhyData_t));
3978 sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
3979 if (!sas_iounit_pg1) {
3980 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3981 ioc->name, __FILE__, __LINE__, __func__);
3982 goto out;
3983 }
3984 if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
3985 sas_iounit_pg1, sz))) {
3986 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3987 ioc->name, __FILE__, __LINE__, __func__);
3988 goto out;
3989 }
3990 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
3991 MPI2_IOCSTATUS_MASK;
3992 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
3993 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
3994 ioc->name, __FILE__, __LINE__, __func__);
3995 goto out;
3996 }
3997
3998 /* device missing delay */
3999 dmd = sas_iounit_pg1->ReportDeviceMissingDelay;
4000 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4001 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4002 else
4003 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4004 dmd_orignal = dmd;
4005 if (device_missing_delay > 0x7F) {
4006 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 :
4007 device_missing_delay;
4008 dmd = dmd / 16;
4009 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16;
4010 } else
4011 dmd = device_missing_delay;
4012 sas_iounit_pg1->ReportDeviceMissingDelay = dmd;
4013
4014 /* io missing delay */
4015 io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay;
4016 sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay;
4017
4018 if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,
4019 sz)) {
4020 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4021 dmd_new = (dmd &
4022 MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4023 else
4024 dmd_new =
4025 dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4026 pr_info(MPT3SAS_FMT "device_missing_delay: old(%d), new(%d)\n",
4027 ioc->name, dmd_orignal, dmd_new);
4028 pr_info(MPT3SAS_FMT "ioc_missing_delay: old(%d), new(%d)\n",
4029 ioc->name, io_missing_delay_original,
4030 io_missing_delay);
4031 ioc->device_missing_delay = dmd_new;
4032 ioc->io_missing_delay = io_missing_delay;
4033 }
4034
4035out:
4036 kfree(sas_iounit_pg1);
4037}
4038/**
4039 * _base_static_config_pages - static start of day config pages
4040 * @ioc: per adapter object
4041 *
4042 * Return nothing.
4043 */
4044static void
4045_base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
4046{
4047 Mpi2ConfigReply_t mpi_reply;
4048 u32 iounit_pg1_flags;
4049
4050 mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
4051 if (ioc->ir_firmware)
4052 mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
4053 &ioc->manu_pg10);
4054
4055 /*
4056 * Ensure correct T10 PI operation if vendor left EEDPTagMode
4057 * flag unset in NVDATA.
4058 */
4059 mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11);
4060 if (ioc->manu_pg11.EEDPTagMode == 0) {
4061 pr_err("%s: overriding NVDATA EEDPTagMode setting\n",
4062 ioc->name);
4063 ioc->manu_pg11.EEDPTagMode &= ~0x3;
4064 ioc->manu_pg11.EEDPTagMode |= 0x1;
4065 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
4066 &ioc->manu_pg11);
4067 }
4068
4069 mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
4070 mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
4071 mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);
4072 mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);
4073 mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
2d8ce8c9 4074 mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
f92363d1
SR		 4075 _base_display_ioc_capabilities(ioc);
4076
4077 /*
4078 * Enable task_set_full handling in iounit_pg1 when the
4079 * facts capabilities indicate that its supported.
4080 */
4081 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4082 if ((ioc->facts.IOCCapabilities &
4083 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING))
4084 iounit_pg1_flags &=
4085 ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4086 else
4087 iounit_pg1_flags |=
4088 MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
4089 ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);
4090 mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
2d8ce8c9
SR		 4091
4092 if (ioc->iounit_pg8.NumSensors)
4093 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
f92363d1
SR		 4094}
4095
4096/**
4097 * _base_release_memory_pools - release memory
4098 * @ioc: per adapter object
4099 *
4100 * Free memory allocated from _base_allocate_memory_pools.
4101 *
4102 * Return nothing.
4103 */
4104static void
4105_base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
4106{
9b05c91a
SR		 4107 int i = 0;
4108 struct reply_post_struct *rps;
f92363d1
SR		 4109
4110 dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4111 __func__));
4112
4113 if (ioc->request) {
4114 pci_free_consistent(ioc->pdev, ioc->request_dma_sz,
4115 ioc->request, ioc->request_dma);
4116 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4117 "request_pool(0x%p): free\n",
4118 ioc->name, ioc->request));
4119 ioc->request = NULL;
4120 }
4121
4122 if (ioc->sense) {
e9d98418
RP		 4123 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
4124 dma_pool_destroy(ioc->sense_dma_pool);
f92363d1
SR		 4125 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4126 "sense_pool(0x%p): free\n",
4127 ioc->name, ioc->sense));
4128 ioc->sense = NULL;
4129 }
4130
4131 if (ioc->reply) {
e9d98418
RP		 4132 dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
4133 dma_pool_destroy(ioc->reply_dma_pool);
f92363d1
SR		 4134 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4135 "reply_pool(0x%p): free\n",
4136 ioc->name, ioc->reply));
4137 ioc->reply = NULL;
4138 }
4139
4140 if (ioc->reply_free) {
e9d98418 4141 dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
f92363d1 4142 ioc->reply_free_dma);
e9d98418 4143 dma_pool_destroy(ioc->reply_free_dma_pool);
f92363d1
SR		 4144 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4145 "reply_free_pool(0x%p): free\n",
4146 ioc->name, ioc->reply_free));
4147 ioc->reply_free = NULL;
4148 }
4149
9b05c91a
SR		 4150 if (ioc->reply_post) {
4151 do {
4152 rps = &ioc->reply_post[i];
4153 if (rps->reply_post_free) {
e9d98418 4154 dma_pool_free(
9b05c91a
SR		 4155 ioc->reply_post_free_dma_pool,
4156 rps->reply_post_free,
4157 rps->reply_post_free_dma);
4158 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4159 "reply_post_free_pool(0x%p): free\n",
4160 ioc->name, rps->reply_post_free));
4161 rps->reply_post_free = NULL;
4162 }
4163 } while (ioc->rdpq_array_enable &&
4164 (++i < ioc->reply_queue_count));
cd33223b
C		 4165 if (ioc->reply_post_free_array &&
4166 ioc->rdpq_array_enable) {
4167 dma_pool_free(ioc->reply_post_free_array_dma_pool,
4168 ioc->reply_post_free_array,
4169 ioc->reply_post_free_array_dma);
4170 ioc->reply_post_free_array = NULL;
4171 }
4172 dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
e9d98418 4173 dma_pool_destroy(ioc->reply_post_free_dma_pool);
9b05c91a 4174 kfree(ioc->reply_post);
f92363d1
SR		 4175 }
4176
016d5c35
SPS		 4177 if (ioc->pcie_sgl_dma_pool) {
4178 for (i = 0; i < ioc->scsiio_depth; i++) {
dbec4c90
SPS		 4179 dma_pool_free(ioc->pcie_sgl_dma_pool,
4180 ioc->pcie_sg_lookup[i].pcie_sgl,
4181 ioc->pcie_sg_lookup[i].pcie_sgl_dma);
016d5c35
SPS		 4182 }
4183 if (ioc->pcie_sgl_dma_pool)
13a06405 4184 dma_pool_destroy(ioc->pcie_sgl_dma_pool);
016d5c35
SPS		 4185 }
4186
f92363d1
SR		 4187 if (ioc->config_page) {
4188 dexitprintk(ioc, pr_info(MPT3SAS_FMT
4189 "config_page(0x%p): free\n", ioc->name,
4190 ioc->config_page));
4191 pci_free_consistent(ioc->pdev, ioc->config_page_sz,
4192 ioc->config_page, ioc->config_page_dma);
4193 }
4194
f92363d1
SR		 4195 kfree(ioc->hpr_lookup);
4196 kfree(ioc->internal_lookup);
4197 if (ioc->chain_lookup) {
4198 for (i = 0; i < ioc->chain_depth; i++) {
4199 if (ioc->chain_lookup[i].chain_buffer)
e9d98418 4200 dma_pool_free(ioc->chain_dma_pool,
f92363d1
SR		 4201 ioc->chain_lookup[i].chain_buffer,
4202 ioc->chain_lookup[i].chain_buffer_dma);
4203 }
e9d98418 4204 dma_pool_destroy(ioc->chain_dma_pool);
f92363d1
SR		 4205 free_pages((ulong)ioc->chain_lookup, ioc->chain_pages);
4206 ioc->chain_lookup = NULL;
4207 }
4208}
4209
4210/**
4211 * _base_allocate_memory_pools - allocate start of day memory pools
4212 * @ioc: per adapter object
f92363d1
SR		 4213 *
4214 * Returns 0 success, anything else error
4215 */
4216static int
98c56ad3 4217_base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 4218{
4219 struct mpt3sas_facts *facts;
4220 u16 max_sge_elements;
4221 u16 chains_needed_per_io;
cd33223b 4222 u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
f92363d1 4223 u32 retry_sz;
016d5c35 4224 u16 max_request_credit, nvme_blocks_needed;
f92363d1
SR		 4225 unsigned short sg_tablesize;
4226 u16 sge_size;
4227 int i;
4228
4229 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4230 __func__));
4231
4232
4233 retry_sz = 0;
4234 facts = &ioc->facts;
4235
4236 /* command line tunables for max sgl entries */
4237 if (max_sgl_entries != -1)
4238 sg_tablesize = max_sgl_entries;
471ef9d4
SR		 4239 else {
4240 if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
4241 sg_tablesize = MPT2SAS_SG_DEPTH;
4242 else
4243 sg_tablesize = MPT3SAS_SG_DEPTH;
4244 }
f92363d1 4245
06f5f976
SR		 4246 /* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */
4247 if (reset_devices)
4248 sg_tablesize = min_t(unsigned short, sg_tablesize,
4249 MPT_KDUMP_MIN_PHYS_SEGMENTS);
4250
0448f019
SPS		 4251 if (ioc->is_mcpu_endpoint)
4252 ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
4253 else {
4254 if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS)
4255 sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
4256 else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) {
4257 sg_tablesize = min_t(unsigned short, sg_tablesize,
4258 SG_MAX_SEGMENTS);
4259 pr_warn(MPT3SAS_FMT
4260 "sg_tablesize(%u) is bigger than kernel "
4261 "defined SG_CHUNK_SIZE(%u)\n", ioc->name,
4262 sg_tablesize, MPT_MAX_PHYS_SEGMENTS);
4263 }
4264 ioc->shost->sg_tablesize = sg_tablesize;
ad666a0f 4265 }
f92363d1 4266
fd0331b3
SS		 4267 ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),
4268 (facts->RequestCredit / 4));
4269 if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {
4270 if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT +
4271 INTERNAL_SCSIIO_CMDS_COUNT)) {
4272 pr_err(MPT3SAS_FMT "IOC doesn't have enough Request \
4273 Credits, it has just %d number of credits\n",
4274 ioc->name, facts->RequestCredit);
4275 return -ENOMEM;
4276 }
4277 ioc->internal_depth = 10;
4278 }
4279
4280 ioc->hi_priority_depth = ioc->internal_depth - (5);
f92363d1
SR		 4281 /* command line tunables for max controller queue depth */
4282 if (max_queue_depth != -1 && max_queue_depth != 0) {
4283 max_request_credit = min_t(u16, max_queue_depth +
fd0331b3 4284 ioc->internal_depth, facts->RequestCredit);
f92363d1
SR		 4285 if (max_request_credit > MAX_HBA_QUEUE_DEPTH)
4286 max_request_credit = MAX_HBA_QUEUE_DEPTH;
06f5f976
SR		 4287 } else if (reset_devices)
4288 max_request_credit = min_t(u16, facts->RequestCredit,
4289 (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));
4290 else
f92363d1
SR		 4291 max_request_credit = min_t(u16, facts->RequestCredit,
4292 MAX_HBA_QUEUE_DEPTH);
4293
fd0331b3
SS		 4294 /* Firmware maintains additional facts->HighPriorityCredit number of
4295 * credits for HiPriprity Request messages, so hba queue depth will be
4296 * sum of max_request_credit and high priority queue depth.
4297 */
4298 ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;
f92363d1
SR		 4299
4300 /* request frame size */
4301 ioc->request_sz = facts->IOCRequestFrameSize * 4;
4302
4303 /* reply frame size */
4304 ioc->reply_sz = facts->ReplyFrameSize * 4;
4305
ebb3024e
SS		 4306 /* chain segment size */
4307 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
4308 if (facts->IOCMaxChainSegmentSize)
4309 ioc->chain_segment_sz =
4310 facts->IOCMaxChainSegmentSize *
4311 MAX_CHAIN_ELEMT_SZ;
4312 else
4313 /* set to 128 bytes size if IOCMaxChainSegmentSize is zero */
4314 ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *
4315 MAX_CHAIN_ELEMT_SZ;
4316 } else
4317 ioc->chain_segment_sz = ioc->request_sz;
4318
f92363d1
SR		 4319 /* calculate the max scatter element size */
4320 sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);
4321
4322 retry_allocation:
4323 total_sz = 0;
4324 /* calculate number of sg elements left over in the 1st frame */
4325 max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -
4326 sizeof(Mpi2SGEIOUnion_t)) + sge_size);
4327 ioc->max_sges_in_main_message = max_sge_elements/sge_size;
4328
4329 /* now do the same for a chain buffer */
ebb3024e 4330 max_sge_elements = ioc->chain_segment_sz - sge_size;
f92363d1
SR		 4331 ioc->max_sges_in_chain_message = max_sge_elements/sge_size;
4332
4333 /*
4334 * MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE
4335 */
4336 chains_needed_per_io = ((ioc->shost->sg_tablesize -
4337 ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)
4338 + 1;
4339 if (chains_needed_per_io > facts->MaxChainDepth) {
4340 chains_needed_per_io = facts->MaxChainDepth;
4341 ioc->shost->sg_tablesize = min_t(u16,
4342 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message
4343 * chains_needed_per_io), ioc->shost->sg_tablesize);
4344 }
4345 ioc->chains_needed_per_io = chains_needed_per_io;
4346
4347 /* reply free queue sizing - taking into account for 64 FW events */
4348 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
4349
0448f019
SPS		 4350 /* mCPU manage single counters for simplicity */
4351 if (ioc->is_mcpu_endpoint)
4352 ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;
4353 else {
4354 /* calculate reply descriptor post queue depth */
4355 ioc->reply_post_queue_depth = ioc->hba_queue_depth +
4356 ioc->reply_free_queue_depth + 1;
4357 /* align the reply post queue on the next 16 count boundary */
4358 if (ioc->reply_post_queue_depth % 16)
4359 ioc->reply_post_queue_depth += 16 -
4360 (ioc->reply_post_queue_depth % 16);
4361 }
f92363d1 4362
f92363d1
SR		 4363 if (ioc->reply_post_queue_depth >
4364 facts->MaxReplyDescriptorPostQueueDepth) {
4365 ioc->reply_post_queue_depth =
4366 facts->MaxReplyDescriptorPostQueueDepth -
4367 (facts->MaxReplyDescriptorPostQueueDepth % 16);
4368 ioc->hba_queue_depth =
4369 ((ioc->reply_post_queue_depth - 64) / 2) - 1;
4370 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
4371 }
4372
4373 dinitprintk(ioc, pr_info(MPT3SAS_FMT "scatter gather: " \
4374 "sge_in_main_msg(%d), sge_per_chain(%d), sge_per_io(%d), "
4375 "chains_per_io(%d)\n", ioc->name, ioc->max_sges_in_main_message,
4376 ioc->max_sges_in_chain_message, ioc->shost->sg_tablesize,
4377 ioc->chains_needed_per_io));
4378
9b05c91a
SR		 4379 /* reply post queue, 16 byte align */
4380 reply_post_free_sz = ioc->reply_post_queue_depth *
4381 sizeof(Mpi2DefaultReplyDescriptor_t);
4382
4383 sz = reply_post_free_sz;
4384 if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
4385 sz *= ioc->reply_queue_count;
4386
4387 ioc->reply_post = kcalloc((ioc->rdpq_array_enable) ?
4388 (ioc->reply_queue_count):1,
4389 sizeof(struct reply_post_struct), GFP_KERNEL);
4390
4391 if (!ioc->reply_post) {
4392 pr_err(MPT3SAS_FMT "reply_post_free pool: kcalloc failed\n",
4393 ioc->name);
4394 goto out;
4395 }
e9d98418
RP		 4396 ioc->reply_post_free_dma_pool = dma_pool_create("reply_post_free pool",
4397 &ioc->pdev->dev, sz, 16, 0);
9b05c91a
SR		 4398 if (!ioc->reply_post_free_dma_pool) {
4399 pr_err(MPT3SAS_FMT
e9d98418 4400 "reply_post_free pool: dma_pool_create failed\n",
9b05c91a
SR		 4401 ioc->name);
4402 goto out;
4403 }
4404 i = 0;
4405 do {
4406 ioc->reply_post[i].reply_post_free =
e9d98418 4407 dma_pool_alloc(ioc->reply_post_free_dma_pool,
9b05c91a
SR		 4408 GFP_KERNEL,
4409 &ioc->reply_post[i].reply_post_free_dma);
4410 if (!ioc->reply_post[i].reply_post_free) {
4411 pr_err(MPT3SAS_FMT
e9d98418 4412 "reply_post_free pool: dma_pool_alloc failed\n",
9b05c91a
SR		 4413 ioc->name);
4414 goto out;
4415 }
4416 memset(ioc->reply_post[i].reply_post_free, 0, sz);
4417 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4418 "reply post free pool (0x%p): depth(%d),"
4419 "element_size(%d), pool_size(%d kB)\n", ioc->name,
4420 ioc->reply_post[i].reply_post_free,
4421 ioc->reply_post_queue_depth, 8, sz/1024));
4422 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4423 "reply_post_free_dma = (0x%llx)\n", ioc->name,
4424 (unsigned long long)
4425 ioc->reply_post[i].reply_post_free_dma));
4426 total_sz += sz;
4427 } while (ioc->rdpq_array_enable && (++i < ioc->reply_queue_count));
4428
4429 if (ioc->dma_mask == 64) {
4430 if (_base_change_consistent_dma_mask(ioc, ioc->pdev) != 0) {
4431 pr_warn(MPT3SAS_FMT
4432 "no suitable consistent DMA mask for %s\n",
4433 ioc->name, pci_name(ioc->pdev));
4434 goto out;
4435 }
4436 }
4437
f92363d1
SR		 4438 ioc->scsiio_depth = ioc->hba_queue_depth -
4439 ioc->hi_priority_depth - ioc->internal_depth;
4440
4441 /* set the scsi host can_queue depth
4442 * with some internal commands that could be outstanding
4443 */
fd0331b3 4444 ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;
f92363d1
SR		 4445 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4446 "scsi host: can_queue depth (%d)\n",
4447 ioc->name, ioc->shost->can_queue));
4448
4449
4450 /* contiguous pool for request and chains, 16 byte align, one extra "
4451 * "frame for smid=0
4452 */
4453 ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;
4454 sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);
4455
4456 /* hi-priority queue */
4457 sz += (ioc->hi_priority_depth * ioc->request_sz);
4458
4459 /* internal queue */
4460 sz += (ioc->internal_depth * ioc->request_sz);
4461
4462 ioc->request_dma_sz = sz;
4463 ioc->request = pci_alloc_consistent(ioc->pdev, sz, &ioc->request_dma);
4464 if (!ioc->request) {
4465 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
4466 "failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
4467 "total(%d kB)\n", ioc->name, ioc->hba_queue_depth,
4468 ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
4469 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)
4470 goto out;
fd0331b3
SS		 4471 retry_sz = 64;
4472 ioc->hba_queue_depth -= retry_sz;
8ff045c9 4473 _base_release_memory_pools(ioc);
f92363d1
SR		 4474 goto retry_allocation;
4475 }
4476
4477 if (retry_sz)
4478 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
4479 "succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
4480 "total(%d kb)\n", ioc->name, ioc->hba_queue_depth,
4481 ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
4482
4483 /* hi-priority queue */
4484 ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *
4485 ioc->request_sz);
4486 ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *
4487 ioc->request_sz);
4488
4489 /* internal queue */
4490 ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *
4491 ioc->request_sz);
4492 ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *
4493 ioc->request_sz);
4494
4495 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4496 "request pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
4497 ioc->name, ioc->request, ioc->hba_queue_depth, ioc->request_sz,
4498 (ioc->hba_queue_depth * ioc->request_sz)/1024));
4499
4500 dinitprintk(ioc, pr_info(MPT3SAS_FMT "request pool: dma(0x%llx)\n",
4501 ioc->name, (unsigned long long) ioc->request_dma));
4502 total_sz += sz;
4503
f92363d1
SR		 4504 dinitprintk(ioc, pr_info(MPT3SAS_FMT "scsiio(0x%p): depth(%d)\n",
4505 ioc->name, ioc->request, ioc->scsiio_depth));
4506
4507 ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
4508 sz = ioc->chain_depth * sizeof(struct chain_tracker);
4509 ioc->chain_pages = get_order(sz);
4510 ioc->chain_lookup = (struct chain_tracker *)__get_free_pages(
4511 GFP_KERNEL, ioc->chain_pages);
4512 if (!ioc->chain_lookup) {
4513 pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages failed\n",
4514 ioc->name);
4515 goto out;
4516 }
e9d98418 4517 ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
ebb3024e 4518 ioc->chain_segment_sz, 16, 0);
f92363d1 4519 if (!ioc->chain_dma_pool) {
e9d98418 4520 pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
f92363d1
SR		 4521 ioc->name);
4522 goto out;
4523 }
4524 for (i = 0; i < ioc->chain_depth; i++) {
e9d98418 4525 ioc->chain_lookup[i].chain_buffer = dma_pool_alloc(
f92363d1
SR		 4526 ioc->chain_dma_pool , GFP_KERNEL,
4527 &ioc->chain_lookup[i].chain_buffer_dma);
4528 if (!ioc->chain_lookup[i].chain_buffer) {
4529 ioc->chain_depth = i;
4530 goto chain_done;
4531 }
ebb3024e 4532 total_sz += ioc->chain_segment_sz;
f92363d1
SR		 4533 }
4534 chain_done:
4535 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4536 "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
ebb3024e
SS		 4537 ioc->name, ioc->chain_depth, ioc->chain_segment_sz,
4538 ((ioc->chain_depth * ioc->chain_segment_sz))/1024));
f92363d1
SR		 4539
4540 /* initialize hi-priority queue smid's */
4541 ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
4542 sizeof(struct request_tracker), GFP_KERNEL);
4543 if (!ioc->hpr_lookup) {
4544 pr_err(MPT3SAS_FMT "hpr_lookup: kcalloc failed\n",
4545 ioc->name);
4546 goto out;
4547 }
4548 ioc->hi_priority_smid = ioc->scsiio_depth + 1;
4549 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4550 "hi_priority(0x%p): depth(%d), start smid(%d)\n",
4551 ioc->name, ioc->hi_priority,
4552 ioc->hi_priority_depth, ioc->hi_priority_smid));
4553
4554 /* initialize internal queue smid's */
4555 ioc->internal_lookup = kcalloc(ioc->internal_depth,
4556 sizeof(struct request_tracker), GFP_KERNEL);
4557 if (!ioc->internal_lookup) {
4558 pr_err(MPT3SAS_FMT "internal_lookup: kcalloc failed\n",
4559 ioc->name);
4560 goto out;
4561 }
4562 ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;
4563 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4564 "internal(0x%p): depth(%d), start smid(%d)\n",
4565 ioc->name, ioc->internal,
4566 ioc->internal_depth, ioc->internal_smid));
016d5c35
SPS		 4567 /*
4568 * The number of NVMe page sized blocks needed is:
4569 * (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1
4570 * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry
4571 * that is placed in the main message frame. 8 is the size of each PRP
4572 * entry or PRP list pointer entry. 8 is subtracted from page_size
4573 * because of the PRP list pointer entry at the end of a page, so this
4574 * is not counted as a PRP entry. The 1 added page is a round up.
4575 *
4576 * To avoid allocation failures due to the amount of memory that could
4577 * be required for NVMe PRP's, only each set of NVMe blocks will be
4578 * contiguous, so a new set is allocated for each possible I/O.
4579 */
4580 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
4581 nvme_blocks_needed =
4582 (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
4583 nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);
4584 nvme_blocks_needed++;
4585
dbec4c90
SPS		 4586 sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;
4587 ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);
4588 if (!ioc->pcie_sg_lookup) {
4589 pr_info(MPT3SAS_FMT
4590 "PCIe SGL lookup: kzalloc failed\n", ioc->name);
4591 goto out;
4592 }
016d5c35
SPS		 4593 sz = nvme_blocks_needed * ioc->page_size;
4594 ioc->pcie_sgl_dma_pool =
13a06405 4595 dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, 16, 0);
016d5c35
SPS		 4596 if (!ioc->pcie_sgl_dma_pool) {
4597 pr_info(MPT3SAS_FMT
13a06405 4598 "PCIe SGL pool: dma_pool_create failed\n",
016d5c35
SPS		 4599 ioc->name);
4600 goto out;
4601 }
4602 for (i = 0; i < ioc->scsiio_depth; i++) {
dbec4c90
SPS		 4603 ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc(
4604 ioc->pcie_sgl_dma_pool, GFP_KERNEL,
4605 &ioc->pcie_sg_lookup[i].pcie_sgl_dma);
4606 if (!ioc->pcie_sg_lookup[i].pcie_sgl) {
016d5c35 4607 pr_info(MPT3SAS_FMT
13a06405 4608 "PCIe SGL pool: dma_pool_alloc failed\n",
016d5c35
SPS		 4609 ioc->name);
4610 goto out;
4611 }
4612 }
f92363d1 4613
016d5c35
SPS		 4614 dinitprintk(ioc, pr_info(MPT3SAS_FMT "PCIe sgl pool depth(%d), "
4615 "element_size(%d), pool_size(%d kB)\n", ioc->name,
4616 ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
4617 total_sz += sz * ioc->scsiio_depth;
4618 }
f92363d1
SR		 4619 /* sense buffers, 4 byte align */
4620 sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
e9d98418
RP		 4621 ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
4622 4, 0);
f92363d1 4623 if (!ioc->sense_dma_pool) {
e9d98418 4624 pr_err(MPT3SAS_FMT "sense pool: dma_pool_create failed\n",
f92363d1
SR		 4625 ioc->name);
4626 goto out;
4627 }
e9d98418 4628 ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
f92363d1
SR		 4629 &ioc->sense_dma);
4630 if (!ioc->sense) {
e9d98418 4631 pr_err(MPT3SAS_FMT "sense pool: dma_pool_alloc failed\n",
f92363d1
SR		 4632 ioc->name);
4633 goto out;
4634 }
4635 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4636 "sense pool(0x%p): depth(%d), element_size(%d), pool_size"
4637 "(%d kB)\n", ioc->name, ioc->sense, ioc->scsiio_depth,
4638 SCSI_SENSE_BUFFERSIZE, sz/1024));
4639 dinitprintk(ioc, pr_info(MPT3SAS_FMT "sense_dma(0x%llx)\n",
4640 ioc->name, (unsigned long long)ioc->sense_dma));
4641 total_sz += sz;
4642
4643 /* reply pool, 4 byte align */
4644 sz = ioc->reply_free_queue_depth * ioc->reply_sz;
e9d98418
RP		 4645 ioc->reply_dma_pool = dma_pool_create("reply pool", &ioc->pdev->dev, sz,
4646 4, 0);
f92363d1 4647 if (!ioc->reply_dma_pool) {
e9d98418 4648 pr_err(MPT3SAS_FMT "reply pool: dma_pool_create failed\n",
f92363d1
SR		 4649 ioc->name);
4650 goto out;
4651 }
e9d98418 4652 ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,
f92363d1
SR		 4653 &ioc->reply_dma);
4654 if (!ioc->reply) {
e9d98418 4655 pr_err(MPT3SAS_FMT "reply pool: dma_pool_alloc failed\n",
f92363d1
SR		 4656 ioc->name);
4657 goto out;
4658 }
4659 ioc->reply_dma_min_address = (u32)(ioc->reply_dma);
4660 ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;
4661 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4662 "reply pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
4663 ioc->name, ioc->reply,
4664 ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024));
4665 dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_dma(0x%llx)\n",
4666 ioc->name, (unsigned long long)ioc->reply_dma));
4667 total_sz += sz;
4668
4669 /* reply free queue, 16 byte align */
4670 sz = ioc->reply_free_queue_depth * 4;
e9d98418
RP		 4671 ioc->reply_free_dma_pool = dma_pool_create("reply_free pool",
4672 &ioc->pdev->dev, sz, 16, 0);
f92363d1 4673 if (!ioc->reply_free_dma_pool) {
e9d98418 4674 pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_create failed\n",
f92363d1
SR		 4675 ioc->name);
4676 goto out;
4677 }
e9d98418 4678 ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, GFP_KERNEL,
f92363d1
SR		 4679 &ioc->reply_free_dma);
4680 if (!ioc->reply_free) {
e9d98418 4681 pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_alloc failed\n",
f92363d1
SR		 4682 ioc->name);
4683 goto out;
4684 }
4685 memset(ioc->reply_free, 0, sz);
4686 dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_free pool(0x%p): " \
4687 "depth(%d), element_size(%d), pool_size(%d kB)\n", ioc->name,
4688 ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));
4689 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4690 "reply_free_dma (0x%llx)\n",
4691 ioc->name, (unsigned long long)ioc->reply_free_dma));
4692 total_sz += sz;
4693
cd33223b
C		 4694 if (ioc->rdpq_array_enable) {
4695 reply_post_free_array_sz = ioc->reply_queue_count *
4696 sizeof(Mpi2IOCInitRDPQArrayEntry);
4697 ioc->reply_post_free_array_dma_pool =
4698 dma_pool_create("reply_post_free_array pool",
4699 &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);
4700 if (!ioc->reply_post_free_array_dma_pool) {
4701 dinitprintk(ioc,
4702 pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
4703 "dma_pool_create failed\n", ioc->name));
4704 goto out;
4705 }
4706 ioc->reply_post_free_array =
4707 dma_pool_alloc(ioc->reply_post_free_array_dma_pool,
4708 GFP_KERNEL, &ioc->reply_post_free_array_dma);
4709 if (!ioc->reply_post_free_array) {
4710 dinitprintk(ioc,
4711 pr_info(MPT3SAS_FMT "reply_post_free_array pool: "
4712 "dma_pool_alloc failed\n", ioc->name));
4713 goto out;
4714 }
4715 }
f92363d1
SR		 4716 ioc->config_page_sz = 512;
4717 ioc->config_page = pci_alloc_consistent(ioc->pdev,
4718 ioc->config_page_sz, &ioc->config_page_dma);
4719 if (!ioc->config_page) {
4720 pr_err(MPT3SAS_FMT
e9d98418 4721 "config page: dma_pool_alloc failed\n",
f92363d1
SR		 4722 ioc->name);
4723 goto out;
4724 }
4725 dinitprintk(ioc, pr_info(MPT3SAS_FMT
4726 "config page(0x%p): size(%d)\n",
4727 ioc->name, ioc->config_page, ioc->config_page_sz));
4728 dinitprintk(ioc, pr_info(MPT3SAS_FMT "config_page_dma(0x%llx)\n",
4729 ioc->name, (unsigned long long)ioc->config_page_dma));
4730 total_sz += ioc->config_page_sz;
4731
4732 pr_info(MPT3SAS_FMT "Allocated physical memory: size(%d kB)\n",
4733 ioc->name, total_sz/1024);
4734 pr_info(MPT3SAS_FMT
4735 "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",
4736 ioc->name, ioc->shost->can_queue, facts->RequestCredit);
4737 pr_info(MPT3SAS_FMT "Scatter Gather Elements per IO(%d)\n",
4738 ioc->name, ioc->shost->sg_tablesize);
4739 return 0;
4740
4741 out:
4742 return -ENOMEM;
4743}
4744
4745/**
4746 * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter.
4747 * @ioc: Pointer to MPT_ADAPTER structure
4748 * @cooked: Request raw or cooked IOC state
4749 *
4750 * Returns all IOC Doorbell register bits if cooked==0, else just the
4751 * Doorbell bits in MPI_IOC_STATE_MASK.
4752 */
4753u32
4754mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
4755{
4756 u32 s, sc;
4757
4758 s = readl(&ioc->chip->Doorbell);
4759 sc = s & MPI2_IOC_STATE_MASK;
4760 return cooked ? sc : s;
4761}
4762
4763/**
4764 * _base_wait_on_iocstate - waiting on a particular ioc state
4765 * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
4766 * @timeout: timeout in second
f92363d1
SR		 4767 *
4768 * Returns 0 for success, non-zero for failure.
4769 */
4770static int
98c56ad3 4771_base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)
f92363d1
SR		 4772{
4773 u32 count, cntdn;
4774 u32 current_state;
4775
4776 count = 0;
98c56ad3 4777 cntdn = 1000 * timeout;
f92363d1
SR		 4778 do {
4779 current_state = mpt3sas_base_get_iocstate(ioc, 1);
4780 if (current_state == ioc_state)
4781 return 0;
4782 if (count && current_state == MPI2_IOC_STATE_FAULT)
4783 break;
98c56ad3
CO		 4784
4785 usleep_range(1000, 1500);
f92363d1
SR		 4786 count++;
4787 } while (--cntdn);
4788
4789 return current_state;
4790}
4791
4792/**
4793 * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
4794 * a write to the doorbell)
4795 * @ioc: per adapter object
4796 * @timeout: timeout in second
f92363d1
SR		 4797 *
4798 * Returns 0 for success, non-zero for failure.
4799 *
4800 * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
4801 */
4dc8c808 4802static int
98c56ad3 4803_base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
4dc8c808 4804
f92363d1 4805static int
98c56ad3 4806_base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
f92363d1
SR		 4807{
4808 u32 cntdn, count;
4809 u32 int_status;
4810
4811 count = 0;
98c56ad3 4812 cntdn = 1000 * timeout;
f92363d1
SR		 4813 do {
4814 int_status = readl(&ioc->chip->HostInterruptStatus);
4815 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
4816 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4817 "%s: successful count(%d), timeout(%d)\n",
4818 ioc->name, __func__, count, timeout));
4819 return 0;
4820 }
98c56ad3
CO		 4821
4822 usleep_range(1000, 1500);
4823 count++;
4824 } while (--cntdn);
4825
4826 pr_err(MPT3SAS_FMT
4827 "%s: failed due to timeout count(%d), int_status(%x)!\n",
4828 ioc->name, __func__, count, int_status);
4829 return -EFAULT;
4830}
4831
4832static int
4833_base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
4834{
4835 u32 cntdn, count;
4836 u32 int_status;
4837
4838 count = 0;
4839 cntdn = 2000 * timeout;
4840 do {
4841 int_status = readl(&ioc->chip->HostInterruptStatus);
4842 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
4843 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4844 "%s: successful count(%d), timeout(%d)\n",
4845 ioc->name, __func__, count, timeout));
4846 return 0;
4847 }
4848
4849 udelay(500);
f92363d1
SR		 4850 count++;
4851 } while (--cntdn);
4852
4853 pr_err(MPT3SAS_FMT
4854 "%s: failed due to timeout count(%d), int_status(%x)!\n",
4855 ioc->name, __func__, count, int_status);
4856 return -EFAULT;
98c56ad3 4857
f92363d1
SR		 4858}
4859
4860/**
4861 * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
4862 * @ioc: per adapter object
4863 * @timeout: timeout in second
f92363d1
SR		 4864 *
4865 * Returns 0 for success, non-zero for failure.
4866 *
4867 * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
4868 * doorbell.
4869 */
4870static int
98c56ad3 4871_base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)
f92363d1
SR		 4872{
4873 u32 cntdn, count;
4874 u32 int_status;
4875 u32 doorbell;
4876
4877 count = 0;
98c56ad3 4878 cntdn = 1000 * timeout;
f92363d1
SR		 4879 do {
4880 int_status = readl(&ioc->chip->HostInterruptStatus);
4881 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
4882 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4883 "%s: successful count(%d), timeout(%d)\n",
4884 ioc->name, __func__, count, timeout));
4885 return 0;
4886 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
4887 doorbell = readl(&ioc->chip->Doorbell);
4888 if ((doorbell & MPI2_IOC_STATE_MASK) ==
4889 MPI2_IOC_STATE_FAULT) {
4890 mpt3sas_base_fault_info(ioc , doorbell);
4891 return -EFAULT;
4892 }
4893 } else if (int_status == 0xFFFFFFFF)
4894 goto out;
4895
98c56ad3 4896 usleep_range(1000, 1500);
f92363d1
SR		 4897 count++;
4898 } while (--cntdn);
4899
4900 out:
4901 pr_err(MPT3SAS_FMT
4902 "%s: failed due to timeout count(%d), int_status(%x)!\n",
4903 ioc->name, __func__, count, int_status);
4904 return -EFAULT;
4905}
4906
4907/**
4908 * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
4909 * @ioc: per adapter object
4910 * @timeout: timeout in second
f92363d1
SR		 4911 *
4912 * Returns 0 for success, non-zero for failure.
4913 *
4914 */
4915static int
98c56ad3 4916_base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)
f92363d1
SR		 4917{
4918 u32 cntdn, count;
4919 u32 doorbell_reg;
4920
4921 count = 0;
98c56ad3 4922 cntdn = 1000 * timeout;
f92363d1
SR		 4923 do {
4924 doorbell_reg = readl(&ioc->chip->Doorbell);
4925 if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
4926 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4927 "%s: successful count(%d), timeout(%d)\n",
4928 ioc->name, __func__, count, timeout));
4929 return 0;
4930 }
98c56ad3
CO		 4931
4932 usleep_range(1000, 1500);
f92363d1
SR		 4933 count++;
4934 } while (--cntdn);
4935
4936 pr_err(MPT3SAS_FMT
4937 "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",
4938 ioc->name, __func__, count, doorbell_reg);
4939 return -EFAULT;
4940}
4941
4942/**
4943 * _base_send_ioc_reset - send doorbell reset
4944 * @ioc: per adapter object
4945 * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
4946 * @timeout: timeout in second
f92363d1
SR		 4947 *
4948 * Returns 0 for success, non-zero for failure.
4949 */
4950static int
98c56ad3 4951_base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)
f92363d1
SR		 4952{
4953 u32 ioc_state;
4954 int r = 0;
4955
4956 if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) {
4957 pr_err(MPT3SAS_FMT "%s: unknown reset_type\n",
4958 ioc->name, __func__);
4959 return -EFAULT;
4960 }
4961
4962 if (!(ioc->facts.IOCCapabilities &
4963 MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY))
4964 return -EFAULT;
4965
4966 pr_info(MPT3SAS_FMT "sending message unit reset !!\n", ioc->name);
4967
4968 writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
4969 &ioc->chip->Doorbell);
98c56ad3 4970 if ((_base_wait_for_doorbell_ack(ioc, 15))) {
f92363d1
SR		 4971 r = -EFAULT;
4972 goto out;
4973 }
98c56ad3 4974 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
f92363d1
SR		 4975 if (ioc_state) {
4976 pr_err(MPT3SAS_FMT
4977 "%s: failed going to ready state (ioc_state=0x%x)\n",
4978 ioc->name, __func__, ioc_state);
4979 r = -EFAULT;
4980 goto out;
4981 }
4982 out:
4983 pr_info(MPT3SAS_FMT "message unit reset: %s\n",
4984 ioc->name, ((r == 0) ? "SUCCESS" : "FAILED"));
4985 return r;
4986}
4987
4988/**
4989 * _base_handshake_req_reply_wait - send request thru doorbell interface
4990 * @ioc: per adapter object
4991 * @request_bytes: request length
4992 * @request: pointer having request payload
4993 * @reply_bytes: reply length
4994 * @reply: pointer to reply payload
4995 * @timeout: timeout in second
f92363d1
SR		 4996 *
4997 * Returns 0 for success, non-zero for failure.
4998 */
4999static int
5000_base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
98c56ad3 5001 u32 *request, int reply_bytes, u16 *reply, int timeout)
f92363d1
SR		 5002{
5003 MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
5004 int i;
5005 u8 failed;
f92363d1
SR		 5006 __le32 *mfp;
5007
5008 /* make sure doorbell is not in use */
5009 if ((readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {
5010 pr_err(MPT3SAS_FMT
5011 "doorbell is in use (line=%d)\n",
5012 ioc->name, __LINE__);
5013 return -EFAULT;
5014 }
5015
5016 /* clear pending doorbell interrupts from previous state changes */
5017 if (readl(&ioc->chip->HostInterruptStatus) &
5018 MPI2_HIS_IOC2SYS_DB_STATUS)
5019 writel(0, &ioc->chip->HostInterruptStatus);
5020
5021 /* send message to ioc */
5022 writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) |
5023 ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
5024 &ioc->chip->Doorbell);
5025
98c56ad3 5026 if ((_base_spin_on_doorbell_int(ioc, 5))) {
f92363d1
SR		 5027 pr_err(MPT3SAS_FMT
5028 "doorbell handshake int failed (line=%d)\n",
5029 ioc->name, __LINE__);
5030 return -EFAULT;
5031 }
5032 writel(0, &ioc->chip->HostInterruptStatus);
5033
98c56ad3 5034 if ((_base_wait_for_doorbell_ack(ioc, 5))) {
f92363d1
SR		 5035 pr_err(MPT3SAS_FMT
5036 "doorbell handshake ack failed (line=%d)\n",
5037 ioc->name, __LINE__);
5038 return -EFAULT;
5039 }
5040
5041 /* send message 32-bits at a time */
5042 for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
cf6bf971 5043 writel((u32)(request[i]), &ioc->chip->Doorbell);
98c56ad3 5044 if ((_base_wait_for_doorbell_ack(ioc, 5)))
f92363d1
SR		 5045 failed = 1;
5046 }
5047
5048 if (failed) {
5049 pr_err(MPT3SAS_FMT
5050 "doorbell handshake sending request failed (line=%d)\n",
5051 ioc->name, __LINE__);
5052 return -EFAULT;
5053 }
5054
5055 /* now wait for the reply */
98c56ad3 5056 if ((_base_wait_for_doorbell_int(ioc, timeout))) {
f92363d1
SR		 5057 pr_err(MPT3SAS_FMT
5058 "doorbell handshake int failed (line=%d)\n",
5059 ioc->name, __LINE__);
5060 return -EFAULT;
5061 }
5062
5063 /* read the first two 16-bits, it gives the total length of the reply */
cf6bf971 5064 reply[0] = (u16)(readl(&ioc->chip->Doorbell)
f92363d1
SR		 5065 & MPI2_DOORBELL_DATA_MASK);
5066 writel(0, &ioc->chip->HostInterruptStatus);
98c56ad3 5067 if ((_base_wait_for_doorbell_int(ioc, 5))) {
f92363d1
SR		 5068 pr_err(MPT3SAS_FMT
5069 "doorbell handshake int failed (line=%d)\n",
5070 ioc->name, __LINE__);
5071 return -EFAULT;
5072 }
cf6bf971 5073 reply[1] = (u16)(readl(&ioc->chip->Doorbell)
f92363d1
SR		 5074 & MPI2_DOORBELL_DATA_MASK);
5075 writel(0, &ioc->chip->HostInterruptStatus);
5076
5077 for (i = 2; i < default_reply->MsgLength * 2; i++) {
98c56ad3 5078 if ((_base_wait_for_doorbell_int(ioc, 5))) {
f92363d1
SR		 5079 pr_err(MPT3SAS_FMT
5080 "doorbell handshake int failed (line=%d)\n",
5081 ioc->name, __LINE__);
5082 return -EFAULT;
5083 }
5084 if (i >= reply_bytes/2) /* overflow case */
8bbb1cf6 5085 readl(&ioc->chip->Doorbell);
f92363d1 5086 else
cf6bf971 5087 reply[i] = (u16)(readl(&ioc->chip->Doorbell)
f92363d1
SR		 5088 & MPI2_DOORBELL_DATA_MASK);
5089 writel(0, &ioc->chip->HostInterruptStatus);
5090 }
5091
98c56ad3
CO		 5092 _base_wait_for_doorbell_int(ioc, 5);
5093 if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {
f92363d1
SR		 5094 dhsprintk(ioc, pr_info(MPT3SAS_FMT
5095 "doorbell is in use (line=%d)\n", ioc->name, __LINE__));
5096 }
5097 writel(0, &ioc->chip->HostInterruptStatus);
5098
5099 if (ioc->logging_level & MPT_DEBUG_INIT) {
5100 mfp = (__le32 *)reply;
5101 pr_info("\toffset:data\n");
5102 for (i = 0; i < reply_bytes/4; i++)
5103 pr_info("\t[0x%02x]:%08x\n", i*4,
5104 le32_to_cpu(mfp[i]));
5105 }
5106 return 0;
5107}
5108
5109/**
5110 * mpt3sas_base_sas_iounit_control - send sas iounit control to FW
5111 * @ioc: per adapter object
5112 * @mpi_reply: the reply payload from FW
5113 * @mpi_request: the request payload sent to FW
5114 *
5115 * The SAS IO Unit Control Request message allows the host to perform low-level
5116 * operations, such as resets on the PHYs of the IO Unit, also allows the host
5117 * to obtain the IOC assigned device handles for a device if it has other
5118 * identifying information about the device, in addition allows the host to
5119 * remove IOC resources associated with the device.
5120 *
5121 * Returns 0 for success, non-zero for failure.
5122 */
5123int
5124mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
5125 Mpi2SasIoUnitControlReply_t *mpi_reply,
5126 Mpi2SasIoUnitControlRequest_t *mpi_request)
5127{
5128 u16 smid;
5129 u32 ioc_state;
eb44552b 5130 bool issue_reset = false;
f92363d1
SR		 5131 int rc;
5132 void *request;
5133 u16 wait_state_count;
5134
5135 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5136 __func__));
5137
5138 mutex_lock(&ioc->base_cmds.mutex);
5139
5140 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
5141 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
5142 ioc->name, __func__);
5143 rc = -EAGAIN;
5144 goto out;
5145 }
5146
5147 wait_state_count = 0;
5148 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5149 while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
5150 if (wait_state_count++ == 10) {
5151 pr_err(MPT3SAS_FMT
5152 "%s: failed due to ioc not operational\n",
5153 ioc->name, __func__);
5154 rc = -EFAULT;
5155 goto out;
5156 }
5157 ssleep(1);
5158 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5159 pr_info(MPT3SAS_FMT
5160 "%s: waiting for operational state(count=%d)\n",
5161 ioc->name, __func__, wait_state_count);
5162 }
5163
5164 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5165 if (!smid) {
5166 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5167 ioc->name, __func__);
5168 rc = -EAGAIN;
5169 goto out;
5170 }
5171
5172 rc = 0;
5173 ioc->base_cmds.status = MPT3_CMD_PENDING;
5174 request = mpt3sas_base_get_msg_frame(ioc, smid);
5175 ioc->base_cmds.smid = smid;
5176 memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t));
5177 if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
5178 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
5179 ioc->ioc_link_reset_in_progress = 1;
5180 init_completion(&ioc->base_cmds.done);
40114bde 5181 mpt3sas_base_put_smid_default(ioc, smid);
8bbb1cf6 5182 wait_for_completion_timeout(&ioc->base_cmds.done,
f92363d1
SR		 5183 msecs_to_jiffies(10000));
5184 if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
5185 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
5186 ioc->ioc_link_reset_in_progress)
5187 ioc->ioc_link_reset_in_progress = 0;
5188 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5189 pr_err(MPT3SAS_FMT "%s: timeout\n",
5190 ioc->name, __func__);
5191 _debug_dump_mf(mpi_request,
5192 sizeof(Mpi2SasIoUnitControlRequest_t)/4);
5193 if (!(ioc->base_cmds.status & MPT3_CMD_RESET))
eb44552b 5194 issue_reset = true;
f92363d1
SR		 5195 goto issue_host_reset;
5196 }
5197 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
5198 memcpy(mpi_reply, ioc->base_cmds.reply,
5199 sizeof(Mpi2SasIoUnitControlReply_t));
5200 else
5201 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t));
5202 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5203 goto out;
5204
5205 issue_host_reset:
5206 if (issue_reset)
98c56ad3 5207 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
f92363d1
SR		 5208 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5209 rc = -EFAULT;
5210 out:
5211 mutex_unlock(&ioc->base_cmds.mutex);
5212 return rc;
5213}
5214
5215/**
5216 * mpt3sas_base_scsi_enclosure_processor - sending request to sep device
5217 * @ioc: per adapter object
5218 * @mpi_reply: the reply payload from FW
5219 * @mpi_request: the request payload sent to FW
5220 *
5221 * The SCSI Enclosure Processor request message causes the IOC to
5222 * communicate with SES devices to control LED status signals.
5223 *
5224 * Returns 0 for success, non-zero for failure.
5225 */
5226int
5227mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
5228 Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request)
5229{
5230 u16 smid;
5231 u32 ioc_state;
eb44552b 5232 bool issue_reset = false;
f92363d1
SR		 5233 int rc;
5234 void *request;
5235 u16 wait_state_count;
5236
5237 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5238 __func__));
5239
5240 mutex_lock(&ioc->base_cmds.mutex);
5241
5242 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
5243 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
5244 ioc->name, __func__);
5245 rc = -EAGAIN;
5246 goto out;
5247 }
5248
5249 wait_state_count = 0;
5250 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5251 while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
5252 if (wait_state_count++ == 10) {
5253 pr_err(MPT3SAS_FMT
5254 "%s: failed due to ioc not operational\n",
5255 ioc->name, __func__);
5256 rc = -EFAULT;
5257 goto out;
5258 }
5259 ssleep(1);
5260 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
5261 pr_info(MPT3SAS_FMT
5262 "%s: waiting for operational state(count=%d)\n",
5263 ioc->name,
5264 __func__, wait_state_count);
5265 }
5266
5267 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5268 if (!smid) {
5269 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5270 ioc->name, __func__);
5271 rc = -EAGAIN;
5272 goto out;
5273 }
5274
5275 rc = 0;
5276 ioc->base_cmds.status = MPT3_CMD_PENDING;
5277 request = mpt3sas_base_get_msg_frame(ioc, smid);
5278 ioc->base_cmds.smid = smid;
5279 memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
5280 init_completion(&ioc->base_cmds.done);
40114bde 5281 mpt3sas_base_put_smid_default(ioc, smid);
8bbb1cf6 5282 wait_for_completion_timeout(&ioc->base_cmds.done,
f92363d1
SR		 5283 msecs_to_jiffies(10000));
5284 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5285 pr_err(MPT3SAS_FMT "%s: timeout\n",
5286 ioc->name, __func__);
5287 _debug_dump_mf(mpi_request,
5288 sizeof(Mpi2SepRequest_t)/4);
5289 if (!(ioc->base_cmds.status & MPT3_CMD_RESET))
eb44552b 5290 issue_reset = false;
f92363d1
SR		 5291 goto issue_host_reset;
5292 }
5293 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
5294 memcpy(mpi_reply, ioc->base_cmds.reply,
5295 sizeof(Mpi2SepReply_t));
5296 else
5297 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t));
5298 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5299 goto out;
5300
5301 issue_host_reset:
5302 if (issue_reset)
98c56ad3 5303 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
f92363d1
SR		 5304 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5305 rc = -EFAULT;
5306 out:
5307 mutex_unlock(&ioc->base_cmds.mutex);
5308 return rc;
5309}
5310
5311/**
5312 * _base_get_port_facts - obtain port facts reply and save in ioc
5313 * @ioc: per adapter object
f92363d1
SR		 5314 *
5315 * Returns 0 for success, non-zero for failure.
5316 */
5317static int
98c56ad3 5318_base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)
f92363d1
SR		 5319{
5320 Mpi2PortFactsRequest_t mpi_request;
5321 Mpi2PortFactsReply_t mpi_reply;
5322 struct mpt3sas_port_facts *pfacts;
5323 int mpi_reply_sz, mpi_request_sz, r;
5324
5325 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5326 __func__));
5327
5328 mpi_reply_sz = sizeof(Mpi2PortFactsReply_t);
5329 mpi_request_sz = sizeof(Mpi2PortFactsRequest_t);
5330 memset(&mpi_request, 0, mpi_request_sz);
5331 mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
5332 mpi_request.PortNumber = port;
5333 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
98c56ad3 5334 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
f92363d1
SR		 5335
5336 if (r != 0) {
5337 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5338 ioc->name, __func__, r);
5339 return r;
5340 }
5341
5342 pfacts = &ioc->pfacts[port];
5343 memset(pfacts, 0, sizeof(struct mpt3sas_port_facts));
5344 pfacts->PortNumber = mpi_reply.PortNumber;
5345 pfacts->VP_ID = mpi_reply.VP_ID;
5346 pfacts->VF_ID = mpi_reply.VF_ID;
5347 pfacts->MaxPostedCmdBuffers =
5348 le16_to_cpu(mpi_reply.MaxPostedCmdBuffers);
5349
5350 return 0;
5351}
5352
4dc8c808
SR		 5353/**
5354 * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
5355 * @ioc: per adapter object
5356 * @timeout:
4dc8c808
SR		 5357 *
5358 * Returns 0 for success, non-zero for failure.
5359 */
5360static int
98c56ad3 5361_base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)
4dc8c808
SR		 5362{
5363 u32 ioc_state;
5364 int rc;
5365
5366 dinitprintk(ioc, printk(MPT3SAS_FMT "%s\n", ioc->name,
5367 __func__));
5368
5369 if (ioc->pci_error_recovery) {
5370 dfailprintk(ioc, printk(MPT3SAS_FMT
5371 "%s: host in pci error recovery\n", ioc->name, __func__));
5372 return -EFAULT;
5373 }
5374
5375 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
5376 dhsprintk(ioc, printk(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
5377 ioc->name, __func__, ioc_state));
5378
5379 if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) ||
5380 (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
5381 return 0;
5382
5383 if (ioc_state & MPI2_DOORBELL_USED) {
5384 dhsprintk(ioc, printk(MPT3SAS_FMT
5385 "unexpected doorbell active!\n", ioc->name));
5386 goto issue_diag_reset;
5387 }
5388
5389 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
5390 mpt3sas_base_fault_info(ioc, ioc_state &
5391 MPI2_DOORBELL_DATA_MASK);
5392 goto issue_diag_reset;
5393 }
5394
98c56ad3 5395 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
4dc8c808
SR		 5396 if (ioc_state) {
5397 dfailprintk(ioc, printk(MPT3SAS_FMT
5398 "%s: failed going to ready state (ioc_state=0x%x)\n",
5399 ioc->name, __func__, ioc_state));
5400 return -EFAULT;
5401 }
5402
5403 issue_diag_reset:
98c56ad3 5404 rc = _base_diag_reset(ioc);
4dc8c808
SR		 5405 return rc;
5406}
5407
f92363d1
SR		 5408/**
5409 * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
5410 * @ioc: per adapter object
f92363d1
SR		 5411 *
5412 * Returns 0 for success, non-zero for failure.
5413 */
5414static int
98c56ad3 5415_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5416{
5417 Mpi2IOCFactsRequest_t mpi_request;
5418 Mpi2IOCFactsReply_t mpi_reply;
5419 struct mpt3sas_facts *facts;
5420 int mpi_reply_sz, mpi_request_sz, r;
5421
5422 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5423 __func__));
5424
98c56ad3 5425 r = _base_wait_for_iocstate(ioc, 10);
4dc8c808
SR		 5426 if (r) {
5427 dfailprintk(ioc, printk(MPT3SAS_FMT
5428 "%s: failed getting to correct state\n",
5429 ioc->name, __func__));
5430 return r;
5431 }
f92363d1
SR		 5432 mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t);
5433 mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t);
5434 memset(&mpi_request, 0, mpi_request_sz);
5435 mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
5436 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
98c56ad3 5437 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
f92363d1
SR		 5438
5439 if (r != 0) {
5440 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5441 ioc->name, __func__, r);
5442 return r;
5443 }
5444
5445 facts = &ioc->facts;
5446 memset(facts, 0, sizeof(struct mpt3sas_facts));
5447 facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion);
5448 facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion);
5449 facts->VP_ID = mpi_reply.VP_ID;
5450 facts->VF_ID = mpi_reply.VF_ID;
5451 facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions);
5452 facts->MaxChainDepth = mpi_reply.MaxChainDepth;
5453 facts->WhoInit = mpi_reply.WhoInit;
5454 facts->NumberOfPorts = mpi_reply.NumberOfPorts;
5455 facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors;
5456 facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit);
5457 facts->MaxReplyDescriptorPostQueueDepth =
5458 le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth);
5459 facts->ProductID = le16_to_cpu(mpi_reply.ProductID);
5460 facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities);
5461 if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID))
5462 ioc->ir_firmware = 1;
9b05c91a 5463 if ((facts->IOCCapabilities &
06f5f976 5464 MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices))
9b05c91a 5465 ioc->rdpq_array_capable = 1;
f92363d1
SR		 5466 facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
5467 facts->IOCRequestFrameSize =
5468 le16_to_cpu(mpi_reply.IOCRequestFrameSize);
ebb3024e
SS		 5469 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
5470 facts->IOCMaxChainSegmentSize =
5471 le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize);
5472 }
f92363d1
SR		 5473 facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators);
5474 facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets);
5475 ioc->shost->max_id = -1;
5476 facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders);
5477 facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures);
5478 facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags);
5479 facts->HighPriorityCredit =
5480 le16_to_cpu(mpi_reply.HighPriorityCredit);
5481 facts->ReplyFrameSize = mpi_reply.ReplyFrameSize;
5482 facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle);
016d5c35
SPS		 5483 facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize;
5484
5485 /*
5486 * Get the Page Size from IOC Facts. If it's 0, default to 4k.
5487 */
5488 ioc->page_size = 1 << facts->CurrentHostPageSize;
5489 if (ioc->page_size == 1) {
5490 pr_info(MPT3SAS_FMT "CurrentHostPageSize is 0: Setting "
5491 "default host page size to 4k\n", ioc->name);
5492 ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;
5493 }
5494 dinitprintk(ioc, pr_info(MPT3SAS_FMT "CurrentHostPageSize(%d)\n",
5495 ioc->name, facts->CurrentHostPageSize));
f92363d1
SR		 5496
5497 dinitprintk(ioc, pr_info(MPT3SAS_FMT
5498 "hba queue depth(%d), max chains per io(%d)\n",
5499 ioc->name, facts->RequestCredit,
5500 facts->MaxChainDepth));
5501 dinitprintk(ioc, pr_info(MPT3SAS_FMT
5502 "request frame size(%d), reply frame size(%d)\n", ioc->name,
5503 facts->IOCRequestFrameSize * 4, facts->ReplyFrameSize * 4));
5504 return 0;
5505}
5506
5507/**
5508 * _base_send_ioc_init - send ioc_init to firmware
5509 * @ioc: per adapter object
f92363d1
SR		 5510 *
5511 * Returns 0 for success, non-zero for failure.
5512 */
5513static int
98c56ad3 5514_base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5515{
5516 Mpi2IOCInitRequest_t mpi_request;
5517 Mpi2IOCInitReply_t mpi_reply;
9b05c91a 5518 int i, r = 0;
23409bd4 5519 ktime_t current_time;
f92363d1 5520 u16 ioc_status;
9b05c91a 5521 u32 reply_post_free_array_sz = 0;
f92363d1
SR		 5522
5523 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5524 __func__));
5525
5526 memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t));
5527 mpi_request.Function = MPI2_FUNCTION_IOC_INIT;
5528 mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
5529 mpi_request.VF_ID = 0; /* TODO */
5530 mpi_request.VP_ID = 0;
d357e84d 5531 mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);
f92363d1 5532 mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
016d5c35 5533 mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K;
f92363d1
SR		 5534
5535 if (_base_is_controller_msix_enabled(ioc))
5536 mpi_request.HostMSIxVectors = ioc->reply_queue_count;
5537 mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);
5538 mpi_request.ReplyDescriptorPostQueueDepth =
5539 cpu_to_le16(ioc->reply_post_queue_depth);
5540 mpi_request.ReplyFreeQueueDepth =
5541 cpu_to_le16(ioc->reply_free_queue_depth);
5542
5543 mpi_request.SenseBufferAddressHigh =
5544 cpu_to_le32((u64)ioc->sense_dma >> 32);
5545 mpi_request.SystemReplyAddressHigh =
5546 cpu_to_le32((u64)ioc->reply_dma >> 32);
5547 mpi_request.SystemRequestFrameBaseAddress =
5548 cpu_to_le64((u64)ioc->request_dma);
5549 mpi_request.ReplyFreeQueueAddress =
5550 cpu_to_le64((u64)ioc->reply_free_dma);
f92363d1 5551
9b05c91a
SR		 5552 if (ioc->rdpq_array_enable) {
5553 reply_post_free_array_sz = ioc->reply_queue_count *
5554 sizeof(Mpi2IOCInitRDPQArrayEntry);
cd33223b 5555 memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);
9b05c91a 5556 for (i = 0; i < ioc->reply_queue_count; i++)
cd33223b 5557 ioc->reply_post_free_array[i].RDPQBaseAddress =
9b05c91a
SR		 5558 cpu_to_le64(
5559 (u64)ioc->reply_post[i].reply_post_free_dma);
5560 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
5561 mpi_request.ReplyDescriptorPostQueueAddress =
cd33223b 5562 cpu_to_le64((u64)ioc->reply_post_free_array_dma);
9b05c91a
SR		 5563 } else {
5564 mpi_request.ReplyDescriptorPostQueueAddress =
5565 cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
5566 }
f92363d1
SR		 5567
5568 /* This time stamp specifies number of milliseconds
5569 * since epoch ~ midnight January 1, 1970.
5570 */
23409bd4
TR		 5571 current_time = ktime_get_real();
5572 mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time));
f92363d1
SR		 5573
5574 if (ioc->logging_level & MPT_DEBUG_INIT) {
5575 __le32 *mfp;
5576 int i;
5577
5578 mfp = (__le32 *)&mpi_request;
5579 pr_info("\toffset:data\n");
5580 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++)
5581 pr_info("\t[0x%02x]:%08x\n", i*4,
5582 le32_to_cpu(mfp[i]));
5583 }
5584
5585 r = _base_handshake_req_reply_wait(ioc,
5586 sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
98c56ad3 5587 sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 10);
f92363d1
SR		 5588
5589 if (r != 0) {
5590 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
5591 ioc->name, __func__, r);
cd33223b 5592 return r;
f92363d1
SR		 5593 }
5594
5595 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
5596 if (ioc_status != MPI2_IOCSTATUS_SUCCESS ||
5597 mpi_reply.IOCLogInfo) {
5598 pr_err(MPT3SAS_FMT "%s: failed\n", ioc->name, __func__);
5599 r = -EIO;
5600 }
5601
9b05c91a 5602 return r;
f92363d1
SR		 5603}
5604
5605/**
5606 * mpt3sas_port_enable_done - command completion routine for port enable
5607 * @ioc: per adapter object
5608 * @smid: system request message index
5609 * @msix_index: MSIX table index supplied by the OS
5610 * @reply: reply message frame(lower 32bit addr)
5611 *
5612 * Return 1 meaning mf should be freed from _base_interrupt
5613 * 0 means the mf is freed from this function.
5614 */
5615u8
5616mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
5617 u32 reply)
5618{
5619 MPI2DefaultReply_t *mpi_reply;
5620 u16 ioc_status;
5621
5622 if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)
5623 return 1;
5624
5625 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
5626 if (!mpi_reply)
5627 return 1;
5628
5629 if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE)
5630 return 1;
5631
5632 ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;
5633 ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;
5634 ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;
5635 memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
5636 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
5637 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5638 ioc->port_enable_failed = 1;
5639
5640 if (ioc->is_driver_loading) {
5641 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
5642 mpt3sas_port_enable_complete(ioc);
5643 return 1;
5644 } else {
5645 ioc->start_scan_failed = ioc_status;
5646 ioc->start_scan = 0;
5647 return 1;
5648 }
5649 }
5650 complete(&ioc->port_enable_cmds.done);
5651 return 1;
5652}
5653
5654/**
5655 * _base_send_port_enable - send port_enable(discovery stuff) to firmware
5656 * @ioc: per adapter object
f92363d1
SR		 5657 *
5658 * Returns 0 for success, non-zero for failure.
5659 */
5660static int
98c56ad3 5661_base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5662{
5663 Mpi2PortEnableRequest_t *mpi_request;
5664 Mpi2PortEnableReply_t *mpi_reply;
f92363d1
SR		 5665 int r = 0;
5666 u16 smid;
5667 u16 ioc_status;
5668
5669 pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
5670
5671 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5672 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5673 ioc->name, __func__);
5674 return -EAGAIN;
5675 }
5676
5677 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
5678 if (!smid) {
5679 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5680 ioc->name, __func__);
5681 return -EAGAIN;
5682 }
5683
5684 ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
5685 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5686 ioc->port_enable_cmds.smid = smid;
5687 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
5688 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
5689
5690 init_completion(&ioc->port_enable_cmds.done);
40114bde 5691 mpt3sas_base_put_smid_default(ioc, smid);
8bbb1cf6 5692 wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
f92363d1
SR		 5693 if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
5694 pr_err(MPT3SAS_FMT "%s: timeout\n",
5695 ioc->name, __func__);
5696 _debug_dump_mf(mpi_request,
5697 sizeof(Mpi2PortEnableRequest_t)/4);
5698 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)
5699 r = -EFAULT;
5700 else
5701 r = -ETIME;
5702 goto out;
5703 }
5704
5705 mpi_reply = ioc->port_enable_cmds.reply;
5706 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
5707 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
5708 pr_err(MPT3SAS_FMT "%s: failed with (ioc_status=0x%08x)\n",
5709 ioc->name, __func__, ioc_status);
5710 r = -EFAULT;
5711 goto out;
5712 }
5713
5714 out:
5715 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
5716 pr_info(MPT3SAS_FMT "port enable: %s\n", ioc->name, ((r == 0) ?
5717 "SUCCESS" : "FAILED"));
5718 return r;
5719}
5720
5721/**
5722 * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply)
5723 * @ioc: per adapter object
5724 *
5725 * Returns 0 for success, non-zero for failure.
5726 */
5727int
5728mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
5729{
5730 Mpi2PortEnableRequest_t *mpi_request;
5731 u16 smid;
5732
5733 pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
5734
5735 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5736 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5737 ioc->name, __func__);
5738 return -EAGAIN;
5739 }
5740
5741 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
5742 if (!smid) {
5743 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5744 ioc->name, __func__);
5745 return -EAGAIN;
5746 }
5747
5748 ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
5749 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5750 ioc->port_enable_cmds.smid = smid;
5751 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
5752 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
5753
40114bde 5754 mpt3sas_base_put_smid_default(ioc, smid);
f92363d1
SR		 5755 return 0;
5756}
5757
5758/**
5759 * _base_determine_wait_on_discovery - desposition
5760 * @ioc: per adapter object
5761 *
5762 * Decide whether to wait on discovery to complete. Used to either
5763 * locate boot device, or report volumes ahead of physical devices.
5764 *
5765 * Returns 1 for wait, 0 for don't wait
5766 */
5767static int
5768_base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)
5769{
5770 /* We wait for discovery to complete if IR firmware is loaded.
5771 * The sas topology events arrive before PD events, so we need time to
5772 * turn on the bit in ioc->pd_handles to indicate PD
5773 * Also, it maybe required to report Volumes ahead of physical
5774 * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set.
5775 */
5776 if (ioc->ir_firmware)
5777 return 1;
5778
5779 /* if no Bios, then we don't need to wait */
5780 if (!ioc->bios_pg3.BiosVersion)
5781 return 0;
5782
5783 /* Bios is present, then we drop down here.
5784 *
5785 * If there any entries in the Bios Page 2, then we wait
5786 * for discovery to complete.
5787 */
5788
5789 /* Current Boot Device */
5790 if ((ioc->bios_pg2.CurrentBootDeviceForm &
5791 MPI2_BIOSPAGE2_FORM_MASK) ==
5792 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
5793 /* Request Boot Device */
5794 (ioc->bios_pg2.ReqBootDeviceForm &
5795 MPI2_BIOSPAGE2_FORM_MASK) ==
5796 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
5797 /* Alternate Request Boot Device */
5798 (ioc->bios_pg2.ReqAltBootDeviceForm &
5799 MPI2_BIOSPAGE2_FORM_MASK) ==
5800 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED)
5801 return 0;
5802
5803 return 1;
5804}
5805
5806/**
5807 * _base_unmask_events - turn on notification for this event
5808 * @ioc: per adapter object
5809 * @event: firmware event
5810 *
5811 * The mask is stored in ioc->event_masks.
5812 */
5813static void
5814_base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
5815{
5816 u32 desired_event;
5817
5818 if (event >= 128)
5819 return;
5820
5821 desired_event = (1 << (event % 32));
5822
5823 if (event < 32)
5824 ioc->event_masks[0] &= ~desired_event;
5825 else if (event < 64)
5826 ioc->event_masks[1] &= ~desired_event;
5827 else if (event < 96)
5828 ioc->event_masks[2] &= ~desired_event;
5829 else if (event < 128)
5830 ioc->event_masks[3] &= ~desired_event;
5831}
5832
5833/**
5834 * _base_event_notification - send event notification
5835 * @ioc: per adapter object
f92363d1
SR		 5836 *
5837 * Returns 0 for success, non-zero for failure.
5838 */
5839static int
98c56ad3 5840_base_event_notification(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5841{
5842 Mpi2EventNotificationRequest_t *mpi_request;
f92363d1
SR		 5843 u16 smid;
5844 int r = 0;
5845 int i;
5846
5847 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5848 __func__));
5849
5850 if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
5851 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
5852 ioc->name, __func__);
5853 return -EAGAIN;
5854 }
5855
5856 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
5857 if (!smid) {
5858 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
5859 ioc->name, __func__);
5860 return -EAGAIN;
5861 }
5862 ioc->base_cmds.status = MPT3_CMD_PENDING;
5863 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5864 ioc->base_cmds.smid = smid;
5865 memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t));
5866 mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
5867 mpi_request->VF_ID = 0; /* TODO */
5868 mpi_request->VP_ID = 0;
5869 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
5870 mpi_request->EventMasks[i] =
5871 cpu_to_le32(ioc->event_masks[i]);
5872 init_completion(&ioc->base_cmds.done);
40114bde 5873 mpt3sas_base_put_smid_default(ioc, smid);
8bbb1cf6 5874 wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
f92363d1
SR		 5875 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
5876 pr_err(MPT3SAS_FMT "%s: timeout\n",
5877 ioc->name, __func__);
5878 _debug_dump_mf(mpi_request,
5879 sizeof(Mpi2EventNotificationRequest_t)/4);
5880 if (ioc->base_cmds.status & MPT3_CMD_RESET)
5881 r = -EFAULT;
5882 else
5883 r = -ETIME;
5884 } else
5885 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s: complete\n",
5886 ioc->name, __func__));
5887 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5888 return r;
5889}
5890
5891/**
5892 * mpt3sas_base_validate_event_type - validating event types
5893 * @ioc: per adapter object
5894 * @event: firmware event
5895 *
5896 * This will turn on firmware event notification when application
5897 * ask for that event. We don't mask events that are already enabled.
5898 */
5899void
5900mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
5901{
5902 int i, j;
5903 u32 event_mask, desired_event;
5904 u8 send_update_to_fw;
5905
5906 for (i = 0, send_update_to_fw = 0; i <
5907 MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) {
5908 event_mask = ~event_type[i];
5909 desired_event = 1;
5910 for (j = 0; j < 32; j++) {
5911 if (!(event_mask & desired_event) &&
5912 (ioc->event_masks[i] & desired_event)) {
5913 ioc->event_masks[i] &= ~desired_event;
5914 send_update_to_fw = 1;
5915 }
5916 desired_event = (desired_event << 1);
5917 }
5918 }
5919
5920 if (!send_update_to_fw)
5921 return;
5922
5923 mutex_lock(&ioc->base_cmds.mutex);
98c56ad3 5924 _base_event_notification(ioc);
f92363d1
SR		 5925 mutex_unlock(&ioc->base_cmds.mutex);
5926}
5927
5928/**
5929 * _base_diag_reset - the "big hammer" start of day reset
5930 * @ioc: per adapter object
f92363d1
SR		 5931 *
5932 * Returns 0 for success, non-zero for failure.
5933 */
5934static int
98c56ad3 5935_base_diag_reset(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 5936{
5937 u32 host_diagnostic;
5938 u32 ioc_state;
5939 u32 count;
5940 u32 hcb_size;
5941
5942 pr_info(MPT3SAS_FMT "sending diag reset !!\n", ioc->name);
5943
5944 drsprintk(ioc, pr_info(MPT3SAS_FMT "clear interrupts\n",
5945 ioc->name));
5946
5947 count = 0;
5948 do {
5949 /* Write magic sequence to WriteSequence register
5950 * Loop until in diagnostic mode
5951 */
5952 drsprintk(ioc, pr_info(MPT3SAS_FMT
5953 "write magic sequence\n", ioc->name));
5954 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
5955 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);
5956 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);
5957 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);
5958 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);
5959 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);
5960 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
5961
5962 /* wait 100 msec */
98c56ad3 5963 msleep(100);
f92363d1
SR		 5964
5965 if (count++ > 20)
5966 goto out;
5967
5968 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
5969 drsprintk(ioc, pr_info(MPT3SAS_FMT
5970 "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",
5971 ioc->name, count, host_diagnostic));
5972
5973 } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0);
5974
5975 hcb_size = readl(&ioc->chip->HCBSize);
5976
5977 drsprintk(ioc, pr_info(MPT3SAS_FMT "diag reset: issued\n",
5978 ioc->name));
5979 writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER,
5980 &ioc->chip->HostDiagnostic);
5981
b453ff84 5982 /*This delay allows the chip PCIe hardware time to finish reset tasks*/
98c56ad3 5983 msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
f92363d1 5984
b453ff84
SR		 5985 /* Approximately 300 second max wait */
5986 for (count = 0; count < (300000000 /
5987 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) {
f92363d1
SR		 5988
5989 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
5990
5991 if (host_diagnostic == 0xFFFFFFFF)
5992 goto out;
5993 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
5994 break;
5995
98c56ad3 5996 msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000);
f92363d1
SR		 5997 }
5998
5999 if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
6000
6001 drsprintk(ioc, pr_info(MPT3SAS_FMT
6002 "restart the adapter assuming the HCB Address points to good F/W\n",
6003 ioc->name));
6004 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK;
6005 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW;
6006 writel(host_diagnostic, &ioc->chip->HostDiagnostic);
6007
6008 drsprintk(ioc, pr_info(MPT3SAS_FMT
6009 "re-enable the HCDW\n", ioc->name));
6010 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE,
6011 &ioc->chip->HCBSize);
6012 }
6013
6014 drsprintk(ioc, pr_info(MPT3SAS_FMT "restart the adapter\n",
6015 ioc->name));
6016 writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET,
6017 &ioc->chip->HostDiagnostic);
6018
6019 drsprintk(ioc, pr_info(MPT3SAS_FMT
6020 "disable writes to the diagnostic register\n", ioc->name));
6021 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
6022
6023 drsprintk(ioc, pr_info(MPT3SAS_FMT
6024 "Wait for FW to go to the READY state\n", ioc->name));
98c56ad3 6025 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);
f92363d1
SR		 6026 if (ioc_state) {
6027 pr_err(MPT3SAS_FMT
6028 "%s: failed going to ready state (ioc_state=0x%x)\n",
6029 ioc->name, __func__, ioc_state);
6030 goto out;
6031 }
6032
6033 pr_info(MPT3SAS_FMT "diag reset: SUCCESS\n", ioc->name);
6034 return 0;
6035
6036 out:
6037 pr_err(MPT3SAS_FMT "diag reset: FAILED\n", ioc->name);
6038 return -EFAULT;
6039}
6040
6041/**
6042 * _base_make_ioc_ready - put controller in READY state
6043 * @ioc: per adapter object
f92363d1
SR		 6044 * @type: FORCE_BIG_HAMMER or SOFT_RESET
6045 *
6046 * Returns 0 for success, non-zero for failure.
6047 */
6048static int
98c56ad3 6049_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)
f92363d1
SR		 6050{
6051 u32 ioc_state;
6052 int rc;
6053 int count;
6054
6055 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6056 __func__));
6057
6058 if (ioc->pci_error_recovery)
6059 return 0;
6060
6061 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6062 dhsprintk(ioc, pr_info(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
6063 ioc->name, __func__, ioc_state));
6064
6065 /* if in RESET state, it should move to READY state shortly */
6066 count = 0;
6067 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
6068 while ((ioc_state & MPI2_IOC_STATE_MASK) !=
6069 MPI2_IOC_STATE_READY) {
6070 if (count++ == 10) {
6071 pr_err(MPT3SAS_FMT
6072 "%s: failed going to ready state (ioc_state=0x%x)\n",
6073 ioc->name, __func__, ioc_state);
6074 return -EFAULT;
6075 }
98c56ad3 6076 ssleep(1);
f92363d1
SR		 6077 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6078 }
6079 }
6080
6081 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY)
6082 return 0;
6083
6084 if (ioc_state & MPI2_DOORBELL_USED) {
6085 dhsprintk(ioc, pr_info(MPT3SAS_FMT
6086 "unexpected doorbell active!\n",
6087 ioc->name));
6088 goto issue_diag_reset;
6089 }
6090
6091 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
6092 mpt3sas_base_fault_info(ioc, ioc_state &
6093 MPI2_DOORBELL_DATA_MASK);
6094 goto issue_diag_reset;
6095 }
6096
6097 if (type == FORCE_BIG_HAMMER)
6098 goto issue_diag_reset;
6099
6100 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
6101 if (!(_base_send_ioc_reset(ioc,
98c56ad3 6102 MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) {
f92363d1
SR		 6103 return 0;
6104 }
6105
6106 issue_diag_reset:
98c56ad3 6107 rc = _base_diag_reset(ioc);
f92363d1
SR		 6108 return rc;
6109}
6110
6111/**
6112 * _base_make_ioc_operational - put controller in OPERATIONAL state
6113 * @ioc: per adapter object
f92363d1
SR		 6114 *
6115 * Returns 0 for success, non-zero for failure.
6116 */
6117static int
98c56ad3 6118_base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
f92363d1 6119{
5ec8a175 6120 int r, i, index;
f92363d1
SR		 6121 unsigned long flags;
6122 u32 reply_address;
6123 u16 smid;
6124 struct _tr_list *delayed_tr, *delayed_tr_next;
fd0331b3
SS		 6125 struct _sc_list *delayed_sc, *delayed_sc_next;
6126 struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next;
7786ab6a 6127 u8 hide_flag;
f92363d1 6128 struct adapter_reply_queue *reply_q;
5ec8a175 6129 Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig;
f92363d1
SR		 6130
6131 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6132 __func__));
6133
6134 /* clean the delayed target reset list */
6135 list_for_each_entry_safe(delayed_tr, delayed_tr_next,
6136 &ioc->delayed_tr_list, list) {
6137 list_del(&delayed_tr->list);
6138 kfree(delayed_tr);
6139 }
6140
6141
6142 list_for_each_entry_safe(delayed_tr, delayed_tr_next,
6143 &ioc->delayed_tr_volume_list, list) {
6144 list_del(&delayed_tr->list);
6145 kfree(delayed_tr);
6146 }
6147
fd0331b3
SS		 6148 list_for_each_entry_safe(delayed_sc, delayed_sc_next,
6149 &ioc->delayed_sc_list, list) {
6150 list_del(&delayed_sc->list);
6151 kfree(delayed_sc);
6152 }
6153
6154 list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next,
6155 &ioc->delayed_event_ack_list, list) {
6156 list_del(&delayed_event_ack->list);
6157 kfree(delayed_event_ack);
6158 }
6159
f92363d1 6160 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
f92363d1
SR		 6161
6162 /* hi-priority queue */
6163 INIT_LIST_HEAD(&ioc->hpr_free_list);
6164 smid = ioc->hi_priority_smid;
6165 for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {
6166 ioc->hpr_lookup[i].cb_idx = 0xFF;
6167 ioc->hpr_lookup[i].smid = smid;
6168 list_add_tail(&ioc->hpr_lookup[i].tracker_list,
6169 &ioc->hpr_free_list);
6170 }
6171
6172 /* internal queue */
6173 INIT_LIST_HEAD(&ioc->internal_free_list);
6174 smid = ioc->internal_smid;
6175 for (i = 0; i < ioc->internal_depth; i++, smid++) {
6176 ioc->internal_lookup[i].cb_idx = 0xFF;
6177 ioc->internal_lookup[i].smid = smid;
6178 list_add_tail(&ioc->internal_lookup[i].tracker_list,
6179 &ioc->internal_free_list);
6180 }
6181
6182 /* chain pool */
6183 INIT_LIST_HEAD(&ioc->free_chain_list);
6184 for (i = 0; i < ioc->chain_depth; i++)
6185 list_add_tail(&ioc->chain_lookup[i].tracker_list,
6186 &ioc->free_chain_list);
6187
6188 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
6189
6190 /* initialize Reply Free Queue */
6191 for (i = 0, reply_address = (u32)ioc->reply_dma ;
6192 i < ioc->reply_free_queue_depth ; i++, reply_address +=
b4472d71 6193 ioc->reply_sz) {
f92363d1 6194 ioc->reply_free[i] = cpu_to_le32(reply_address);
b4472d71
SPS		 6195 if (ioc->is_mcpu_endpoint)
6196 _base_clone_reply_to_sys_mem(ioc,
cf6bf971 6197 reply_address, i);
b4472d71 6198 }
f92363d1
SR		 6199
6200 /* initialize reply queues */
6201 if (ioc->is_driver_loading)
6202 _base_assign_reply_queues(ioc);
6203
6204 /* initialize Reply Post Free Queue */
5ec8a175
CO		 6205 index = 0;
6206 reply_post_free_contig = ioc->reply_post[0].reply_post_free;
f92363d1 6207 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
5ec8a175
CO		 6208 /*
6209 * If RDPQ is enabled, switch to the next allocation.
6210 * Otherwise advance within the contiguous region.
6211 */
6212 if (ioc->rdpq_array_enable) {
6213 reply_q->reply_post_free =
6214 ioc->reply_post[index++].reply_post_free;
6215 } else {
6216 reply_q->reply_post_free = reply_post_free_contig;
6217 reply_post_free_contig += ioc->reply_post_queue_depth;
6218 }
6219
f92363d1 6220 reply_q->reply_post_host_index = 0;
f92363d1
SR		 6221 for (i = 0; i < ioc->reply_post_queue_depth; i++)
6222 reply_q->reply_post_free[i].Words =
6223 cpu_to_le64(ULLONG_MAX);
6224 if (!_base_is_controller_msix_enabled(ioc))
6225 goto skip_init_reply_post_free_queue;
f92363d1
SR		 6226 }
6227 skip_init_reply_post_free_queue:
6228
98c56ad3 6229 r = _base_send_ioc_init(ioc);
f92363d1
SR		 6230 if (r)
6231 return r;
6232
6233 /* initialize reply free host index */
6234 ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;
6235 writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);
6236
6237 /* initialize reply post host index */
6238 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
0bb337c9 6239 if (ioc->combined_reply_queue)
fb77bb53
SR		 6240 writel((reply_q->msix_index & 7)<<
6241 MPI2_RPHI_MSIX_INDEX_SHIFT,
6242 ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
6243 else
6244 writel(reply_q->msix_index <<
6245 MPI2_RPHI_MSIX_INDEX_SHIFT,
6246 &ioc->chip->ReplyPostHostIndex);
6247
f92363d1
SR		 6248 if (!_base_is_controller_msix_enabled(ioc))
6249 goto skip_init_reply_post_host_index;
6250 }
6251
6252 skip_init_reply_post_host_index:
6253
6254 _base_unmask_interrupts(ioc);
98c56ad3 6255 r = _base_event_notification(ioc);
f92363d1
SR		 6256 if (r)
6257 return r;
6258
98c56ad3 6259 _base_static_config_pages(ioc);
f92363d1
SR		 6260
6261 if (ioc->is_driver_loading) {
7786ab6a
SR		 6262
6263 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
6264 == 0x80) {
6265 hide_flag = (u8) (
6266 le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &
6267 MFG_PAGE10_HIDE_SSDS_MASK);
6268 if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK)
6269 ioc->mfg_pg10_hide_flag = hide_flag;
6270 }
6271
f92363d1
SR		 6272 ioc->wait_for_discovery_to_complete =
6273 _base_determine_wait_on_discovery(ioc);
6274
6275 return r; /* scan_start and scan_finished support */
6276 }
6277
98c56ad3 6278 r = _base_send_port_enable(ioc);
f92363d1
SR		 6279 if (r)
6280 return r;
6281
6282 return r;
6283}
6284
6285/**
6286 * mpt3sas_base_free_resources - free resources controller resources
6287 * @ioc: per adapter object
6288 *
6289 * Return nothing.
6290 */
6291void
6292mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
6293{
f92363d1
SR		 6294 dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6295 __func__));
6296
08c4d550
SR		 6297 /* synchronizing freeing resource with pci_access_mutex lock */
6298 mutex_lock(&ioc->pci_access_mutex);
cf9bd21a
JL		 6299 if (ioc->chip_phys && ioc->chip) {
6300 _base_mask_interrupts(ioc);
6301 ioc->shost_recovery = 1;
98c56ad3 6302 _base_make_ioc_ready(ioc, SOFT_RESET);
cf9bd21a
JL		 6303 ioc->shost_recovery = 0;
6304 }
6305
580d4e31 6306 mpt3sas_base_unmap_resources(ioc);
08c4d550 6307 mutex_unlock(&ioc->pci_access_mutex);
f92363d1
SR		 6308 return;
6309}
6310
6311/**
6312 * mpt3sas_base_attach - attach controller instance
6313 * @ioc: per adapter object
6314 *
6315 * Returns 0 for success, non-zero for failure.
6316 */
6317int
6318mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
6319{
6320 int r, i;
6321 int cpu_id, last_cpu_id = 0;
6322
6323 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6324 __func__));
6325
6326 /* setup cpu_msix_table */
6327 ioc->cpu_count = num_online_cpus();
6328 for_each_online_cpu(cpu_id)
6329 last_cpu_id = cpu_id;
6330 ioc->cpu_msix_table_sz = last_cpu_id + 1;
6331 ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);
6332 ioc->reply_queue_count = 1;
6333 if (!ioc->cpu_msix_table) {
6334 dfailprintk(ioc, pr_info(MPT3SAS_FMT
6335 "allocation for cpu_msix_table failed!!!\n",
6336 ioc->name));
6337 r = -ENOMEM;
6338 goto out_free_resources;
6339 }
6340
7786ab6a
SR		 6341 if (ioc->is_warpdrive) {
6342 ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,
6343 sizeof(resource_size_t *), GFP_KERNEL);
6344 if (!ioc->reply_post_host_index) {
6345 dfailprintk(ioc, pr_info(MPT3SAS_FMT "allocation "
1d55abc0
HR		 6346 "for reply_post_host_index failed!!!\n",
6347 ioc->name));
7786ab6a
SR		 6348 r = -ENOMEM;
6349 goto out_free_resources;
6350 }
6351 }
6352
9b05c91a
SR		 6353 ioc->rdpq_array_enable_assigned = 0;
6354 ioc->dma_mask = 0;
f92363d1
SR		 6355 r = mpt3sas_base_map_resources(ioc);
6356 if (r)
6357 goto out_free_resources;
6358
f92363d1 6359 pci_set_drvdata(ioc->pdev, ioc->shost);
98c56ad3 6360 r = _base_get_ioc_facts(ioc);
f92363d1
SR		 6361 if (r)
6362 goto out_free_resources;
6363
471ef9d4
SR		 6364 switch (ioc->hba_mpi_version_belonged) {
6365 case MPI2_VERSION:
6366 ioc->build_sg_scmd = &_base_build_sg_scmd;
6367 ioc->build_sg = &_base_build_sg;
6368 ioc->build_zero_len_sge = &_base_build_zero_len_sge;
6369 break;
6370 case MPI25_VERSION:
b130b0d5 6371 case MPI26_VERSION:
471ef9d4
SR		 6372 /*
6373 * In SAS3.0,
6374 * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and
6375 * Target Status - all require the IEEE formated scatter gather
6376 * elements.
6377 */
6378 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;
6379 ioc->build_sg = &_base_build_sg_ieee;
aff39e61 6380 ioc->build_nvme_prp = &_base_build_nvme_prp;
471ef9d4
SR		 6381 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
6382 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
81c16f83 6383
471ef9d4
SR		 6384 break;
6385 }
f92363d1 6386
40114bde
SP		 6387 if (ioc->is_mcpu_endpoint)
6388 ioc->put_smid_scsi_io = &_base_put_smid_mpi_ep_scsi_io;
6389 else
6390 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;
81c16f83 6391
f92363d1
SR		 6392 /*
6393 * These function pointers for other requests that don't
6394 * the require IEEE scatter gather elements.
6395 *
6396 * For example Configuration Pages and SAS IOUNIT Control don't.
6397 */
6398 ioc->build_sg_mpi = &_base_build_sg;
6399 ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
6400
98c56ad3 6401 r = _base_make_ioc_ready(ioc, SOFT_RESET);
f92363d1
SR		 6402 if (r)
6403 goto out_free_resources;
6404
6405 ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,
6406 sizeof(struct mpt3sas_port_facts), GFP_KERNEL);
6407 if (!ioc->pfacts) {
6408 r = -ENOMEM;
6409 goto out_free_resources;
6410 }
6411
6412 for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
98c56ad3 6413 r = _base_get_port_facts(ioc, i);
f92363d1
SR		 6414 if (r)
6415 goto out_free_resources;
6416 }
6417
98c56ad3 6418 r = _base_allocate_memory_pools(ioc);
f92363d1
SR		 6419 if (r)
6420 goto out_free_resources;
6421
6422 init_waitqueue_head(&ioc->reset_wq);
6423
6424 /* allocate memory pd handle bitmask list */
6425 ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
6426 if (ioc->facts.MaxDevHandle % 8)
6427 ioc->pd_handles_sz++;
6428 ioc->pd_handles = kzalloc(ioc->pd_handles_sz,
6429 GFP_KERNEL);
6430 if (!ioc->pd_handles) {
6431 r = -ENOMEM;
6432 goto out_free_resources;
6433 }
6434 ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,
6435 GFP_KERNEL);
6436 if (!ioc->blocking_handles) {
6437 r = -ENOMEM;
6438 goto out_free_resources;
6439 }
6440
c696f7b8
SPS		 6441 /* allocate memory for pending OS device add list */
6442 ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);
6443 if (ioc->facts.MaxDevHandle % 8)
6444 ioc->pend_os_device_add_sz++;
6445 ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,
6446 GFP_KERNEL);
6447 if (!ioc->pend_os_device_add)
6448 goto out_free_resources;
6449
6450 ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;
6451 ioc->device_remove_in_progress =
6452 kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);
6453 if (!ioc->device_remove_in_progress)
6454 goto out_free_resources;
6455
f92363d1
SR		 6456 ioc->fwfault_debug = mpt3sas_fwfault_debug;
6457
6458 /* base internal command bits */
6459 mutex_init(&ioc->base_cmds.mutex);
6460 ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6461 ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6462
6463 /* port_enable command bits */
6464 ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6465 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
6466
6467 /* transport internal command bits */
6468 ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6469 ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
6470 mutex_init(&ioc->transport_cmds.mutex);
6471
6472 /* scsih internal command bits */
6473 ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6474 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
6475 mutex_init(&ioc->scsih_cmds.mutex);
6476
6477 /* task management internal command bits */
6478 ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6479 ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
6480 mutex_init(&ioc->tm_cmds.mutex);
6481
6482 /* config page internal command bits */
6483 ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6484 ioc->config_cmds.status = MPT3_CMD_NOT_USED;
6485 mutex_init(&ioc->config_cmds.mutex);
6486
6487 /* ctl module internal command bits */
6488 ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
6489 ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
6490 ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
6491 mutex_init(&ioc->ctl_cmds.mutex);
6492
a5dd7efd
CJ		 6493 if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||
6494 !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||
6495 !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||
6496 !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {
f92363d1
SR		 6497 r = -ENOMEM;
6498 goto out_free_resources;
6499 }
6500
6501 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
6502 ioc->event_masks[i] = -1;
6503
6504 /* here we enable the events we care about */
6505 _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);
6506 _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
6507 _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
6508 _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
6509 _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
6510 _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
6511 _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);
6512 _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);
6513 _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);
6514 _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
2d8ce8c9 6515 _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
b99b1993 6516 _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
4318c734
SPS		 6517 if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
6518 if (ioc->is_gen35_ioc) {
6519 _base_unmask_events(ioc,
6520 MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
6521 _base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);
6522 _base_unmask_events(ioc,
6523 MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
6524 }
6525 }
98c56ad3 6526 r = _base_make_ioc_operational(ioc);
f92363d1
SR		 6527 if (r)
6528 goto out_free_resources;
6529
16e179bd 6530 ioc->non_operational_loop = 0;
459325c4 6531 ioc->got_task_abort_from_ioctl = 0;
f92363d1
SR		 6532 return 0;
6533
6534 out_free_resources:
6535
6536 ioc->remove_host = 1;
6537
6538 mpt3sas_base_free_resources(ioc);
6539 _base_release_memory_pools(ioc);
6540 pci_set_drvdata(ioc->pdev, NULL);
6541 kfree(ioc->cpu_msix_table);
7786ab6a
SR		 6542 if (ioc->is_warpdrive)
6543 kfree(ioc->reply_post_host_index);
f92363d1
SR		 6544 kfree(ioc->pd_handles);
6545 kfree(ioc->blocking_handles);
c696f7b8
SPS		 6546 kfree(ioc->device_remove_in_progress);
6547 kfree(ioc->pend_os_device_add);
f92363d1
SR		 6548 kfree(ioc->tm_cmds.reply);
6549 kfree(ioc->transport_cmds.reply);
6550 kfree(ioc->scsih_cmds.reply);
6551 kfree(ioc->config_cmds.reply);
6552 kfree(ioc->base_cmds.reply);
6553 kfree(ioc->port_enable_cmds.reply);
6554 kfree(ioc->ctl_cmds.reply);
6555 kfree(ioc->ctl_cmds.sense);
6556 kfree(ioc->pfacts);
6557 ioc->ctl_cmds.reply = NULL;
6558 ioc->base_cmds.reply = NULL;
6559 ioc->tm_cmds.reply = NULL;
6560 ioc->scsih_cmds.reply = NULL;
6561 ioc->transport_cmds.reply = NULL;
6562 ioc->config_cmds.reply = NULL;
6563 ioc->pfacts = NULL;
6564 return r;
6565}
6566
6567
6568/**
6569 * mpt3sas_base_detach - remove controller instance
6570 * @ioc: per adapter object
6571 *
6572 * Return nothing.
6573 */
6574void
6575mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
6576{
6577 dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
6578 __func__));
6579
6580 mpt3sas_base_stop_watchdog(ioc);
6581 mpt3sas_base_free_resources(ioc);
6582 _base_release_memory_pools(ioc);
6583 pci_set_drvdata(ioc->pdev, NULL);
6584 kfree(ioc->cpu_msix_table);
7786ab6a
SR		 6585 if (ioc->is_warpdrive)
6586 kfree(ioc->reply_post_host_index);
f92363d1
SR		 6587 kfree(ioc->pd_handles);
6588 kfree(ioc->blocking_handles);
c696f7b8
SPS		 6589 kfree(ioc->device_remove_in_progress);
6590 kfree(ioc->pend_os_device_add);
f92363d1
SR		 6591 kfree(ioc->pfacts);
6592 kfree(ioc->ctl_cmds.reply);
6593 kfree(ioc->ctl_cmds.sense);
6594 kfree(ioc->base_cmds.reply);
6595 kfree(ioc->port_enable_cmds.reply);
6596 kfree(ioc->tm_cmds.reply);
6597 kfree(ioc->transport_cmds.reply);
6598 kfree(ioc->scsih_cmds.reply);
6599 kfree(ioc->config_cmds.reply);
6600}
6601
6602/**
6603 * _base_reset_handler - reset callback handler (for base)
6604 * @ioc: per adapter object
6605 * @reset_phase: phase
6606 *
6607 * The handler for doing any required cleanup or initialization.
6608 *
6609 * The reset phase can be MPT3_IOC_PRE_RESET, MPT3_IOC_AFTER_RESET,
6610 * MPT3_IOC_DONE_RESET
6611 *
6612 * Return nothing.
6613 */
6614static void
6615_base_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
6616{
6617 mpt3sas_scsih_reset_handler(ioc, reset_phase);
6618 mpt3sas_ctl_reset_handler(ioc, reset_phase);
6619 switch (reset_phase) {
6620 case MPT3_IOC_PRE_RESET:
6621 dtmprintk(ioc, pr_info(MPT3SAS_FMT
6622 "%s: MPT3_IOC_PRE_RESET\n", ioc->name, __func__));
6623 break;
6624 case MPT3_IOC_AFTER_RESET:
6625 dtmprintk(ioc, pr_info(MPT3SAS_FMT
6626 "%s: MPT3_IOC_AFTER_RESET\n", ioc->name, __func__));
6627 if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {
6628 ioc->transport_cmds.status |= MPT3_CMD_RESET;
6629 mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);
6630 complete(&ioc->transport_cmds.done);
6631 }
6632 if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
6633 ioc->base_cmds.status |= MPT3_CMD_RESET;
6634 mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);
6635 complete(&ioc->base_cmds.done);
6636 }
6637 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
6638 ioc->port_enable_failed = 1;
6639 ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
6640 mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);
6641 if (ioc->is_driver_loading) {
6642 ioc->start_scan_failed =
6643 MPI2_IOCSTATUS_INTERNAL_ERROR;
6644 ioc->start_scan = 0;
6645 ioc->port_enable_cmds.status =
6646 MPT3_CMD_NOT_USED;
6647 } else
6648 complete(&ioc->port_enable_cmds.done);
6649 }
6650 if (ioc->config_cmds.status & MPT3_CMD_PENDING) {
6651 ioc->config_cmds.status |= MPT3_CMD_RESET;
6652 mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);
6653 ioc->config_cmds.smid = USHRT_MAX;
6654 complete(&ioc->config_cmds.done);
6655 }
6656 break;
6657 case MPT3_IOC_DONE_RESET:
6658 dtmprintk(ioc, pr_info(MPT3SAS_FMT
6659 "%s: MPT3_IOC_DONE_RESET\n", ioc->name, __func__));
6660 break;
6661 }
6662}
6663
6664/**
c666d3be 6665 * mpt3sas_wait_for_commands_to_complete - reset controller
f92363d1 6666 * @ioc: Pointer to MPT_ADAPTER structure
f92363d1 6667 *
272e253c 6668 * This function is waiting 10s for all pending commands to complete
f92363d1
SR		 6669 * prior to putting controller in reset.
6670 */
c666d3be
SR		 6671void
6672mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)
f92363d1
SR		 6673{
6674 u32 ioc_state;
f92363d1
SR		 6675
6676 ioc->pending_io_count = 0;
f92363d1
SR		 6677
6678 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6679 if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
6680 return;
6681
6682 /* pending command count */
272e253c 6683 ioc->pending_io_count = atomic_read(&ioc->shost->host_busy);
f92363d1
SR		 6684
6685 if (!ioc->pending_io_count)
6686 return;
6687
6688 /* wait for pending commands to complete */
6689 wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);
6690}
6691
6692/**
6693 * mpt3sas_base_hard_reset_handler - reset controller
6694 * @ioc: Pointer to MPT_ADAPTER structure
f92363d1
SR		 6695 * @type: FORCE_BIG_HAMMER or SOFT_RESET
6696 *
6697 * Returns 0 for success, non-zero for failure.
6698 */
6699int
98c56ad3 6700mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
f92363d1
SR		 6701 enum reset_type type)
6702{
6703 int r;
6704 unsigned long flags;
6705 u32 ioc_state;
6706 u8 is_fault = 0, is_trigger = 0;
6707
6708 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: enter\n", ioc->name,
6709 __func__));
6710
6711 if (ioc->pci_error_recovery) {
6712 pr_err(MPT3SAS_FMT "%s: pci error recovery reset\n",
6713 ioc->name, __func__);
6714 r = 0;
6715 goto out_unlocked;
6716 }
6717
6718 if (mpt3sas_fwfault_debug)
6719 mpt3sas_halt_firmware(ioc);
6720
f92363d1
SR		 6721 /* wait for an active reset in progress to complete */
6722 if (!mutex_trylock(&ioc->reset_in_progress_mutex)) {
6723 do {
6724 ssleep(1);
6725 } while (ioc->shost_recovery == 1);
6726 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
6727 __func__));
6728 return ioc->ioc_reset_in_progress_status;
6729 }
6730
6731 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6732 ioc->shost_recovery = 1;
6733 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6734
6735 if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
6736 MPT3_DIAG_BUFFER_IS_REGISTERED) &&
6737 (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
6738 MPT3_DIAG_BUFFER_IS_RELEASED))) {
6739 is_trigger = 1;
6740 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6741 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
6742 is_fault = 1;
6743 }
6744 _base_reset_handler(ioc, MPT3_IOC_PRE_RESET);
c666d3be 6745 mpt3sas_wait_for_commands_to_complete(ioc);
f92363d1 6746 _base_mask_interrupts(ioc);
98c56ad3 6747 r = _base_make_ioc_ready(ioc, type);
f92363d1
SR		 6748 if (r)
6749 goto out;
6750 _base_reset_handler(ioc, MPT3_IOC_AFTER_RESET);
6751
6752 /* If this hard reset is called while port enable is active, then
6753 * there is no reason to call make_ioc_operational
6754 */
6755 if (ioc->is_driver_loading && ioc->port_enable_failed) {
6756 ioc->remove_host = 1;
6757 r = -EFAULT;
6758 goto out;
6759 }
98c56ad3 6760 r = _base_get_ioc_facts(ioc);
f92363d1
SR		 6761 if (r)
6762 goto out;
9b05c91a
SR		 6763
6764 if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)
6765 panic("%s: Issue occurred with flashing controller firmware."
6766 "Please reboot the system and ensure that the correct"
6767 " firmware version is running\n", ioc->name);
6768
98c56ad3 6769 r = _base_make_ioc_operational(ioc);
f92363d1
SR		 6770 if (!r)
6771 _base_reset_handler(ioc, MPT3_IOC_DONE_RESET);
6772
6773 out:
6774 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: %s\n",
6775 ioc->name, __func__, ((r == 0) ? "SUCCESS" : "FAILED")));
6776
6777 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6778 ioc->ioc_reset_in_progress_status = r;
6779 ioc->shost_recovery = 0;
6780 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6781 ioc->ioc_reset_count++;
6782 mutex_unlock(&ioc->reset_in_progress_mutex);
6783
6784 out_unlocked:
6785 if ((r == 0) && is_trigger) {
6786 if (is_fault)
6787 mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);
6788 else
6789 mpt3sas_trigger_master(ioc,
6790 MASTER_TRIGGER_ADAPTER_RESET);
6791 }
6792 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
6793 __func__));
6794 return r;
6795}
Linux 6.x block layer and io_uring tree(s)