drivers/scsi/mpt3sas/mpt3sas_base.c

   1 /*
   2  * This is the Fusion MPT base driver providing common API layer interface
   3  * for access to MPT (Message Passing Technology) firmware.
   4  *
   5  * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c
   6  * Copyright (C) 2012-2014  LSI Corporation
   7  * Copyright (C) 2013-2014 Avago Technologies
   8  *  (mailto: MPT-FusionLinux.pdl@avagotech.com)
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; either version 2
  13  * of the License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * NO WARRANTY
  21  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
  22  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
  23  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
  24  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
  25  * solely responsible for determining the appropriateness of using and
  26  * distributing the Program and assumes all risks associated with its
  27  * exercise of rights under this Agreement, including but not limited to
  28  * the risks and costs of program errors, damage to or loss of data,
  29  * programs or equipment, and unavailability or interruption of operations.
  30
  31  * DISCLAIMER OF LIABILITY
  32  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
  33  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  34  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
  35  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  36  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  37  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
  38  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
  39
  40  * You should have received a copy of the GNU General Public License
  41  * along with this program; if not, write to the Free Software
  42  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  43  * USA.
  44  */
  45
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/errno.h>
  49 #include <linux/init.h>
  50 #include <linux/slab.h>
  51 #include <linux/types.h>
  52 #include <linux/pci.h>
  53 #include <linux/kdev_t.h>
  54 #include <linux/blkdev.h>
  55 #include <linux/delay.h>
  56 #include <linux/interrupt.h>
  57 #include <linux/dma-mapping.h>
  58 #include <linux/io.h>
  59 #include <linux/time.h>
  60 #include <linux/kthread.h>
  61 #include <linux/aer.h>
  62
  63
  64 #include "mpt3sas_base.h"
  65
  66 static MPT_CALLBACK     mpt_callbacks[MPT_MAX_CALLBACKS];
  67
  68
  69 #define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */
  70
  71  /* maximum controller queue depth */
  72 #define MAX_HBA_QUEUE_DEPTH     30000
  73 #define MAX_CHAIN_DEPTH         100000
  74 static int max_queue_depth = -1;
  75 module_param(max_queue_depth, int, 0);
  76 MODULE_PARM_DESC(max_queue_depth, " max controller queue depth ");
  77
  78 static int max_sgl_entries = -1;
  79 module_param(max_sgl_entries, int, 0);
  80 MODULE_PARM_DESC(max_sgl_entries, " max sg entries ");
  81
  82 static int msix_disable = -1;
  83 module_param(msix_disable, int, 0);
  84 MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
  85
  86 static int max_msix_vectors = -1;
  87 module_param(max_msix_vectors, int, 0);
  88 MODULE_PARM_DESC(max_msix_vectors,
  89         " max msix vectors");
  90
  91 static int mpt3sas_fwfault_debug;
  92 MODULE_PARM_DESC(mpt3sas_fwfault_debug,
  93         " enable detection of firmware fault and halt firmware - (default=0)");
  94
  95 static int
  96 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag);
  97
  98 /**
  99  * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
 100  *
 101  */
 102 static int
 103 _scsih_set_fwfault_debug(const char *val, struct kernel_param *kp)
 104 {
 105         int ret = param_set_int(val, kp);
 106         struct MPT3SAS_ADAPTER *ioc;
 107
 108         if (ret)
 109                 return ret;
 110
 111         pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug);
 112         list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
 113                 ioc->fwfault_debug = mpt3sas_fwfault_debug;
 114         return 0;
 115 }
 116 module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug,
 117         param_get_int, &mpt3sas_fwfault_debug, 0644);
 118
 119 /**
 120  *  mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc
 121  * @arg: input argument, used to derive ioc
 122  *
 123  * Return 0 if controller is removed from pci subsystem.
 124  * Return -1 for other case.
 125  */
 126 static int mpt3sas_remove_dead_ioc_func(void *arg)
 127 {
 128         struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;
 129         struct pci_dev *pdev;
 130
 131         if ((ioc == NULL))
 132                 return -1;
 133
 134         pdev = ioc->pdev;
 135         if ((pdev == NULL))
 136                 return -1;
 137         pci_stop_and_remove_bus_device_locked(pdev);
 138         return 0;
 139 }
 140
 141 /**
 142  * _base_fault_reset_work - workq handling ioc fault conditions
 143  * @work: input argument, used to derive ioc
 144  * Context: sleep.
 145  *
 146  * Return nothing.
 147  */
 148 static void
 149 _base_fault_reset_work(struct work_struct *work)
 150 {
 151         struct MPT3SAS_ADAPTER *ioc =
 152             container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work);
 153         unsigned long    flags;
 154         u32 doorbell;
 155         int rc;
 156         struct task_struct *p;
 157
 158
 159         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 160         if (ioc->shost_recovery)
 161                 goto rearm_timer;
 162         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 163
 164         doorbell = mpt3sas_base_get_iocstate(ioc, 0);
 165         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) {
 166                 pr_err(MPT3SAS_FMT "SAS host is non-operational !!!!\n",
 167                     ioc->name);
 168
 169                 /*
 170                  * Call _scsih_flush_pending_cmds callback so that we flush all
 171                  * pending commands back to OS. This call is required to aovid
 172                  * deadlock at block layer. Dead IOC will fail to do diag reset,
 173                  * and this call is safe since dead ioc will never return any
 174                  * command back from HW.
 175                  */
 176                 ioc->schedule_dead_ioc_flush_running_cmds(ioc);
 177                 /*
 178                  * Set remove_host flag early since kernel thread will
 179                  * take some time to execute.
 180                  */
 181                 ioc->remove_host = 1;
 182                 /*Remove the Dead Host */
 183                 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,
 184                     "mpt3sas_dead_ioc_%d", ioc->id);
 185                 if (IS_ERR(p))
 186                         pr_err(MPT3SAS_FMT
 187                         "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",
 188                         ioc->name, __func__);
 189                 else
 190                         pr_err(MPT3SAS_FMT
 191                         "%s: Running mpt3sas_dead_ioc thread success !!!!\n",
 192                         ioc->name, __func__);
 193                 return; /* don't rearm timer */
 194         }
 195
 196         if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
 197                 rc = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
 198                     FORCE_BIG_HAMMER);
 199                 pr_warn(MPT3SAS_FMT "%s: hard reset: %s\n", ioc->name,
 200                     __func__, (rc == 0) ? "success" : "failed");
 201                 doorbell = mpt3sas_base_get_iocstate(ioc, 0);
 202                 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
 203                         mpt3sas_base_fault_info(ioc, doorbell &
 204                             MPI2_DOORBELL_DATA_MASK);
 205                 if (rc && (doorbell & MPI2_IOC_STATE_MASK) !=
 206                     MPI2_IOC_STATE_OPERATIONAL)
 207                         return; /* don't rearm timer */
 208         }
 209
 210         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 211  rearm_timer:
 212         if (ioc->fault_reset_work_q)
 213                 queue_delayed_work(ioc->fault_reset_work_q,
 214                     &ioc->fault_reset_work,
 215                     msecs_to_jiffies(FAULT_POLLING_INTERVAL));
 216         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 217 }
 218
 219 /**
 220  * mpt3sas_base_start_watchdog - start the fault_reset_work_q
 221  * @ioc: per adapter object
 222  * Context: sleep.
 223  *
 224  * Return nothing.
 225  */
 226 void
 227 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
 228 {
 229         unsigned long    flags;
 230
 231         if (ioc->fault_reset_work_q)
 232                 return;
 233
 234         /* initialize fault polling */
 235
 236         INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
 237         snprintf(ioc->fault_reset_work_q_name,
 238             sizeof(ioc->fault_reset_work_q_name), "poll_%d_status", ioc->id);
 239         ioc->fault_reset_work_q =
 240                 create_singlethread_workqueue(ioc->fault_reset_work_q_name);
 241         if (!ioc->fault_reset_work_q) {
 242                 pr_err(MPT3SAS_FMT "%s: failed (line=%d)\n",
 243                     ioc->name, __func__, __LINE__);
 244                         return;
 245         }
 246         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 247         if (ioc->fault_reset_work_q)
 248                 queue_delayed_work(ioc->fault_reset_work_q,
 249                     &ioc->fault_reset_work,
 250                     msecs_to_jiffies(FAULT_POLLING_INTERVAL));
 251         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 252 }
 253
 254 /**
 255  * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q
 256  * @ioc: per adapter object
 257  * Context: sleep.
 258  *
 259  * Return nothing.
 260  */
 261 void
 262 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)
 263 {
 264         unsigned long flags;
 265         struct workqueue_struct *wq;
 266
 267         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
 268         wq = ioc->fault_reset_work_q;
 269         ioc->fault_reset_work_q = NULL;
 270         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
 271         if (wq) {
 272                 if (!cancel_delayed_work_sync(&ioc->fault_reset_work))
 273                         flush_workqueue(wq);
 274                 destroy_workqueue(wq);
 275         }
 276 }
 277
 278 /**
 279  * mpt3sas_base_fault_info - verbose translation of firmware FAULT code
 280  * @ioc: per adapter object
 281  * @fault_code: fault code
 282  *
 283  * Return nothing.
 284  */
 285 void
 286 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code)
 287 {
 288         pr_err(MPT3SAS_FMT "fault_state(0x%04x)!\n",
 289             ioc->name, fault_code);
 290 }
 291
 292 /**
 293  * mpt3sas_halt_firmware - halt's mpt controller firmware
 294  * @ioc: per adapter object
 295  *
 296  * For debugging timeout related issues.  Writing 0xCOFFEE00
 297  * to the doorbell register will halt controller firmware. With
 298  * the purpose to stop both driver and firmware, the enduser can
 299  * obtain a ring buffer from controller UART.
 300  */
 301 void
 302 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)
 303 {
 304         u32 doorbell;
 305
 306         if (!ioc->fwfault_debug)
 307                 return;
 308
 309         dump_stack();
 310
 311         doorbell = readl(&ioc->chip->Doorbell);
 312         if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
 313                 mpt3sas_base_fault_info(ioc , doorbell);
 314         else {
 315                 writel(0xC0FFEE00, &ioc->chip->Doorbell);
 316                 pr_err(MPT3SAS_FMT "Firmware is halted due to command timeout\n",
 317                         ioc->name);
 318         }
 319
 320         if (ioc->fwfault_debug == 2)
 321                 for (;;)
 322                         ;
 323         else
 324                 panic("panic in %s\n", __func__);
 325 }
 326
 327 /**
 328  * _base_sas_ioc_info - verbose translation of the ioc status
 329  * @ioc: per adapter object
 330  * @mpi_reply: reply mf payload returned from firmware
 331  * @request_hdr: request mf
 332  *
 333  * Return nothing.
 334  */
 335 static void
 336 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,
 337         MPI2RequestHeader_t *request_hdr)
 338 {
 339         u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
 340             MPI2_IOCSTATUS_MASK;
 341         char *desc = NULL;
 342         u16 frame_sz;
 343         char *func_str = NULL;
 344
 345         /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */
 346         if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
 347             request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
 348             request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION)
 349                 return;
 350
 351         if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
 352                 return;
 353
 354         switch (ioc_status) {
 355
 356 /****************************************************************************
 357 *  Common IOCStatus values for all replies
 358 ****************************************************************************/
 359
 360         case MPI2_IOCSTATUS_INVALID_FUNCTION:
 361                 desc = "invalid function";
 362                 break;
 363         case MPI2_IOCSTATUS_BUSY:
 364                 desc = "busy";
 365                 break;
 366         case MPI2_IOCSTATUS_INVALID_SGL:
 367                 desc = "invalid sgl";
 368                 break;
 369         case MPI2_IOCSTATUS_INTERNAL_ERROR:
 370                 desc = "internal error";
 371                 break;
 372         case MPI2_IOCSTATUS_INVALID_VPID:
 373                 desc = "invalid vpid";
 374                 break;
 375         case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
 376                 desc = "insufficient resources";
 377                 break;
 378         case MPI2_IOCSTATUS_INVALID_FIELD:
 379                 desc = "invalid field";
 380                 break;
 381         case MPI2_IOCSTATUS_INVALID_STATE:
 382                 desc = "invalid state";
 383                 break;
 384         case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
 385                 desc = "op state not supported";
 386                 break;
 387
 388 /****************************************************************************
 389 *  Config IOCStatus values
 390 ****************************************************************************/
 391
 392         case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION:
 393                 desc = "config invalid action";
 394                 break;
 395         case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE:
 396                 desc = "config invalid type";
 397                 break;
 398         case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE:
 399                 desc = "config invalid page";
 400                 break;
 401         case MPI2_IOCSTATUS_CONFIG_INVALID_DATA:
 402                 desc = "config invalid data";
 403                 break;
 404         case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS:
 405                 desc = "config no defaults";
 406                 break;
 407         case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT:
 408                 desc = "config cant commit";
 409                 break;
 410
 411 /****************************************************************************
 412 *  SCSI IO Reply
 413 ****************************************************************************/
 414
 415         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
 416         case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
 417         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
 418         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
 419         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
 420         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
 421         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
 422         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
 423         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
 424         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
 425         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
 426         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
 427                 break;
 428
 429 /****************************************************************************
 430 *  For use by SCSI Initiator and SCSI Target end-to-end data protection
 431 ****************************************************************************/
 432
 433         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
 434                 desc = "eedp guard error";
 435                 break;
 436         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
 437                 desc = "eedp ref tag error";
 438                 break;
 439         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
 440                 desc = "eedp app tag error";
 441                 break;
 442
 443 /****************************************************************************
 444 *  SCSI Target values
 445 ****************************************************************************/
 446
 447         case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX:
 448                 desc = "target invalid io index";
 449                 break;
 450         case MPI2_IOCSTATUS_TARGET_ABORTED:
 451                 desc = "target aborted";
 452                 break;
 453         case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE:
 454                 desc = "target no conn retryable";
 455                 break;
 456         case MPI2_IOCSTATUS_TARGET_NO_CONNECTION:
 457                 desc = "target no connection";
 458                 break;
 459         case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH:
 460                 desc = "target xfer count mismatch";
 461                 break;
 462         case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR:
 463                 desc = "target data offset error";
 464                 break;
 465         case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA:
 466                 desc = "target too much write data";
 467                 break;
 468         case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT:
 469                 desc = "target iu too short";
 470                 break;
 471         case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT:
 472                 desc = "target ack nak timeout";
 473                 break;
 474         case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED:
 475                 desc = "target nak received";
 476                 break;
 477
 478 /****************************************************************************
 479 *  Serial Attached SCSI values
 480 ****************************************************************************/
 481
 482         case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED:
 483                 desc = "smp request failed";
 484                 break;
 485         case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN:
 486                 desc = "smp data overrun";
 487                 break;
 488
 489 /****************************************************************************
 490 *  Diagnostic Buffer Post / Diagnostic Release values
 491 ****************************************************************************/
 492
 493         case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED:
 494                 desc = "diagnostic released";
 495                 break;
 496         default:
 497                 break;
 498         }
 499
 500         if (!desc)
 501                 return;
 502
 503         switch (request_hdr->Function) {
 504         case MPI2_FUNCTION_CONFIG:
 505                 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;
 506                 func_str = "config_page";
 507                 break;
 508         case MPI2_FUNCTION_SCSI_TASK_MGMT:
 509                 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t);
 510                 func_str = "task_mgmt";
 511                 break;
 512         case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
 513                 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t);
 514                 func_str = "sas_iounit_ctl";
 515                 break;
 516         case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR:
 517                 frame_sz = sizeof(Mpi2SepRequest_t);
 518                 func_str = "enclosure";
 519                 break;
 520         case MPI2_FUNCTION_IOC_INIT:
 521                 frame_sz = sizeof(Mpi2IOCInitRequest_t);
 522                 func_str = "ioc_init";
 523                 break;
 524         case MPI2_FUNCTION_PORT_ENABLE:
 525                 frame_sz = sizeof(Mpi2PortEnableRequest_t);
 526                 func_str = "port_enable";
 527                 break;
 528         case MPI2_FUNCTION_SMP_PASSTHROUGH:
 529                 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;
 530                 func_str = "smp_passthru";
 531                 break;
 532         default:
 533                 frame_sz = 32;
 534                 func_str = "unknown";
 535                 break;
 536         }
 537
 538         pr_warn(MPT3SAS_FMT "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",
 539                 ioc->name, desc, ioc_status, request_hdr, func_str);
 540
 541         _debug_dump_mf(request_hdr, frame_sz/4);
 542 }
 543
 544 /**
 545  * _base_display_event_data - verbose translation of firmware asyn events
 546  * @ioc: per adapter object
 547  * @mpi_reply: reply mf payload returned from firmware
 548  *
 549  * Return nothing.
 550  */
 551 static void
 552 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
 553         Mpi2EventNotificationReply_t *mpi_reply)
 554 {
 555         char *desc = NULL;
 556         u16 event;
 557
 558         if (!(ioc->logging_level & MPT_DEBUG_EVENTS))
 559                 return;
 560
 561         event = le16_to_cpu(mpi_reply->Event);
 562
 563         switch (event) {
 564         case MPI2_EVENT_LOG_DATA:
 565                 desc = "Log Data";
 566                 break;
 567         case MPI2_EVENT_STATE_CHANGE:
 568                 desc = "Status Change";
 569                 break;
 570         case MPI2_EVENT_HARD_RESET_RECEIVED:
 571                 desc = "Hard Reset Received";
 572                 break;
 573         case MPI2_EVENT_EVENT_CHANGE:
 574                 desc = "Event Change";
 575                 break;
 576         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
 577                 desc = "Device Status Change";
 578                 break;
 579         case MPI2_EVENT_IR_OPERATION_STATUS:
 580                 desc = "IR Operation Status";
 581                 break;
 582         case MPI2_EVENT_SAS_DISCOVERY:
 583         {
 584                 Mpi2EventDataSasDiscovery_t *event_data =
 585                     (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData;
 586                 pr_info(MPT3SAS_FMT "Discovery: (%s)", ioc->name,
 587                     (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED) ?
 588                     "start" : "stop");
 589                 if (event_data->DiscoveryStatus)
 590                         pr_info("discovery_status(0x%08x)",
 591                             le32_to_cpu(event_data->DiscoveryStatus));
 592                         pr_info("\n");
 593                 return;
 594         }
 595         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
 596                 desc = "SAS Broadcast Primitive";
 597                 break;
 598         case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 599                 desc = "SAS Init Device Status Change";
 600                 break;
 601         case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW:
 602                 desc = "SAS Init Table Overflow";
 603                 break;
 604         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 605                 desc = "SAS Topology Change List";
 606                 break;
 607         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
 608                 desc = "SAS Enclosure Device Status Change";
 609                 break;
 610         case MPI2_EVENT_IR_VOLUME:
 611                 desc = "IR Volume";
 612                 break;
 613         case MPI2_EVENT_IR_PHYSICAL_DISK:
 614                 desc = "IR Physical Disk";
 615                 break;
 616         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
 617                 desc = "IR Configuration Change List";
 618                 break;
 619         case MPI2_EVENT_LOG_ENTRY_ADDED:
 620                 desc = "Log Entry Added";
 621                 break;
 622         case MPI2_EVENT_TEMP_THRESHOLD:
 623                 desc = "Temperature Threshold";
 624                 break;
 625         }
 626
 627         if (!desc)
 628                 return;
 629
 630         pr_info(MPT3SAS_FMT "%s\n", ioc->name, desc);
 631 }
 632
 633 /**
 634  * _base_sas_log_info - verbose translation of firmware log info
 635  * @ioc: per adapter object
 636  * @log_info: log info
 637  *
 638  * Return nothing.
 639  */
 640 static void
 641 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info)
 642 {
 643         union loginfo_type {
 644                 u32     loginfo;
 645                 struct {
 646                         u32     subcode:16;
 647                         u32     code:8;
 648                         u32     originator:4;
 649                         u32     bus_type:4;
 650                 } dw;
 651         };
 652         union loginfo_type sas_loginfo;
 653         char *originator_str = NULL;
 654
 655         sas_loginfo.loginfo = log_info;
 656         if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
 657                 return;
 658
 659         /* each nexus loss loginfo */
 660         if (log_info == 0x31170000)
 661                 return;
 662
 663         /* eat the loginfos associated with task aborts */
 664         if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==
 665             0x31140000 || log_info == 0x31130000))
 666                 return;
 667
 668         switch (sas_loginfo.dw.originator) {
 669         case 0:
 670                 originator_str = "IOP";
 671                 break;
 672         case 1:
 673                 originator_str = "PL";
 674                 break;
 675         case 2:
 676                 originator_str = "IR";
 677                 break;
 678         }
 679
 680         pr_warn(MPT3SAS_FMT
 681                 "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",
 682                 ioc->name, log_info,
 683              originator_str, sas_loginfo.dw.code,
 684              sas_loginfo.dw.subcode);
 685 }
 686
 687 /**
 688  * _base_display_reply_info -
 689  * @ioc: per adapter object
 690  * @smid: system request message index
 691  * @msix_index: MSIX table index supplied by the OS
 692  * @reply: reply message frame(lower 32bit addr)
 693  *
 694  * Return nothing.
 695  */
 696 static void
 697 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 698         u32 reply)
 699 {
 700         MPI2DefaultReply_t *mpi_reply;
 701         u16 ioc_status;
 702         u32 loginfo = 0;
 703
 704         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
 705         if (unlikely(!mpi_reply)) {
 706                 pr_err(MPT3SAS_FMT "mpi_reply not valid at %s:%d/%s()!\n",
 707                     ioc->name, __FILE__, __LINE__, __func__);
 708                 return;
 709         }
 710         ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
 711
 712         if ((ioc_status & MPI2_IOCSTATUS_MASK) &&
 713             (ioc->logging_level & MPT_DEBUG_REPLY)) {
 714                 _base_sas_ioc_info(ioc , mpi_reply,
 715                    mpt3sas_base_get_msg_frame(ioc, smid));
 716         }
 717
 718         if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
 719                 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo);
 720                 _base_sas_log_info(ioc, loginfo);
 721         }
 722
 723         if (ioc_status || loginfo) {
 724                 ioc_status &= MPI2_IOCSTATUS_MASK;
 725                 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);
 726         }
 727 }
 728
 729 /**
 730  * mpt3sas_base_done - base internal command completion routine
 731  * @ioc: per adapter object
 732  * @smid: system request message index
 733  * @msix_index: MSIX table index supplied by the OS
 734  * @reply: reply message frame(lower 32bit addr)
 735  *
 736  * Return 1 meaning mf should be freed from _base_interrupt
 737  *        0 means the mf is freed from this function.
 738  */
 739 u8
 740 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 741         u32 reply)
 742 {
 743         MPI2DefaultReply_t *mpi_reply;
 744
 745         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
 746         if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK)
 747                 return 1;
 748
 749         if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)
 750                 return 1;
 751
 752         ioc->base_cmds.status |= MPT3_CMD_COMPLETE;
 753         if (mpi_reply) {
 754                 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;
 755                 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
 756         }
 757         ioc->base_cmds.status &= ~MPT3_CMD_PENDING;
 758
 759         complete(&ioc->base_cmds.done);
 760         return 1;
 761 }
 762
 763 /**
 764  * _base_async_event - main callback handler for firmware asyn events
 765  * @ioc: per adapter object
 766  * @msix_index: MSIX table index supplied by the OS
 767  * @reply: reply message frame(lower 32bit addr)
 768  *
 769  * Return 1 meaning mf should be freed from _base_interrupt
 770  *        0 means the mf is freed from this function.
 771  */
 772 static u8
 773 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
 774 {
 775         Mpi2EventNotificationReply_t *mpi_reply;
 776         Mpi2EventAckRequest_t *ack_request;
 777         u16 smid;
 778
 779         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
 780         if (!mpi_reply)
 781                 return 1;
 782         if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION)
 783                 return 1;
 784
 785         _base_display_event_data(ioc, mpi_reply);
 786
 787         if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED))
 788                 goto out;
 789         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
 790         if (!smid) {
 791                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
 792                     ioc->name, __func__);
 793                 goto out;
 794         }
 795
 796         ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
 797         memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
 798         ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
 799         ack_request->Event = mpi_reply->Event;
 800         ack_request->EventContext = mpi_reply->EventContext;
 801         ack_request->VF_ID = 0;  /* TODO */
 802         ack_request->VP_ID = 0;
 803         mpt3sas_base_put_smid_default(ioc, smid);
 804
 805  out:
 806
 807         /* scsih callback handler */
 808         mpt3sas_scsih_event_callback(ioc, msix_index, reply);
 809
 810         /* ctl callback handler */
 811         mpt3sas_ctl_event_callback(ioc, msix_index, reply);
 812
 813         return 1;
 814 }
 815
 816 /**
 817  * _base_get_cb_idx - obtain the callback index
 818  * @ioc: per adapter object
 819  * @smid: system request message index
 820  *
 821  * Return callback index.
 822  */
 823 static u8
 824 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 825 {
 826         int i;
 827         u8 cb_idx;
 828
 829         if (smid < ioc->hi_priority_smid) {
 830                 i = smid - 1;
 831                 cb_idx = ioc->scsi_lookup[i].cb_idx;
 832         } else if (smid < ioc->internal_smid) {
 833                 i = smid - ioc->hi_priority_smid;
 834                 cb_idx = ioc->hpr_lookup[i].cb_idx;
 835         } else if (smid <= ioc->hba_queue_depth) {
 836                 i = smid - ioc->internal_smid;
 837                 cb_idx = ioc->internal_lookup[i].cb_idx;
 838         } else
 839                 cb_idx = 0xFF;
 840         return cb_idx;
 841 }
 842
 843 /**
 844  * _base_mask_interrupts - disable interrupts
 845  * @ioc: per adapter object
 846  *
 847  * Disabling ResetIRQ, Reply and Doorbell Interrupts
 848  *
 849  * Return nothing.
 850  */
 851 static void
 852 _base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)
 853 {
 854         u32 him_register;
 855
 856         ioc->mask_interrupts = 1;
 857         him_register = readl(&ioc->chip->HostInterruptMask);
 858         him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK;
 859         writel(him_register, &ioc->chip->HostInterruptMask);
 860         readl(&ioc->chip->HostInterruptMask);
 861 }
 862
 863 /**
 864  * _base_unmask_interrupts - enable interrupts
 865  * @ioc: per adapter object
 866  *
 867  * Enabling only Reply Interrupts
 868  *
 869  * Return nothing.
 870  */
 871 static void
 872 _base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)
 873 {
 874         u32 him_register;
 875
 876         him_register = readl(&ioc->chip->HostInterruptMask);
 877         him_register &= ~MPI2_HIM_RIM;
 878         writel(him_register, &ioc->chip->HostInterruptMask);
 879         ioc->mask_interrupts = 0;
 880 }
 881
 882 union reply_descriptor {
 883         u64 word;
 884         struct {
 885                 u32 low;
 886                 u32 high;
 887         } u;
 888 };
 889
 890 /**
 891  * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
 892  * @irq: irq number (not used)
 893  * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
 894  * @r: pt_regs pointer (not used)
 895  *
 896  * Return IRQ_HANDLE if processed, else IRQ_NONE.
 897  */
 898 static irqreturn_t
 899 _base_interrupt(int irq, void *bus_id)
 900 {
 901         struct adapter_reply_queue *reply_q = bus_id;
 902         union reply_descriptor rd;
 903         u32 completed_cmds;
 904         u8 request_desript_type;
 905         u16 smid;
 906         u8 cb_idx;
 907         u32 reply;
 908         u8 msix_index = reply_q->msix_index;
 909         struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
 910         Mpi2ReplyDescriptorsUnion_t *rpf;
 911         u8 rc;
 912
 913         if (ioc->mask_interrupts)
 914                 return IRQ_NONE;
 915
 916         if (!atomic_add_unless(&reply_q->busy, 1, 1))
 917                 return IRQ_NONE;
 918
 919         rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
 920         request_desript_type = rpf->Default.ReplyFlags
 921              & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
 922         if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
 923                 atomic_dec(&reply_q->busy);
 924                 return IRQ_NONE;
 925         }
 926
 927         completed_cmds = 0;
 928         cb_idx = 0xFF;
 929         do {
 930                 rd.word = le64_to_cpu(rpf->Words);
 931                 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX)
 932                         goto out;
 933                 reply = 0;
 934                 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
 935                 if (request_desript_type ==
 936                     MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
 937                     request_desript_type ==
 938                     MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS) {
 939                         cb_idx = _base_get_cb_idx(ioc, smid);
 940                         if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
 941                             (likely(mpt_callbacks[cb_idx] != NULL))) {
 942                                 rc = mpt_callbacks[cb_idx](ioc, smid,
 943                                     msix_index, 0);
 944                                 if (rc)
 945                                         mpt3sas_base_free_smid(ioc, smid);
 946                         }
 947                 } else if (request_desript_type ==
 948                     MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
 949                         reply = le32_to_cpu(
 950                             rpf->AddressReply.ReplyFrameAddress);
 951                         if (reply > ioc->reply_dma_max_address ||
 952                             reply < ioc->reply_dma_min_address)
 953                                 reply = 0;
 954                         if (smid) {
 955                                 cb_idx = _base_get_cb_idx(ioc, smid);
 956                                 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
 957                                     (likely(mpt_callbacks[cb_idx] != NULL))) {
 958                                         rc = mpt_callbacks[cb_idx](ioc, smid,
 959                                             msix_index, reply);
 960                                         if (reply)
 961                                                 _base_display_reply_info(ioc,
 962                                                     smid, msix_index, reply);
 963                                         if (rc)
 964                                                 mpt3sas_base_free_smid(ioc,
 965                                                     smid);
 966                                 }
 967                         } else {
 968                                 _base_async_event(ioc, msix_index, reply);
 969                         }
 970
 971                         /* reply free queue handling */
 972                         if (reply) {
 973                                 ioc->reply_free_host_index =
 974                                     (ioc->reply_free_host_index ==
 975                                     (ioc->reply_free_queue_depth - 1)) ?
 976                                     0 : ioc->reply_free_host_index + 1;
 977                                 ioc->reply_free[ioc->reply_free_host_index] =
 978                                     cpu_to_le32(reply);
 979                                 wmb();
 980                                 writel(ioc->reply_free_host_index,
 981                                     &ioc->chip->ReplyFreeHostIndex);
 982                         }
 983                 }
 984
 985                 rpf->Words = cpu_to_le64(ULLONG_MAX);
 986                 reply_q->reply_post_host_index =
 987                     (reply_q->reply_post_host_index ==
 988                     (ioc->reply_post_queue_depth - 1)) ? 0 :
 989                     reply_q->reply_post_host_index + 1;
 990                 request_desript_type =
 991                     reply_q->reply_post_free[reply_q->reply_post_host_index].
 992                     Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
 993                 completed_cmds++;
 994                 if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
 995                         goto out;
 996                 if (!reply_q->reply_post_host_index)
 997                         rpf = reply_q->reply_post_free;
 998                 else
 999                         rpf++;
1000         } while (1);
1001
1002  out:
1003
1004         if (!completed_cmds) {
1005                 atomic_dec(&reply_q->busy);
1006                 return IRQ_NONE;
1007         }
1008
1009         wmb();
1010
1011         /* Update Reply Post Host Index.
1012          * For those HBA's which support combined reply queue feature
1013          * 1. Get the correct Supplemental Reply Post Host Index Register.
1014          *    i.e. (msix_index / 8)th entry from Supplemental Reply Post Host
1015          *    Index Register address bank i.e replyPostRegisterIndex[],
1016          * 2. Then update this register with new reply host index value
1017          *    in ReplyPostIndex field and the MSIxIndex field with
1018          *    msix_index value reduced to a value between 0 and 7,
1019          *    using a modulo 8 operation. Since each Supplemental Reply Post
1020          *    Host Index Register supports 8 MSI-X vectors.
1021          *
1022          * For other HBA's just update the Reply Post Host Index register with
1023          * new reply host index value in ReplyPostIndex Field and msix_index
1024          * value in MSIxIndex field.
1025          */
1026         if (ioc->msix96_vector)
1027                 writel(reply_q->reply_post_host_index | ((msix_index  & 7) <<
1028                         MPI2_RPHI_MSIX_INDEX_SHIFT),
1029                         ioc->replyPostRegisterIndex[msix_index/8]);
1030         else
1031                 writel(reply_q->reply_post_host_index | (msix_index <<
1032                         MPI2_RPHI_MSIX_INDEX_SHIFT),
1033                         &ioc->chip->ReplyPostHostIndex);
1034         atomic_dec(&reply_q->busy);
1035         return IRQ_HANDLED;
1036 }
1037
1038 /**
1039  * _base_is_controller_msix_enabled - is controller support muli-reply queues
1040  * @ioc: per adapter object
1041  *
1042  */
1043 static inline int
1044 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)
1045 {
1046         return (ioc->facts.IOCCapabilities &
1047             MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;
1048 }
1049
1050 /**
1051  * mpt3sas_base_flush_reply_queues - flushing the MSIX reply queues
1052  * @ioc: per adapter object
1053  * Context: ISR conext
1054  *
1055  * Called when a Task Management request has completed. We want
1056  * to flush the other reply queues so all the outstanding IO has been
1057  * completed back to OS before we process the TM completetion.
1058  *
1059  * Return nothing.
1060  */
1061 void
1062 mpt3sas_base_flush_reply_queues(struct MPT3SAS_ADAPTER *ioc)
1063 {
1064         struct adapter_reply_queue *reply_q;
1065
1066         /* If MSIX capability is turned off
1067          * then multi-queues are not enabled
1068          */
1069         if (!_base_is_controller_msix_enabled(ioc))
1070                 return;
1071
1072         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
1073                 if (ioc->shost_recovery)
1074                         return;
1075                 /* TMs are on msix_index == 0 */
1076                 if (reply_q->msix_index == 0)
1077                         continue;
1078                 _base_interrupt(reply_q->vector, (void *)reply_q);
1079         }
1080 }
1081
1082 /**
1083  * mpt3sas_base_release_callback_handler - clear interrupt callback handler
1084  * @cb_idx: callback index
1085  *
1086  * Return nothing.
1087  */
1088 void
1089 mpt3sas_base_release_callback_handler(u8 cb_idx)
1090 {
1091         mpt_callbacks[cb_idx] = NULL;
1092 }
1093
1094 /**
1095  * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler
1096  * @cb_func: callback function
1097  *
1098  * Returns cb_func.
1099  */
1100 u8
1101 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func)
1102 {
1103         u8 cb_idx;
1104
1105         for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--)
1106                 if (mpt_callbacks[cb_idx] == NULL)
1107                         break;
1108
1109         mpt_callbacks[cb_idx] = cb_func;
1110         return cb_idx;
1111 }
1112
1113 /**
1114  * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler
1115  *
1116  * Return nothing.
1117  */
1118 void
1119 mpt3sas_base_initialize_callback_handler(void)
1120 {
1121         u8 cb_idx;
1122
1123         for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++)
1124                 mpt3sas_base_release_callback_handler(cb_idx);
1125 }
1126
1127
1128 /**
1129  * _base_build_zero_len_sge - build zero length sg entry
1130  * @ioc: per adapter object
1131  * @paddr: virtual address for SGE
1132  *
1133  * Create a zero length scatter gather entry to insure the IOCs hardware has
1134  * something to use if the target device goes brain dead and tries
1135  * to send data even when none is asked for.
1136  *
1137  * Return nothing.
1138  */
1139 static void
1140 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1141 {
1142         u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT |
1143             MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST |
1144             MPI2_SGE_FLAGS_SIMPLE_ELEMENT) <<
1145             MPI2_SGE_FLAGS_SHIFT);
1146         ioc->base_add_sg_single(paddr, flags_length, -1);
1147 }
1148
1149 /**
1150  * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr.
1151  * @paddr: virtual address for SGE
1152  * @flags_length: SGE flags and data transfer length
1153  * @dma_addr: Physical address
1154  *
1155  * Return nothing.
1156  */
1157 static void
1158 _base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1159 {
1160         Mpi2SGESimple32_t *sgel = paddr;
1161
1162         flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING |
1163             MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1164         sgel->FlagsLength = cpu_to_le32(flags_length);
1165         sgel->Address = cpu_to_le32(dma_addr);
1166 }
1167
1168
1169 /**
1170  * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr.
1171  * @paddr: virtual address for SGE
1172  * @flags_length: SGE flags and data transfer length
1173  * @dma_addr: Physical address
1174  *
1175  * Return nothing.
1176  */
1177 static void
1178 _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1179 {
1180         Mpi2SGESimple64_t *sgel = paddr;
1181
1182         flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
1183             MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1184         sgel->FlagsLength = cpu_to_le32(flags_length);
1185         sgel->Address = cpu_to_le64(dma_addr);
1186 }
1187
1188 /**
1189  * _base_get_chain_buffer_tracker - obtain chain tracker
1190  * @ioc: per adapter object
1191  * @smid: smid associated to an IO request
1192  *
1193  * Returns chain tracker(from ioc->free_chain_list)
1194  */
1195 static struct chain_tracker *
1196 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1197 {
1198         struct chain_tracker *chain_req;
1199         unsigned long flags;
1200
1201         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
1202         if (list_empty(&ioc->free_chain_list)) {
1203                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
1204                 dfailprintk(ioc, pr_warn(MPT3SAS_FMT
1205                         "chain buffers not available\n", ioc->name));
1206                 return NULL;
1207         }
1208         chain_req = list_entry(ioc->free_chain_list.next,
1209             struct chain_tracker, tracker_list);
1210         list_del_init(&chain_req->tracker_list);
1211         list_add_tail(&chain_req->tracker_list,
1212             &ioc->scsi_lookup[smid - 1].chain_list);
1213         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
1214         return chain_req;
1215 }
1216
1217
1218 /**
1219  * _base_build_sg - build generic sg
1220  * @ioc: per adapter object
1221  * @psge: virtual address for SGE
1222  * @data_out_dma: physical address for WRITES
1223  * @data_out_sz: data xfer size for WRITES
1224  * @data_in_dma: physical address for READS
1225  * @data_in_sz: data xfer size for READS
1226  *
1227  * Return nothing.
1228  */
1229 static void
1230 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,
1231         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1232         size_t data_in_sz)
1233 {
1234         u32 sgl_flags;
1235
1236         if (!data_out_sz && !data_in_sz) {
1237                 _base_build_zero_len_sge(ioc, psge);
1238                 return;
1239         }
1240
1241         if (data_out_sz && data_in_sz) {
1242                 /* WRITE sgel first */
1243                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1244                     MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC);
1245                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1246                 ioc->base_add_sg_single(psge, sgl_flags |
1247                     data_out_sz, data_out_dma);
1248
1249                 /* incr sgel */
1250                 psge += ioc->sge_size;
1251
1252                 /* READ sgel last */
1253                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1254                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1255                     MPI2_SGE_FLAGS_END_OF_LIST);
1256                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1257                 ioc->base_add_sg_single(psge, sgl_flags |
1258                     data_in_sz, data_in_dma);
1259         } else if (data_out_sz) /* WRITE */ {
1260                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1261                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1262                     MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC);
1263                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1264                 ioc->base_add_sg_single(psge, sgl_flags |
1265                     data_out_sz, data_out_dma);
1266         } else if (data_in_sz) /* READ */ {
1267                 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
1268                     MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
1269                     MPI2_SGE_FLAGS_END_OF_LIST);
1270                 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1271                 ioc->base_add_sg_single(psge, sgl_flags |
1272                     data_in_sz, data_in_dma);
1273         }
1274 }
1275
1276 /* IEEE format sgls */
1277
1278 /**
1279  * _base_add_sg_single_ieee - add sg element for IEEE format
1280  * @paddr: virtual address for SGE
1281  * @flags: SGE flags
1282  * @chain_offset: number of 128 byte elements from start of segment
1283  * @length: data transfer length
1284  * @dma_addr: Physical address
1285  *
1286  * Return nothing.
1287  */
1288 static void
1289 _base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length,
1290         dma_addr_t dma_addr)
1291 {
1292         Mpi25IeeeSgeChain64_t *sgel = paddr;
1293
1294         sgel->Flags = flags;
1295         sgel->NextChainOffset = chain_offset;
1296         sgel->Length = cpu_to_le32(length);
1297         sgel->Address = cpu_to_le64(dma_addr);
1298 }
1299
1300 /**
1301  * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format
1302  * @ioc: per adapter object
1303  * @paddr: virtual address for SGE
1304  *
1305  * Create a zero length scatter gather entry to insure the IOCs hardware has
1306  * something to use if the target device goes brain dead and tries
1307  * to send data even when none is asked for.
1308  *
1309  * Return nothing.
1310  */
1311 static void
1312 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1313 {
1314         u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
1315                 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
1316                 MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
1317         _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1);
1318 }
1319
1320 /**
1321  * _base_build_sg_scmd - main sg creation routine
1322  * @ioc: per adapter object
1323  * @scmd: scsi command
1324  * @smid: system request message index
1325  * Context: none.
1326  *
1327  * The main routine that builds scatter gather table from a given
1328  * scsi request sent via the .queuecommand main handler.
1329  *
1330  * Returns 0 success, anything else error
1331  */
1332 static int
1333 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,
1334                 struct scsi_cmnd *scmd, u16 smid)
1335 {
1336         Mpi2SCSIIORequest_t *mpi_request;
1337         dma_addr_t chain_dma;
1338         struct scatterlist *sg_scmd;
1339         void *sg_local, *chain;
1340         u32 chain_offset;
1341         u32 chain_length;
1342         u32 chain_flags;
1343         int sges_left;
1344         u32 sges_in_segment;
1345         u32 sgl_flags;
1346         u32 sgl_flags_last_element;
1347         u32 sgl_flags_end_buffer;
1348         struct chain_tracker *chain_req;
1349
1350         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
1351
1352         /* init scatter gather flags */
1353         sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT;
1354         if (scmd->sc_data_direction == DMA_TO_DEVICE)
1355                 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
1356         sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT)
1357             << MPI2_SGE_FLAGS_SHIFT;
1358         sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT |
1359             MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST)
1360             << MPI2_SGE_FLAGS_SHIFT;
1361         sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
1362
1363         sg_scmd = scsi_sglist(scmd);
1364         sges_left = scsi_dma_map(scmd);
1365         if (sges_left < 0) {
1366                 sdev_printk(KERN_ERR, scmd->device,
1367                  "pci_map_sg failed: request for %d bytes!\n",
1368                  scsi_bufflen(scmd));
1369                 return -ENOMEM;
1370         }
1371
1372         sg_local = &mpi_request->SGL;
1373         sges_in_segment = ioc->max_sges_in_main_message;
1374         if (sges_left <= sges_in_segment)
1375                 goto fill_in_last_segment;
1376
1377         mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) +
1378             (sges_in_segment * ioc->sge_size))/4;
1379
1380         /* fill in main message segment when there is a chain following */
1381         while (sges_in_segment) {
1382                 if (sges_in_segment == 1)
1383                         ioc->base_add_sg_single(sg_local,
1384                             sgl_flags_last_element | sg_dma_len(sg_scmd),
1385                             sg_dma_address(sg_scmd));
1386                 else
1387                         ioc->base_add_sg_single(sg_local, sgl_flags |
1388                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
1389                 sg_scmd = sg_next(sg_scmd);
1390                 sg_local += ioc->sge_size;
1391                 sges_left--;
1392                 sges_in_segment--;
1393         }
1394
1395         /* initializing the chain flags and pointers */
1396         chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT;
1397         chain_req = _base_get_chain_buffer_tracker(ioc, smid);
1398         if (!chain_req)
1399                 return -1;
1400         chain = chain_req->chain_buffer;
1401         chain_dma = chain_req->chain_buffer_dma;
1402         do {
1403                 sges_in_segment = (sges_left <=
1404                     ioc->max_sges_in_chain_message) ? sges_left :
1405                     ioc->max_sges_in_chain_message;
1406                 chain_offset = (sges_left == sges_in_segment) ?
1407                     0 : (sges_in_segment * ioc->sge_size)/4;
1408                 chain_length = sges_in_segment * ioc->sge_size;
1409                 if (chain_offset) {
1410                         chain_offset = chain_offset <<
1411                             MPI2_SGE_CHAIN_OFFSET_SHIFT;
1412                         chain_length += ioc->sge_size;
1413                 }
1414                 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |
1415                     chain_length, chain_dma);
1416                 sg_local = chain;
1417                 if (!chain_offset)
1418                         goto fill_in_last_segment;
1419
1420                 /* fill in chain segments */
1421                 while (sges_in_segment) {
1422                         if (sges_in_segment == 1)
1423                                 ioc->base_add_sg_single(sg_local,
1424                                     sgl_flags_last_element |
1425                                     sg_dma_len(sg_scmd),
1426                                     sg_dma_address(sg_scmd));
1427                         else
1428                                 ioc->base_add_sg_single(sg_local, sgl_flags |
1429                                     sg_dma_len(sg_scmd),
1430                                     sg_dma_address(sg_scmd));
1431                         sg_scmd = sg_next(sg_scmd);
1432                         sg_local += ioc->sge_size;
1433                         sges_left--;
1434                         sges_in_segment--;
1435                 }
1436
1437                 chain_req = _base_get_chain_buffer_tracker(ioc, smid);
1438                 if (!chain_req)
1439                         return -1;
1440                 chain = chain_req->chain_buffer;
1441                 chain_dma = chain_req->chain_buffer_dma;
1442         } while (1);
1443
1444
1445  fill_in_last_segment:
1446
1447         /* fill the last segment */
1448         while (sges_left) {
1449                 if (sges_left == 1)
1450                         ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |
1451                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
1452                 else
1453                         ioc->base_add_sg_single(sg_local, sgl_flags |
1454                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
1455                 sg_scmd = sg_next(sg_scmd);
1456                 sg_local += ioc->sge_size;
1457                 sges_left--;
1458         }
1459
1460         return 0;
1461 }
1462
1463 /**
1464  * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format
1465  * @ioc: per adapter object
1466  * @scmd: scsi command
1467  * @smid: system request message index
1468  * Context: none.
1469  *
1470  * The main routine that builds scatter gather table from a given
1471  * scsi request sent via the .queuecommand main handler.
1472  *
1473  * Returns 0 success, anything else error
1474  */
1475 static int
1476 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,
1477         struct scsi_cmnd *scmd, u16 smid)
1478 {
1479         Mpi2SCSIIORequest_t *mpi_request;
1480         dma_addr_t chain_dma;
1481         struct scatterlist *sg_scmd;
1482         void *sg_local, *chain;
1483         u32 chain_offset;
1484         u32 chain_length;
1485         int sges_left;
1486         u32 sges_in_segment;
1487         u8 simple_sgl_flags;
1488         u8 simple_sgl_flags_last;
1489         u8 chain_sgl_flags;
1490         struct chain_tracker *chain_req;
1491
1492         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
1493
1494         /* init scatter gather flags */
1495         simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
1496             MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
1497         simple_sgl_flags_last = simple_sgl_flags |
1498             MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
1499         chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
1500             MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
1501
1502         sg_scmd = scsi_sglist(scmd);
1503         sges_left = scsi_dma_map(scmd);
1504         if (sges_left < 0) {
1505                 sdev_printk(KERN_ERR, scmd->device,
1506                         "pci_map_sg failed: request for %d bytes!\n",
1507                         scsi_bufflen(scmd));
1508                 return -ENOMEM;
1509         }
1510
1511         sg_local = &mpi_request->SGL;
1512         sges_in_segment = (ioc->request_sz -
1513             offsetof(Mpi2SCSIIORequest_t, SGL))/ioc->sge_size_ieee;
1514         if (sges_left <= sges_in_segment)
1515                 goto fill_in_last_segment;
1516
1517         mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) +
1518             (offsetof(Mpi2SCSIIORequest_t, SGL)/ioc->sge_size_ieee);
1519
1520         /* fill in main message segment when there is a chain following */
1521         while (sges_in_segment > 1) {
1522                 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
1523                     sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
1524                 sg_scmd = sg_next(sg_scmd);
1525                 sg_local += ioc->sge_size_ieee;
1526                 sges_left--;
1527                 sges_in_segment--;
1528         }
1529
1530         /* initializing the pointers */
1531         chain_req = _base_get_chain_buffer_tracker(ioc, smid);
1532         if (!chain_req)
1533                 return -1;
1534         chain = chain_req->chain_buffer;
1535         chain_dma = chain_req->chain_buffer_dma;
1536         do {
1537                 sges_in_segment = (sges_left <=
1538                     ioc->max_sges_in_chain_message) ? sges_left :
1539                     ioc->max_sges_in_chain_message;
1540                 chain_offset = (sges_left == sges_in_segment) ?
1541                     0 : sges_in_segment;
1542                 chain_length = sges_in_segment * ioc->sge_size_ieee;
1543                 if (chain_offset)
1544                         chain_length += ioc->sge_size_ieee;
1545                 _base_add_sg_single_ieee(sg_local, chain_sgl_flags,
1546                     chain_offset, chain_length, chain_dma);
1547
1548                 sg_local = chain;
1549                 if (!chain_offset)
1550                         goto fill_in_last_segment;
1551
1552                 /* fill in chain segments */
1553                 while (sges_in_segment) {
1554                         _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
1555                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
1556                         sg_scmd = sg_next(sg_scmd);
1557                         sg_local += ioc->sge_size_ieee;
1558                         sges_left--;
1559                         sges_in_segment--;
1560                 }
1561
1562                 chain_req = _base_get_chain_buffer_tracker(ioc, smid);
1563                 if (!chain_req)
1564                         return -1;
1565                 chain = chain_req->chain_buffer;
1566                 chain_dma = chain_req->chain_buffer_dma;
1567         } while (1);
1568
1569
1570  fill_in_last_segment:
1571
1572         /* fill the last segment */
1573         while (sges_left > 0) {
1574                 if (sges_left == 1)
1575                         _base_add_sg_single_ieee(sg_local,
1576                             simple_sgl_flags_last, 0, sg_dma_len(sg_scmd),
1577                             sg_dma_address(sg_scmd));
1578                 else
1579                         _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
1580                             sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
1581                 sg_scmd = sg_next(sg_scmd);
1582                 sg_local += ioc->sge_size_ieee;
1583                 sges_left--;
1584         }
1585
1586         return 0;
1587 }
1588
1589 /**
1590  * _base_build_sg_ieee - build generic sg for IEEE format
1591  * @ioc: per adapter object
1592  * @psge: virtual address for SGE
1593  * @data_out_dma: physical address for WRITES
1594  * @data_out_sz: data xfer size for WRITES
1595  * @data_in_dma: physical address for READS
1596  * @data_in_sz: data xfer size for READS
1597  *
1598  * Return nothing.
1599  */
1600 static void
1601 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
1602         dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
1603         size_t data_in_sz)
1604 {
1605         u8 sgl_flags;
1606
1607         if (!data_out_sz && !data_in_sz) {
1608                 _base_build_zero_len_sge_ieee(ioc, psge);
1609                 return;
1610         }
1611
1612         if (data_out_sz && data_in_sz) {
1613                 /* WRITE sgel first */
1614                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
1615                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
1616                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
1617                     data_out_dma);
1618
1619                 /* incr sgel */
1620                 psge += ioc->sge_size_ieee;
1621
1622                 /* READ sgel last */
1623                 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
1624                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
1625                     data_in_dma);
1626         } else if (data_out_sz) /* WRITE */ {
1627                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
1628                     MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
1629                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
1630                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
1631                     data_out_dma);
1632         } else if (data_in_sz) /* READ */ {
1633                 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
1634                     MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
1635                     MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
1636                 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
1637                     data_in_dma);
1638         }
1639 }
1640
1641 #define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10))
1642
1643 /**
1644  * _base_config_dma_addressing - set dma addressing
1645  * @ioc: per adapter object
1646  * @pdev: PCI device struct
1647  *
1648  * Returns 0 for success, non-zero for failure.
1649  */
1650 static int
1651 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
1652 {
1653         struct sysinfo s;
1654         u64 consistent_dma_mask;
1655
1656         if (ioc->dma_mask)
1657                 consistent_dma_mask = DMA_BIT_MASK(64);
1658         else
1659                 consistent_dma_mask = DMA_BIT_MASK(32);
1660
1661         if (sizeof(dma_addr_t) > 4) {
1662                 const uint64_t required_mask =
1663                     dma_get_required_mask(&pdev->dev);
1664                 if ((required_mask > DMA_BIT_MASK(32)) &&
1665                     !pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) &&
1666                     !pci_set_consistent_dma_mask(pdev, consistent_dma_mask)) {
1667                         ioc->base_add_sg_single = &_base_add_sg_single_64;
1668                         ioc->sge_size = sizeof(Mpi2SGESimple64_t);
1669                         ioc->dma_mask = 64;
1670                         goto out;
1671                 }
1672         }
1673
1674         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))
1675             && !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
1676                 ioc->base_add_sg_single = &_base_add_sg_single_32;
1677                 ioc->sge_size = sizeof(Mpi2SGESimple32_t);
1678                 ioc->dma_mask = 32;
1679         } else
1680                 return -ENODEV;
1681
1682  out:
1683         si_meminfo(&s);
1684         pr_info(MPT3SAS_FMT
1685                 "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
1686                 ioc->name, ioc->dma_mask, convert_to_kb(s.totalram));
1687
1688         return 0;
1689 }
1690
1691 static int
1692 _base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc,
1693                                       struct pci_dev *pdev)
1694 {
1695         if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
1696                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
1697                         return -ENODEV;
1698         }
1699         return 0;
1700 }
1701
1702 /**
1703  * _base_check_enable_msix - checks MSIX capabable.
1704  * @ioc: per adapter object
1705  *
1706  * Check to see if card is capable of MSIX, and set number
1707  * of available msix vectors
1708  */
1709 static int
1710 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)
1711 {
1712         int base;
1713         u16 message_control;
1714
1715         base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);
1716         if (!base) {
1717                 dfailprintk(ioc, pr_info(MPT3SAS_FMT "msix not supported\n",
1718                         ioc->name));
1719                 return -EINVAL;
1720         }
1721
1722         /* get msix vector count */
1723
1724         pci_read_config_word(ioc->pdev, base + 2, &message_control);
1725         ioc->msix_vector_count = (message_control & 0x3FF) + 1;
1726         dinitprintk(ioc, pr_info(MPT3SAS_FMT
1727                 "msix is supported, vector_count(%d)\n",
1728                 ioc->name, ioc->msix_vector_count));
1729         return 0;
1730 }
1731
1732 /**
1733  * _base_free_irq - free irq
1734  * @ioc: per adapter object
1735  *
1736  * Freeing respective reply_queue from the list.
1737  */
1738 static void
1739 _base_free_irq(struct MPT3SAS_ADAPTER *ioc)
1740 {
1741         struct adapter_reply_queue *reply_q, *next;
1742
1743         if (list_empty(&ioc->reply_queue_list))
1744                 return;
1745
1746         list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
1747                 list_del(&reply_q->list);
1748                 irq_set_affinity_hint(reply_q->vector, NULL);
1749                 free_cpumask_var(reply_q->affinity_hint);
1750                 synchronize_irq(reply_q->vector);
1751                 free_irq(reply_q->vector, reply_q);
1752                 kfree(reply_q);
1753         }
1754 }
1755
1756 /**
1757  * _base_request_irq - request irq
1758  * @ioc: per adapter object
1759  * @index: msix index into vector table
1760  * @vector: irq vector
1761  *
1762  * Inserting respective reply_queue into the list.
1763  */
1764 static int
1765 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index, u32 vector)
1766 {
1767         struct adapter_reply_queue *reply_q;
1768         int r;
1769
1770         reply_q =  kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL);
1771         if (!reply_q) {
1772                 pr_err(MPT3SAS_FMT "unable to allocate memory %d!\n",
1773                     ioc->name, (int)sizeof(struct adapter_reply_queue));
1774                 return -ENOMEM;
1775         }
1776         reply_q->ioc = ioc;
1777         reply_q->msix_index = index;
1778         reply_q->vector = vector;
1779
1780         if (!alloc_cpumask_var(&reply_q->affinity_hint, GFP_KERNEL))
1781                 return -ENOMEM;
1782         cpumask_clear(reply_q->affinity_hint);
1783
1784         atomic_set(&reply_q->busy, 0);
1785         if (ioc->msix_enable)
1786                 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
1787                     driver_name, ioc->id, index);
1788         else
1789                 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
1790                     driver_name, ioc->id);
1791         r = request_irq(vector, _base_interrupt, IRQF_SHARED, reply_q->name,
1792             reply_q);
1793         if (r) {
1794                 pr_err(MPT3SAS_FMT "unable to allocate interrupt %d!\n",
1795                     reply_q->name, vector);
1796                 kfree(reply_q);
1797                 return -EBUSY;
1798         }
1799
1800         INIT_LIST_HEAD(&reply_q->list);
1801         list_add_tail(&reply_q->list, &ioc->reply_queue_list);
1802         return 0;
1803 }
1804
1805 /**
1806  * _base_assign_reply_queues - assigning msix index for each cpu
1807  * @ioc: per adapter object
1808  *
1809  * The enduser would need to set the affinity via /proc/irq/#/smp_affinity
1810  *
1811  * It would nice if we could call irq_set_affinity, however it is not
1812  * an exported symbol
1813  */
1814 static void
1815 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
1816 {
1817         unsigned int cpu, nr_cpus, nr_msix, index = 0;
1818         struct adapter_reply_queue *reply_q;
1819
1820         if (!_base_is_controller_msix_enabled(ioc))
1821                 return;
1822
1823         memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
1824
1825         nr_cpus = num_online_cpus();
1826         nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,
1827                                                ioc->facts.MaxMSIxVectors);
1828         if (!nr_msix)
1829                 return;
1830
1831         cpu = cpumask_first(cpu_online_mask);
1832
1833         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
1834
1835                 unsigned int i, group = nr_cpus / nr_msix;
1836
1837                 if (cpu >= nr_cpus)
1838                         break;
1839
1840                 if (index < nr_cpus % nr_msix)
1841                         group++;
1842
1843                 for (i = 0 ; i < group ; i++) {
1844                         ioc->cpu_msix_table[cpu] = index;
1845                         cpumask_or(reply_q->affinity_hint,
1846                                    reply_q->affinity_hint, get_cpu_mask(cpu));
1847                         cpu = cpumask_next(cpu, cpu_online_mask);
1848                 }
1849
1850                 if (irq_set_affinity_hint(reply_q->vector,
1851                                            reply_q->affinity_hint))
1852                         dinitprintk(ioc, pr_info(MPT3SAS_FMT
1853                             "error setting affinity hint for irq vector %d\n",
1854                             ioc->name, reply_q->vector));
1855                 index++;
1856         }
1857 }
1858
1859 /**
1860  * _base_disable_msix - disables msix
1861  * @ioc: per adapter object
1862  *
1863  */
1864 static void
1865 _base_disable_msix(struct MPT3SAS_ADAPTER *ioc)
1866 {
1867         if (!ioc->msix_enable)
1868                 return;
1869         pci_disable_msix(ioc->pdev);
1870         ioc->msix_enable = 0;
1871 }
1872
1873 /**
1874  * _base_enable_msix - enables msix, failback to io_apic
1875  * @ioc: per adapter object
1876  *
1877  */
1878 static int
1879 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
1880 {
1881         struct msix_entry *entries, *a;
1882         int r;
1883         int i;
1884         u8 try_msix = 0;
1885
1886         if (msix_disable == -1 || msix_disable == 0)
1887                 try_msix = 1;
1888
1889         if (!try_msix)
1890                 goto try_ioapic;
1891
1892         if (_base_check_enable_msix(ioc) != 0)
1893                 goto try_ioapic;
1894
1895         ioc->reply_queue_count = min_t(int, ioc->cpu_count,
1896             ioc->msix_vector_count);
1897
1898         printk(MPT3SAS_FMT "MSI-X vectors supported: %d, no of cores"
1899           ": %d, max_msix_vectors: %d\n", ioc->name, ioc->msix_vector_count,
1900           ioc->cpu_count, max_msix_vectors);
1901
1902         if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
1903                 max_msix_vectors = 8;
1904
1905         if (max_msix_vectors > 0) {
1906                 ioc->reply_queue_count = min_t(int, max_msix_vectors,
1907                         ioc->reply_queue_count);
1908                 ioc->msix_vector_count = ioc->reply_queue_count;
1909         } else if (max_msix_vectors == 0)
1910                 goto try_ioapic;
1911
1912         entries = kcalloc(ioc->reply_queue_count, sizeof(struct msix_entry),
1913             GFP_KERNEL);
1914         if (!entries) {
1915                 dfailprintk(ioc, pr_info(MPT3SAS_FMT
1916                         "kcalloc failed @ at %s:%d/%s() !!!\n",
1917                         ioc->name, __FILE__, __LINE__, __func__));
1918                 goto try_ioapic;
1919         }
1920
1921         for (i = 0, a = entries; i < ioc->reply_queue_count; i++, a++)
1922                 a->entry = i;
1923
1924         r = pci_enable_msix_exact(ioc->pdev, entries, ioc->reply_queue_count);
1925         if (r) {
1926                 dfailprintk(ioc, pr_info(MPT3SAS_FMT
1927                         "pci_enable_msix_exact failed (r=%d) !!!\n",
1928                         ioc->name, r));
1929                 kfree(entries);
1930                 goto try_ioapic;
1931         }
1932
1933         ioc->msix_enable = 1;
1934         for (i = 0, a = entries; i < ioc->reply_queue_count; i++, a++) {
1935                 r = _base_request_irq(ioc, i, a->vector);
1936                 if (r) {
1937                         _base_free_irq(ioc);
1938                         _base_disable_msix(ioc);
1939                         kfree(entries);
1940                         goto try_ioapic;
1941                 }
1942         }
1943
1944         kfree(entries);
1945         return 0;
1946
1947 /* failback to io_apic interrupt routing */
1948  try_ioapic:
1949
1950         ioc->reply_queue_count = 1;
1951         r = _base_request_irq(ioc, 0, ioc->pdev->irq);
1952
1953         return r;
1954 }
1955
1956 /**
1957  * mpt3sas_base_unmap_resources - free controller resources
1958  * @ioc: per adapter object
1959  */
1960 void
1961 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
1962 {
1963         struct pci_dev *pdev = ioc->pdev;
1964
1965         dexitprintk(ioc, printk(MPT3SAS_FMT "%s\n",
1966                 ioc->name, __func__));
1967
1968         _base_free_irq(ioc);
1969         _base_disable_msix(ioc);
1970
1971         if (ioc->msix96_vector)
1972                 kfree(ioc->replyPostRegisterIndex);
1973
1974         if (ioc->chip_phys) {
1975                 iounmap(ioc->chip);
1976                 ioc->chip_phys = 0;
1977         }
1978
1979         if (pci_is_enabled(pdev)) {
1980                 pci_release_selected_regions(ioc->pdev, ioc->bars);
1981                 pci_disable_pcie_error_reporting(pdev);
1982                 pci_disable_device(pdev);
1983         }
1984 }
1985
1986 /**
1987  * mpt3sas_base_map_resources - map in controller resources (io/irq/memap)
1988  * @ioc: per adapter object
1989  *
1990  * Returns 0 for success, non-zero for failure.
1991  */
1992 int
1993 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
1994 {
1995         struct pci_dev *pdev = ioc->pdev;
1996         u32 memap_sz;
1997         u32 pio_sz;
1998         int i, r = 0;
1999         u64 pio_chip = 0;
2000         u64 chip_phys = 0;
2001         struct adapter_reply_queue *reply_q;
2002
2003         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n",
2004             ioc->name, __func__));
2005
2006         ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
2007         if (pci_enable_device_mem(pdev)) {
2008                 pr_warn(MPT3SAS_FMT "pci_enable_device_mem: failed\n",
2009                         ioc->name);
2010                 ioc->bars = 0;
2011                 return -ENODEV;
2012         }
2013
2014
2015         if (pci_request_selected_regions(pdev, ioc->bars,
2016             driver_name)) {
2017                 pr_warn(MPT3SAS_FMT "pci_request_selected_regions: failed\n",
2018                         ioc->name);
2019                 ioc->bars = 0;
2020                 r = -ENODEV;
2021                 goto out_fail;
2022         }
2023
2024 /* AER (Advanced Error Reporting) hooks */
2025         pci_enable_pcie_error_reporting(pdev);
2026
2027         pci_set_master(pdev);
2028
2029
2030         if (_base_config_dma_addressing(ioc, pdev) != 0) {
2031                 pr_warn(MPT3SAS_FMT "no suitable DMA mask for %s\n",
2032                     ioc->name, pci_name(pdev));
2033                 r = -ENODEV;
2034                 goto out_fail;
2035         }
2036
2037         for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) &&
2038              (!memap_sz || !pio_sz); i++) {
2039                 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
2040                         if (pio_sz)
2041                                 continue;
2042                         pio_chip = (u64)pci_resource_start(pdev, i);
2043                         pio_sz = pci_resource_len(pdev, i);
2044                 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
2045                         if (memap_sz)
2046                                 continue;
2047                         ioc->chip_phys = pci_resource_start(pdev, i);
2048                         chip_phys = (u64)ioc->chip_phys;
2049                         memap_sz = pci_resource_len(pdev, i);
2050                         ioc->chip = ioremap(ioc->chip_phys, memap_sz);
2051                 }
2052         }
2053
2054         if (ioc->chip == NULL) {
2055                 pr_err(MPT3SAS_FMT "unable to map adapter memory! "
2056                         " or resource not found\n", ioc->name);
2057                 r = -EINVAL;
2058                 goto out_fail;
2059         }
2060
2061         _base_mask_interrupts(ioc);
2062
2063         r = _base_get_ioc_facts(ioc, CAN_SLEEP);
2064         if (r)
2065                 goto out_fail;
2066
2067         if (!ioc->rdpq_array_enable_assigned) {
2068                 ioc->rdpq_array_enable = ioc->rdpq_array_capable;
2069                 ioc->rdpq_array_enable_assigned = 1;
2070         }
2071
2072         r = _base_enable_msix(ioc);
2073         if (r)
2074                 goto out_fail;
2075
2076         /* Use the Combined reply queue feature only for SAS3 C0 & higher
2077          * revision HBAs and also only when reply queue count is greater than 8
2078          */
2079         if (ioc->msix96_vector && ioc->reply_queue_count > 8) {
2080                 /* Determine the Supplemental Reply Post Host Index Registers
2081                  * Addresse. Supplemental Reply Post Host Index Registers
2082                  * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
2083                  * each register is at offset bytes of
2084                  * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
2085                  */
2086                 ioc->replyPostRegisterIndex = kcalloc(
2087                      MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT,
2088                      sizeof(resource_size_t *), GFP_KERNEL);
2089                 if (!ioc->replyPostRegisterIndex) {
2090                         dfailprintk(ioc, printk(MPT3SAS_FMT
2091                         "allocation for reply Post Register Index failed!!!\n",
2092                                                                    ioc->name));
2093                         r = -ENOMEM;
2094                         goto out_fail;
2095                 }
2096
2097                 for (i = 0; i < MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT; i++) {
2098                         ioc->replyPostRegisterIndex[i] = (resource_size_t *)
2099                              ((u8 *)&ioc->chip->Doorbell +
2100                              MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
2101                              (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
2102                 }
2103         } else
2104                 ioc->msix96_vector = 0;
2105
2106         list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
2107                 pr_info(MPT3SAS_FMT "%s: IRQ %d\n",
2108                     reply_q->name,  ((ioc->msix_enable) ? "PCI-MSI-X enabled" :
2109                     "IO-APIC enabled"), reply_q->vector);
2110
2111         pr_info(MPT3SAS_FMT "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
2112             ioc->name, (unsigned long long)chip_phys, ioc->chip, memap_sz);
2113         pr_info(MPT3SAS_FMT "ioport(0x%016llx), size(%d)\n",
2114             ioc->name, (unsigned long long)pio_chip, pio_sz);
2115
2116         /* Save PCI configuration state for recovery from PCI AER/EEH errors */
2117         pci_save_state(pdev);
2118         return 0;
2119
2120  out_fail:
2121         mpt3sas_base_unmap_resources(ioc);
2122         return r;
2123 }
2124
2125 /**
2126  * mpt3sas_base_get_msg_frame - obtain request mf pointer
2127  * @ioc: per adapter object
2128  * @smid: system request message index(smid zero is invalid)
2129  *
2130  * Returns virt pointer to message frame.
2131  */
2132 void *
2133 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)
2134 {
2135         return (void *)(ioc->request + (smid * ioc->request_sz));
2136 }
2137
2138 /**
2139  * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr
2140  * @ioc: per adapter object
2141  * @smid: system request message index
2142  *
2143  * Returns virt pointer to sense buffer.
2144  */
2145 void *
2146 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)
2147 {
2148         return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));
2149 }
2150
2151 /**
2152  * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr
2153  * @ioc: per adapter object
2154  * @smid: system request message index
2155  *
2156  * Returns phys pointer to the low 32bit address of the sense buffer.
2157  */
2158 __le32
2159 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
2160 {
2161         return cpu_to_le32(ioc->sense_dma + ((smid - 1) *
2162             SCSI_SENSE_BUFFERSIZE));
2163 }
2164
2165 /**
2166  * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address
2167  * @ioc: per adapter object
2168  * @phys_addr: lower 32 physical addr of the reply
2169  *
2170  * Converts 32bit lower physical addr into a virt address.
2171  */
2172 void *
2173 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
2174 {
2175         if (!phys_addr)
2176                 return NULL;
2177         return ioc->reply + (phys_addr - (u32)ioc->reply_dma);
2178 }
2179
2180 /**
2181  * mpt3sas_base_get_smid - obtain a free smid from internal queue
2182  * @ioc: per adapter object
2183  * @cb_idx: callback index
2184  *
2185  * Returns smid (zero is invalid)
2186  */
2187 u16
2188 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
2189 {
2190         unsigned long flags;
2191         struct request_tracker *request;
2192         u16 smid;
2193
2194         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
2195         if (list_empty(&ioc->internal_free_list)) {
2196                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2197                 pr_err(MPT3SAS_FMT "%s: smid not available\n",
2198                     ioc->name, __func__);
2199                 return 0;
2200         }
2201
2202         request = list_entry(ioc->internal_free_list.next,
2203             struct request_tracker, tracker_list);
2204         request->cb_idx = cb_idx;
2205         smid = request->smid;
2206         list_del(&request->tracker_list);
2207         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2208         return smid;
2209 }
2210
2211 /**
2212  * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue
2213  * @ioc: per adapter object
2214  * @cb_idx: callback index
2215  * @scmd: pointer to scsi command object
2216  *
2217  * Returns smid (zero is invalid)
2218  */
2219 u16
2220 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
2221         struct scsi_cmnd *scmd)
2222 {
2223         unsigned long flags;
2224         struct scsiio_tracker *request;
2225         u16 smid;
2226
2227         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
2228         if (list_empty(&ioc->free_list)) {
2229                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2230                 pr_err(MPT3SAS_FMT "%s: smid not available\n",
2231                     ioc->name, __func__);
2232                 return 0;
2233         }
2234
2235         request = list_entry(ioc->free_list.next,
2236             struct scsiio_tracker, tracker_list);
2237         request->scmd = scmd;
2238         request->cb_idx = cb_idx;
2239         smid = request->smid;
2240         list_del(&request->tracker_list);
2241         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2242         return smid;
2243 }
2244
2245 /**
2246  * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue
2247  * @ioc: per adapter object
2248  * @cb_idx: callback index
2249  *
2250  * Returns smid (zero is invalid)
2251  */
2252 u16
2253 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
2254 {
2255         unsigned long flags;
2256         struct request_tracker *request;
2257         u16 smid;
2258
2259         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
2260         if (list_empty(&ioc->hpr_free_list)) {
2261                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2262                 return 0;
2263         }
2264
2265         request = list_entry(ioc->hpr_free_list.next,
2266             struct request_tracker, tracker_list);
2267         request->cb_idx = cb_idx;
2268         smid = request->smid;
2269         list_del(&request->tracker_list);
2270         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2271         return smid;
2272 }
2273
2274 /**
2275  * mpt3sas_base_free_smid - put smid back on free_list
2276  * @ioc: per adapter object
2277  * @smid: system request message index
2278  *
2279  * Return nothing.
2280  */
2281 void
2282 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
2283 {
2284         unsigned long flags;
2285         int i;
2286         struct chain_tracker *chain_req, *next;
2287
2288         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
2289         if (smid < ioc->hi_priority_smid) {
2290                 /* scsiio queue */
2291                 i = smid - 1;
2292                 if (!list_empty(&ioc->scsi_lookup[i].chain_list)) {
2293                         list_for_each_entry_safe(chain_req, next,
2294                             &ioc->scsi_lookup[i].chain_list, tracker_list) {
2295                                 list_del_init(&chain_req->tracker_list);
2296                                 list_add(&chain_req->tracker_list,
2297                                     &ioc->free_chain_list);
2298                         }
2299                 }
2300                 ioc->scsi_lookup[i].cb_idx = 0xFF;
2301                 ioc->scsi_lookup[i].scmd = NULL;
2302                 list_add(&ioc->scsi_lookup[i].tracker_list, &ioc->free_list);
2303                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2304
2305                 /*
2306                  * See _wait_for_commands_to_complete() call with regards
2307                  * to this code.
2308                  */
2309                 if (ioc->shost_recovery && ioc->pending_io_count) {
2310                         if (ioc->pending_io_count == 1)
2311                                 wake_up(&ioc->reset_wq);
2312                         ioc->pending_io_count--;
2313                 }
2314                 return;
2315         } else if (smid < ioc->internal_smid) {
2316                 /* hi-priority */
2317                 i = smid - ioc->hi_priority_smid;
2318                 ioc->hpr_lookup[i].cb_idx = 0xFF;
2319                 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);
2320         } else if (smid <= ioc->hba_queue_depth) {
2321                 /* internal queue */
2322                 i = smid - ioc->internal_smid;
2323                 ioc->internal_lookup[i].cb_idx = 0xFF;
2324                 list_add(&ioc->internal_lookup[i].tracker_list,
2325                     &ioc->internal_free_list);
2326         }
2327         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
2328 }
2329
2330 /**
2331  * _base_writeq - 64 bit write to MMIO
2332  * @ioc: per adapter object
2333  * @b: data payload
2334  * @addr: address in MMIO space
2335  * @writeq_lock: spin lock
2336  *
2337  * Glue for handling an atomic 64 bit word to MMIO. This special handling takes
2338  * care of 32 bit environment where its not quarenteed to send the entire word
2339  * in one transfer.
2340  */
2341 #if defined(writeq) && defined(CONFIG_64BIT)
2342 static inline void
2343 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
2344 {
2345         writeq(cpu_to_le64(b), addr);
2346 }
2347 #else
2348 static inline void
2349 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
2350 {
2351         unsigned long flags;
2352         __u64 data_out = cpu_to_le64(b);
2353
2354         spin_lock_irqsave(writeq_lock, flags);
2355         writel((u32)(data_out), addr);
2356         writel((u32)(data_out >> 32), (addr + 4));
2357         spin_unlock_irqrestore(writeq_lock, flags);
2358 }
2359 #endif
2360
2361 static inline u8
2362 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc)
2363 {
2364         return ioc->cpu_msix_table[raw_smp_processor_id()];
2365 }
2366
2367 /**
2368  * mpt3sas_base_put_smid_scsi_io - send SCSI_IO request to firmware
2369  * @ioc: per adapter object
2370  * @smid: system request message index
2371  * @handle: device handle
2372  *
2373  * Return nothing.
2374  */
2375 void
2376 mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
2377 {
2378         Mpi2RequestDescriptorUnion_t descriptor;
2379         u64 *request = (u64 *)&descriptor;
2380
2381
2382         descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
2383         descriptor.SCSIIO.MSIxIndex =  _base_get_msix_index(ioc);
2384         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
2385         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
2386         descriptor.SCSIIO.LMID = 0;
2387         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
2388             &ioc->scsi_lookup_lock);
2389 }
2390
2391 /**
2392  * mpt3sas_base_put_smid_fast_path - send fast path request to firmware
2393  * @ioc: per adapter object
2394  * @smid: system request message index
2395  * @handle: device handle
2396  *
2397  * Return nothing.
2398  */
2399 void
2400 mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
2401         u16 handle)
2402 {
2403         Mpi2RequestDescriptorUnion_t descriptor;
2404         u64 *request = (u64 *)&descriptor;
2405
2406         descriptor.SCSIIO.RequestFlags =
2407             MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
2408         descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc);
2409         descriptor.SCSIIO.SMID = cpu_to_le16(smid);
2410         descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
2411         descriptor.SCSIIO.LMID = 0;
2412         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
2413             &ioc->scsi_lookup_lock);
2414 }
2415
2416 /**
2417  * mpt3sas_base_put_smid_hi_priority - send Task Managment request to firmware
2418  * @ioc: per adapter object
2419  * @smid: system request message index
2420  *
2421  * Return nothing.
2422  */
2423 void
2424 mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid)
2425 {
2426         Mpi2RequestDescriptorUnion_t descriptor;
2427         u64 *request = (u64 *)&descriptor;
2428
2429         descriptor.HighPriority.RequestFlags =
2430             MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
2431         descriptor.HighPriority.MSIxIndex =  0;
2432         descriptor.HighPriority.SMID = cpu_to_le16(smid);
2433         descriptor.HighPriority.LMID = 0;
2434         descriptor.HighPriority.Reserved1 = 0;
2435         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
2436             &ioc->scsi_lookup_lock);
2437 }
2438
2439 /**
2440  * mpt3sas_base_put_smid_default - Default, primarily used for config pages
2441  * @ioc: per adapter object
2442  * @smid: system request message index
2443  *
2444  * Return nothing.
2445  */
2446 void
2447 mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
2448 {
2449         Mpi2RequestDescriptorUnion_t descriptor;
2450         u64 *request = (u64 *)&descriptor;
2451
2452         descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2453         descriptor.Default.MSIxIndex =  _base_get_msix_index(ioc);
2454         descriptor.Default.SMID = cpu_to_le16(smid);
2455         descriptor.Default.LMID = 0;
2456         descriptor.Default.DescriptorTypeDependent = 0;
2457         _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
2458             &ioc->scsi_lookup_lock);
2459 }
2460
2461 /**
2462  * _base_display_intel_branding - Display branding string
2463  * @ioc: per adapter object
2464  *
2465  * Return nothing.
2466  */
2467 static void
2468 _base_display_intel_branding(struct MPT3SAS_ADAPTER *ioc)
2469 {
2470         if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
2471                 return;
2472
2473         switch (ioc->pdev->device) {
2474         case MPI25_MFGPAGE_DEVID_SAS3008:
2475                 switch (ioc->pdev->subsystem_device) {
2476                 case MPT3SAS_INTEL_RMS3JC080_SSDID:
2477                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2478                                 MPT3SAS_INTEL_RMS3JC080_BRANDING);
2479                         break;
2480
2481                 case MPT3SAS_INTEL_RS3GC008_SSDID:
2482                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2483                                 MPT3SAS_INTEL_RS3GC008_BRANDING);
2484                         break;
2485                 case MPT3SAS_INTEL_RS3FC044_SSDID:
2486                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2487                                 MPT3SAS_INTEL_RS3FC044_BRANDING);
2488                         break;
2489                 case MPT3SAS_INTEL_RS3UC080_SSDID:
2490                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2491                                 MPT3SAS_INTEL_RS3UC080_BRANDING);
2492                         break;
2493                 default:
2494                         pr_info(MPT3SAS_FMT
2495                                 "Intel(R) Controller: Subsystem ID: 0x%X\n",
2496                                 ioc->name, ioc->pdev->subsystem_device);
2497                         break;
2498                 }
2499                 break;
2500         default:
2501                 pr_info(MPT3SAS_FMT
2502                         "Intel(R) Controller: Subsystem ID: 0x%X\n",
2503                         ioc->name, ioc->pdev->subsystem_device);
2504                 break;
2505         }
2506 }
2507
2508
2509
2510 /**
2511  * _base_display_dell_branding - Display branding string
2512  * @ioc: per adapter object
2513  *
2514  * Return nothing.
2515  */
2516 static void
2517 _base_display_dell_branding(struct MPT3SAS_ADAPTER *ioc)
2518 {
2519         if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_DELL)
2520                 return;
2521
2522         switch (ioc->pdev->device) {
2523         case MPI25_MFGPAGE_DEVID_SAS3008:
2524                 switch (ioc->pdev->subsystem_device) {
2525                 case MPT3SAS_DELL_12G_HBA_SSDID:
2526                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2527                                 MPT3SAS_DELL_12G_HBA_BRANDING);
2528                         break;
2529                 default:
2530                         pr_info(MPT3SAS_FMT
2531                            "Dell 12Gbps HBA: Subsystem ID: 0x%X\n", ioc->name,
2532                            ioc->pdev->subsystem_device);
2533                         break;
2534                 }
2535                 break;
2536         default:
2537                 pr_info(MPT3SAS_FMT
2538                         "Dell 12Gbps HBA: Subsystem ID: 0x%X\n", ioc->name,
2539                         ioc->pdev->subsystem_device);
2540                 break;
2541         }
2542 }
2543
2544 /**
2545  * _base_display_cisco_branding - Display branding string
2546  * @ioc: per adapter object
2547  *
2548  * Return nothing.
2549  */
2550 static void
2551 _base_display_cisco_branding(struct MPT3SAS_ADAPTER *ioc)
2552 {
2553         if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_CISCO)
2554                 return;
2555
2556         switch (ioc->pdev->device) {
2557         case MPI25_MFGPAGE_DEVID_SAS3008:
2558                 switch (ioc->pdev->subsystem_device) {
2559                 case MPT3SAS_CISCO_12G_8E_HBA_SSDID:
2560                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2561                                 MPT3SAS_CISCO_12G_8E_HBA_BRANDING);
2562                         break;
2563                 case MPT3SAS_CISCO_12G_8I_HBA_SSDID:
2564                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2565                                 MPT3SAS_CISCO_12G_8I_HBA_BRANDING);
2566                         break;
2567                 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
2568                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2569                                 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
2570                         break;
2571                 default:
2572                         pr_info(MPT3SAS_FMT
2573                           "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
2574                           ioc->name, ioc->pdev->subsystem_device);
2575                         break;
2576                 }
2577                 break;
2578         case MPI25_MFGPAGE_DEVID_SAS3108_1:
2579                 switch (ioc->pdev->subsystem_device) {
2580                 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
2581                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2582                         MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
2583                         break;
2584                 case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID:
2585                         pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2586                         MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING);
2587                         break;
2588                 default:
2589                         pr_info(MPT3SAS_FMT
2590                          "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
2591                          ioc->name, ioc->pdev->subsystem_device);
2592                         break;
2593                 }
2594                 break;
2595         default:
2596                  pr_info(MPT3SAS_FMT
2597                         "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
2598                         ioc->name, ioc->pdev->subsystem_device);
2599                 break;
2600         }
2601 }
2602
2603 /**
2604  * _base_display_ioc_capabilities - Disply IOC's capabilities.
2605  * @ioc: per adapter object
2606  *
2607  * Return nothing.
2608  */
2609 static void
2610 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
2611 {
2612         int i = 0;
2613         char desc[16];
2614         u32 iounit_pg1_flags;
2615         u32 bios_version;
2616
2617         bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
2618         strncpy(desc, ioc->manu_pg0.ChipName, 16);
2619         pr_info(MPT3SAS_FMT "%s: FWVersion(%02d.%02d.%02d.%02d), "\
2620            "ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
2621             ioc->name, desc,
2622            (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
2623            (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
2624            (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
2625            ioc->facts.FWVersion.Word & 0x000000FF,
2626            ioc->pdev->revision,
2627            (bios_version & 0xFF000000) >> 24,
2628            (bios_version & 0x00FF0000) >> 16,
2629            (bios_version & 0x0000FF00) >> 8,
2630             bios_version & 0x000000FF);
2631
2632         _base_display_intel_branding(ioc);
2633         _base_display_dell_branding(ioc);
2634         _base_display_cisco_branding(ioc);
2635
2636         pr_info(MPT3SAS_FMT "Protocol=(", ioc->name);
2637
2638         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {
2639                 pr_info("Initiator");
2640                 i++;
2641         }
2642
2643         if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {
2644                 pr_info("%sTarget", i ? "," : "");
2645                 i++;
2646         }
2647
2648         i = 0;
2649         pr_info("), ");
2650         pr_info("Capabilities=(");
2651
2652         if (ioc->facts.IOCCapabilities &
2653                     MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) {
2654                         pr_info("Raid");
2655                         i++;
2656         }
2657
2658         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {
2659                 pr_info("%sTLR", i ? "," : "");
2660                 i++;
2661         }
2662
2663         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {
2664                 pr_info("%sMulticast", i ? "," : "");
2665                 i++;
2666         }
2667
2668         if (ioc->facts.IOCCapabilities &
2669             MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) {
2670                 pr_info("%sBIDI Target", i ? "," : "");
2671                 i++;
2672         }
2673
2674         if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {
2675                 pr_info("%sEEDP", i ? "," : "");
2676                 i++;
2677         }
2678
2679         if (ioc->facts.IOCCapabilities &
2680             MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) {
2681                 pr_info("%sSnapshot Buffer", i ? "," : "");
2682                 i++;
2683         }
2684
2685         if (ioc->facts.IOCCapabilities &
2686             MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) {
2687                 pr_info("%sDiag Trace Buffer", i ? "," : "");
2688                 i++;
2689         }
2690
2691         if (ioc->facts.IOCCapabilities &
2692             MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) {
2693                 pr_info("%sDiag Extended Buffer", i ? "," : "");
2694                 i++;
2695         }
2696
2697         if (ioc->facts.IOCCapabilities &
2698             MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) {
2699                 pr_info("%sTask Set Full", i ? "," : "");
2700                 i++;
2701         }
2702
2703         iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
2704         if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) {
2705                 pr_info("%sNCQ", i ? "," : "");
2706                 i++;
2707         }
2708
2709         pr_info(")\n");
2710 }
2711
2712 /**
2713  * mpt3sas_base_update_missing_delay - change the missing delay timers
2714  * @ioc: per adapter object
2715  * @device_missing_delay: amount of time till device is reported missing
2716  * @io_missing_delay: interval IO is returned when there is a missing device
2717  *
2718  * Return nothing.
2719  *
2720  * Passed on the command line, this function will modify the device missing
2721  * delay, as well as the io missing delay. This should be called at driver
2722  * load time.
2723  */
2724 void
2725 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
2726         u16 device_missing_delay, u8 io_missing_delay)
2727 {
2728         u16 dmd, dmd_new, dmd_orignal;
2729         u8 io_missing_delay_original;
2730         u16 sz;
2731         Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
2732         Mpi2ConfigReply_t mpi_reply;
2733         u8 num_phys = 0;
2734         u16 ioc_status;
2735
2736         mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
2737         if (!num_phys)
2738                 return;
2739
2740         sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
2741             sizeof(Mpi2SasIOUnit1PhyData_t));
2742         sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
2743         if (!sas_iounit_pg1) {
2744                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
2745                     ioc->name, __FILE__, __LINE__, __func__);
2746                 goto out;
2747         }
2748         if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
2749             sas_iounit_pg1, sz))) {
2750                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
2751                     ioc->name, __FILE__, __LINE__, __func__);
2752                 goto out;
2753         }
2754         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
2755             MPI2_IOCSTATUS_MASK;
2756         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
2757                 pr_err(MPT3SAS_FMT "failure at %s:%d/%s()!\n",
2758                     ioc->name, __FILE__, __LINE__, __func__);
2759                 goto out;
2760         }
2761
2762         /* device missing delay */
2763         dmd = sas_iounit_pg1->ReportDeviceMissingDelay;
2764         if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
2765                 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
2766         else
2767                 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
2768         dmd_orignal = dmd;
2769         if (device_missing_delay > 0x7F) {
2770                 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 :
2771                     device_missing_delay;
2772                 dmd = dmd / 16;
2773                 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16;
2774         } else
2775                 dmd = device_missing_delay;
2776         sas_iounit_pg1->ReportDeviceMissingDelay = dmd;
2777
2778         /* io missing delay */
2779         io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay;
2780         sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay;
2781
2782         if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,
2783             sz)) {
2784                 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
2785                         dmd_new = (dmd &
2786                             MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
2787                 else
2788                         dmd_new =
2789                     dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
2790                 pr_info(MPT3SAS_FMT "device_missing_delay: old(%d), new(%d)\n",
2791                         ioc->name, dmd_orignal, dmd_new);
2792                 pr_info(MPT3SAS_FMT "ioc_missing_delay: old(%d), new(%d)\n",
2793                         ioc->name, io_missing_delay_original,
2794                     io_missing_delay);
2795                 ioc->device_missing_delay = dmd_new;
2796                 ioc->io_missing_delay = io_missing_delay;
2797         }
2798
2799 out:
2800         kfree(sas_iounit_pg1);
2801 }
2802 /**
2803  * _base_static_config_pages - static start of day config pages
2804  * @ioc: per adapter object
2805  *
2806  * Return nothing.
2807  */
2808 static void
2809 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
2810 {
2811         Mpi2ConfigReply_t mpi_reply;
2812         u32 iounit_pg1_flags;
2813
2814         mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0);
2815         if (ioc->ir_firmware)
2816                 mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
2817                     &ioc->manu_pg10);
2818
2819         /*
2820          * Ensure correct T10 PI operation if vendor left EEDPTagMode
2821          * flag unset in NVDATA.
2822          */
2823         mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11);
2824         if (ioc->manu_pg11.EEDPTagMode == 0) {
2825                 pr_err("%s: overriding NVDATA EEDPTagMode setting\n",
2826                     ioc->name);
2827                 ioc->manu_pg11.EEDPTagMode &= ~0x3;
2828                 ioc->manu_pg11.EEDPTagMode |= 0x1;
2829                 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
2830                     &ioc->manu_pg11);
2831         }
2832
2833         mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
2834         mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
2835         mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);
2836         mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);
2837         mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
2838         mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
2839         _base_display_ioc_capabilities(ioc);
2840
2841         /*
2842          * Enable task_set_full handling in iounit_pg1 when the
2843          * facts capabilities indicate that its supported.
2844          */
2845         iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
2846         if ((ioc->facts.IOCCapabilities &
2847             MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING))
2848                 iounit_pg1_flags &=
2849                     ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
2850         else
2851                 iounit_pg1_flags |=
2852                     MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
2853         ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);
2854         mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
2855
2856         if (ioc->iounit_pg8.NumSensors)
2857                 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
2858 }
2859
2860 /**
2861  * _base_release_memory_pools - release memory
2862  * @ioc: per adapter object
2863  *
2864  * Free memory allocated from _base_allocate_memory_pools.
2865  *
2866  * Return nothing.
2867  */
2868 static void
2869 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
2870 {
2871         int i = 0;
2872         struct reply_post_struct *rps;
2873
2874         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2875             __func__));
2876
2877         if (ioc->request) {
2878                 pci_free_consistent(ioc->pdev, ioc->request_dma_sz,
2879                     ioc->request,  ioc->request_dma);
2880                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
2881                         "request_pool(0x%p): free\n",
2882                         ioc->name, ioc->request));
2883                 ioc->request = NULL;
2884         }
2885
2886         if (ioc->sense) {
2887                 pci_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
2888                 if (ioc->sense_dma_pool)
2889                         pci_pool_destroy(ioc->sense_dma_pool);
2890                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
2891                         "sense_pool(0x%p): free\n",
2892                         ioc->name, ioc->sense));
2893                 ioc->sense = NULL;
2894         }
2895
2896         if (ioc->reply) {
2897                 pci_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
2898                 if (ioc->reply_dma_pool)
2899                         pci_pool_destroy(ioc->reply_dma_pool);
2900                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
2901                         "reply_pool(0x%p): free\n",
2902                         ioc->name, ioc->reply));
2903                 ioc->reply = NULL;
2904         }
2905
2906         if (ioc->reply_free) {
2907                 pci_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
2908                     ioc->reply_free_dma);
2909                 if (ioc->reply_free_dma_pool)
2910                         pci_pool_destroy(ioc->reply_free_dma_pool);
2911                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
2912                         "reply_free_pool(0x%p): free\n",
2913                         ioc->name, ioc->reply_free));
2914                 ioc->reply_free = NULL;
2915         }
2916
2917         if (ioc->reply_post) {
2918                 do {
2919                         rps = &ioc->reply_post[i];
2920                         if (rps->reply_post_free) {
2921                                 pci_pool_free(
2922                                     ioc->reply_post_free_dma_pool,
2923                                     rps->reply_post_free,
2924                                     rps->reply_post_free_dma);
2925                                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
2926                                     "reply_post_free_pool(0x%p): free\n",
2927                                     ioc->name, rps->reply_post_free));
2928                                 rps->reply_post_free = NULL;
2929                         }
2930                 } while (ioc->rdpq_array_enable &&
2931                            (++i < ioc->reply_queue_count));
2932
2933                 if (ioc->reply_post_free_dma_pool)
2934                         pci_pool_destroy(ioc->reply_post_free_dma_pool);
2935                 kfree(ioc->reply_post);
2936         }
2937
2938         if (ioc->config_page) {
2939                 dexitprintk(ioc, pr_info(MPT3SAS_FMT
2940                     "config_page(0x%p): free\n", ioc->name,
2941                     ioc->config_page));
2942                 pci_free_consistent(ioc->pdev, ioc->config_page_sz,
2943                     ioc->config_page, ioc->config_page_dma);
2944         }
2945
2946         if (ioc->scsi_lookup) {
2947                 free_pages((ulong)ioc->scsi_lookup, ioc->scsi_lookup_pages);
2948                 ioc->scsi_lookup = NULL;
2949         }
2950         kfree(ioc->hpr_lookup);
2951         kfree(ioc->internal_lookup);
2952         if (ioc->chain_lookup) {
2953                 for (i = 0; i < ioc->chain_depth; i++) {
2954                         if (ioc->chain_lookup[i].chain_buffer)
2955                                 pci_pool_free(ioc->chain_dma_pool,
2956                                     ioc->chain_lookup[i].chain_buffer,
2957                                     ioc->chain_lookup[i].chain_buffer_dma);
2958                 }
2959                 if (ioc->chain_dma_pool)
2960                         pci_pool_destroy(ioc->chain_dma_pool);
2961                 free_pages((ulong)ioc->chain_lookup, ioc->chain_pages);
2962                 ioc->chain_lookup = NULL;
2963         }
2964 }
2965
2966 /**
2967  * _base_allocate_memory_pools - allocate start of day memory pools
2968  * @ioc: per adapter object
2969  * @sleep_flag: CAN_SLEEP or NO_SLEEP
2970  *
2971  * Returns 0 success, anything else error
2972  */
2973 static int
2974 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc,  int sleep_flag)
2975 {
2976         struct mpt3sas_facts *facts;
2977         u16 max_sge_elements;
2978         u16 chains_needed_per_io;
2979         u32 sz, total_sz, reply_post_free_sz;
2980         u32 retry_sz;
2981         u16 max_request_credit;
2982         unsigned short sg_tablesize;
2983         u16 sge_size;
2984         int i;
2985
2986         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
2987             __func__));
2988
2989
2990         retry_sz = 0;
2991         facts = &ioc->facts;
2992
2993         /* command line tunables for max sgl entries */
2994         if (max_sgl_entries != -1)
2995                 sg_tablesize = max_sgl_entries;
2996         else {
2997                 if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
2998                         sg_tablesize = MPT2SAS_SG_DEPTH;
2999                 else
3000                         sg_tablesize = MPT3SAS_SG_DEPTH;
3001         }
3002
3003         if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS)
3004                 sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
3005         else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) {
3006                 sg_tablesize = min_t(unsigned short, sg_tablesize,
3007                                       SCSI_MAX_SG_CHAIN_SEGMENTS);
3008                 pr_warn(MPT3SAS_FMT
3009                  "sg_tablesize(%u) is bigger than kernel"
3010                  " defined SCSI_MAX_SG_SEGMENTS(%u)\n", ioc->name,
3011                  sg_tablesize, MPT_MAX_PHYS_SEGMENTS);
3012         }
3013         ioc->shost->sg_tablesize = sg_tablesize;
3014
3015         ioc->hi_priority_depth = facts->HighPriorityCredit;
3016         ioc->internal_depth = ioc->hi_priority_depth + (5);
3017         /* command line tunables  for max controller queue depth */
3018         if (max_queue_depth != -1 && max_queue_depth != 0) {
3019                 max_request_credit = min_t(u16, max_queue_depth +
3020                     ioc->hi_priority_depth + ioc->internal_depth,
3021                     facts->RequestCredit);
3022                 if (max_request_credit > MAX_HBA_QUEUE_DEPTH)
3023                         max_request_credit =  MAX_HBA_QUEUE_DEPTH;
3024         } else
3025                 max_request_credit = min_t(u16, facts->RequestCredit,
3026                     MAX_HBA_QUEUE_DEPTH);
3027
3028         ioc->hba_queue_depth = max_request_credit;
3029
3030         /* request frame size */
3031         ioc->request_sz = facts->IOCRequestFrameSize * 4;
3032
3033         /* reply frame size */
3034         ioc->reply_sz = facts->ReplyFrameSize * 4;
3035
3036         /* calculate the max scatter element size */
3037         sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);
3038
3039  retry_allocation:
3040         total_sz = 0;
3041         /* calculate number of sg elements left over in the 1st frame */
3042         max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -
3043             sizeof(Mpi2SGEIOUnion_t)) + sge_size);
3044         ioc->max_sges_in_main_message = max_sge_elements/sge_size;
3045
3046         /* now do the same for a chain buffer */
3047         max_sge_elements = ioc->request_sz - sge_size;
3048         ioc->max_sges_in_chain_message = max_sge_elements/sge_size;
3049
3050         /*
3051          *  MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE
3052          */
3053         chains_needed_per_io = ((ioc->shost->sg_tablesize -
3054            ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)
3055             + 1;
3056         if (chains_needed_per_io > facts->MaxChainDepth) {
3057                 chains_needed_per_io = facts->MaxChainDepth;
3058                 ioc->shost->sg_tablesize = min_t(u16,
3059                 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message
3060                 * chains_needed_per_io), ioc->shost->sg_tablesize);
3061         }
3062         ioc->chains_needed_per_io = chains_needed_per_io;
3063
3064         /* reply free queue sizing - taking into account for 64 FW events */
3065         ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
3066
3067         /* calculate reply descriptor post queue depth */
3068         ioc->reply_post_queue_depth = ioc->hba_queue_depth +
3069                                 ioc->reply_free_queue_depth +  1 ;
3070         /* align the reply post queue on the next 16 count boundary */
3071         if (ioc->reply_post_queue_depth % 16)
3072                 ioc->reply_post_queue_depth += 16 -
3073                 (ioc->reply_post_queue_depth % 16);
3074
3075
3076         if (ioc->reply_post_queue_depth >
3077             facts->MaxReplyDescriptorPostQueueDepth) {
3078                 ioc->reply_post_queue_depth =
3079                                 facts->MaxReplyDescriptorPostQueueDepth -
3080                     (facts->MaxReplyDescriptorPostQueueDepth % 16);
3081                 ioc->hba_queue_depth =
3082                                 ((ioc->reply_post_queue_depth - 64) / 2) - 1;
3083                 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
3084         }
3085
3086         dinitprintk(ioc, pr_info(MPT3SAS_FMT "scatter gather: " \
3087             "sge_in_main_msg(%d), sge_per_chain(%d), sge_per_io(%d), "
3088             "chains_per_io(%d)\n", ioc->name, ioc->max_sges_in_main_message,
3089             ioc->max_sges_in_chain_message, ioc->shost->sg_tablesize,
3090             ioc->chains_needed_per_io));
3091
3092         /* reply post queue, 16 byte align */
3093         reply_post_free_sz = ioc->reply_post_queue_depth *
3094             sizeof(Mpi2DefaultReplyDescriptor_t);
3095
3096         sz = reply_post_free_sz;
3097         if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
3098                 sz *= ioc->reply_queue_count;
3099
3100         ioc->reply_post = kcalloc((ioc->rdpq_array_enable) ?
3101             (ioc->reply_queue_count):1,
3102             sizeof(struct reply_post_struct), GFP_KERNEL);
3103
3104         if (!ioc->reply_post) {
3105                 pr_err(MPT3SAS_FMT "reply_post_free pool: kcalloc failed\n",
3106                         ioc->name);
3107                 goto out;
3108         }
3109         ioc->reply_post_free_dma_pool = pci_pool_create("reply_post_free pool",
3110             ioc->pdev, sz, 16, 0);
3111         if (!ioc->reply_post_free_dma_pool) {
3112                 pr_err(MPT3SAS_FMT
3113                  "reply_post_free pool: pci_pool_create failed\n",
3114                  ioc->name);
3115                 goto out;
3116         }
3117         i = 0;
3118         do {
3119                 ioc->reply_post[i].reply_post_free =
3120                     pci_pool_alloc(ioc->reply_post_free_dma_pool,
3121                     GFP_KERNEL,
3122                     &ioc->reply_post[i].reply_post_free_dma);
3123                 if (!ioc->reply_post[i].reply_post_free) {
3124                         pr_err(MPT3SAS_FMT
3125                         "reply_post_free pool: pci_pool_alloc failed\n",
3126                         ioc->name);
3127                         goto out;
3128                 }
3129                 memset(ioc->reply_post[i].reply_post_free, 0, sz);
3130                 dinitprintk(ioc, pr_info(MPT3SAS_FMT
3131                     "reply post free pool (0x%p): depth(%d),"
3132                     "element_size(%d), pool_size(%d kB)\n", ioc->name,
3133                     ioc->reply_post[i].reply_post_free,
3134                     ioc->reply_post_queue_depth, 8, sz/1024));
3135                 dinitprintk(ioc, pr_info(MPT3SAS_FMT
3136                     "reply_post_free_dma = (0x%llx)\n", ioc->name,
3137                     (unsigned long long)
3138                     ioc->reply_post[i].reply_post_free_dma));
3139                 total_sz += sz;
3140         } while (ioc->rdpq_array_enable && (++i < ioc->reply_queue_count));
3141
3142         if (ioc->dma_mask == 64) {
3143                 if (_base_change_consistent_dma_mask(ioc, ioc->pdev) != 0) {
3144                         pr_warn(MPT3SAS_FMT
3145                             "no suitable consistent DMA mask for %s\n",
3146                             ioc->name, pci_name(ioc->pdev));
3147                         goto out;
3148                 }
3149         }
3150
3151         ioc->scsiio_depth = ioc->hba_queue_depth -
3152             ioc->hi_priority_depth - ioc->internal_depth;
3153
3154         /* set the scsi host can_queue depth
3155          * with some internal commands that could be outstanding
3156          */
3157         ioc->shost->can_queue = ioc->scsiio_depth;
3158         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3159                 "scsi host: can_queue depth (%d)\n",
3160                 ioc->name, ioc->shost->can_queue));
3161
3162
3163         /* contiguous pool for request and chains, 16 byte align, one extra "
3164          * "frame for smid=0
3165          */
3166         ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;
3167         sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);
3168
3169         /* hi-priority queue */
3170         sz += (ioc->hi_priority_depth * ioc->request_sz);
3171
3172         /* internal queue */
3173         sz += (ioc->internal_depth * ioc->request_sz);
3174
3175         ioc->request_dma_sz = sz;
3176         ioc->request = pci_alloc_consistent(ioc->pdev, sz, &ioc->request_dma);
3177         if (!ioc->request) {
3178                 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
3179                     "failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
3180                     "total(%d kB)\n", ioc->name, ioc->hba_queue_depth,
3181                     ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
3182                 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)
3183                         goto out;
3184                 retry_sz += 64;
3185                 ioc->hba_queue_depth = max_request_credit - retry_sz;
3186                 goto retry_allocation;
3187         }
3188
3189         if (retry_sz)
3190                 pr_err(MPT3SAS_FMT "request pool: pci_alloc_consistent " \
3191                     "succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), "
3192                     "total(%d kb)\n", ioc->name, ioc->hba_queue_depth,
3193                     ioc->chains_needed_per_io, ioc->request_sz, sz/1024);
3194
3195         /* hi-priority queue */
3196         ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *
3197             ioc->request_sz);
3198         ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *
3199             ioc->request_sz);
3200
3201         /* internal queue */
3202         ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *
3203             ioc->request_sz);
3204         ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *
3205             ioc->request_sz);
3206
3207         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3208                 "request pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
3209                 ioc->name, ioc->request, ioc->hba_queue_depth, ioc->request_sz,
3210             (ioc->hba_queue_depth * ioc->request_sz)/1024));
3211
3212         dinitprintk(ioc, pr_info(MPT3SAS_FMT "request pool: dma(0x%llx)\n",
3213             ioc->name, (unsigned long long) ioc->request_dma));
3214         total_sz += sz;
3215
3216         sz = ioc->scsiio_depth * sizeof(struct scsiio_tracker);
3217         ioc->scsi_lookup_pages = get_order(sz);
3218         ioc->scsi_lookup = (struct scsiio_tracker *)__get_free_pages(
3219             GFP_KERNEL, ioc->scsi_lookup_pages);
3220         if (!ioc->scsi_lookup) {
3221                 pr_err(MPT3SAS_FMT "scsi_lookup: get_free_pages failed, sz(%d)\n",
3222                         ioc->name, (int)sz);
3223                 goto out;
3224         }
3225
3226         dinitprintk(ioc, pr_info(MPT3SAS_FMT "scsiio(0x%p): depth(%d)\n",
3227                 ioc->name, ioc->request, ioc->scsiio_depth));
3228
3229         ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
3230         sz = ioc->chain_depth * sizeof(struct chain_tracker);
3231         ioc->chain_pages = get_order(sz);
3232         ioc->chain_lookup = (struct chain_tracker *)__get_free_pages(
3233             GFP_KERNEL, ioc->chain_pages);
3234         if (!ioc->chain_lookup) {
3235                 pr_err(MPT3SAS_FMT "chain_lookup: __get_free_pages failed\n",
3236                         ioc->name);
3237                 goto out;
3238         }
3239         ioc->chain_dma_pool = pci_pool_create("chain pool", ioc->pdev,
3240             ioc->request_sz, 16, 0);
3241         if (!ioc->chain_dma_pool) {
3242                 pr_err(MPT3SAS_FMT "chain_dma_pool: pci_pool_create failed\n",
3243                         ioc->name);
3244                 goto out;
3245         }
3246         for (i = 0; i < ioc->chain_depth; i++) {
3247                 ioc->chain_lookup[i].chain_buffer = pci_pool_alloc(
3248                     ioc->chain_dma_pool , GFP_KERNEL,
3249                     &ioc->chain_lookup[i].chain_buffer_dma);
3250                 if (!ioc->chain_lookup[i].chain_buffer) {
3251                         ioc->chain_depth = i;
3252                         goto chain_done;
3253                 }
3254                 total_sz += ioc->request_sz;
3255         }
3256  chain_done:
3257         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3258                 "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
3259                 ioc->name, ioc->chain_depth, ioc->request_sz,
3260                 ((ioc->chain_depth *  ioc->request_sz))/1024));
3261
3262         /* initialize hi-priority queue smid's */
3263         ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
3264             sizeof(struct request_tracker), GFP_KERNEL);
3265         if (!ioc->hpr_lookup) {
3266                 pr_err(MPT3SAS_FMT "hpr_lookup: kcalloc failed\n",
3267                     ioc->name);
3268                 goto out;
3269         }
3270         ioc->hi_priority_smid = ioc->scsiio_depth + 1;
3271         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3272                 "hi_priority(0x%p): depth(%d), start smid(%d)\n",
3273                 ioc->name, ioc->hi_priority,
3274             ioc->hi_priority_depth, ioc->hi_priority_smid));
3275
3276         /* initialize internal queue smid's */
3277         ioc->internal_lookup = kcalloc(ioc->internal_depth,
3278             sizeof(struct request_tracker), GFP_KERNEL);
3279         if (!ioc->internal_lookup) {
3280                 pr_err(MPT3SAS_FMT "internal_lookup: kcalloc failed\n",
3281                     ioc->name);
3282                 goto out;
3283         }
3284         ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;
3285         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3286                 "internal(0x%p): depth(%d), start smid(%d)\n",
3287                 ioc->name, ioc->internal,
3288             ioc->internal_depth, ioc->internal_smid));
3289
3290         /* sense buffers, 4 byte align */
3291         sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
3292         ioc->sense_dma_pool = pci_pool_create("sense pool", ioc->pdev, sz, 4,
3293             0);
3294         if (!ioc->sense_dma_pool) {
3295                 pr_err(MPT3SAS_FMT "sense pool: pci_pool_create failed\n",
3296                     ioc->name);
3297                 goto out;
3298         }
3299         ioc->sense = pci_pool_alloc(ioc->sense_dma_pool , GFP_KERNEL,
3300             &ioc->sense_dma);
3301         if (!ioc->sense) {
3302                 pr_err(MPT3SAS_FMT "sense pool: pci_pool_alloc failed\n",
3303                     ioc->name);
3304                 goto out;
3305         }
3306         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3307             "sense pool(0x%p): depth(%d), element_size(%d), pool_size"
3308             "(%d kB)\n", ioc->name, ioc->sense, ioc->scsiio_depth,
3309             SCSI_SENSE_BUFFERSIZE, sz/1024));
3310         dinitprintk(ioc, pr_info(MPT3SAS_FMT "sense_dma(0x%llx)\n",
3311             ioc->name, (unsigned long long)ioc->sense_dma));
3312         total_sz += sz;
3313
3314         /* reply pool, 4 byte align */
3315         sz = ioc->reply_free_queue_depth * ioc->reply_sz;
3316         ioc->reply_dma_pool = pci_pool_create("reply pool", ioc->pdev, sz, 4,
3317             0);
3318         if (!ioc->reply_dma_pool) {
3319                 pr_err(MPT3SAS_FMT "reply pool: pci_pool_create failed\n",
3320                     ioc->name);
3321                 goto out;
3322         }
3323         ioc->reply = pci_pool_alloc(ioc->reply_dma_pool , GFP_KERNEL,
3324             &ioc->reply_dma);
3325         if (!ioc->reply) {
3326                 pr_err(MPT3SAS_FMT "reply pool: pci_pool_alloc failed\n",
3327                     ioc->name);
3328                 goto out;
3329         }
3330         ioc->reply_dma_min_address = (u32)(ioc->reply_dma);
3331         ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;
3332         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3333                 "reply pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n",
3334                 ioc->name, ioc->reply,
3335             ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024));
3336         dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_dma(0x%llx)\n",
3337             ioc->name, (unsigned long long)ioc->reply_dma));
3338         total_sz += sz;
3339
3340         /* reply free queue, 16 byte align */
3341         sz = ioc->reply_free_queue_depth * 4;
3342         ioc->reply_free_dma_pool = pci_pool_create("reply_free pool",
3343             ioc->pdev, sz, 16, 0);
3344         if (!ioc->reply_free_dma_pool) {
3345                 pr_err(MPT3SAS_FMT "reply_free pool: pci_pool_create failed\n",
3346                         ioc->name);
3347                 goto out;
3348         }
3349         ioc->reply_free = pci_pool_alloc(ioc->reply_free_dma_pool , GFP_KERNEL,
3350             &ioc->reply_free_dma);
3351         if (!ioc->reply_free) {
3352                 pr_err(MPT3SAS_FMT "reply_free pool: pci_pool_alloc failed\n",
3353                         ioc->name);
3354                 goto out;
3355         }
3356         memset(ioc->reply_free, 0, sz);
3357         dinitprintk(ioc, pr_info(MPT3SAS_FMT "reply_free pool(0x%p): " \
3358             "depth(%d), element_size(%d), pool_size(%d kB)\n", ioc->name,
3359             ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));
3360         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3361                 "reply_free_dma (0x%llx)\n",
3362                 ioc->name, (unsigned long long)ioc->reply_free_dma));
3363         total_sz += sz;
3364
3365         ioc->config_page_sz = 512;
3366         ioc->config_page = pci_alloc_consistent(ioc->pdev,
3367             ioc->config_page_sz, &ioc->config_page_dma);
3368         if (!ioc->config_page) {
3369                 pr_err(MPT3SAS_FMT
3370                         "config page: pci_pool_alloc failed\n",
3371                         ioc->name);
3372                 goto out;
3373         }
3374         dinitprintk(ioc, pr_info(MPT3SAS_FMT
3375                 "config page(0x%p): size(%d)\n",
3376                 ioc->name, ioc->config_page, ioc->config_page_sz));
3377         dinitprintk(ioc, pr_info(MPT3SAS_FMT "config_page_dma(0x%llx)\n",
3378                 ioc->name, (unsigned long long)ioc->config_page_dma));
3379         total_sz += ioc->config_page_sz;
3380
3381         pr_info(MPT3SAS_FMT "Allocated physical memory: size(%d kB)\n",
3382             ioc->name, total_sz/1024);
3383         pr_info(MPT3SAS_FMT
3384                 "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",
3385             ioc->name, ioc->shost->can_queue, facts->RequestCredit);
3386         pr_info(MPT3SAS_FMT "Scatter Gather Elements per IO(%d)\n",
3387             ioc->name, ioc->shost->sg_tablesize);
3388         return 0;
3389
3390  out:
3391         return -ENOMEM;
3392 }
3393
3394 /**
3395  * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter.
3396  * @ioc: Pointer to MPT_ADAPTER structure
3397  * @cooked: Request raw or cooked IOC state
3398  *
3399  * Returns all IOC Doorbell register bits if cooked==0, else just the
3400  * Doorbell bits in MPI_IOC_STATE_MASK.
3401  */
3402 u32
3403 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
3404 {
3405         u32 s, sc;
3406
3407         s = readl(&ioc->chip->Doorbell);
3408         sc = s & MPI2_IOC_STATE_MASK;
3409         return cooked ? sc : s;
3410 }
3411
3412 /**
3413  * _base_wait_on_iocstate - waiting on a particular ioc state
3414  * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
3415  * @timeout: timeout in second
3416  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3417  *
3418  * Returns 0 for success, non-zero for failure.
3419  */
3420 static int
3421 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout,
3422         int sleep_flag)
3423 {
3424         u32 count, cntdn;
3425         u32 current_state;
3426
3427         count = 0;
3428         cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
3429         do {
3430                 current_state = mpt3sas_base_get_iocstate(ioc, 1);
3431                 if (current_state == ioc_state)
3432                         return 0;
3433                 if (count && current_state == MPI2_IOC_STATE_FAULT)
3434                         break;
3435                 if (sleep_flag == CAN_SLEEP)
3436                         usleep_range(1000, 1500);
3437                 else
3438                         udelay(500);
3439                 count++;
3440         } while (--cntdn);
3441
3442         return current_state;
3443 }
3444
3445 /**
3446  * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
3447  * a write to the doorbell)
3448  * @ioc: per adapter object
3449  * @timeout: timeout in second
3450  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3451  *
3452  * Returns 0 for success, non-zero for failure.
3453  *
3454  * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
3455  */
3456 static int
3457 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag);
3458
3459 static int
3460 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
3461         int sleep_flag)
3462 {
3463         u32 cntdn, count;
3464         u32 int_status;
3465
3466         count = 0;
3467         cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
3468         do {
3469                 int_status = readl(&ioc->chip->HostInterruptStatus);
3470                 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
3471                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
3472                                 "%s: successful count(%d), timeout(%d)\n",
3473                                 ioc->name, __func__, count, timeout));
3474                         return 0;
3475                 }
3476                 if (sleep_flag == CAN_SLEEP)
3477                         usleep_range(1000, 1500);
3478                 else
3479                         udelay(500);
3480                 count++;
3481         } while (--cntdn);
3482
3483         pr_err(MPT3SAS_FMT
3484                 "%s: failed due to timeout count(%d), int_status(%x)!\n",
3485                 ioc->name, __func__, count, int_status);
3486         return -EFAULT;
3487 }
3488
3489 /**
3490  * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
3491  * @ioc: per adapter object
3492  * @timeout: timeout in second
3493  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3494  *
3495  * Returns 0 for success, non-zero for failure.
3496  *
3497  * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
3498  * doorbell.
3499  */
3500 static int
3501 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout,
3502         int sleep_flag)
3503 {
3504         u32 cntdn, count;
3505         u32 int_status;
3506         u32 doorbell;
3507
3508         count = 0;
3509         cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
3510         do {
3511                 int_status = readl(&ioc->chip->HostInterruptStatus);
3512                 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
3513                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
3514                                 "%s: successful count(%d), timeout(%d)\n",
3515                                 ioc->name, __func__, count, timeout));
3516                         return 0;
3517                 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
3518                         doorbell = readl(&ioc->chip->Doorbell);
3519                         if ((doorbell & MPI2_IOC_STATE_MASK) ==
3520                             MPI2_IOC_STATE_FAULT) {
3521                                 mpt3sas_base_fault_info(ioc , doorbell);
3522                                 return -EFAULT;
3523                         }
3524                 } else if (int_status == 0xFFFFFFFF)
3525                         goto out;
3526
3527                 if (sleep_flag == CAN_SLEEP)
3528                         usleep_range(1000, 1500);
3529                 else
3530                         udelay(500);
3531                 count++;
3532         } while (--cntdn);
3533
3534  out:
3535         pr_err(MPT3SAS_FMT
3536          "%s: failed due to timeout count(%d), int_status(%x)!\n",
3537          ioc->name, __func__, count, int_status);
3538         return -EFAULT;
3539 }
3540
3541 /**
3542  * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
3543  * @ioc: per adapter object
3544  * @timeout: timeout in second
3545  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3546  *
3547  * Returns 0 for success, non-zero for failure.
3548  *
3549  */
3550 static int
3551 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout,
3552         int sleep_flag)
3553 {
3554         u32 cntdn, count;
3555         u32 doorbell_reg;
3556
3557         count = 0;
3558         cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
3559         do {
3560                 doorbell_reg = readl(&ioc->chip->Doorbell);
3561                 if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
3562                         dhsprintk(ioc, pr_info(MPT3SAS_FMT
3563                                 "%s: successful count(%d), timeout(%d)\n",
3564                                 ioc->name, __func__, count, timeout));
3565                         return 0;
3566                 }
3567                 if (sleep_flag == CAN_SLEEP)
3568                         usleep_range(1000, 1500);
3569                 else
3570                         udelay(500);
3571                 count++;
3572         } while (--cntdn);
3573
3574         pr_err(MPT3SAS_FMT
3575                 "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",
3576                 ioc->name, __func__, count, doorbell_reg);
3577         return -EFAULT;
3578 }
3579
3580 /**
3581  * _base_send_ioc_reset - send doorbell reset
3582  * @ioc: per adapter object
3583  * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
3584  * @timeout: timeout in second
3585  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3586  *
3587  * Returns 0 for success, non-zero for failure.
3588  */
3589 static int
3590 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout,
3591         int sleep_flag)
3592 {
3593         u32 ioc_state;
3594         int r = 0;
3595
3596         if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) {
3597                 pr_err(MPT3SAS_FMT "%s: unknown reset_type\n",
3598                     ioc->name, __func__);
3599                 return -EFAULT;
3600         }
3601
3602         if (!(ioc->facts.IOCCapabilities &
3603            MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY))
3604                 return -EFAULT;
3605
3606         pr_info(MPT3SAS_FMT "sending message unit reset !!\n", ioc->name);
3607
3608         writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
3609             &ioc->chip->Doorbell);
3610         if ((_base_wait_for_doorbell_ack(ioc, 15, sleep_flag))) {
3611                 r = -EFAULT;
3612                 goto out;
3613         }
3614         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY,
3615             timeout, sleep_flag);
3616         if (ioc_state) {
3617                 pr_err(MPT3SAS_FMT
3618                         "%s: failed going to ready state (ioc_state=0x%x)\n",
3619                         ioc->name, __func__, ioc_state);
3620                 r = -EFAULT;
3621                 goto out;
3622         }
3623  out:
3624         pr_info(MPT3SAS_FMT "message unit reset: %s\n",
3625             ioc->name, ((r == 0) ? "SUCCESS" : "FAILED"));
3626         return r;
3627 }
3628
3629 /**
3630  * _base_handshake_req_reply_wait - send request thru doorbell interface
3631  * @ioc: per adapter object
3632  * @request_bytes: request length
3633  * @request: pointer having request payload
3634  * @reply_bytes: reply length
3635  * @reply: pointer to reply payload
3636  * @timeout: timeout in second
3637  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3638  *
3639  * Returns 0 for success, non-zero for failure.
3640  */
3641 static int
3642 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
3643         u32 *request, int reply_bytes, u16 *reply, int timeout, int sleep_flag)
3644 {
3645         MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
3646         int i;
3647         u8 failed;
3648         u16 dummy;
3649         __le32 *mfp;
3650
3651         /* make sure doorbell is not in use */
3652         if ((readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {
3653                 pr_err(MPT3SAS_FMT
3654                         "doorbell is in use (line=%d)\n",
3655                         ioc->name, __LINE__);
3656                 return -EFAULT;
3657         }
3658
3659         /* clear pending doorbell interrupts from previous state changes */
3660         if (readl(&ioc->chip->HostInterruptStatus) &
3661             MPI2_HIS_IOC2SYS_DB_STATUS)
3662                 writel(0, &ioc->chip->HostInterruptStatus);
3663
3664         /* send message to ioc */
3665         writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) |
3666             ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
3667             &ioc->chip->Doorbell);
3668
3669         if ((_base_wait_for_doorbell_int(ioc, 5, NO_SLEEP))) {
3670                 pr_err(MPT3SAS_FMT
3671                         "doorbell handshake int failed (line=%d)\n",
3672                         ioc->name, __LINE__);
3673                 return -EFAULT;
3674         }
3675         writel(0, &ioc->chip->HostInterruptStatus);
3676
3677         if ((_base_wait_for_doorbell_ack(ioc, 5, sleep_flag))) {
3678                 pr_err(MPT3SAS_FMT
3679                         "doorbell handshake ack failed (line=%d)\n",
3680                         ioc->name, __LINE__);
3681                 return -EFAULT;
3682         }
3683
3684         /* send message 32-bits at a time */
3685         for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
3686                 writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
3687                 if ((_base_wait_for_doorbell_ack(ioc, 5, sleep_flag)))
3688                         failed = 1;
3689         }
3690
3691         if (failed) {
3692                 pr_err(MPT3SAS_FMT
3693                         "doorbell handshake sending request failed (line=%d)\n",
3694                         ioc->name, __LINE__);
3695                 return -EFAULT;
3696         }
3697
3698         /* now wait for the reply */
3699         if ((_base_wait_for_doorbell_int(ioc, timeout, sleep_flag))) {
3700                 pr_err(MPT3SAS_FMT
3701                         "doorbell handshake int failed (line=%d)\n",
3702                         ioc->name, __LINE__);
3703                 return -EFAULT;
3704         }
3705
3706         /* read the first two 16-bits, it gives the total length of the reply */
3707         reply[0] = le16_to_cpu(readl(&ioc->chip->Doorbell)
3708             & MPI2_DOORBELL_DATA_MASK);
3709         writel(0, &ioc->chip->HostInterruptStatus);
3710         if ((_base_wait_for_doorbell_int(ioc, 5, sleep_flag))) {
3711                 pr_err(MPT3SAS_FMT
3712                         "doorbell handshake int failed (line=%d)\n",
3713                         ioc->name, __LINE__);
3714                 return -EFAULT;
3715         }
3716         reply[1] = le16_to_cpu(readl(&ioc->chip->Doorbell)
3717             & MPI2_DOORBELL_DATA_MASK);
3718         writel(0, &ioc->chip->HostInterruptStatus);
3719
3720         for (i = 2; i < default_reply->MsgLength * 2; i++)  {
3721                 if ((_base_wait_for_doorbell_int(ioc, 5, sleep_flag))) {
3722                         pr_err(MPT3SAS_FMT
3723                                 "doorbell handshake int failed (line=%d)\n",
3724                                 ioc->name, __LINE__);
3725                         return -EFAULT;
3726                 }
3727                 if (i >=  reply_bytes/2) /* overflow case */
3728                         dummy = readl(&ioc->chip->Doorbell);
3729                 else
3730                         reply[i] = le16_to_cpu(readl(&ioc->chip->Doorbell)
3731                             & MPI2_DOORBELL_DATA_MASK);
3732                 writel(0, &ioc->chip->HostInterruptStatus);
3733         }
3734
3735         _base_wait_for_doorbell_int(ioc, 5, sleep_flag);
3736         if (_base_wait_for_doorbell_not_used(ioc, 5, sleep_flag) != 0) {
3737                 dhsprintk(ioc, pr_info(MPT3SAS_FMT
3738                         "doorbell is in use (line=%d)\n", ioc->name, __LINE__));
3739         }
3740         writel(0, &ioc->chip->HostInterruptStatus);
3741
3742         if (ioc->logging_level & MPT_DEBUG_INIT) {
3743                 mfp = (__le32 *)reply;
3744                 pr_info("\toffset:data\n");
3745                 for (i = 0; i < reply_bytes/4; i++)
3746                         pr_info("\t[0x%02x]:%08x\n", i*4,
3747                             le32_to_cpu(mfp[i]));
3748         }
3749         return 0;
3750 }
3751
3752 /**
3753  * mpt3sas_base_sas_iounit_control - send sas iounit control to FW
3754  * @ioc: per adapter object
3755  * @mpi_reply: the reply payload from FW
3756  * @mpi_request: the request payload sent to FW
3757  *
3758  * The SAS IO Unit Control Request message allows the host to perform low-level
3759  * operations, such as resets on the PHYs of the IO Unit, also allows the host
3760  * to obtain the IOC assigned device handles for a device if it has other
3761  * identifying information about the device, in addition allows the host to
3762  * remove IOC resources associated with the device.
3763  *
3764  * Returns 0 for success, non-zero for failure.
3765  */
3766 int
3767 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
3768         Mpi2SasIoUnitControlReply_t *mpi_reply,
3769         Mpi2SasIoUnitControlRequest_t *mpi_request)
3770 {
3771         u16 smid;
3772         u32 ioc_state;
3773         unsigned long timeleft;
3774         bool issue_reset = false;
3775         int rc;
3776         void *request;
3777         u16 wait_state_count;
3778
3779         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3780             __func__));
3781
3782         mutex_lock(&ioc->base_cmds.mutex);
3783
3784         if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
3785                 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
3786                     ioc->name, __func__);
3787                 rc = -EAGAIN;
3788                 goto out;
3789         }
3790
3791         wait_state_count = 0;
3792         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
3793         while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
3794                 if (wait_state_count++ == 10) {
3795                         pr_err(MPT3SAS_FMT
3796                             "%s: failed due to ioc not operational\n",
3797                             ioc->name, __func__);
3798                         rc = -EFAULT;
3799                         goto out;
3800                 }
3801                 ssleep(1);
3802                 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
3803                 pr_info(MPT3SAS_FMT
3804                         "%s: waiting for operational state(count=%d)\n",
3805                         ioc->name, __func__, wait_state_count);
3806         }
3807
3808         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
3809         if (!smid) {
3810                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
3811                     ioc->name, __func__);
3812                 rc = -EAGAIN;
3813                 goto out;
3814         }
3815
3816         rc = 0;
3817         ioc->base_cmds.status = MPT3_CMD_PENDING;
3818         request = mpt3sas_base_get_msg_frame(ioc, smid);
3819         ioc->base_cmds.smid = smid;
3820         memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t));
3821         if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
3822             mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
3823                 ioc->ioc_link_reset_in_progress = 1;
3824         init_completion(&ioc->base_cmds.done);
3825         mpt3sas_base_put_smid_default(ioc, smid);
3826         timeleft = wait_for_completion_timeout(&ioc->base_cmds.done,
3827             msecs_to_jiffies(10000));
3828         if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
3829             mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
3830             ioc->ioc_link_reset_in_progress)
3831                 ioc->ioc_link_reset_in_progress = 0;
3832         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
3833                 pr_err(MPT3SAS_FMT "%s: timeout\n",
3834                     ioc->name, __func__);
3835                 _debug_dump_mf(mpi_request,
3836                     sizeof(Mpi2SasIoUnitControlRequest_t)/4);
3837                 if (!(ioc->base_cmds.status & MPT3_CMD_RESET))
3838                         issue_reset = true;
3839                 goto issue_host_reset;
3840         }
3841         if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
3842                 memcpy(mpi_reply, ioc->base_cmds.reply,
3843                     sizeof(Mpi2SasIoUnitControlReply_t));
3844         else
3845                 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t));
3846         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
3847         goto out;
3848
3849  issue_host_reset:
3850         if (issue_reset)
3851                 mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
3852                     FORCE_BIG_HAMMER);
3853         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
3854         rc = -EFAULT;
3855  out:
3856         mutex_unlock(&ioc->base_cmds.mutex);
3857         return rc;
3858 }
3859
3860 /**
3861  * mpt3sas_base_scsi_enclosure_processor - sending request to sep device
3862  * @ioc: per adapter object
3863  * @mpi_reply: the reply payload from FW
3864  * @mpi_request: the request payload sent to FW
3865  *
3866  * The SCSI Enclosure Processor request message causes the IOC to
3867  * communicate with SES devices to control LED status signals.
3868  *
3869  * Returns 0 for success, non-zero for failure.
3870  */
3871 int
3872 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
3873         Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request)
3874 {
3875         u16 smid;
3876         u32 ioc_state;
3877         unsigned long timeleft;
3878         bool issue_reset = false;
3879         int rc;
3880         void *request;
3881         u16 wait_state_count;
3882
3883         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3884             __func__));
3885
3886         mutex_lock(&ioc->base_cmds.mutex);
3887
3888         if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
3889                 pr_err(MPT3SAS_FMT "%s: base_cmd in use\n",
3890                     ioc->name, __func__);
3891                 rc = -EAGAIN;
3892                 goto out;
3893         }
3894
3895         wait_state_count = 0;
3896         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
3897         while (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
3898                 if (wait_state_count++ == 10) {
3899                         pr_err(MPT3SAS_FMT
3900                             "%s: failed due to ioc not operational\n",
3901                             ioc->name, __func__);
3902                         rc = -EFAULT;
3903                         goto out;
3904                 }
3905                 ssleep(1);
3906                 ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
3907                 pr_info(MPT3SAS_FMT
3908                         "%s: waiting for operational state(count=%d)\n",
3909                         ioc->name,
3910                     __func__, wait_state_count);
3911         }
3912
3913         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
3914         if (!smid) {
3915                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
3916                     ioc->name, __func__);
3917                 rc = -EAGAIN;
3918                 goto out;
3919         }
3920
3921         rc = 0;
3922         ioc->base_cmds.status = MPT3_CMD_PENDING;
3923         request = mpt3sas_base_get_msg_frame(ioc, smid);
3924         ioc->base_cmds.smid = smid;
3925         memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
3926         init_completion(&ioc->base_cmds.done);
3927         mpt3sas_base_put_smid_default(ioc, smid);
3928         timeleft = wait_for_completion_timeout(&ioc->base_cmds.done,
3929             msecs_to_jiffies(10000));
3930         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
3931                 pr_err(MPT3SAS_FMT "%s: timeout\n",
3932                     ioc->name, __func__);
3933                 _debug_dump_mf(mpi_request,
3934                     sizeof(Mpi2SepRequest_t)/4);
3935                 if (!(ioc->base_cmds.status & MPT3_CMD_RESET))
3936                         issue_reset = false;
3937                 goto issue_host_reset;
3938         }
3939         if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
3940                 memcpy(mpi_reply, ioc->base_cmds.reply,
3941                     sizeof(Mpi2SepReply_t));
3942         else
3943                 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t));
3944         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
3945         goto out;
3946
3947  issue_host_reset:
3948         if (issue_reset)
3949                 mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
3950                     FORCE_BIG_HAMMER);
3951         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
3952         rc = -EFAULT;
3953  out:
3954         mutex_unlock(&ioc->base_cmds.mutex);
3955         return rc;
3956 }
3957
3958 /**
3959  * _base_get_port_facts - obtain port facts reply and save in ioc
3960  * @ioc: per adapter object
3961  * @sleep_flag: CAN_SLEEP or NO_SLEEP
3962  *
3963  * Returns 0 for success, non-zero for failure.
3964  */
3965 static int
3966 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port, int sleep_flag)
3967 {
3968         Mpi2PortFactsRequest_t mpi_request;
3969         Mpi2PortFactsReply_t mpi_reply;
3970         struct mpt3sas_port_facts *pfacts;
3971         int mpi_reply_sz, mpi_request_sz, r;
3972
3973         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
3974             __func__));
3975
3976         mpi_reply_sz = sizeof(Mpi2PortFactsReply_t);
3977         mpi_request_sz = sizeof(Mpi2PortFactsRequest_t);
3978         memset(&mpi_request, 0, mpi_request_sz);
3979         mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
3980         mpi_request.PortNumber = port;
3981         r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
3982             (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5, CAN_SLEEP);
3983
3984         if (r != 0) {
3985                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
3986                     ioc->name, __func__, r);
3987                 return r;
3988         }
3989
3990         pfacts = &ioc->pfacts[port];
3991         memset(pfacts, 0, sizeof(struct mpt3sas_port_facts));
3992         pfacts->PortNumber = mpi_reply.PortNumber;
3993         pfacts->VP_ID = mpi_reply.VP_ID;
3994         pfacts->VF_ID = mpi_reply.VF_ID;
3995         pfacts->MaxPostedCmdBuffers =
3996             le16_to_cpu(mpi_reply.MaxPostedCmdBuffers);
3997
3998         return 0;
3999 }
4000
4001 /**
4002  * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
4003  * @ioc: per adapter object
4004  * @timeout:
4005  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4006  *
4007  * Returns 0 for success, non-zero for failure.
4008  */
4009 static int
4010 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout,
4011         int sleep_flag)
4012 {
4013         u32 ioc_state;
4014         int rc;
4015
4016         dinitprintk(ioc, printk(MPT3SAS_FMT "%s\n", ioc->name,
4017             __func__));
4018
4019         if (ioc->pci_error_recovery) {
4020                 dfailprintk(ioc, printk(MPT3SAS_FMT
4021                     "%s: host in pci error recovery\n", ioc->name, __func__));
4022                 return -EFAULT;
4023         }
4024
4025         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
4026         dhsprintk(ioc, printk(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
4027             ioc->name, __func__, ioc_state));
4028
4029         if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) ||
4030             (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
4031                 return 0;
4032
4033         if (ioc_state & MPI2_DOORBELL_USED) {
4034                 dhsprintk(ioc, printk(MPT3SAS_FMT
4035                     "unexpected doorbell active!\n", ioc->name));
4036                 goto issue_diag_reset;
4037         }
4038
4039         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
4040                 mpt3sas_base_fault_info(ioc, ioc_state &
4041                     MPI2_DOORBELL_DATA_MASK);
4042                 goto issue_diag_reset;
4043         }
4044
4045         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY,
4046             timeout, sleep_flag);
4047         if (ioc_state) {
4048                 dfailprintk(ioc, printk(MPT3SAS_FMT
4049                     "%s: failed going to ready state (ioc_state=0x%x)\n",
4050                     ioc->name, __func__, ioc_state));
4051                 return -EFAULT;
4052         }
4053
4054  issue_diag_reset:
4055         rc = _base_diag_reset(ioc, sleep_flag);
4056         return rc;
4057 }
4058
4059 /**
4060  * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
4061  * @ioc: per adapter object
4062  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4063  *
4064  * Returns 0 for success, non-zero for failure.
4065  */
4066 static int
4067 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
4068 {
4069         Mpi2IOCFactsRequest_t mpi_request;
4070         Mpi2IOCFactsReply_t mpi_reply;
4071         struct mpt3sas_facts *facts;
4072         int mpi_reply_sz, mpi_request_sz, r;
4073
4074         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4075             __func__));
4076
4077         r = _base_wait_for_iocstate(ioc, 10, sleep_flag);
4078         if (r) {
4079                 dfailprintk(ioc, printk(MPT3SAS_FMT
4080                     "%s: failed getting to correct state\n",
4081                     ioc->name, __func__));
4082                 return r;
4083         }
4084         mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t);
4085         mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t);
4086         memset(&mpi_request, 0, mpi_request_sz);
4087         mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
4088         r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
4089             (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5, CAN_SLEEP);
4090
4091         if (r != 0) {
4092                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
4093                     ioc->name, __func__, r);
4094                 return r;
4095         }
4096
4097         facts = &ioc->facts;
4098         memset(facts, 0, sizeof(struct mpt3sas_facts));
4099         facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion);
4100         facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion);
4101         facts->VP_ID = mpi_reply.VP_ID;
4102         facts->VF_ID = mpi_reply.VF_ID;
4103         facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions);
4104         facts->MaxChainDepth = mpi_reply.MaxChainDepth;
4105         facts->WhoInit = mpi_reply.WhoInit;
4106         facts->NumberOfPorts = mpi_reply.NumberOfPorts;
4107         facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors;
4108         facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit);
4109         facts->MaxReplyDescriptorPostQueueDepth =
4110             le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth);
4111         facts->ProductID = le16_to_cpu(mpi_reply.ProductID);
4112         facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities);
4113         if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID))
4114                 ioc->ir_firmware = 1;
4115         if ((facts->IOCCapabilities &
4116               MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE))
4117                 ioc->rdpq_array_capable = 1;
4118         facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
4119         facts->IOCRequestFrameSize =
4120             le16_to_cpu(mpi_reply.IOCRequestFrameSize);
4121         facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators);
4122         facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets);
4123         ioc->shost->max_id = -1;
4124         facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders);
4125         facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures);
4126         facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags);
4127         facts->HighPriorityCredit =
4128             le16_to_cpu(mpi_reply.HighPriorityCredit);
4129         facts->ReplyFrameSize = mpi_reply.ReplyFrameSize;
4130         facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle);
4131
4132         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4133                 "hba queue depth(%d), max chains per io(%d)\n",
4134                 ioc->name, facts->RequestCredit,
4135             facts->MaxChainDepth));
4136         dinitprintk(ioc, pr_info(MPT3SAS_FMT
4137                 "request frame size(%d), reply frame size(%d)\n", ioc->name,
4138             facts->IOCRequestFrameSize * 4, facts->ReplyFrameSize * 4));
4139         return 0;
4140 }
4141
4142 /**
4143  * _base_send_ioc_init - send ioc_init to firmware
4144  * @ioc: per adapter object
4145  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4146  *
4147  * Returns 0 for success, non-zero for failure.
4148  */
4149 static int
4150 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
4151 {
4152         Mpi2IOCInitRequest_t mpi_request;
4153         Mpi2IOCInitReply_t mpi_reply;
4154         int i, r = 0;
4155         struct timeval current_time;
4156         u16 ioc_status;
4157         u32 reply_post_free_array_sz = 0;
4158         Mpi2IOCInitRDPQArrayEntry *reply_post_free_array = NULL;
4159         dma_addr_t reply_post_free_array_dma;
4160
4161         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4162             __func__));
4163
4164         memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t));
4165         mpi_request.Function = MPI2_FUNCTION_IOC_INIT;
4166         mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
4167         mpi_request.VF_ID = 0; /* TODO */
4168         mpi_request.VP_ID = 0;
4169         mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);
4170         mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
4171
4172         if (_base_is_controller_msix_enabled(ioc))
4173                 mpi_request.HostMSIxVectors = ioc->reply_queue_count;
4174         mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);
4175         mpi_request.ReplyDescriptorPostQueueDepth =
4176             cpu_to_le16(ioc->reply_post_queue_depth);
4177         mpi_request.ReplyFreeQueueDepth =
4178             cpu_to_le16(ioc->reply_free_queue_depth);
4179
4180         mpi_request.SenseBufferAddressHigh =
4181             cpu_to_le32((u64)ioc->sense_dma >> 32);
4182         mpi_request.SystemReplyAddressHigh =
4183             cpu_to_le32((u64)ioc->reply_dma >> 32);
4184         mpi_request.SystemRequestFrameBaseAddress =
4185             cpu_to_le64((u64)ioc->request_dma);
4186         mpi_request.ReplyFreeQueueAddress =
4187             cpu_to_le64((u64)ioc->reply_free_dma);
4188
4189         if (ioc->rdpq_array_enable) {
4190                 reply_post_free_array_sz = ioc->reply_queue_count *
4191                     sizeof(Mpi2IOCInitRDPQArrayEntry);
4192                 reply_post_free_array = pci_alloc_consistent(ioc->pdev,
4193                         reply_post_free_array_sz, &reply_post_free_array_dma);
4194                 if (!reply_post_free_array) {
4195                         pr_err(MPT3SAS_FMT
4196                         "reply_post_free_array: pci_alloc_consistent failed\n",
4197                         ioc->name);
4198                         r = -ENOMEM;
4199                         goto out;
4200                 }
4201                 memset(reply_post_free_array, 0, reply_post_free_array_sz);
4202                 for (i = 0; i < ioc->reply_queue_count; i++)
4203                         reply_post_free_array[i].RDPQBaseAddress =
4204                             cpu_to_le64(
4205                                 (u64)ioc->reply_post[i].reply_post_free_dma);
4206                 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
4207                 mpi_request.ReplyDescriptorPostQueueAddress =
4208                     cpu_to_le64((u64)reply_post_free_array_dma);
4209         } else {
4210                 mpi_request.ReplyDescriptorPostQueueAddress =
4211                     cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
4212         }
4213
4214         /* This time stamp specifies number of milliseconds
4215          * since epoch ~ midnight January 1, 1970.
4216          */
4217         do_gettimeofday(&current_time);
4218         mpi_request.TimeStamp = cpu_to_le64((u64)current_time.tv_sec * 1000 +
4219             (current_time.tv_usec / 1000));
4220
4221         if (ioc->logging_level & MPT_DEBUG_INIT) {
4222                 __le32 *mfp;
4223                 int i;
4224
4225                 mfp = (__le32 *)&mpi_request;
4226                 pr_info("\toffset:data\n");
4227                 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++)
4228                         pr_info("\t[0x%02x]:%08x\n", i*4,
4229                             le32_to_cpu(mfp[i]));
4230         }
4231
4232         r = _base_handshake_req_reply_wait(ioc,
4233             sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
4234             sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 10,
4235             sleep_flag);
4236
4237         if (r != 0) {
4238                 pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
4239                     ioc->name, __func__, r);
4240                 goto out;
4241         }
4242
4243         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
4244         if (ioc_status != MPI2_IOCSTATUS_SUCCESS ||
4245             mpi_reply.IOCLogInfo) {
4246                 pr_err(MPT3SAS_FMT "%s: failed\n", ioc->name, __func__);
4247                 r = -EIO;
4248         }
4249
4250 out:
4251         if (reply_post_free_array)
4252                 pci_free_consistent(ioc->pdev, reply_post_free_array_sz,
4253                                     reply_post_free_array,
4254                                     reply_post_free_array_dma);
4255         return r;
4256 }
4257
4258 /**
4259  * mpt3sas_port_enable_done - command completion routine for port enable
4260  * @ioc: per adapter object
4261  * @smid: system request message index
4262  * @msix_index: MSIX table index supplied by the OS
4263  * @reply: reply message frame(lower 32bit addr)
4264  *
4265  * Return 1 meaning mf should be freed from _base_interrupt
4266  *        0 means the mf is freed from this function.
4267  */
4268 u8
4269 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
4270         u32 reply)
4271 {
4272         MPI2DefaultReply_t *mpi_reply;
4273         u16 ioc_status;
4274
4275         if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)
4276                 return 1;
4277
4278         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
4279         if (!mpi_reply)
4280                 return 1;
4281
4282         if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE)
4283                 return 1;
4284
4285         ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;
4286         ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;
4287         ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;
4288         memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
4289         ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
4290         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
4291                 ioc->port_enable_failed = 1;
4292
4293         if (ioc->is_driver_loading) {
4294                 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
4295                         mpt3sas_port_enable_complete(ioc);
4296                         return 1;
4297                 } else {
4298                         ioc->start_scan_failed = ioc_status;
4299                         ioc->start_scan = 0;
4300                         return 1;
4301                 }
4302         }
4303         complete(&ioc->port_enable_cmds.done);
4304         return 1;
4305 }
4306
4307 /**
4308  * _base_send_port_enable - send port_enable(discovery stuff) to firmware
4309  * @ioc: per adapter object
4310  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4311  *
4312  * Returns 0 for success, non-zero for failure.
4313  */
4314 static int
4315 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
4316 {
4317         Mpi2PortEnableRequest_t *mpi_request;
4318         Mpi2PortEnableReply_t *mpi_reply;
4319         unsigned long timeleft;
4320         int r = 0;
4321         u16 smid;
4322         u16 ioc_status;
4323
4324         pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
4325
4326         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
4327                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
4328                     ioc->name, __func__);
4329                 return -EAGAIN;
4330         }
4331
4332         smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
4333         if (!smid) {
4334                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
4335                     ioc->name, __func__);
4336                 return -EAGAIN;
4337         }
4338
4339         ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
4340         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4341         ioc->port_enable_cmds.smid = smid;
4342         memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
4343         mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
4344
4345         init_completion(&ioc->port_enable_cmds.done);
4346         mpt3sas_base_put_smid_default(ioc, smid);
4347         timeleft = wait_for_completion_timeout(&ioc->port_enable_cmds.done,
4348             300*HZ);
4349         if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
4350                 pr_err(MPT3SAS_FMT "%s: timeout\n",
4351                     ioc->name, __func__);
4352                 _debug_dump_mf(mpi_request,
4353                     sizeof(Mpi2PortEnableRequest_t)/4);
4354                 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)
4355                         r = -EFAULT;
4356                 else
4357                         r = -ETIME;
4358                 goto out;
4359         }
4360
4361         mpi_reply = ioc->port_enable_cmds.reply;
4362         ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
4363         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4364                 pr_err(MPT3SAS_FMT "%s: failed with (ioc_status=0x%08x)\n",
4365                     ioc->name, __func__, ioc_status);
4366                 r = -EFAULT;
4367                 goto out;
4368         }
4369
4370  out:
4371         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
4372         pr_info(MPT3SAS_FMT "port enable: %s\n", ioc->name, ((r == 0) ?
4373             "SUCCESS" : "FAILED"));
4374         return r;
4375 }
4376
4377 /**
4378  * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply)
4379  * @ioc: per adapter object
4380  *
4381  * Returns 0 for success, non-zero for failure.
4382  */
4383 int
4384 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
4385 {
4386         Mpi2PortEnableRequest_t *mpi_request;
4387         u16 smid;
4388
4389         pr_info(MPT3SAS_FMT "sending port enable !!\n", ioc->name);
4390
4391         if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
4392                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
4393                     ioc->name, __func__);
4394                 return -EAGAIN;
4395         }
4396
4397         smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
4398         if (!smid) {
4399                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
4400                     ioc->name, __func__);
4401                 return -EAGAIN;
4402         }
4403
4404         ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
4405         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4406         ioc->port_enable_cmds.smid = smid;
4407         memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
4408         mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
4409
4410         mpt3sas_base_put_smid_default(ioc, smid);
4411         return 0;
4412 }
4413
4414 /**
4415  * _base_determine_wait_on_discovery - desposition
4416  * @ioc: per adapter object
4417  *
4418  * Decide whether to wait on discovery to complete. Used to either
4419  * locate boot device, or report volumes ahead of physical devices.
4420  *
4421  * Returns 1 for wait, 0 for don't wait
4422  */
4423 static int
4424 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)
4425 {
4426         /* We wait for discovery to complete if IR firmware is loaded.
4427          * The sas topology events arrive before PD events, so we need time to
4428          * turn on the bit in ioc->pd_handles to indicate PD
4429          * Also, it maybe required to report Volumes ahead of physical
4430          * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set.
4431          */
4432         if (ioc->ir_firmware)
4433                 return 1;
4434
4435         /* if no Bios, then we don't need to wait */
4436         if (!ioc->bios_pg3.BiosVersion)
4437                 return 0;
4438
4439         /* Bios is present, then we drop down here.
4440          *
4441          * If there any entries in the Bios Page 2, then we wait
4442          * for discovery to complete.
4443          */
4444
4445         /* Current Boot Device */
4446         if ((ioc->bios_pg2.CurrentBootDeviceForm &
4447             MPI2_BIOSPAGE2_FORM_MASK) ==
4448             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
4449         /* Request Boot Device */
4450            (ioc->bios_pg2.ReqBootDeviceForm &
4451             MPI2_BIOSPAGE2_FORM_MASK) ==
4452             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
4453         /* Alternate Request Boot Device */
4454            (ioc->bios_pg2.ReqAltBootDeviceForm &
4455             MPI2_BIOSPAGE2_FORM_MASK) ==
4456             MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED)
4457                 return 0;
4458
4459         return 1;
4460 }
4461
4462 /**
4463  * _base_unmask_events - turn on notification for this event
4464  * @ioc: per adapter object
4465  * @event: firmware event
4466  *
4467  * The mask is stored in ioc->event_masks.
4468  */
4469 static void
4470 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
4471 {
4472         u32 desired_event;
4473
4474         if (event >= 128)
4475                 return;
4476
4477         desired_event = (1 << (event % 32));
4478
4479         if (event < 32)
4480                 ioc->event_masks[0] &= ~desired_event;
4481         else if (event < 64)
4482                 ioc->event_masks[1] &= ~desired_event;
4483         else if (event < 96)
4484                 ioc->event_masks[2] &= ~desired_event;
4485         else if (event < 128)
4486                 ioc->event_masks[3] &= ~desired_event;
4487 }
4488
4489 /**
4490  * _base_event_notification - send event notification
4491  * @ioc: per adapter object
4492  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4493  *
4494  * Returns 0 for success, non-zero for failure.
4495  */
4496 static int
4497 _base_event_notification(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
4498 {
4499         Mpi2EventNotificationRequest_t *mpi_request;
4500         unsigned long timeleft;
4501         u16 smid;
4502         int r = 0;
4503         int i;
4504
4505         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4506             __func__));
4507
4508         if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
4509                 pr_err(MPT3SAS_FMT "%s: internal command already in use\n",
4510                     ioc->name, __func__);
4511                 return -EAGAIN;
4512         }
4513
4514         smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
4515         if (!smid) {
4516                 pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
4517                     ioc->name, __func__);
4518                 return -EAGAIN;
4519         }
4520         ioc->base_cmds.status = MPT3_CMD_PENDING;
4521         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4522         ioc->base_cmds.smid = smid;
4523         memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t));
4524         mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
4525         mpi_request->VF_ID = 0; /* TODO */
4526         mpi_request->VP_ID = 0;
4527         for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
4528                 mpi_request->EventMasks[i] =
4529                     cpu_to_le32(ioc->event_masks[i]);
4530         init_completion(&ioc->base_cmds.done);
4531         mpt3sas_base_put_smid_default(ioc, smid);
4532         timeleft = wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
4533         if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
4534                 pr_err(MPT3SAS_FMT "%s: timeout\n",
4535                     ioc->name, __func__);
4536                 _debug_dump_mf(mpi_request,
4537                     sizeof(Mpi2EventNotificationRequest_t)/4);
4538                 if (ioc->base_cmds.status & MPT3_CMD_RESET)
4539                         r = -EFAULT;
4540                 else
4541                         r = -ETIME;
4542         } else
4543                 dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s: complete\n",
4544                     ioc->name, __func__));
4545         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
4546         return r;
4547 }
4548
4549 /**
4550  * mpt3sas_base_validate_event_type - validating event types
4551  * @ioc: per adapter object
4552  * @event: firmware event
4553  *
4554  * This will turn on firmware event notification when application
4555  * ask for that event. We don't mask events that are already enabled.
4556  */
4557 void
4558 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
4559 {
4560         int i, j;
4561         u32 event_mask, desired_event;
4562         u8 send_update_to_fw;
4563
4564         for (i = 0, send_update_to_fw = 0; i <
4565             MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) {
4566                 event_mask = ~event_type[i];
4567                 desired_event = 1;
4568                 for (j = 0; j < 32; j++) {
4569                         if (!(event_mask & desired_event) &&
4570                             (ioc->event_masks[i] & desired_event)) {
4571                                 ioc->event_masks[i] &= ~desired_event;
4572                                 send_update_to_fw = 1;
4573                         }
4574                         desired_event = (desired_event << 1);
4575                 }
4576         }
4577
4578         if (!send_update_to_fw)
4579                 return;
4580
4581         mutex_lock(&ioc->base_cmds.mutex);
4582         _base_event_notification(ioc, CAN_SLEEP);
4583         mutex_unlock(&ioc->base_cmds.mutex);
4584 }
4585
4586 /**
4587  * _base_diag_reset - the "big hammer" start of day reset
4588  * @ioc: per adapter object
4589  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4590  *
4591  * Returns 0 for success, non-zero for failure.
4592  */
4593 static int
4594 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
4595 {
4596         u32 host_diagnostic;
4597         u32 ioc_state;
4598         u32 count;
4599         u32 hcb_size;
4600
4601         pr_info(MPT3SAS_FMT "sending diag reset !!\n", ioc->name);
4602
4603         drsprintk(ioc, pr_info(MPT3SAS_FMT "clear interrupts\n",
4604             ioc->name));
4605
4606         count = 0;
4607         do {
4608                 /* Write magic sequence to WriteSequence register
4609                  * Loop until in diagnostic mode
4610                  */
4611                 drsprintk(ioc, pr_info(MPT3SAS_FMT
4612                         "write magic sequence\n", ioc->name));
4613                 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
4614                 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);
4615                 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);
4616                 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);
4617                 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);
4618                 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);
4619                 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
4620
4621                 /* wait 100 msec */
4622                 if (sleep_flag == CAN_SLEEP)
4623                         msleep(100);
4624                 else
4625                         mdelay(100);
4626
4627                 if (count++ > 20)
4628                         goto out;
4629
4630                 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
4631                 drsprintk(ioc, pr_info(MPT3SAS_FMT
4632                         "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",
4633                     ioc->name, count, host_diagnostic));
4634
4635         } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0);
4636
4637         hcb_size = readl(&ioc->chip->HCBSize);
4638
4639         drsprintk(ioc, pr_info(MPT3SAS_FMT "diag reset: issued\n",
4640             ioc->name));
4641         writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER,
4642              &ioc->chip->HostDiagnostic);
4643
4644         /*This delay allows the chip PCIe hardware time to finish reset tasks*/
4645         if (sleep_flag == CAN_SLEEP)
4646                 msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
4647         else
4648                 mdelay(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
4649
4650         /* Approximately 300 second max wait */
4651         for (count = 0; count < (300000000 /
4652                 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) {
4653
4654                 host_diagnostic = readl(&ioc->chip->HostDiagnostic);
4655
4656                 if (host_diagnostic == 0xFFFFFFFF)
4657                         goto out;
4658                 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
4659                         break;
4660
4661                 /* Wait to pass the second read delay window */
4662                 if (sleep_flag == CAN_SLEEP)
4663                         msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC
4664                                                                 / 1000);
4665                 else
4666                         mdelay(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC
4667                                                                 / 1000);
4668         }
4669
4670         if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
4671
4672                 drsprintk(ioc, pr_info(MPT3SAS_FMT
4673                 "restart the adapter assuming the HCB Address points to good F/W\n",
4674                     ioc->name));
4675                 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK;
4676                 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW;
4677                 writel(host_diagnostic, &ioc->chip->HostDiagnostic);
4678
4679                 drsprintk(ioc, pr_info(MPT3SAS_FMT
4680                     "re-enable the HCDW\n", ioc->name));
4681                 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE,
4682                     &ioc->chip->HCBSize);
4683         }
4684
4685         drsprintk(ioc, pr_info(MPT3SAS_FMT "restart the adapter\n",
4686             ioc->name));
4687         writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET,
4688             &ioc->chip->HostDiagnostic);
4689
4690         drsprintk(ioc, pr_info(MPT3SAS_FMT
4691                 "disable writes to the diagnostic register\n", ioc->name));
4692         writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
4693
4694         drsprintk(ioc, pr_info(MPT3SAS_FMT
4695                 "Wait for FW to go to the READY state\n", ioc->name));
4696         ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20,
4697             sleep_flag);
4698         if (ioc_state) {
4699                 pr_err(MPT3SAS_FMT
4700                         "%s: failed going to ready state (ioc_state=0x%x)\n",
4701                         ioc->name, __func__, ioc_state);
4702                 goto out;
4703         }
4704
4705         pr_info(MPT3SAS_FMT "diag reset: SUCCESS\n", ioc->name);
4706         return 0;
4707
4708  out:
4709         pr_err(MPT3SAS_FMT "diag reset: FAILED\n", ioc->name);
4710         return -EFAULT;
4711 }
4712
4713 /**
4714  * _base_make_ioc_ready - put controller in READY state
4715  * @ioc: per adapter object
4716  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4717  * @type: FORCE_BIG_HAMMER or SOFT_RESET
4718  *
4719  * Returns 0 for success, non-zero for failure.
4720  */
4721 static int
4722 _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
4723         enum reset_type type)
4724 {
4725         u32 ioc_state;
4726         int rc;
4727         int count;
4728
4729         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4730             __func__));
4731
4732         if (ioc->pci_error_recovery)
4733                 return 0;
4734
4735         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
4736         dhsprintk(ioc, pr_info(MPT3SAS_FMT "%s: ioc_state(0x%08x)\n",
4737             ioc->name, __func__, ioc_state));
4738
4739         /* if in RESET state, it should move to READY state shortly */
4740         count = 0;
4741         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
4742                 while ((ioc_state & MPI2_IOC_STATE_MASK) !=
4743                     MPI2_IOC_STATE_READY) {
4744                         if (count++ == 10) {
4745                                 pr_err(MPT3SAS_FMT
4746                                         "%s: failed going to ready state (ioc_state=0x%x)\n",
4747                                     ioc->name, __func__, ioc_state);
4748                                 return -EFAULT;
4749                         }
4750                         if (sleep_flag == CAN_SLEEP)
4751                                 ssleep(1);
4752                         else
4753                                 mdelay(1000);
4754                         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
4755                 }
4756         }
4757
4758         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY)
4759                 return 0;
4760
4761         if (ioc_state & MPI2_DOORBELL_USED) {
4762                 dhsprintk(ioc, pr_info(MPT3SAS_FMT
4763                         "unexpected doorbell active!\n",
4764                         ioc->name));
4765                 goto issue_diag_reset;
4766         }
4767
4768         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
4769                 mpt3sas_base_fault_info(ioc, ioc_state &
4770                     MPI2_DOORBELL_DATA_MASK);
4771                 goto issue_diag_reset;
4772         }
4773
4774         if (type == FORCE_BIG_HAMMER)
4775                 goto issue_diag_reset;
4776
4777         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
4778                 if (!(_base_send_ioc_reset(ioc,
4779                     MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15, CAN_SLEEP))) {
4780                         return 0;
4781         }
4782
4783  issue_diag_reset:
4784         rc = _base_diag_reset(ioc, CAN_SLEEP);
4785         return rc;
4786 }
4787
4788 /**
4789  * _base_make_ioc_operational - put controller in OPERATIONAL state
4790  * @ioc: per adapter object
4791  * @sleep_flag: CAN_SLEEP or NO_SLEEP
4792  *
4793  * Returns 0 for success, non-zero for failure.
4794  */
4795 static int
4796 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
4797 {
4798         int r, i;
4799         unsigned long   flags;
4800         u32 reply_address;
4801         u16 smid;
4802         struct _tr_list *delayed_tr, *delayed_tr_next;
4803         struct adapter_reply_queue *reply_q;
4804         long reply_post_free;
4805         u32 reply_post_free_sz, index = 0;
4806
4807         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4808             __func__));
4809
4810         /* clean the delayed target reset list */
4811         list_for_each_entry_safe(delayed_tr, delayed_tr_next,
4812             &ioc->delayed_tr_list, list) {
4813                 list_del(&delayed_tr->list);
4814                 kfree(delayed_tr);
4815         }
4816
4817
4818         list_for_each_entry_safe(delayed_tr, delayed_tr_next,
4819             &ioc->delayed_tr_volume_list, list) {
4820                 list_del(&delayed_tr->list);
4821                 kfree(delayed_tr);
4822         }
4823
4824         /* initialize the scsi lookup free list */
4825         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4826         INIT_LIST_HEAD(&ioc->free_list);
4827         smid = 1;
4828         for (i = 0; i < ioc->scsiio_depth; i++, smid++) {
4829                 INIT_LIST_HEAD(&ioc->scsi_lookup[i].chain_list);
4830                 ioc->scsi_lookup[i].cb_idx = 0xFF;
4831                 ioc->scsi_lookup[i].smid = smid;
4832                 ioc->scsi_lookup[i].scmd = NULL;
4833                 list_add_tail(&ioc->scsi_lookup[i].tracker_list,
4834                     &ioc->free_list);
4835         }
4836
4837         /* hi-priority queue */
4838         INIT_LIST_HEAD(&ioc->hpr_free_list);
4839         smid = ioc->hi_priority_smid;
4840         for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {
4841                 ioc->hpr_lookup[i].cb_idx = 0xFF;
4842                 ioc->hpr_lookup[i].smid = smid;
4843                 list_add_tail(&ioc->hpr_lookup[i].tracker_list,
4844                     &ioc->hpr_free_list);
4845         }
4846
4847         /* internal queue */
4848         INIT_LIST_HEAD(&ioc->internal_free_list);
4849         smid = ioc->internal_smid;
4850         for (i = 0; i < ioc->internal_depth; i++, smid++) {
4851                 ioc->internal_lookup[i].cb_idx = 0xFF;
4852                 ioc->internal_lookup[i].smid = smid;
4853                 list_add_tail(&ioc->internal_lookup[i].tracker_list,
4854                     &ioc->internal_free_list);
4855         }
4856
4857         /* chain pool */
4858         INIT_LIST_HEAD(&ioc->free_chain_list);
4859         for (i = 0; i < ioc->chain_depth; i++)
4860                 list_add_tail(&ioc->chain_lookup[i].tracker_list,
4861                     &ioc->free_chain_list);
4862
4863         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
4864
4865         /* initialize Reply Free Queue */
4866         for (i = 0, reply_address = (u32)ioc->reply_dma ;
4867             i < ioc->reply_free_queue_depth ; i++, reply_address +=
4868             ioc->reply_sz)
4869                 ioc->reply_free[i] = cpu_to_le32(reply_address);
4870
4871         /* initialize reply queues */
4872         if (ioc->is_driver_loading)
4873                 _base_assign_reply_queues(ioc);
4874
4875         /* initialize Reply Post Free Queue */
4876         reply_post_free_sz = ioc->reply_post_queue_depth *
4877             sizeof(Mpi2DefaultReplyDescriptor_t);
4878         reply_post_free = (long)ioc->reply_post[index].reply_post_free;
4879         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
4880                 reply_q->reply_post_host_index = 0;
4881                 reply_q->reply_post_free = (Mpi2ReplyDescriptorsUnion_t *)
4882                     reply_post_free;
4883                 for (i = 0; i < ioc->reply_post_queue_depth; i++)
4884                         reply_q->reply_post_free[i].Words =
4885                             cpu_to_le64(ULLONG_MAX);
4886                 if (!_base_is_controller_msix_enabled(ioc))
4887                         goto skip_init_reply_post_free_queue;
4888                 /*
4889                  * If RDPQ is enabled, switch to the next allocation.
4890                  * Otherwise advance within the contiguous region.
4891                  */
4892                 if (ioc->rdpq_array_enable)
4893                         reply_post_free = (long)
4894                             ioc->reply_post[++index].reply_post_free;
4895                 else
4896                         reply_post_free += reply_post_free_sz;
4897         }
4898  skip_init_reply_post_free_queue:
4899
4900         r = _base_send_ioc_init(ioc, sleep_flag);
4901         if (r)
4902                 return r;
4903
4904         /* initialize reply free host index */
4905         ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;
4906         writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);
4907
4908         /* initialize reply post host index */
4909         list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
4910                 if (ioc->msix96_vector)
4911                         writel((reply_q->msix_index & 7)<<
4912                            MPI2_RPHI_MSIX_INDEX_SHIFT,
4913                            ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
4914                 else
4915                         writel(reply_q->msix_index <<
4916                                 MPI2_RPHI_MSIX_INDEX_SHIFT,
4917                                 &ioc->chip->ReplyPostHostIndex);
4918
4919                 if (!_base_is_controller_msix_enabled(ioc))
4920                         goto skip_init_reply_post_host_index;
4921         }
4922
4923  skip_init_reply_post_host_index:
4924
4925         _base_unmask_interrupts(ioc);
4926         r = _base_event_notification(ioc, sleep_flag);
4927         if (r)
4928                 return r;
4929
4930         if (sleep_flag == CAN_SLEEP)
4931                 _base_static_config_pages(ioc);
4932
4933
4934         if (ioc->is_driver_loading) {
4935                 ioc->wait_for_discovery_to_complete =
4936                     _base_determine_wait_on_discovery(ioc);
4937
4938                 return r; /* scan_start and scan_finished support */
4939         }
4940
4941         r = _base_send_port_enable(ioc, sleep_flag);
4942         if (r)
4943                 return r;
4944
4945         return r;
4946 }
4947
4948 /**
4949  * mpt3sas_base_free_resources - free resources controller resources
4950  * @ioc: per adapter object
4951  *
4952  * Return nothing.
4953  */
4954 void
4955 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
4956 {
4957         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4958             __func__));
4959
4960         if (ioc->chip_phys && ioc->chip) {
4961                 _base_mask_interrupts(ioc);
4962                 ioc->shost_recovery = 1;
4963                 _base_make_ioc_ready(ioc, CAN_SLEEP, SOFT_RESET);
4964                 ioc->shost_recovery = 0;
4965         }
4966
4967         mpt3sas_base_unmap_resources(ioc);
4968         return;
4969 }
4970
4971 /**
4972  * mpt3sas_base_attach - attach controller instance
4973  * @ioc: per adapter object
4974  *
4975  * Returns 0 for success, non-zero for failure.
4976  */
4977 int
4978 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
4979 {
4980         int r, i;
4981         int cpu_id, last_cpu_id = 0;
4982         u8 revision;
4983
4984         dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
4985             __func__));
4986
4987         /* setup cpu_msix_table */
4988         ioc->cpu_count = num_online_cpus();
4989         for_each_online_cpu(cpu_id)
4990                 last_cpu_id = cpu_id;
4991         ioc->cpu_msix_table_sz = last_cpu_id + 1;
4992         ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);
4993         ioc->reply_queue_count = 1;
4994         if (!ioc->cpu_msix_table) {
4995                 dfailprintk(ioc, pr_info(MPT3SAS_FMT
4996                         "allocation for cpu_msix_table failed!!!\n",
4997                         ioc->name));
4998                 r = -ENOMEM;
4999                 goto out_free_resources;
5000         }
5001
5002         /* Check whether the controller revision is C0 or above.
5003          * only C0 and above revision controllers support 96 MSI-X vectors.
5004          */
5005         revision = ioc->pdev->revision;
5006
5007         if ((ioc->pdev->device == MPI25_MFGPAGE_DEVID_SAS3004 ||
5008              ioc->pdev->device == MPI25_MFGPAGE_DEVID_SAS3008 ||
5009              ioc->pdev->device == MPI25_MFGPAGE_DEVID_SAS3108_1 ||
5010              ioc->pdev->device == MPI25_MFGPAGE_DEVID_SAS3108_2 ||
5011              ioc->pdev->device == MPI25_MFGPAGE_DEVID_SAS3108_5 ||
5012              ioc->pdev->device == MPI25_MFGPAGE_DEVID_SAS3108_6) &&
5013              (revision >= 0x02))
5014                 ioc->msix96_vector = 1;
5015
5016         ioc->rdpq_array_enable_assigned = 0;
5017         ioc->dma_mask = 0;
5018         r = mpt3sas_base_map_resources(ioc);
5019         if (r)
5020                 goto out_free_resources;
5021
5022
5023         pci_set_drvdata(ioc->pdev, ioc->shost);
5024         r = _base_get_ioc_facts(ioc, CAN_SLEEP);
5025         if (r)
5026                 goto out_free_resources;
5027
5028         switch (ioc->hba_mpi_version_belonged) {
5029         case MPI2_VERSION:
5030                 ioc->build_sg_scmd = &_base_build_sg_scmd;
5031                 ioc->build_sg = &_base_build_sg;
5032                 ioc->build_zero_len_sge = &_base_build_zero_len_sge;
5033                 break;
5034         case MPI25_VERSION:
5035                 /*
5036                  * In SAS3.0,
5037                  * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and
5038                  * Target Status - all require the IEEE formated scatter gather
5039                  * elements.
5040                  */
5041                 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;
5042                 ioc->build_sg = &_base_build_sg_ieee;
5043                 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
5044                 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
5045                 break;
5046         }
5047
5048         /*
5049          * These function pointers for other requests that don't
5050          * the require IEEE scatter gather elements.
5051          *
5052          * For example Configuration Pages and SAS IOUNIT Control don't.
5053          */
5054         ioc->build_sg_mpi = &_base_build_sg;
5055         ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
5056
5057         r = _base_make_ioc_ready(ioc, CAN_SLEEP, SOFT_RESET);
5058         if (r)
5059                 goto out_free_resources;
5060
5061         ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,
5062             sizeof(struct mpt3sas_port_facts), GFP_KERNEL);
5063         if (!ioc->pfacts) {
5064                 r = -ENOMEM;
5065                 goto out_free_resources;
5066         }
5067
5068         for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
5069                 r = _base_get_port_facts(ioc, i, CAN_SLEEP);
5070                 if (r)
5071                         goto out_free_resources;
5072         }
5073
5074         r = _base_allocate_memory_pools(ioc, CAN_SLEEP);
5075         if (r)
5076                 goto out_free_resources;
5077
5078         init_waitqueue_head(&ioc->reset_wq);
5079
5080         /* allocate memory pd handle bitmask list */
5081         ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
5082         if (ioc->facts.MaxDevHandle % 8)
5083                 ioc->pd_handles_sz++;
5084         ioc->pd_handles = kzalloc(ioc->pd_handles_sz,
5085             GFP_KERNEL);
5086         if (!ioc->pd_handles) {
5087                 r = -ENOMEM;
5088                 goto out_free_resources;
5089         }
5090         ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,
5091             GFP_KERNEL);
5092         if (!ioc->blocking_handles) {
5093                 r = -ENOMEM;
5094                 goto out_free_resources;
5095         }
5096
5097         ioc->fwfault_debug = mpt3sas_fwfault_debug;
5098
5099         /* base internal command bits */
5100         mutex_init(&ioc->base_cmds.mutex);
5101         ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5102         ioc->base_cmds.status = MPT3_CMD_NOT_USED;
5103
5104         /* port_enable command bits */
5105         ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5106         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
5107
5108         /* transport internal command bits */
5109         ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5110         ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
5111         mutex_init(&ioc->transport_cmds.mutex);
5112
5113         /* scsih internal command bits */
5114         ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5115         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
5116         mutex_init(&ioc->scsih_cmds.mutex);
5117
5118         /* task management internal command bits */
5119         ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5120         ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
5121         mutex_init(&ioc->tm_cmds.mutex);
5122
5123         /* config page internal command bits */
5124         ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5125         ioc->config_cmds.status = MPT3_CMD_NOT_USED;
5126         mutex_init(&ioc->config_cmds.mutex);
5127
5128         /* ctl module internal command bits */
5129         ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
5130         ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
5131         ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
5132         mutex_init(&ioc->ctl_cmds.mutex);
5133
5134         if (!ioc->base_cmds.reply || !ioc->transport_cmds.reply ||
5135             !ioc->scsih_cmds.reply || !ioc->tm_cmds.reply ||
5136             !ioc->config_cmds.reply || !ioc->ctl_cmds.reply ||
5137             !ioc->ctl_cmds.sense) {
5138                 r = -ENOMEM;
5139                 goto out_free_resources;
5140         }
5141
5142         for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
5143                 ioc->event_masks[i] = -1;
5144
5145         /* here we enable the events we care about */
5146         _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);
5147         _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
5148         _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
5149         _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
5150         _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
5151         _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
5152         _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);
5153         _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);
5154         _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);
5155         _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
5156         _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
5157
5158         r = _base_make_ioc_operational(ioc, CAN_SLEEP);
5159         if (r)
5160                 goto out_free_resources;
5161
5162         return 0;
5163
5164  out_free_resources:
5165
5166         ioc->remove_host = 1;
5167
5168         mpt3sas_base_free_resources(ioc);
5169         _base_release_memory_pools(ioc);
5170         pci_set_drvdata(ioc->pdev, NULL);
5171         kfree(ioc->cpu_msix_table);
5172         kfree(ioc->pd_handles);
5173         kfree(ioc->blocking_handles);
5174         kfree(ioc->tm_cmds.reply);
5175         kfree(ioc->transport_cmds.reply);
5176         kfree(ioc->scsih_cmds.reply);
5177         kfree(ioc->config_cmds.reply);
5178         kfree(ioc->base_cmds.reply);
5179         kfree(ioc->port_enable_cmds.reply);
5180         kfree(ioc->ctl_cmds.reply);
5181         kfree(ioc->ctl_cmds.sense);
5182         kfree(ioc->pfacts);
5183         ioc->ctl_cmds.reply = NULL;
5184         ioc->base_cmds.reply = NULL;
5185         ioc->tm_cmds.reply = NULL;
5186         ioc->scsih_cmds.reply = NULL;
5187         ioc->transport_cmds.reply = NULL;
5188         ioc->config_cmds.reply = NULL;
5189         ioc->pfacts = NULL;
5190         return r;
5191 }
5192
5193
5194 /**
5195  * mpt3sas_base_detach - remove controller instance
5196  * @ioc: per adapter object
5197  *
5198  * Return nothing.
5199  */
5200 void
5201 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
5202 {
5203         dexitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
5204             __func__));
5205
5206         mpt3sas_base_stop_watchdog(ioc);
5207         mpt3sas_base_free_resources(ioc);
5208         _base_release_memory_pools(ioc);
5209         pci_set_drvdata(ioc->pdev, NULL);
5210         kfree(ioc->cpu_msix_table);
5211         kfree(ioc->pd_handles);
5212         kfree(ioc->blocking_handles);
5213         kfree(ioc->pfacts);
5214         kfree(ioc->ctl_cmds.reply);
5215         kfree(ioc->ctl_cmds.sense);
5216         kfree(ioc->base_cmds.reply);
5217         kfree(ioc->port_enable_cmds.reply);
5218         kfree(ioc->tm_cmds.reply);
5219         kfree(ioc->transport_cmds.reply);
5220         kfree(ioc->scsih_cmds.reply);
5221         kfree(ioc->config_cmds.reply);
5222 }
5223
5224 /**
5225  * _base_reset_handler - reset callback handler (for base)
5226  * @ioc: per adapter object
5227  * @reset_phase: phase
5228  *
5229  * The handler for doing any required cleanup or initialization.
5230  *
5231  * The reset phase can be MPT3_IOC_PRE_RESET, MPT3_IOC_AFTER_RESET,
5232  * MPT3_IOC_DONE_RESET
5233  *
5234  * Return nothing.
5235  */
5236 static void
5237 _base_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
5238 {
5239         mpt3sas_scsih_reset_handler(ioc, reset_phase);
5240         mpt3sas_ctl_reset_handler(ioc, reset_phase);
5241         switch (reset_phase) {
5242         case MPT3_IOC_PRE_RESET:
5243                 dtmprintk(ioc, pr_info(MPT3SAS_FMT
5244                 "%s: MPT3_IOC_PRE_RESET\n", ioc->name, __func__));
5245                 break;
5246         case MPT3_IOC_AFTER_RESET:
5247                 dtmprintk(ioc, pr_info(MPT3SAS_FMT
5248                 "%s: MPT3_IOC_AFTER_RESET\n", ioc->name, __func__));
5249                 if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {
5250                         ioc->transport_cmds.status |= MPT3_CMD_RESET;
5251                         mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);
5252                         complete(&ioc->transport_cmds.done);
5253                 }
5254                 if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
5255                         ioc->base_cmds.status |= MPT3_CMD_RESET;
5256                         mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);
5257                         complete(&ioc->base_cmds.done);
5258                 }
5259                 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
5260                         ioc->port_enable_failed = 1;
5261                         ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
5262                         mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);
5263                         if (ioc->is_driver_loading) {
5264                                 ioc->start_scan_failed =
5265                                     MPI2_IOCSTATUS_INTERNAL_ERROR;
5266                                 ioc->start_scan = 0;
5267                                 ioc->port_enable_cmds.status =
5268                                     MPT3_CMD_NOT_USED;
5269                         } else
5270                                 complete(&ioc->port_enable_cmds.done);
5271                 }
5272                 if (ioc->config_cmds.status & MPT3_CMD_PENDING) {
5273                         ioc->config_cmds.status |= MPT3_CMD_RESET;
5274                         mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);
5275                         ioc->config_cmds.smid = USHRT_MAX;
5276                         complete(&ioc->config_cmds.done);
5277                 }
5278                 break;
5279         case MPT3_IOC_DONE_RESET:
5280                 dtmprintk(ioc, pr_info(MPT3SAS_FMT
5281                         "%s: MPT3_IOC_DONE_RESET\n", ioc->name, __func__));
5282                 break;
5283         }
5284 }
5285
5286 /**
5287  * _wait_for_commands_to_complete - reset controller
5288  * @ioc: Pointer to MPT_ADAPTER structure
5289  * @sleep_flag: CAN_SLEEP or NO_SLEEP
5290  *
5291  * This function waiting(3s) for all pending commands to complete
5292  * prior to putting controller in reset.
5293  */
5294 static void
5295 _wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
5296 {
5297         u32 ioc_state;
5298         unsigned long flags;
5299         u16 i;
5300
5301         ioc->pending_io_count = 0;
5302         if (sleep_flag != CAN_SLEEP)
5303                 return;
5304
5305         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
5306         if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
5307                 return;
5308
5309         /* pending command count */
5310         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
5311         for (i = 0; i < ioc->scsiio_depth; i++)
5312                 if (ioc->scsi_lookup[i].cb_idx != 0xFF)
5313                         ioc->pending_io_count++;
5314         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
5315
5316         if (!ioc->pending_io_count)
5317                 return;
5318
5319         /* wait for pending commands to complete */
5320         wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);
5321 }
5322
5323 /**
5324  * mpt3sas_base_hard_reset_handler - reset controller
5325  * @ioc: Pointer to MPT_ADAPTER structure
5326  * @sleep_flag: CAN_SLEEP or NO_SLEEP
5327  * @type: FORCE_BIG_HAMMER or SOFT_RESET
5328  *
5329  * Returns 0 for success, non-zero for failure.
5330  */
5331 int
5332 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
5333         enum reset_type type)
5334 {
5335         int r;
5336         unsigned long flags;
5337         u32 ioc_state;
5338         u8 is_fault = 0, is_trigger = 0;
5339
5340         dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: enter\n", ioc->name,
5341             __func__));
5342
5343         if (ioc->pci_error_recovery) {
5344                 pr_err(MPT3SAS_FMT "%s: pci error recovery reset\n",
5345                     ioc->name, __func__);
5346                 r = 0;
5347                 goto out_unlocked;
5348         }
5349
5350         if (mpt3sas_fwfault_debug)
5351                 mpt3sas_halt_firmware(ioc);
5352
5353         /* TODO - What we really should be doing is pulling
5354          * out all the code associated with NO_SLEEP; its never used.
5355          * That is legacy code from mpt fusion driver, ported over.
5356          * I will leave this BUG_ON here for now till its been resolved.
5357          */
5358         BUG_ON(sleep_flag == NO_SLEEP);
5359
5360         /* wait for an active reset in progress to complete */
5361         if (!mutex_trylock(&ioc->reset_in_progress_mutex)) {
5362                 do {
5363                         ssleep(1);
5364                 } while (ioc->shost_recovery == 1);
5365                 dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
5366                     __func__));
5367                 return ioc->ioc_reset_in_progress_status;
5368         }
5369
5370         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
5371         ioc->shost_recovery = 1;
5372         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
5373
5374         if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
5375             MPT3_DIAG_BUFFER_IS_REGISTERED) &&
5376             (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
5377             MPT3_DIAG_BUFFER_IS_RELEASED))) {
5378                 is_trigger = 1;
5379                 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
5380                 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT)
5381                         is_fault = 1;
5382         }
5383         _base_reset_handler(ioc, MPT3_IOC_PRE_RESET);
5384         _wait_for_commands_to_complete(ioc, sleep_flag);
5385         _base_mask_interrupts(ioc);
5386         r = _base_make_ioc_ready(ioc, sleep_flag, type);
5387         if (r)
5388                 goto out;
5389         _base_reset_handler(ioc, MPT3_IOC_AFTER_RESET);
5390
5391         /* If this hard reset is called while port enable is active, then
5392          * there is no reason to call make_ioc_operational
5393          */
5394         if (ioc->is_driver_loading && ioc->port_enable_failed) {
5395                 ioc->remove_host = 1;
5396                 r = -EFAULT;
5397                 goto out;
5398         }
5399         r = _base_get_ioc_facts(ioc, CAN_SLEEP);
5400         if (r)
5401                 goto out;
5402
5403         if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)
5404                 panic("%s: Issue occurred with flashing controller firmware."
5405                       "Please reboot the system and ensure that the correct"
5406                       " firmware version is running\n", ioc->name);
5407
5408         r = _base_make_ioc_operational(ioc, sleep_flag);
5409         if (!r)
5410                 _base_reset_handler(ioc, MPT3_IOC_DONE_RESET);
5411
5412  out:
5413         dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: %s\n",
5414             ioc->name, __func__, ((r == 0) ? "SUCCESS" : "FAILED")));
5415
5416         spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
5417         ioc->ioc_reset_in_progress_status = r;
5418         ioc->shost_recovery = 0;
5419         spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
5420         ioc->ioc_reset_count++;
5421         mutex_unlock(&ioc->reset_in_progress_mutex);
5422
5423  out_unlocked:
5424         if ((r == 0) && is_trigger) {
5425                 if (is_fault)
5426                         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);
5427                 else
5428                         mpt3sas_trigger_master(ioc,
5429                             MASTER_TRIGGER_ADAPTER_RESET);
5430         }
5431         dtmprintk(ioc, pr_info(MPT3SAS_FMT "%s: exit\n", ioc->name,
5432             __func__));
5433         return r;
5434 }